def game_result_receiver():
"""
This task register callback function on getting user results.
:return:
"""
rabbit = get_rabbit()
init_rabbit(rabbit)
rabbit_channel = rabbit.channel()
rabbit_channel.basic_qos(prefetch_count=1)
rabbit_channel.basic_consume(
game_result_receiver_callback,
queue="game.result"
)
rabbit_channel.start_consuming()
def game_spider_checker():
"""
This function checks if a user should be crawled.
:return:
"""
global game_spider_checker_pub_channel
rabbit = get_rabbit()
init_rabbit(rabbit)
game_spider_checker_pub_channel = rabbit.channel()
rabbit_channel = rabbit.channel()
rabbit_channel.basic_qos(prefetch_count=1)
rabbit_channel.basic_consume(
game_spider_checker_callback,
queue="game.order.pre"
)
rabbit_channel.start_consuming()
def review_spider_issuer():
"""
This task issue orders to the review spider to crawl reviews.
:return:
"""
while True:
dolphin = get_dolphin()
rabbit = get_rabbit()
init_rabbit(rabbit)
query_sql = "SELECT `appid`, `crawled_at` FROM `apps` WHERE `crawled_at` < %s AND `is_concerned` = 1"
update_sql = "UPDATE `apps` SET `crawled_at` = %s WHERE `appid` = %s"
issue_interval = args.review_interval
this_time = int(time.time())
time_since = this_time - issue_interval
review_language = args.review_language
try:
with dolphin.cursor() as cursor:
# Query apps haven't be crawled in some time.
cursor.execute(query_sql, (time_since,))
results = cursor.fetchall()
with rabbit.channel() as channel:
# Publish review crawl orders.
for app in results:
app["language"] = review_language
app["last_crawled"] = app["crawled_at"]
channel.basic_publish(
exchange="ubi",
routing_key="review.order",
body=json.dumps(app),
properties=pika.BasicProperties(delivery_mode=2, )
)
logging.info(
"Issued a review order of app [%d] since [%d] in [%s]",
app["appid"],
app["last_crawled"],
app["language"]
)
with dolphin.cursor() as cursor:
# Update "last_crawled".
cursor.execute(update_sql, (this_time, app["appid"]))
dolphin.commit()
except Exception as e:
logging.error("An error occurred while issuing review orders: %s", e)
finally:
dolphin.close()
# Let's sleep.
time.sleep(issue_interval // 2)
def main():
global rabbit
# Initialise logging.
init_logging("game")
# Parse arguments.
args = parse_arguments()
# Establish MQ connection.
rabbit_params = pika.URLParameters(args.mq_url)
rabbit_params.heartbeat = 0
rabbit = pika.BlockingConnection(rabbit_params)
# Declare exchange & queues.
init_rabbit(rabbit)
# Set up subscription and start consuming.
rabbit_channel = rabbit.channel()
rabbit_channel.basic_qos(prefetch_count=1)
rabbit_channel.basic_consume(order_callback, queue="game.order")
rabbit_channel.start_consuming()