def run():
redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT)
amqp_client = AMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL,
SCRAPE_NEWS_TASK_QUEUE_NAME)
amqp_client.connect()
try:
while True:
news_list = news_client.get_news_from_sources(NEWS_SOURCES)
num_news = 0
for news in news_list:
digest = hashlib.md5(news['title'].encode('utf-8')).hexdigest()
if redis_client.get(digest):
continue
num_news += 1
news['digest'] = digest
redis_client.set(digest, True)
redis_client.expire(digest, NEWS_TIME_OUT_IN_SECONDS)
print(news)
amqp_client.send_message(news)
print('News Monitor: fectched {} news'.format(num_news))
amqp_client.sleep(SLEEP_TIME_IN_SECONDS)
except KeyboardInterrupt:
print('keyboard interrupt')
# except SigTerm
finally:
amqp_client.close()
def run(redis_host=REDIS_HOST,
redis_port=REDIS_PORT,
scrape_queue_url=SCRAPE_NEWS_TASK_QUEUE_URL,
scrape_queue_name=SCRAPE_NEWS_TASK_QUEUE_NAME):
redis_client = redis.StrictRedis(redis_host, redis_port)
amqp_client = AMQPClient(scrape_queue_url, scrape_queue_name)
amqp_client.connect()
while True:
logger.debug('News monitor: iter..')
news_list = news_client.get_news_from_sources(NEWS_SOURCES)
num_news = 0
for news in news_list:
digest = hashlib.md5(news['title'].encode('utf-8')).hexdigest()
if redis_client.get(digest):
continue
num_news += 1
news['digest'] = digest
redis_client.set(digest, True)
redis_client.expire(digest, NEWS_TIME_OUT_IN_SECONDS)
logger.debug('News Monitor: got news {}'.format(news))
amqp_client.send_message(news)
logger.info('News Monitor: fectched {} news'.format(num_news))
amqp_client.sleep(SLEEP_TIME_IN_SECONDS)
def run(scrape_queue_url=SCRAPE_QUEUE_URL, scrape_queue_name=SCRAPE_NEWS_TASK_QUEUE_NAME,
dedupe_queue_url=DEDUPE_QUEUE_URL, dedupe_queue_name=DEDUPE_NEWS_TASK_QUEUE_NAME,
times=-1):
scrape_queue_client = AMQPClient(scrape_queue_url, scrape_queue_name)
scrape_queue_client.connect()
dedupe_queue_client = AMQPClient(dedupe_queue_url, dedupe_queue_name)
dedupe_queue_client.connect()
assert scrape_queue_client.is_connected()
assert dedupe_queue_client.is_connected()
while True:
logger.debug('News fetcher: iter..')
msg = scrape_queue_client.get_message()
if msg is not None:
try:
handle_message(msg)
dedupe_queue_client.send_message(msg)
logger.info('News Fetcher: message sent to dedupe queue (url: {})'
.format(msg.get('url')))
except Exception as e:
logger.warning('News fetcher: handling error: {}'.format(e))
# if decreas count here, weird behavior, decreasing happens before processing message
scrape_queue_client.sleep(SLEEP_TIME_IN_SECONDS)
if times > 0: times -= 1
if times == 0: break
def run(times=-1):
click_queue_client = AMQPClient(USER_CLICK_QUEUE_URL,
USER_CLICK_QUEUE_NAME)
click_queue_client.connect()
assert click_queue_client.is_connected()
print('Click Handler: my queue name: {}'.format(click_queue_client))
while True:
message = click_queue_client.get_message()
try:
handle_message(message)
except Exception as e:
raise e
click_queue_client.sleep(SLEEP_TIME_IN_SECONDS)
if times > 0: times -= 1
if times == 0: break
def run(times=-1):
click_queue_client = AMQPClient(USER_CLICK_QUEUE_URL,
USER_CLICK_QUEUE_NAME)
click_queue_client.connect()
news_collection = mongodb_client.get_news_collection()
pref_collection = (mongodb_client.get_db(PREF_DB_NAME).get_collection(
PREF_COLLECTION_NAME))
assert click_queue_client.is_connected()
while True:
msg = click_queue_client.get_message()
try:
handle_message(msg, news_collection, pref_collection)
except Exception as e:
raise e
click_queue_client.sleep(SLEEP_TIME_IN_SECONDS)
if times > 0: times -= 1
if times == 0: break