def run(): redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT) amqp_client = AMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL, SCRAPE_NEWS_TASK_QUEUE_NAME) amqp_client.connect() try: while True: news_list = news_client.get_news_from_sources(NEWS_SOURCES) num_news = 0 for news in news_list: digest = hashlib.md5(news['title'].encode('utf-8')).hexdigest() if redis_client.get(digest): continue num_news += 1 news['digest'] = digest redis_client.set(digest, True) redis_client.expire(digest, NEWS_TIME_OUT_IN_SECONDS) print(news) amqp_client.send_message(news) print('News Monitor: fectched {} news'.format(num_news)) amqp_client.sleep(SLEEP_TIME_IN_SECONDS) except KeyboardInterrupt: print('keyboard interrupt') # except SigTerm finally: amqp_client.close()
def run(redis_host=REDIS_HOST, redis_port=REDIS_PORT, scrape_queue_url=SCRAPE_NEWS_TASK_QUEUE_URL, scrape_queue_name=SCRAPE_NEWS_TASK_QUEUE_NAME): redis_client = redis.StrictRedis(redis_host, redis_port) amqp_client = AMQPClient(scrape_queue_url, scrape_queue_name) amqp_client.connect() while True: logger.debug('News monitor: iter..') news_list = news_client.get_news_from_sources(NEWS_SOURCES) num_news = 0 for news in news_list: digest = hashlib.md5(news['title'].encode('utf-8')).hexdigest() if redis_client.get(digest): continue num_news += 1 news['digest'] = digest redis_client.set(digest, True) redis_client.expire(digest, NEWS_TIME_OUT_IN_SECONDS) logger.debug('News Monitor: got news {}'.format(news)) amqp_client.send_message(news) logger.info('News Monitor: fectched {} news'.format(num_news)) amqp_client.sleep(SLEEP_TIME_IN_SECONDS)
def run(scrape_queue_url=SCRAPE_QUEUE_URL, scrape_queue_name=SCRAPE_NEWS_TASK_QUEUE_NAME, dedupe_queue_url=DEDUPE_QUEUE_URL, dedupe_queue_name=DEDUPE_NEWS_TASK_QUEUE_NAME, times=-1): scrape_queue_client = AMQPClient(scrape_queue_url, scrape_queue_name) scrape_queue_client.connect() dedupe_queue_client = AMQPClient(dedupe_queue_url, dedupe_queue_name) dedupe_queue_client.connect() assert scrape_queue_client.is_connected() assert dedupe_queue_client.is_connected() while True: logger.debug('News fetcher: iter..') msg = scrape_queue_client.get_message() if msg is not None: try: handle_message(msg) dedupe_queue_client.send_message(msg) logger.info('News Fetcher: message sent to dedupe queue (url: {})' .format(msg.get('url'))) except Exception as e: logger.warning('News fetcher: handling error: {}'.format(e)) # if decreas count here, weird behavior, decreasing happens before processing message scrape_queue_client.sleep(SLEEP_TIME_IN_SECONDS) if times > 0: times -= 1 if times == 0: break
def run(times=-1): click_queue_client = AMQPClient(USER_CLICK_QUEUE_URL, USER_CLICK_QUEUE_NAME) click_queue_client.connect() assert click_queue_client.is_connected() print('Click Handler: my queue name: {}'.format(click_queue_client)) while True: message = click_queue_client.get_message() try: handle_message(message) except Exception as e: raise e click_queue_client.sleep(SLEEP_TIME_IN_SECONDS) if times > 0: times -= 1 if times == 0: break
def run(times=-1): click_queue_client = AMQPClient(USER_CLICK_QUEUE_URL, USER_CLICK_QUEUE_NAME) click_queue_client.connect() news_collection = mongodb_client.get_news_collection() pref_collection = (mongodb_client.get_db(PREF_DB_NAME).get_collection( PREF_COLLECTION_NAME)) assert click_queue_client.is_connected() while True: msg = click_queue_client.get_message() try: handle_message(msg, news_collection, pref_collection) except Exception as e: raise e click_queue_client.sleep(SLEEP_TIME_IN_SECONDS) if times > 0: times -= 1 if times == 0: break