def test_basic(): client = CloudAMQPClient(TEST_CLOUDAMQP_URL, TEST_QUEUE_NAME) sentMsg = {'test': 'test'} client.send_message(sentMsg) receivedMsg = client.get_message() assert sentMsg == receivedMsg print('test_basic passed.')
def test_basic(): """"Test method""" client = CloudAMQPClient(CLOUDAMQP_URL, QUEUE_NAME) sent_message = {'test': 'test'} client.send_message(sent_message) received_message = client.get_message() assert sent_message == received_message print('test_basic passed.')
def test_basic(): ''' test cloudAMQP_client ''' client = CloudAMQPClient(CLOUDAMQP_URL, TEST_QUEUE_NAME) sent_msg = {'test': 'test'} client.send_message(sent_msg) received_msg = client.get_message() assert sent_msg == received_msg print 'test_basic passed.'
def clearQueue(queue_url, queue_name): scrape_news_queue_client = CloudAMQPClient(queue_url, queue_name) num_of_messages = 0 while True: if scrape_news_queue_client is not None: msg = scrape_news_queue_client.get_message() if msg is None: print "Clear %d messages." % num_of_messages return num_of_messages += 1
def test_basic(): """ test unit """ client = CloudAMQPClient(TEST_CLOUDAMQP_URL, TEST_QUEUE_NAME) sent_msg = {'test': 'test'} client.send_message(sent_msg) client.sleep(5) receive_msg = client.get_message() assert sent_msg == receive_msg print('test_basic passed!')
text = None # if task['source'] == 'cnn': # print 'scraping CNN news' # text = cnn_news_scraper.extract_news(task['url']) # else: # print 'news source [%s] is not supported.' % task['source'] # task['text'] = text article = Article(task['url']) article.download() article.parse() # article.text is unicode, need to encode it to utf-8 task['text'] = article.text.encode('utf-8') dedupe_news_queue_client.send_message(task) while True: if scrape_news_queue_client is not None: msg = scrape_news_queue_client.get_message() if msg is not None: try: handle_message(msg) except Exception as e: print e pass scrape_news_queue_client.sleep(SLEEP_TIME_IN_SECONDS)
rows, _ = pairwise_sim.shape for row in range(1, rows): if pairwise_sim[row, 0] > SAME_NEWS_SIMILARITY_THRESHOLD: print 'Duplicated news. Ignore.' return task['publishedAt'] = parser.parse(task['publishedAt']) # Classify news description = task['description'] if description is not None: topic = news_topic_modeling_service_client.classify(description) task['class'] = topic db[NEWS_TABLE_NAME].replace_one({'digest': task['digest']}, task, upsert=True) while True: if cloudAMQP_client is not None: msg = cloudAMQP_client.get_message() if msg is not None: # Parse and process the task try: handle_message(msg) except Exception as e: print e pass cloudAMQP_client.sleep(SLEEP_TIME_IN_SECONDS)
import os import sys import news_deduper as deduper # import common package in parent directory sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) from cloudAMQP_client import CloudAMQPClient DEDUPE_NEWS_TASK_QUEUE_URL = "amqp://*****:*****@otter.rmq.cloudamqp.com/xemwnewz" DEDUPE_NEWS_TASK_QUEUE_NAME = "tap-news-dedupe-news-task-queue" cloudAMQP_client = CloudAMQPClient(DEDUPE_NEWS_TASK_QUEUE_URL, DEDUPE_NEWS_TASK_QUEUE_NAME) TEST_MSG1 = "" TEST_MSG2 = cloudAMQP_client.get_message() def test_basic(): deduper.handle_message(TEST_MSG1) deduper.handle_message(TEST_MSG2) print('test_basic passed!') if __name__ == "__main__": test_basic()
import os import sys import news_fetcher as fetcher # import common package in parent directory sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) from cloudAMQP_client import CloudAMQPClient SCRAPE_NEWS_TASK_QUEUE_URL = "amqp://*****:*****@otter.rmq.cloudamqp.com/jxasfgzt" SCRAPE_NEWS_TASK_QUEUE_NAME = "tap-news-scrape-news-task-queue" scrape_news_queue_client = CloudAMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL, SCRAPE_NEWS_TASK_QUEUE_NAME) TEST_MSG1 = "" TEST_MSG2 = scrape_news_queue_client.get_message() def test_basic(): fetcher.handle_message(TEST_MSG1) fetcher.handle_message(TEST_MSG2) print('test_basic passed!') if __name__ == "__main__": test_basic()