def test_basic(): client = CloudAMQPClient(TEST_CLOUDAMQP_URL, TEST_QUEUE_NAME) sentMsg = {'test': 'test'} client.send_message(sentMsg) receivedMsg = client.get_message() assert sentMsg == receivedMsg print('test_basic passed.')
def test_basic(): """"Test method""" client = CloudAMQPClient(CLOUDAMQP_URL, QUEUE_NAME) sent_message = {'test': 'test'} client.send_message(sent_message) received_message = client.get_message() assert sent_message == received_message print('test_basic passed.')
def test_basic(): ''' test cloudAMQP_client ''' client = CloudAMQPClient(CLOUDAMQP_URL, TEST_QUEUE_NAME) sent_msg = {'test': 'test'} client.send_message(sent_msg) received_msg = client.get_message() assert sent_msg == received_msg print 'test_basic passed.'
def test_basic(): """ test unit """ client = CloudAMQPClient(TEST_CLOUDAMQP_URL, TEST_QUEUE_NAME) sent_msg = {'test': 'test'} client.send_message(sent_msg) client.sleep(5) receive_msg = client.get_message() assert sent_msg == receive_msg print('test_basic passed!')
# REDIS_PORT = 6379 REDIS_CLIENT = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=0) while True: news_list = news_api_client.get_news_from_url(NEWS_SOURCE, SORT_BY) for news in news_list: news_digest = hashlib.md5( news['title'].encode('utf-8')).digest().encode('base64') if REDIS_CLIENT.get(news_digest) is None: print 'we have new news incoming~~~~~~~~~~~' news[ 'digest'] = news_digest #1.populate the news['digest'] by news_digest NUMBER_OF_NEWS = NUMBER_OF_NEWS + 1 # 2. #news++ if news['publishedAt'] is None: #3. populate news['publishAt'] news['publishedAt'] = datetime.now().strftime( '%Y-%m-%dT%H:%M:%SZ') #4. put the news_digest into redis for later duplidated removal REDIS_CLIENT.set(news_digest, 'True') REDIS_CLIENT.expire(news_digest, EXPIRE_TIME) MQ_CLIENT.send_message(news) else: print 'no new news for now~~~~~~~~~~~~~~' print "Till now, Fetched %d news" % NUMBER_OF_NEWS MQ_CLIENT.sleep(SLEEP_SECONDS)
redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT) cloudAMQP_client = CloudAMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL, SCRAPE_NEWS_TASK_QUEUE_NAME) while True: news_list = news_api_client.getNewsFromSource(sources=NEWS_SOURCES, api_name=EVERYTHING_API, sort_by=SORT_BY_TOP, pageSize=PAGE_SIZE) num_of_new_news = 0 for news in news_list: news_digest = hashlib.md5( news['title'].encode('utf-8')).digest().encode('base64') if redis_client.get(news_digest) is None: num_of_new_news += 1 news['digest'] = news_digest if news['publishedAt'] is None: news['publishedAt'] = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%SZ") redis_client.set(news_digest, "True") redis_client.expire(news_digest, NEWS_TIME_OUT_IN_SECONDS) cloudAMQP_client.send_message(news) print("Fetched {} news.".format(num_of_new_news)) cloudAMQP_client.sleep(SLEEP_TIME_IN_SECONDS)