def test_basic():
client = CloudAMQPClient(TEST_CLOUDAMQP_URL, TEST_QUEUE_NAME)
sentMsg = {'test': 'test'}
client.send_message(sentMsg)
receivedMsg = client.get_message()
assert sentMsg == receivedMsg
print('test_basic passed.')
def test_basic():
""""Test method"""
client = CloudAMQPClient(CLOUDAMQP_URL, QUEUE_NAME)
sent_message = {'test': 'test'}
client.send_message(sent_message)
received_message = client.get_message()
assert sent_message == received_message
print('test_basic passed.')
def test_basic():
''' test cloudAMQP_client '''
client = CloudAMQPClient(CLOUDAMQP_URL, TEST_QUEUE_NAME)
sent_msg = {'test': 'test'}
client.send_message(sent_msg)
received_msg = client.get_message()
assert sent_msg == received_msg
print 'test_basic passed.'
def test_basic():
""" test unit """
client = CloudAMQPClient(TEST_CLOUDAMQP_URL, TEST_QUEUE_NAME)
sent_msg = {'test': 'test'}
client.send_message(sent_msg)
client.sleep(5)
receive_msg = client.get_message()
assert sent_msg == receive_msg
print('test_basic passed!')
# REDIS_PORT = 6379
REDIS_CLIENT = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=0)
while True:
news_list = news_api_client.get_news_from_url(NEWS_SOURCE, SORT_BY)
for news in news_list:
news_digest = hashlib.md5(
news['title'].encode('utf-8')).digest().encode('base64')
if REDIS_CLIENT.get(news_digest) is None:
print 'we have new news incoming~~~~~~~~~~~'
news[
'digest'] = news_digest #1.populate the news['digest'] by news_digest
NUMBER_OF_NEWS = NUMBER_OF_NEWS + 1 # 2. #news++
if news['publishedAt'] is None: #3. populate news['publishAt']
news['publishedAt'] = datetime.now().strftime(
'%Y-%m-%dT%H:%M:%SZ')
#4. put the news_digest into redis for later duplidated removal
REDIS_CLIENT.set(news_digest, 'True')
REDIS_CLIENT.expire(news_digest, EXPIRE_TIME)
MQ_CLIENT.send_message(news)
else:
print 'no new news for now~~~~~~~~~~~~~~'
print "Till now, Fetched %d news" % NUMBER_OF_NEWS
MQ_CLIENT.sleep(SLEEP_SECONDS)
redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT)
cloudAMQP_client = CloudAMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL,
SCRAPE_NEWS_TASK_QUEUE_NAME)
while True:
news_list = news_api_client.getNewsFromSource(sources=NEWS_SOURCES,
api_name=EVERYTHING_API,
sort_by=SORT_BY_TOP,
pageSize=PAGE_SIZE)
num_of_new_news = 0
for news in news_list:
news_digest = hashlib.md5(
news['title'].encode('utf-8')).digest().encode('base64')
if redis_client.get(news_digest) is None:
num_of_new_news += 1
news['digest'] = news_digest
if news['publishedAt'] is None:
news['publishedAt'] = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%SZ")
redis_client.set(news_digest, "True")
redis_client.expire(news_digest, NEWS_TIME_OUT_IN_SECONDS)
cloudAMQP_client.send_message(news)
print("Fetched {} news.".format(num_of_new_news))
cloudAMQP_client.sleep(SLEEP_TIME_IN_SECONDS)
