def test_basic():
print('news_fetcher_test: cleaning all queues...')
queue_cleaner.clear_all()
print('flushing all cache in Redis')
redis_client.flushall() # TODO dangerous to run when deployed
scrape_queue_client = AMQPClient(SCRAPE_QUEUE_URL,
SCRAPE_NEWS_TASK_QUEUE_NAME)
scrape_queue_client.connect()
assert scrape_queue_client.is_connected()
print('test_fetcher_basic: adding news onto scrape queue...')
for message in TEST_SCRAPE_TASK:
scrape_queue_client.send_message(message)
print('getting messages from the queue and process...')
news_fetcher.SLEEP_TIME_IN_SECONDS = 1
news_fetcher.run(len(TEST_SCRAPE_TASK))
should_be_empty_msg = scrape_queue_client.get_message()
print('news_fetcher_test(expecting None):', should_be_empty_msg)
assert should_be_empty_msg is None
scrape_queue_client.close()
queue_cleaner.clear_queue(DEDUPE_QUEUE_URL, DEDUPE_NEWS_TASK_QUEUE_NAME)
print('news_fetcher test passed')
def run():
redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT)
amqp_client = AMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL,
SCRAPE_NEWS_TASK_QUEUE_NAME)
amqp_client.connect()
try:
while True:
news_list = news_client.get_news_from_sources(NEWS_SOURCES)
num_news = 0
for news in news_list:
digest = hashlib.md5(news['title'].encode('utf-8')).hexdigest()
if redis_client.get(digest):
continue
num_news += 1
news['digest'] = digest
redis_client.set(digest, True)
redis_client.expire(digest, NEWS_TIME_OUT_IN_SECONDS)
print(news)
amqp_client.send_message(news)
print('News Monitor: fectched {} news'.format(num_news))
amqp_client.sleep(SLEEP_TIME_IN_SECONDS)
except KeyboardInterrupt:
print('keyboard interrupt')
# except SigTerm
finally:
amqp_client.close()
def test_basic():
# print(amqp_url + "({})".format(type(amqp_url)))
client = AMQPClient(AMQP_URL, 'my_queue')
client.connect()
assert client.is_connected()
assert client.get_message() is None
client.send_message('hello world')
assert client.get_message() == 'hello world'
obj = {"hello": "world"}
client.send_message(obj)
assert client.get_message() == obj
assert client.get_message() is None
client.cancel_queue()
client.close()
print('[x] cloud amqp_client test passed')
def test_basic():
queue_client = AMQPClient(USER_CLICK_QUEUE_URL, USER_CLICK_QUEUE_NAME)
user_id, digest = 'click tester', 'digest'
clickdilivery.send_click(queue_client, user_id, digest)
message = queue_client.get_message()
print(message)
assert message['userId'] == user_id
assert message['newsDigest'] == digest
queue_client.close()
print('click test passed')
def clear_queue(queue_url, queue_name):
count = 0
amqp_client = AMQPClient(queue_url, queue_name)
amqp_client.connect()
print('xxx cleaning queue "{}"'.format(queue_name))
while True:
message = amqp_client.get_message()
if message is None:
break
print(message)
count += 1
amqp_client.close()
print('xxx cleaned {} message on {}'.format(count, queue_name))