def test_basic():
print('news_fetcher_test: cleaning all queues...')
queue_cleaner.clear_all()
print('flushing all cache in Redis')
redis_client.flushall() # TODO dangerous to run when deployed
scrape_queue_client = AMQPClient(SCRAPE_QUEUE_URL,
SCRAPE_NEWS_TASK_QUEUE_NAME)
scrape_queue_client.connect()
assert scrape_queue_client.is_connected()
print('test_fetcher_basic: adding news onto scrape queue...')
for message in TEST_SCRAPE_TASK:
scrape_queue_client.send_message(message)
print('getting messages from the queue and process...')
news_fetcher.SLEEP_TIME_IN_SECONDS = 1
news_fetcher.run(len(TEST_SCRAPE_TASK))
should_be_empty_msg = scrape_queue_client.get_message()
print('news_fetcher_test(expecting None):', should_be_empty_msg)
assert should_be_empty_msg is None
scrape_queue_client.close()
queue_cleaner.clear_queue(DEDUPE_QUEUE_URL, DEDUPE_NEWS_TASK_QUEUE_NAME)
print('news_fetcher test passed')
def test_monitor_basic():
news_monitor.NEWS_SOURCES = news_client.MOCK_SOURCES
MOCK_DATA = news_client.MOCK_DATA
print('test_monitor_basic: cleaning queue "{}" first---------'.format(QUEUE_NAME))
clear_queue(QUEUE_URL, QUEUE_NAME)
# TODO: redis server flush all
redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT)
redis_client.flushall()
print('test_monitor_basic: adding message to queue "{}"--------'.format(QUEUE_NAME))
amqp_client = AMQPClient(QUEUE_URL, QUEUE_NAME)
amqp_client.connect()
proc = Process(target=news_monitor.run, name='monitor_run',
args=(REDIS_HOST, REDIS_PORT, QUEUE_URL, QUEUE_NAME))
proc.start()
print('test_monitor_basic: executing... (wait for 2 seconds to cut)')
time.sleep(2)
proc.terminate()
for i in range(len(MOCK_DATA)):
message = amqp_client.get_message()
del message['digest']
print(message, MOCK_DATA[i])
assert message == MOCK_DATA[i]
print('test_monitor_basic: [x] test_monitor_basic test passed')
def test_basic():
clear_queue(QUEUE_URL, QUEUE_NAME)
dedupe_queue_client = cloud_amqp_client.AMQPClient(QUEUE_URL, QUEUE_NAME)
dedupe_queue_client.connect()
print('news_deduper_test: adding task onto the dedupe queue...')
for task in TEST_DEDUPE_TASK:
dedupe_queue_client.send_message(task)
# some setting for the deduper module, so mind that the global variables in the module
# are not tested
news_deduper.DEDUPE_QUEUE_URL = QUEUE_URL
news_deduper.DEDUPE_QUEUE_NAME = QUEUE_NAME
news_deduper.COLLECTION_NAME = COLLECTION_NAME
news_deduper.dedupe_queue_client = dedupe_queue_client
news_deduper.SLEEP_TIME_IN_SECONDS = 1
news_deduper.run(len(TEST_DEDUPE_TASK))
test_collection = mongodb_client.get_db(DB_NAME).get_collection(
COLLECTION_NAME)
# for news in test_collection.find():
# print(news)
print('news deduper: number of documents(expecting 2)',
test_collection.count())
assert test_collection.count() == 2
print('news deduper test passed')
dedupe_queue_client.close()
def test_basic():
# not a good idea to test, but a way I can think of seperating the queue
# in test and production
rpc_operations.USER_CLICK_QUEUE_URL = USER_CLICK_QUEUE_URL
rpc_operations.USER_CLICK_QUEUE_NAME = USER_CLICK_QUEUE_NAME
rpc_operations.init()
clickhandler.USER_CLICK_QUEUE_URL = USER_CLICK_QUEUE_URL
clickhandler.USER_CLICK_QUEUE_NAME = USER_CLICK_QUEUE_NAME
print('Click Handler test: cleaning up click queue...')
clear_queue(USER_CLICK_QUEUE_URL, USER_CLICK_QUEUE_NAME)
print('Click Handler test: logging clicks...')
for pair in TEST_CLICK_DATA:
rpc_operations.log_click(*pair)
print(print('Click Handler test: handling clicks...'))
clickhandler.run(len(TEST_CLICK_DATA))
queue_client = AMQPClient(USER_CLICK_QUEUE_URL, USER_CLICK_QUEUE_NAME)
queue_client.connect()
should_be_empty = queue_client.get_message()
print('Click Handler Test: {} should be None'.format(should_be_empty))
assert should_be_empty is None
queue_client.close
print('Click Handler: test passed')
def test_basic():
# print('Click Learner tester: clearing queue {}'.format(CLICK_QUEUE_NAME))
clear_queue(clicklearner.USER_CLICK_QUEUE_URL, CLICK_QUEUE_NAME)
clicklearner.PREF_COLLECTION_NAME = PREF_COLLECTION_NAME
clicklearner.USER_CLICK_QUEUE_NAME = CLICK_QUEUE_NAME
clicklearner.SLEEP_TIME_IN_SECONDS = 1
click_queue_client = AMQPClient(clicklearner.USER_CLICK_QUEUE_URL,
CLICK_QUEUE_NAME)
click_queue_client.connect()
assert click_queue_client.is_connected()
print('Click Learner tester: clearing collection "{}" in db "{}"...'.format(PREF_COLLECTION_NAME,
PREF_DB_NAME))
pref_collection.remove()
print('Click Learner tester: sending click logs')
for click in TEST_SEQUENCE:
click_queue_client.send_message(click)
print('Click Learner tester: start handling clicks')
pref_model_ref = {'userId': TEST_USER_NAME}
for cat in NEWS_CATEGORIES:
pref_model_ref[cat] = 1 / len(NEWS_CATEGORIES)
for click_log in TEST_SEQUENCE:
clicklearner.run(1)
pref_model = pref_collection.find_one({'userId': TEST_USER_NAME})
selected = news_collection.find_one({'digest': (click_log['newsDigest'])})['category']
pref_model_ref[selected] = (1 - ALPHA) * pref_model_ref[selected] + ALPHA
for cat in NEWS_CATEGORIES:
if cat != selected:
pref_model_ref[cat] = (1 - ALPHA) * pref_model_ref[cat]
del pref_model['_id']
print('Click Learner tester: expecting {} == {}'.format(pref_model, pref_model_ref))
assert pref_model == pref_model_ref
print('xx Click Learner test passed')