def test__run(self, mocked_logger, mock_get_from_public, mocked_save_bulk):
self.queue = Queue()
self.retry_queue = Queue()
self.api_clients_queue = Queue()
queue_item = (1, uuid.uuid4().hex)
doc = {
'id': queue_item[1],
'_rev': '1-{}'.format(uuid.uuid4().hex),
'dateModified': datetime.datetime.utcnow().isoformat(),
'doc_type': 'Tender'
}
client = MagicMock()
api_client_dict = {
'id': uuid.uuid4().hex,
'client': client,
'request_interval': 0
}
client.session.headers = {'User-Agent': 'Test-Agent'}
self.api_clients_info = {
api_client_dict['id']: {
'drop_cookies': False,
'request_durations': []
}
}
self.db = MagicMock()
worker = ResourceItemWorker(
api_clients_queue=self.api_clients_queue,
resource_items_queue=self.queue,
retry_resource_items_queue=self.retry_queue,
db=self.db,
api_clients_info=self.api_clients_info,
config_dict=self.worker_config)
worker.exit = MagicMock()
worker.exit.__nonzero__.side_effect = [False, True]
# Try get api client from clients queue
self.assertEqual(self.queue.qsize(), 0)
worker._run()
self.assertEqual(self.queue.qsize(), 0)
mocked_logger.debug.assert_called_once_with(
'API clients queue is empty.')
# Try get item from resource items queue
self.api_clients_queue.put(api_client_dict)
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(mocked_logger.debug.call_args_list[1:], [
call('GET API CLIENT: {} {} with requests interval: {}'.format(
api_client_dict['id'],
api_client_dict['client'].session.headers['User-Agent'],
api_client_dict['request_interval']),
extra={
'REQUESTS_TIMEOUT': 0,
'MESSAGE_ID': 'get_client'
}),
call('PUT API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'put_client'}),
call('Resource items queue is empty.')
])
# Try get resource item from local storage
self.queue.put(queue_item)
mock_get_from_public.return_value = doc
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(mocked_logger.debug.call_args_list[4:], [
call('GET API CLIENT: {} {} with requests interval: {}'.format(
api_client_dict['id'],
api_client_dict['client'].session.headers['User-Agent'],
api_client_dict['request_interval']),
extra={
'REQUESTS_TIMEOUT': 0,
'MESSAGE_ID': 'get_client'
}),
call('Get tender {} from main queue.'.format(doc['id'])),
call('Put in bulk tender {} {}'.format(doc['id'],
doc['dateModified']),
extra={'MESSAGE_ID': 'add_to_save_bulk'})
])
# Try get local_resource_item with Exception
self.api_clients_queue.put(api_client_dict)
self.queue.put(queue_item)
mock_get_from_public.return_value = doc
self.db.get.side_effect = [Exception('Database Error')]
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(mocked_logger.debug.call_args_list[7:], [
call('GET API CLIENT: {} {} with requests interval: {}'.format(
api_client_dict['id'],
api_client_dict['client'].session.headers['User-Agent'],
api_client_dict['request_interval']),
extra={
'REQUESTS_TIMEOUT': 0,
'MESSAGE_ID': 'get_client'
}),
call('Get tender {} from main queue.'.format(doc['id'])),
call('PUT API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'put_client'})
])
mocked_logger.error.assert_called_once_with(
"Error while getting resource item from couchdb:"
" Exception('Database Error',)",
extra={'MESSAGE_ID': 'exceptions'})
self.api_clients_queue.put(api_client_dict)
self.queue.put(queue_item)
mock_get_from_public.return_value = None
self.db.get.side_effect = [doc]
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(mocked_logger.debug.call_args_list[10:], [
call('GET API CLIENT: {} {} with requests interval: {}'.format(
api_client_dict['id'],
api_client_dict['client'].session.headers['User-Agent'],
api_client_dict['request_interval']),
extra={
'REQUESTS_TIMEOUT': 0,
'MESSAGE_ID': 'get_client'
}),
call('Get tender {} from main queue.'.format(doc['id'])),
])
self.assertEqual(mocked_save_bulk.call_count, 1)
def test__run(self, mocked_logger, mock_get_from_public, mocked_save_bulk):
self.queue = Queue()
self.retry_queue = Queue()
self.api_clients_queue = Queue()
queue_item = {
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()
}
doc = {
'id': queue_item['id'],
'_rev': '1-{}'.format(uuid.uuid4().hex),
'dateModified': datetime.datetime.utcnow().isoformat(),
'doc_type': 'Tender'
}
client = MagicMock()
api_client_dict = {
'id': uuid.uuid4().hex,
'client': client,
'request_interval': 0
}
client.session.headers = {'User-Agent': 'Test-Agent'}
self.api_clients_info = {
api_client_dict['id']: {
'drop_cookies': False, 'request_durations': []
}
}
self.db = MagicMock()
worker = ResourceItemWorker(
api_clients_queue=self.api_clients_queue,
resource_items_queue=self.queue,
retry_resource_items_queue=self.retry_queue,
db=self.db, api_clients_info=self.api_clients_info,
config_dict=self.worker_config
)
worker.exit = MagicMock()
worker.exit.__nonzero__.side_effect = [False, True]
# Try get api client from clients queue
self.assertEqual(self.queue.qsize(), 0)
worker._run()
self.assertEqual(self.queue.qsize(), 0)
mocked_logger.debug.assert_called_once_with(
'API clients queue is empty.')
# Try get item from resource items queue
self.api_clients_queue.put(api_client_dict)
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[1:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'], client.session.headers['User-Agent']
)),
call('PUT API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'put_client'}),
call('Resource items queue is empty.')
]
)
# Try get resource item from local storage
self.queue.put(queue_item)
mock_get_from_public.return_value = doc
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[6:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id'])),
call('Put in bulk tender {} {}'.format(doc['id'],
doc['dateModified']),
extra={'MESSAGE_ID': 'add_to_save_bulk'})
]
)
# queue_resource_item dateModified is None and None public doc
self.api_clients_queue.put(api_client_dict)
self.queue.put({'id': doc['id'], 'dateModified': None})
mock_get_from_public.return_value = None
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[11:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id']))
]
)
# queue_resource_item dateModified is None and not None public doc
self.api_clients_queue.put(api_client_dict)
self.api_clients_queue.put(api_client_dict)
self.queue.put({'id': doc['id'], 'dateModified': None})
mock_get_from_public.return_value = doc
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[15:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id'])),
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Ignored duplicate tender {} in bulk: previous {}, '
'current {}'.format(
doc['id'], doc['dateModified'], doc['dateModified']),
extra={'MESSAGE_ID': 'skipped'})
]
)
# Add to retry queue
self.api_clients_queue.put(api_client_dict)
self.queue.put({'id': doc['id'], 'dateModified': None})
mock_get_from_public.return_value = doc
self.db.get_doc.return_value = doc
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[23:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id']))
]
)
mocked_logger.info.assert_called_once_with(
'Put tender {} to \'retries_queue\''.format(doc['id']),
extra={'MESSAGE_ID': 'add_to_retry'}
)
# Skip doc
self.api_clients_queue.put(api_client_dict)
self.api_clients_queue.put(api_client_dict)
self.queue.put({'id': doc['id'], 'dateModified': None})
mock_get_from_public.return_value = doc
self.db.get_doc.return_value = doc
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[27:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id'])),
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Ignored duplicate tender {} in bulk: previous {}, '
'current {}'.format(
doc['id'], doc['dateModified'], doc['dateModified']),
extra={'MESSAGE_ID': 'skipped'})
]
)
self.assertEqual(mocked_logger.info.call_count, 1)
# Skip doc with raise exception
self.api_clients_queue.put(api_client_dict)
self.api_clients_queue.put(api_client_dict)
self.queue.put({'id': doc['id'], 'dateModified': None})
mock_get_from_public.side_effect = Exception('test')
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[35:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id'])),
call('PUT API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'put_client'})
]
)
mocked_logger.error.assert_called_once_with(
'Error while getting resource item from couchdb: {}'.format(repr(
Exception('test'))),
extra={'MESSAGE_ID': 'exceptions'}
)
self.assertEqual(
mocked_logger.info.call_args_list[1],
call('Put tender {} to \'retries_queue\''.format(doc['id']),
extra={'MESSAGE_ID': 'add_to_retry'})
)
# Try get resource item from public server with None public doc
new_date_modified = datetime.datetime.utcnow().isoformat()
self.queue.put({'id': doc['id'], 'dateModified': new_date_modified})
mock_get_from_public.return_value = None
mock_get_from_public.side_effect = None
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[40:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id']))
]
)
self.assertEqual(mocked_logger.info.call_count, 2)
self.assertEqual(mocked_logger.error.call_count, 1)
# Try get resource item from public server
new_date_modified = datetime.datetime.utcnow().isoformat()
self.queue.put({'id': doc['id'], 'dateModified': new_date_modified})
mock_get_from_public.return_value = doc
mock_get_from_public.side_effect = None
worker.exit.__nonzero__.side_effect = [False, True]
worker._run()
self.assertEqual(
mocked_logger.debug.call_args_list[44:],
[
call('GET API CLIENT: {}'.format(api_client_dict['id']),
extra={'MESSAGE_ID': 'get_client'}),
call('SLEEP before return client: 0'),
call('Got api_client ID: {} {}'.format(
api_client_dict['id'],
client.session.headers['User-Agent'])),
call('Get tender {} from main queue.'.format(
doc['id'])),
call('Ignored duplicate tender {} in bulk: previous {}, '
'current {}'.format(
doc['id'], doc['dateModified'], doc['dateModified']),
extra={'MESSAGE_ID': 'skipped'})
]
)
self.assertEqual(mocked_logger.info.call_count, 2)
self.assertEqual(mocked_logger.error.call_count, 1)