def test__get_api_client_dict(self):
api_clients_queue = Queue()
client = MagicMock()
client_dict = {'client': client, 'request_interval': 0}
api_clients_queue.put(client_dict)
# Success test
worker = ArchiveWorker(api_clients_queue=api_clients_queue,
config_dict=self.worker_config,
log_dict=self.log_dict)
self.assertEqual(worker.api_clients_queue.qsize(), 1)
api_client = worker._get_api_client_dict()
self.assertEqual(api_client, client_dict)
# Empty queue test
api_client = worker._get_api_client_dict()
self.assertEqual(api_client, None)
del worker
def test__get_resource_item_from_queue(self):
items_queue = Queue()
item = {
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat(),
'resource': 'tenders'}
items_queue.put(item)
# Success test
worker = ArchiveWorker(resource_items_queue=items_queue, config_dict=self.worker_config,
log_dict=self.log_dict)
self.assertEqual(worker.resource_items_queue.qsize(), 1)
resource_item = worker._get_resource_item_from_queue()
self.assertEqual(resource_item, item)
self.assertEqual(worker.resource_items_queue.qsize(), 0)
# Empty queue test
resource_item = worker._get_resource_item_from_queue()
self.assertEqual(resource_item, None)
del worker
def test_init(self):
worker = ArchiveWorker()
self.assertEqual(worker.exit, False)
self.assertEqual(worker.update_doc, False)
self.assertEqual(worker.db, None)
self.assertEqual(worker.archive_db, None)
self.assertEqual(worker.secret_archive_db, None)
self.assertEqual(worker.config, None)
self.assertEqual(worker.log_dict, None)
self.assertEqual(worker.api_clients_queue, None)
self.assertEqual(worker.resource_items_queue, None)
self.assertEqual(worker.retry_resource_items_queue, None)
self.assertGreater(datetime.datetime.now().isoformat(), worker.start_time.isoformat())
def test_add_to_retry_queue(self):
retry_items_queue = Queue()
worker = ArchiveWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_items_queue,
log_dict=self.log_dict)
retry_item = {
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat(),
'resource': 'tenders'
}
self.assertEqual(retry_items_queue.qsize(), 0)
self.assertEqual(worker.log_dict['add_to_retry'], 0)
# Add to retry_resource_items_queue
worker.add_to_retry_queue(retry_item)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(retry_items_queue.qsize(), 1)
self.assertEqual(worker.log_dict['add_to_retry'], 1)
retry_item_from_queue = retry_items_queue.get()
self.assertEqual(retry_item_from_queue['retries_count'], 1)
self.assertEqual(retry_item_from_queue['timeout'],
worker.config['retry_default_timeout'] * 2)
# Add to retry_resource_items_queue with status_code '429'
worker.add_to_retry_queue(retry_item, status_code=429)
retry_item_from_queue = retry_items_queue.get()
self.assertEqual(retry_item_from_queue['retries_count'], 1)
self.assertEqual(retry_item_from_queue['timeout'],
worker.config['retry_default_timeout'] * 2)
# Drop from retry_resource_items_queue
retry_item['retries_count'] = 6
self.assertEqual(worker.log_dict['droped'], 0)
worker.add_to_retry_queue(retry_item)
self.assertEqual(worker.log_dict['droped'], 1)
self.assertEqual(retry_items_queue.qsize(), 0)
del worker
def test_storage(self, mock_api_client):
s3_key = 'key'
s3_secret_key = 'secret'
s3_bucket = 'bucket'
queue = Queue()
retry_queue = Queue()
api_clients_queue = Queue()
api_client_dict = {
'client': mock_api_client,
'request_interval': 0
}
queue_resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
archive_doc = {
'id': resource_item['id'],
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex,
'rev': '1-' + uuid.uuid4().hex
}
db = MagicMock()
archive_db = MagicMock()
conn = MockConnection(s3_key, s3_secret_key)
conn.create_bucket(s3_bucket)
secret_archive = S3Storage(conn, s3_bucket)
bridge = ArchiveWorker(config_dict=self.worker_config, log_dict=self.log_dict,
resource_items_queue=queue, retry_resource_items_queue=retry_queue,
api_clients_queue=api_clients_queue, db=db, archive_db=archive_db,
secret_archive_db=secret_archive)
bridge.db.get.side_effect = [resource_item, resource_item]
bridge.archive_db.get.side_effect = [munchify(archive_doc), munchify(archive_doc)]
bridge._get_api_client_dict = MagicMock(side_effect=[api_client_dict, api_client_dict])
bridge.archive_db.save = MagicMock()
# Try get resource item dump from cdb
resource_item['_rev'] = '1-' + uuid.uuid4().hex
secret_doc = {
'id': resource_item['id']
}
bridge._action_resource_item_from_cdb = MagicMock(side_effect=[secret_doc, secret_doc])
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 0)
self.assertEqual(bridge.log_dict['add_to_retry'], 0)
data = bridge.secret_archive_db.get(queue_resource_item['id'])
self.assertEqual(secret_doc, data.get('data'))
# Test invalid key
data = bridge.secret_archive_db.get('invalid')
self.assertTrue(data is None)
def test_shutdown(self):
worker = ArchiveWorker()
self.assertEqual(worker.exit, False)
worker.shutdown()
self.assertEqual(worker.exit, True)
def test__run(self, mock_api_client):
queue = Queue()
retry_queue = Queue()
api_clients_queue = Queue()
api_client_dict = {
'client': mock_api_client,
'request_interval': 0
}
queue_resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
archive_doc = {
'id': resource_item['id'],
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex,
'rev': '1-' + uuid.uuid4().hex
}
db = MagicMock()
archive_db = MagicMock()
secret_archive_db = MagicMock()
bridge = ArchiveWorker(config_dict=self.worker_config, log_dict=self.log_dict,
resource_items_queue=queue, retry_resource_items_queue=retry_queue,
api_clients_queue=api_clients_queue, db=db, archive_db=archive_db,
secret_archive_db=secret_archive_db)
# Try get item from resource items queue
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 0)
self.assertEqual(bridge.log_dict['add_to_retry'], 0)
# Get resource from edge db
bridge.db.get.side_effect = [Exception('DB exception'), None, resource_item, resource_item,
resource_item, resource_item, resource_item, resource_item,
resource_item, resource_item, resource_item, resource_item]
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 1)
self.assertEqual(bridge.log_dict['add_to_retry'], 1)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 1)
self.assertEqual(bridge.log_dict['add_to_retry'], 1)
resource_item['dateModified'] = datetime.datetime.now().isoformat()
bridge.archive_db.get.side_effect = [Exception('Archive DB exception'), None,
munchify(archive_doc), munchify(archive_doc),
munchify(archive_doc), munchify(archive_doc),
munchify(archive_doc), munchify(archive_doc),
munchify(archive_doc)]
bridge.archive_db.save = MagicMock()
# Put resource to public db
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 2)
self.assertEqual(bridge.log_dict['add_to_retry'], 2)
# Try get api client from clients queue
queue.put(queue_resource_item)
bridge._get_api_client_dict = MagicMock(side_effect=[None, api_client_dict, api_client_dict,
api_client_dict, None, api_client_dict,
api_client_dict, api_client_dict,
api_client_dict, api_client_dict,
api_client_dict])
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 2)
self.assertEqual(bridge.log_dict['add_to_retry'], 3)
# Try get resource item dump from cdb
resource_item['_rev'] = '1-' + uuid.uuid4().hex
secret_doc = {
'id': resource_item['id']
}
bridge._action_resource_item_from_cdb = MagicMock(side_effect=[Exception('From CDB exception'),
secret_doc, secret_doc,
secret_doc, Exception('Delete'),
secret_doc, secret_doc,
secret_doc, secret_doc])
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 3)
self.assertEqual(bridge.log_dict['add_to_retry'], 4)
bridge.secret_archive_db.get.side_effect = [Exception('Secret DB exception'), None, secret_doc,
secret_doc, secret_doc, secret_doc, secret_doc]
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 4)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 4)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 5)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
# Delete resource from edge db
queue.put(queue_resource_item)
bridge.db.save.side_effect = [True, Exception('Delete from edge')]
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 5)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 6)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
def test__action_resource_item_from_cdb(self, mock_api_client):
item = {
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat(),
'resource': 'tenders'
}
api_clients_queue = Queue()
api_clients_queue.put({
'client': mock_api_client,
'request_interval': 0.02})
retry_queue = Queue()
return_dict = {
'data': {
'id': item['id'],
'dateModified': datetime.datetime.utcnow().isoformat()
}
}
mock_api_client.get_resource_dump.return_value = return_dict
worker = ArchiveWorker(api_clients_queue=api_clients_queue,
config_dict=self.worker_config,
retry_resource_items_queue=retry_queue,
log_dict=self.log_dict)
# Success test
self.assertEqual(worker.api_clients_queue.qsize(), 1)
api_client = worker._get_api_client_dict()
self.assertEqual(api_client['request_interval'], 0.02)
self.assertEqual(worker.api_clients_queue.qsize(), 0)
public_item = worker._action_resource_item_from_cdb(api_client, item)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 0)
self.assertEqual(public_item, return_dict['data'])
# InvalidResponse
mock_api_client.get_resource_dump.side_effect = InvalidResponse('invalid response')
self.assertEqual(self.log_dict['exceptions_count'], 0)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 0)
api_client = worker._get_api_client_dict()
self.assertEqual(worker.api_clients_queue.qsize(), 0)
public_item = worker._action_resource_item_from_cdb(api_client, item)
self.assertEqual(public_item, None)
self.assertEqual(worker.log_dict['exceptions_count'], 1)
self.assertEqual(worker.log_dict['add_to_retry'], 1)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 1)
self.assertEqual(worker.api_clients_queue.qsize(), 1)
# RequestFailed status_code=429
mock_api_client.get_resource_dump.side_effect = RequestFailed(
munchify({'status_code': 429}))
api_client = worker._get_api_client_dict()
self.assertEqual(worker.api_clients_queue.qsize(), 0)
self.assertEqual(api_client['request_interval'], 0)
public_item = worker._action_resource_item_from_cdb(api_client, item)
self.assertEqual(public_item, None)
self.assertEqual(worker.log_dict['exceptions_count'], 2)
self.assertEqual(worker.log_dict['add_to_retry'], 2)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 2)
self.assertEqual(worker.api_clients_queue.qsize(), 1)
api_client = worker._get_api_client_dict()
self.assertEqual(worker.api_clients_queue.qsize(), 0)
self.assertEqual(api_client['request_interval'], worker.config['client_inc_step_timeout'])
# RequestFailed status_code=429 with drop cookies
api_client['request_interval'] = 2
public_item = worker._action_resource_item_from_cdb(api_client, item)
sleep(api_client['request_interval'])
self.assertEqual(worker.api_clients_queue.qsize(), 1)
self.assertEqual(public_item, None)
self.assertEqual(api_client['request_interval'], 0)
self.assertEqual(worker.log_dict['exceptions_count'], 3)
self.assertEqual(worker.log_dict['add_to_retry'], 3)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 3)
# RequestFailed with status_code not equal 429
mock_api_client.get_resource_dump.side_effect = RequestFailed(
munchify({'status_code': 404}))
api_client = worker._get_api_client_dict()
self.assertEqual(worker.api_clients_queue.qsize(), 0)
public_item = worker._action_resource_item_from_cdb(api_client, item)
self.assertEqual(public_item, None)
self.assertEqual(worker.api_clients_queue.qsize(), 1)
self.assertEqual(api_client['request_interval'], 0)
self.assertEqual(worker.log_dict['exceptions_count'], 4)
self.assertEqual(worker.log_dict['add_to_retry'], 4)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 4)
# ResourceNotFound
mock_api_client.get_resource_dump.side_effect = RNF(
munchify({'status_code': 404}))
api_client = worker._get_api_client_dict()
self.assertEqual(worker.api_clients_queue.qsize(), 0)
public_item = worker._action_resource_item_from_cdb(api_client, item)
self.assertEqual(public_item, None)
self.assertEqual(worker.api_clients_queue.qsize(), 1)
self.assertEqual(api_client['request_interval'], 0)
self.assertEqual(worker.log_dict['exceptions_count'], 4)
self.assertEqual(worker.log_dict['add_to_retry'], 4)
self.assertEqual(worker.log_dict['not_found_count'], 1)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 4)
# Exception
api_client = worker._get_api_client_dict()
mock_api_client.get_resource_dump.side_effect = Exception('text except')
public_item = worker._action_resource_item_from_cdb(api_client, item)
self.assertEqual(public_item, None)
self.assertEqual(api_client['request_interval'], 0)
self.assertEqual(worker.log_dict['exceptions_count'], 5)
self.assertEqual(worker.log_dict['add_to_retry'], 5)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 5)
del worker
def test_storage(self, mock_api_client):
s3_key = 'key'
s3_secret_key = 'secret'
s3_bucket = 'bucket'
queue = Queue()
retry_queue = Queue()
api_clients_queue = Queue()
api_client_dict = {'client': mock_api_client, 'request_interval': 0}
queue_resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
archive_doc = {
'id': resource_item['id'],
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex,
'rev': '1-' + uuid.uuid4().hex
}
db = MagicMock()
archive_db = MagicMock()
conn = MockConnection(s3_key, s3_secret_key)
conn.create_bucket(s3_bucket)
secret_archive = S3Storage(conn, s3_bucket)
bridge = ArchiveWorker(config_dict=self.worker_config,
log_dict=self.log_dict,
resource_items_queue=queue,
retry_resource_items_queue=retry_queue,
api_clients_queue=api_clients_queue,
db=db,
archive_db=archive_db,
secret_archive_db=secret_archive)
bridge.db.get.side_effect = [resource_item, resource_item]
bridge.archive_db.get.side_effect = [
munchify(archive_doc),
munchify(archive_doc)
]
bridge._get_api_client_dict = MagicMock(
side_effect=[api_client_dict, api_client_dict])
bridge.archive_db.save = MagicMock()
# Try get resource item dump from cdb
resource_item['_rev'] = '1-' + uuid.uuid4().hex
secret_doc = {
'data': {
'dateModified': datetime.datetime.now().isoformat(),
'versions': 'versions',
'tender': {
'item': 'item',
'pubkey': 'pubkey'
}
},
'_id': uuid.uuid4().hex
}
bridge._action_resource_item_from_cdb = MagicMock(
side_effect=[secret_doc, secret_doc])
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 0)
self.assertEqual(bridge.log_dict['add_to_retry'], 0)
data = bridge.secret_archive_db.get(queue_resource_item['id'])
self.assertIn('_rev', data)
self.assertEqual(data.get('_rev'), 1)
self.assertEqual(data.get('dateModified'),
queue_resource_item['dateModified'])
self.assertEqual(secret_doc, data.get('data'))
# Test invalid key
data = bridge.secret_archive_db.get('invalid')
self.assertTrue(data is None)
data = bridge.secret_archive_db.get('/')
self.assertEqual(data, None)
key = '/'.join([
format(i, 'x') for i in uuid.UUID(queue_resource_item['id']).fields
])
data = bridge.secret_archive_db.get(key)
self.assertEqual(data.get('data'), secret_doc)
secret_doc_updated = secret_doc.copy()
secret_doc_updated['data']['tender'] = {
'item': 'item2',
'pubkey': 'pubkey'
}
bridge.db.get.side_effect = [resource_item, resource_item]
bridge.archive_db.get.side_effect = [
munchify(archive_doc),
munchify(archive_doc)
]
bridge._get_api_client_dict = MagicMock(
side_effect=[api_client_dict, api_client_dict])
bridge._action_resource_item_from_cdb = MagicMock(
side_effect=[secret_doc_updated, secret_doc_updated])
queue_resource_item_updated = queue_resource_item
queue_resource_item_updated['dateModified'] = (
datetime.datetime.now() + timedelta(days=1)).isoformat()
queue.put(queue_resource_item)
bridge._run()
data = bridge.secret_archive_db.get(queue_resource_item['id'])
self.assertEqual(data.get('_rev'), 2)
self.assertEqual(data.get('data'), secret_doc_updated)
