def test__run(self, mock_api_client):
queue = Queue()
retry_queue = Queue()
api_clients_queue = Queue()
api_client_dict = {
'client': mock_api_client,
'request_interval': 0
}
queue_resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
archive_doc = {
'id': resource_item['id'],
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex,
'rev': '1-' + uuid.uuid4().hex
}
db = MagicMock()
archive_db = MagicMock()
secret_archive_db = MagicMock()
bridge = ArchiveWorker(config_dict=self.worker_config, log_dict=self.log_dict,
resource_items_queue=queue, retry_resource_items_queue=retry_queue,
api_clients_queue=api_clients_queue, db=db, archive_db=archive_db,
secret_archive_db=secret_archive_db)
# Try get item from resource items queue
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 0)
self.assertEqual(bridge.log_dict['add_to_retry'], 0)
# Get resource from edge db
bridge.db.get.side_effect = [Exception('DB exception'), None, resource_item, resource_item,
resource_item, resource_item, resource_item, resource_item,
resource_item, resource_item, resource_item, resource_item]
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 1)
self.assertEqual(bridge.log_dict['add_to_retry'], 1)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 1)
self.assertEqual(bridge.log_dict['add_to_retry'], 1)
resource_item['dateModified'] = datetime.datetime.now().isoformat()
bridge.archive_db.get.side_effect = [Exception('Archive DB exception'), None,
munchify(archive_doc), munchify(archive_doc),
munchify(archive_doc), munchify(archive_doc),
munchify(archive_doc), munchify(archive_doc),
munchify(archive_doc)]
bridge.archive_db.save = MagicMock()
# Put resource to public db
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 2)
self.assertEqual(bridge.log_dict['add_to_retry'], 2)
# Try get api client from clients queue
queue.put(queue_resource_item)
bridge._get_api_client_dict = MagicMock(side_effect=[None, api_client_dict, api_client_dict,
api_client_dict, None, api_client_dict,
api_client_dict, api_client_dict,
api_client_dict, api_client_dict,
api_client_dict])
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 2)
self.assertEqual(bridge.log_dict['add_to_retry'], 3)
# Try get resource item dump from cdb
resource_item['_rev'] = '1-' + uuid.uuid4().hex
secret_doc = {
'id': resource_item['id']
}
bridge._action_resource_item_from_cdb = MagicMock(side_effect=[Exception('From CDB exception'),
secret_doc, secret_doc,
secret_doc, Exception('Delete'),
secret_doc, secret_doc,
secret_doc, secret_doc])
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 3)
self.assertEqual(bridge.log_dict['add_to_retry'], 4)
bridge.secret_archive_db.get.side_effect = [Exception('Secret DB exception'), None, secret_doc,
secret_doc, secret_doc, secret_doc, secret_doc]
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 4)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 4)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 5)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
# Delete resource from edge db
queue.put(queue_resource_item)
bridge.db.save.side_effect = [True, Exception('Delete from edge')]
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 5)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 6)
self.assertEqual(bridge.log_dict['add_to_retry'], 5)
def test_storage(self, mock_api_client):
s3_key = 'key'
s3_secret_key = 'secret'
s3_bucket = 'bucket'
queue = Queue()
retry_queue = Queue()
api_clients_queue = Queue()
api_client_dict = {
'client': mock_api_client,
'request_interval': 0
}
queue_resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
archive_doc = {
'id': resource_item['id'],
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex,
'rev': '1-' + uuid.uuid4().hex
}
db = MagicMock()
archive_db = MagicMock()
conn = MockConnection(s3_key, s3_secret_key)
conn.create_bucket(s3_bucket)
secret_archive = S3Storage(conn, s3_bucket)
bridge = ArchiveWorker(config_dict=self.worker_config, log_dict=self.log_dict,
resource_items_queue=queue, retry_resource_items_queue=retry_queue,
api_clients_queue=api_clients_queue, db=db, archive_db=archive_db,
secret_archive_db=secret_archive)
bridge.db.get.side_effect = [resource_item, resource_item]
bridge.archive_db.get.side_effect = [munchify(archive_doc), munchify(archive_doc)]
bridge._get_api_client_dict = MagicMock(side_effect=[api_client_dict, api_client_dict])
bridge.archive_db.save = MagicMock()
# Try get resource item dump from cdb
resource_item['_rev'] = '1-' + uuid.uuid4().hex
secret_doc = {
'id': resource_item['id']
}
bridge._action_resource_item_from_cdb = MagicMock(side_effect=[secret_doc, secret_doc])
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 0)
self.assertEqual(bridge.log_dict['add_to_retry'], 0)
data = bridge.secret_archive_db.get(queue_resource_item['id'])
self.assertEqual(secret_doc, data.get('data'))
# Test invalid key
data = bridge.secret_archive_db.get('invalid')
self.assertTrue(data is None)
def test_storage(self, mock_api_client):
s3_key = 'key'
s3_secret_key = 'secret'
s3_bucket = 'bucket'
queue = Queue()
retry_queue = Queue()
api_clients_queue = Queue()
api_client_dict = {'client': mock_api_client, 'request_interval': 0}
queue_resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
resource_item = {
'resource': 'tenders',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex
}
archive_doc = {
'id': resource_item['id'],
'dateModified': datetime.datetime.now().isoformat(),
'_rev': '1-' + uuid.uuid4().hex,
'rev': '1-' + uuid.uuid4().hex
}
db = MagicMock()
archive_db = MagicMock()
conn = MockConnection(s3_key, s3_secret_key)
conn.create_bucket(s3_bucket)
secret_archive = S3Storage(conn, s3_bucket)
bridge = ArchiveWorker(config_dict=self.worker_config,
log_dict=self.log_dict,
resource_items_queue=queue,
retry_resource_items_queue=retry_queue,
api_clients_queue=api_clients_queue,
db=db,
archive_db=archive_db,
secret_archive_db=secret_archive)
bridge.db.get.side_effect = [resource_item, resource_item]
bridge.archive_db.get.side_effect = [
munchify(archive_doc),
munchify(archive_doc)
]
bridge._get_api_client_dict = MagicMock(
side_effect=[api_client_dict, api_client_dict])
bridge.archive_db.save = MagicMock()
# Try get resource item dump from cdb
resource_item['_rev'] = '1-' + uuid.uuid4().hex
secret_doc = {
'data': {
'dateModified': datetime.datetime.now().isoformat(),
'versions': 'versions',
'tender': {
'item': 'item',
'pubkey': 'pubkey'
}
},
'_id': uuid.uuid4().hex
}
bridge._action_resource_item_from_cdb = MagicMock(
side_effect=[secret_doc, secret_doc])
queue.put(queue_resource_item)
bridge._run()
self.assertEqual(bridge.log_dict['exceptions_count'], 0)
self.assertEqual(bridge.log_dict['add_to_retry'], 0)
data = bridge.secret_archive_db.get(queue_resource_item['id'])
self.assertIn('_rev', data)
self.assertEqual(data.get('_rev'), 1)
self.assertEqual(data.get('dateModified'),
queue_resource_item['dateModified'])
self.assertEqual(secret_doc, data.get('data'))
# Test invalid key
data = bridge.secret_archive_db.get('invalid')
self.assertTrue(data is None)
data = bridge.secret_archive_db.get('/')
self.assertEqual(data, None)
key = '/'.join([
format(i, 'x') for i in uuid.UUID(queue_resource_item['id']).fields
])
data = bridge.secret_archive_db.get(key)
self.assertEqual(data.get('data'), secret_doc)
secret_doc_updated = secret_doc.copy()
secret_doc_updated['data']['tender'] = {
'item': 'item2',
'pubkey': 'pubkey'
}
bridge.db.get.side_effect = [resource_item, resource_item]
bridge.archive_db.get.side_effect = [
munchify(archive_doc),
munchify(archive_doc)
]
bridge._get_api_client_dict = MagicMock(
side_effect=[api_client_dict, api_client_dict])
bridge._action_resource_item_from_cdb = MagicMock(
side_effect=[secret_doc_updated, secret_doc_updated])
queue_resource_item_updated = queue_resource_item
queue_resource_item_updated['dateModified'] = (
datetime.datetime.now() + timedelta(days=1)).isoformat()
queue.put(queue_resource_item)
bridge._run()
data = bridge.secret_archive_db.get(queue_resource_item['id'])
self.assertEqual(data.get('_rev'), 2)
self.assertEqual(data.get('data'), secret_doc_updated)