def test__run(self, mock_api_client): queue = Queue() retry_queue = Queue() api_clients_queue = Queue() api_client_dict = { 'client': mock_api_client, 'request_interval': 0 } queue_resource_item = { 'resource': 'tenders', 'id': uuid.uuid4().hex, 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex } resource_item = { 'resource': 'tenders', 'id': uuid.uuid4().hex, 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex } archive_doc = { 'id': resource_item['id'], 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex, 'rev': '1-' + uuid.uuid4().hex } db = MagicMock() archive_db = MagicMock() secret_archive_db = MagicMock() bridge = ArchiveWorker(config_dict=self.worker_config, log_dict=self.log_dict, resource_items_queue=queue, retry_resource_items_queue=retry_queue, api_clients_queue=api_clients_queue, db=db, archive_db=archive_db, secret_archive_db=secret_archive_db) # Try get item from resource items queue bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 0) self.assertEqual(bridge.log_dict['add_to_retry'], 0) # Get resource from edge db bridge.db.get.side_effect = [Exception('DB exception'), None, resource_item, resource_item, resource_item, resource_item, resource_item, resource_item, resource_item, resource_item, resource_item, resource_item] queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 1) self.assertEqual(bridge.log_dict['add_to_retry'], 1) queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 1) self.assertEqual(bridge.log_dict['add_to_retry'], 1) resource_item['dateModified'] = datetime.datetime.now().isoformat() bridge.archive_db.get.side_effect = [Exception('Archive DB exception'), None, munchify(archive_doc), munchify(archive_doc), munchify(archive_doc), munchify(archive_doc), munchify(archive_doc), munchify(archive_doc), munchify(archive_doc)] bridge.archive_db.save = MagicMock() # Put resource to public db queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 2) self.assertEqual(bridge.log_dict['add_to_retry'], 2) # Try get api client from clients queue queue.put(queue_resource_item) bridge._get_api_client_dict = MagicMock(side_effect=[None, api_client_dict, api_client_dict, api_client_dict, None, api_client_dict, api_client_dict, api_client_dict, api_client_dict, api_client_dict, api_client_dict]) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 2) self.assertEqual(bridge.log_dict['add_to_retry'], 3) # Try get resource item dump from cdb resource_item['_rev'] = '1-' + uuid.uuid4().hex secret_doc = { 'id': resource_item['id'] } bridge._action_resource_item_from_cdb = MagicMock(side_effect=[Exception('From CDB exception'), secret_doc, secret_doc, secret_doc, Exception('Delete'), secret_doc, secret_doc, secret_doc, secret_doc]) queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 3) self.assertEqual(bridge.log_dict['add_to_retry'], 4) bridge.secret_archive_db.get.side_effect = [Exception('Secret DB exception'), None, secret_doc, secret_doc, secret_doc, secret_doc, secret_doc] queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 4) self.assertEqual(bridge.log_dict['add_to_retry'], 5) queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 4) self.assertEqual(bridge.log_dict['add_to_retry'], 5) queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 5) self.assertEqual(bridge.log_dict['add_to_retry'], 5) # Delete resource from edge db queue.put(queue_resource_item) bridge.db.save.side_effect = [True, Exception('Delete from edge')] bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 5) self.assertEqual(bridge.log_dict['add_to_retry'], 5) queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 6) self.assertEqual(bridge.log_dict['add_to_retry'], 5)
def test_storage(self, mock_api_client): s3_key = 'key' s3_secret_key = 'secret' s3_bucket = 'bucket' queue = Queue() retry_queue = Queue() api_clients_queue = Queue() api_client_dict = { 'client': mock_api_client, 'request_interval': 0 } queue_resource_item = { 'resource': 'tenders', 'id': uuid.uuid4().hex, 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex } resource_item = { 'resource': 'tenders', 'id': uuid.uuid4().hex, 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex } archive_doc = { 'id': resource_item['id'], 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex, 'rev': '1-' + uuid.uuid4().hex } db = MagicMock() archive_db = MagicMock() conn = MockConnection(s3_key, s3_secret_key) conn.create_bucket(s3_bucket) secret_archive = S3Storage(conn, s3_bucket) bridge = ArchiveWorker(config_dict=self.worker_config, log_dict=self.log_dict, resource_items_queue=queue, retry_resource_items_queue=retry_queue, api_clients_queue=api_clients_queue, db=db, archive_db=archive_db, secret_archive_db=secret_archive) bridge.db.get.side_effect = [resource_item, resource_item] bridge.archive_db.get.side_effect = [munchify(archive_doc), munchify(archive_doc)] bridge._get_api_client_dict = MagicMock(side_effect=[api_client_dict, api_client_dict]) bridge.archive_db.save = MagicMock() # Try get resource item dump from cdb resource_item['_rev'] = '1-' + uuid.uuid4().hex secret_doc = { 'id': resource_item['id'] } bridge._action_resource_item_from_cdb = MagicMock(side_effect=[secret_doc, secret_doc]) queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 0) self.assertEqual(bridge.log_dict['add_to_retry'], 0) data = bridge.secret_archive_db.get(queue_resource_item['id']) self.assertEqual(secret_doc, data.get('data')) # Test invalid key data = bridge.secret_archive_db.get('invalid') self.assertTrue(data is None)
def test_storage(self, mock_api_client): s3_key = 'key' s3_secret_key = 'secret' s3_bucket = 'bucket' queue = Queue() retry_queue = Queue() api_clients_queue = Queue() api_client_dict = {'client': mock_api_client, 'request_interval': 0} queue_resource_item = { 'resource': 'tenders', 'id': uuid.uuid4().hex, 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex } resource_item = { 'resource': 'tenders', 'id': uuid.uuid4().hex, 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex } archive_doc = { 'id': resource_item['id'], 'dateModified': datetime.datetime.now().isoformat(), '_rev': '1-' + uuid.uuid4().hex, 'rev': '1-' + uuid.uuid4().hex } db = MagicMock() archive_db = MagicMock() conn = MockConnection(s3_key, s3_secret_key) conn.create_bucket(s3_bucket) secret_archive = S3Storage(conn, s3_bucket) bridge = ArchiveWorker(config_dict=self.worker_config, log_dict=self.log_dict, resource_items_queue=queue, retry_resource_items_queue=retry_queue, api_clients_queue=api_clients_queue, db=db, archive_db=archive_db, secret_archive_db=secret_archive) bridge.db.get.side_effect = [resource_item, resource_item] bridge.archive_db.get.side_effect = [ munchify(archive_doc), munchify(archive_doc) ] bridge._get_api_client_dict = MagicMock( side_effect=[api_client_dict, api_client_dict]) bridge.archive_db.save = MagicMock() # Try get resource item dump from cdb resource_item['_rev'] = '1-' + uuid.uuid4().hex secret_doc = { 'data': { 'dateModified': datetime.datetime.now().isoformat(), 'versions': 'versions', 'tender': { 'item': 'item', 'pubkey': 'pubkey' } }, '_id': uuid.uuid4().hex } bridge._action_resource_item_from_cdb = MagicMock( side_effect=[secret_doc, secret_doc]) queue.put(queue_resource_item) bridge._run() self.assertEqual(bridge.log_dict['exceptions_count'], 0) self.assertEqual(bridge.log_dict['add_to_retry'], 0) data = bridge.secret_archive_db.get(queue_resource_item['id']) self.assertIn('_rev', data) self.assertEqual(data.get('_rev'), 1) self.assertEqual(data.get('dateModified'), queue_resource_item['dateModified']) self.assertEqual(secret_doc, data.get('data')) # Test invalid key data = bridge.secret_archive_db.get('invalid') self.assertTrue(data is None) data = bridge.secret_archive_db.get('/') self.assertEqual(data, None) key = '/'.join([ format(i, 'x') for i in uuid.UUID(queue_resource_item['id']).fields ]) data = bridge.secret_archive_db.get(key) self.assertEqual(data.get('data'), secret_doc) secret_doc_updated = secret_doc.copy() secret_doc_updated['data']['tender'] = { 'item': 'item2', 'pubkey': 'pubkey' } bridge.db.get.side_effect = [resource_item, resource_item] bridge.archive_db.get.side_effect = [ munchify(archive_doc), munchify(archive_doc) ] bridge._get_api_client_dict = MagicMock( side_effect=[api_client_dict, api_client_dict]) bridge._action_resource_item_from_cdb = MagicMock( side_effect=[secret_doc_updated, secret_doc_updated]) queue_resource_item_updated = queue_resource_item queue_resource_item_updated['dateModified'] = ( datetime.datetime.now() + timedelta(days=1)).isoformat() queue.put(queue_resource_item) bridge._run() data = bridge.secret_archive_db.get(queue_resource_item['id']) self.assertEqual(data.get('_rev'), 2) self.assertEqual(data.get('data'), secret_doc_updated)