def test_import_rows(self, mock_Reader): mock_reader = MagicMock() mock_Reader.return_value = mock_reader rows = ((1, 2), (3, 4)) mock_reader.read.return_value = rows progress = MagicMock() write = MagicMock() _self = MagicMock() _self.logger = MagicMock() _self.injector.inject = MagicMock() _self.merger = MagicMock() _self.converter = MagicMock() entity = 'Entity' _self.converter.convert.return_value = entity _self.validator = MagicMock() ImportClient.import_rows(_self, write, progress) _self.logger.info.assert_called() self.assertEqual(_self.injector.inject.call_args_list, [call(c) for c in rows]) self.assertEqual(_self.merger.merge.call_args_list, [call(c, write) for c in rows]) self.assertEqual(_self.converter.convert.call_args_list, [call(c) for c in rows]) self.assertEqual(_self.validator.validate.call_args_list, [call(entity) for c in rows]) self.assertEqual(write.call_args_list, [call(entity) for c in rows]) _self.validator.result.called_once_with() self.assertEqual(len(_self.logger.info.call_args_list), 3)
def test_publish(self): logger = MagicMock() self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger) self.import_client.n_rows = 10 self.import_client.filename = "filename" msg = self.import_client.get_result_msg() self.assertEqual(msg['contents_ref'], 'filename') self.assertEqual(msg['summary']['num_records'], 10) self.assertEqual(msg['header']['version'], 0.1)
def test_publish_delete_mode(self): logger = MagicMock() self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger, ImportMode.DELETE) self.import_client.n_rows = 0 self.import_client.filename = "filename" app = self.mock_dataset['source']['application'] entity = self.mock_dataset['entity'] msg1 = f"Import dataset {entity} from {app} (mode = DELETE) started" logger.info.assert_called_with(msg1) self.import_client.get_result_msg() msg2 = f"Import dataset {entity} from {app} completed. " \ "0 records imported, all known entities will be marked as deleted." logger.info.assert_called_with(msg2, kwargs={'data': {'num_records': 0}})
def test_import_dataset_mode_delete(self, mock_progress_ticker): _self = MagicMock() _self.mode = ImportMode.DELETE with patch("builtins.open", mock_open(read_data="")) as m: ImportClient.import_dataset(_self) m.assert_called_once() m.mock_calls[1] = call().write("[") m.mock_calls[2] = call().write("]") m.mock_calls[3] = call().close() _self.merger.assert_not_called() _self.import_rows.assert_not_called() _self.entity_validator.assert_not_called() _self.get_result_msg.assert_called_once()
def test_import_row_too_few_records(self, mock_Reader): reader = MagicMock() mock_Reader.return_value = reader rows = set() reader.read.return_value = rows progress = MagicMock() write = MagicMock() _self = MagicMock() _self.mode = ImportMode.FULL _self.dataset = {} ImportClient.import_rows(_self, write, progress) _self.validator.result.assert_called_once_with() self.assertEqual(len(_self.logger.info.call_args_list), 3) self.assertEqual(len(_self.logger.error.call_args_list), 1)
def test_import_dataset_exception(self, mock_traceback, mock_ProgressTicker, mock_ContentsWriter): _self = MagicMock() _self.get_result_msg.return_value = 'res' writer = MagicMock() writer.side_effect = Exception('Boom') mock_ContentsWriter.return_value.__enter__ = writer res = ImportClient.import_dataset(_self) self.assertEqual(res, 'res') self.assertEqual(len(_self.logger.error.call_args_list), 2) mock_traceback.format_exc.assert_called_once_with(limit=-5)
def handle_import_msg(msg): dataset = extract_dataset_from_msg(msg) msg['header'] |= { 'source': dataset['source']['name'], 'application': dataset['source']['application'], 'catalogue': dataset['catalogue'], 'entity': dataset['entity'], } logger.configure(msg, "IMPORT") header = msg.get('header', {}) # Create a new import client and start the process mode = ImportMode(header.get('mode', ImportMode.FULL.value)) import_client = ImportClient(dataset=dataset, msg=msg, mode=mode, logger=logger) return import_client.import_dataset()
def test_import_dataset(self, mock_ProgressTicker, mock_ContentsWriter): _self = MagicMock() _self.get_result_msg.return_value = 'res' writer = MagicMock() mock_ContentsWriter.return_value.__enter__.return_value = writer filename = 'fname' writer.filename = filename writer.write = 'write' progress = MagicMock() mock_ProgressTicker.return_value.__enter__.return_value = progress res = ImportClient.import_dataset(_self) self.assertEqual(res, 'res') mock_ProgressTicker.called_once() _self.merger.prepare.assert_called_once_with(progress) self.assertEqual(_self.filename, filename) _self.import_rows.assert_called_once_with('write', progress) _self.merger.finish.assert_called_once_with('write') _self.entity_validator.result.assert_called_once()
def test_init(self): logger = MagicMock() self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger) logger.info.assert_called()
class TestImportClient(TestCase): import_client = None def setUp(self): self.mock_dataset = { 'source': { 'entity_id': fixtures.random_string(), 'application': fixtures.random_string(), 'name': fixtures.random_string(), 'type': 'file', 'config': {}, 'query': fixtures.random_string(), }, 'version': 0.1, 'catalogue': fixtures.random_string(), 'entity': fixtures.random_string(), 'gob_mapping': {} } self.mock_msg = { 'header': {} } def test_init(self): logger = MagicMock() self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger) logger.info.assert_called() def test_publish(self): logger = MagicMock() self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger) self.import_client.n_rows = 10 self.import_client.filename = "filename" msg = self.import_client.get_result_msg() self.assertEqual(msg['contents_ref'], 'filename') self.assertEqual(msg['summary']['num_records'], 10) self.assertEqual(msg['header']['version'], 0.1) def test_publish_delete_mode(self): logger = MagicMock() self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger, ImportMode.DELETE) self.import_client.n_rows = 0 self.import_client.filename = "filename" app = self.mock_dataset['source']['application'] entity = self.mock_dataset['entity'] msg1 = f"Import dataset {entity} from {app} (mode = DELETE) started" logger.info.assert_called_with(msg1) self.import_client.get_result_msg() msg2 = f"Import dataset {entity} from {app} completed. " \ "0 records imported, all known entities will be marked as deleted." logger.info.assert_called_with(msg2, kwargs={'data': {'num_records': 0}}) @patch('gobimport.import_client.Reader') def test_import_rows(self, mock_Reader): mock_reader = MagicMock() mock_Reader.return_value = mock_reader rows = ((1, 2), (3, 4)) mock_reader.read.return_value = rows progress = MagicMock() write = MagicMock() _self = MagicMock() _self.logger = MagicMock() _self.injector.inject = MagicMock() _self.merger = MagicMock() _self.converter = MagicMock() entity = 'Entity' _self.converter.convert.return_value = entity _self.validator = MagicMock() ImportClient.import_rows(_self, write, progress) _self.logger.info.assert_called() self.assertEqual(_self.injector.inject.call_args_list, [call(c) for c in rows]) self.assertEqual(_self.merger.merge.call_args_list, [call(c, write) for c in rows]) self.assertEqual(_self.converter.convert.call_args_list, [call(c) for c in rows]) self.assertEqual(_self.validator.validate.call_args_list, [call(entity) for c in rows]) self.assertEqual(write.call_args_list, [call(entity) for c in rows]) _self.validator.result.called_once_with() self.assertEqual(len(_self.logger.info.call_args_list), 3) @patch('gobimport.import_client.Reader') def test_import_row_too_few_records(self, mock_Reader): reader = MagicMock() mock_Reader.return_value = reader rows = set() reader.read.return_value = rows progress = MagicMock() write = MagicMock() _self = MagicMock() _self.mode = ImportMode.FULL _self.dataset = {} ImportClient.import_rows(_self, write, progress) _self.validator.result.assert_called_once_with() self.assertEqual(len(_self.logger.info.call_args_list), 3) self.assertEqual(len(_self.logger.error.call_args_list), 1) @patch('gobimport.import_client.ContentsWriter') @patch('gobimport.import_client.ProgressTicker') def test_import_dataset(self, mock_ProgressTicker, mock_ContentsWriter): _self = MagicMock() _self.get_result_msg.return_value = 'res' writer = MagicMock() mock_ContentsWriter.return_value.__enter__.return_value = writer filename = 'fname' writer.filename = filename writer.write = 'write' progress = MagicMock() mock_ProgressTicker.return_value.__enter__.return_value = progress res = ImportClient.import_dataset(_self) self.assertEqual(res, 'res') mock_ProgressTicker.called_once() _self.merger.prepare.assert_called_once_with(progress) self.assertEqual(_self.filename, filename) _self.import_rows.assert_called_once_with('write', progress) _self.merger.finish.assert_called_once_with('write') _self.entity_validator.result.assert_called_once() @patch("gobimport.import_client.ProgressTicker") def test_import_dataset_mode_delete(self, mock_progress_ticker): _self = MagicMock() _self.mode = ImportMode.DELETE with patch("builtins.open", mock_open(read_data="")) as m: ImportClient.import_dataset(_self) m.assert_called_once() m.mock_calls[1] = call().write("[") m.mock_calls[2] = call().write("]") m.mock_calls[3] = call().close() _self.merger.assert_not_called() _self.import_rows.assert_not_called() _self.entity_validator.assert_not_called() _self.get_result_msg.assert_called_once() @patch('gobimport.import_client.ContentsWriter') @patch('gobimport.import_client.ProgressTicker') @patch('gobimport.import_client.traceback') def test_import_dataset_exception(self, mock_traceback, mock_ProgressTicker, mock_ContentsWriter): _self = MagicMock() _self.get_result_msg.return_value = 'res' writer = MagicMock() writer.side_effect = Exception('Boom') mock_ContentsWriter.return_value.__enter__ = writer res = ImportClient.import_dataset(_self) self.assertEqual(res, 'res') self.assertEqual(len(_self.logger.error.call_args_list), 2) mock_traceback.format_exc.assert_called_once_with(limit=-5)