def test_import_rows(self, mock_Reader):
mock_reader = MagicMock()
mock_Reader.return_value = mock_reader
rows = ((1, 2), (3, 4))
mock_reader.read.return_value = rows
progress = MagicMock()
write = MagicMock()
_self = MagicMock()
_self.logger = MagicMock()
_self.injector.inject = MagicMock()
_self.merger = MagicMock()
_self.converter = MagicMock()
entity = 'Entity'
_self.converter.convert.return_value = entity
_self.validator = MagicMock()
ImportClient.import_rows(_self, write, progress)
_self.logger.info.assert_called()
self.assertEqual(_self.injector.inject.call_args_list, [call(c) for c in rows])
self.assertEqual(_self.merger.merge.call_args_list, [call(c, write) for c in rows])
self.assertEqual(_self.converter.convert.call_args_list, [call(c) for c in rows])
self.assertEqual(_self.validator.validate.call_args_list, [call(entity) for c in rows])
self.assertEqual(write.call_args_list, [call(entity) for c in rows])
_self.validator.result.called_once_with()
self.assertEqual(len(_self.logger.info.call_args_list), 3)
def test_publish(self):
logger = MagicMock()
self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger)
self.import_client.n_rows = 10
self.import_client.filename = "filename"
msg = self.import_client.get_result_msg()
self.assertEqual(msg['contents_ref'], 'filename')
self.assertEqual(msg['summary']['num_records'], 10)
self.assertEqual(msg['header']['version'], 0.1)
def test_publish_delete_mode(self):
logger = MagicMock()
self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger, ImportMode.DELETE)
self.import_client.n_rows = 0
self.import_client.filename = "filename"
app = self.mock_dataset['source']['application']
entity = self.mock_dataset['entity']
msg1 = f"Import dataset {entity} from {app} (mode = DELETE) started"
logger.info.assert_called_with(msg1)
self.import_client.get_result_msg()
msg2 = f"Import dataset {entity} from {app} completed. " \
"0 records imported, all known entities will be marked as deleted."
logger.info.assert_called_with(msg2, kwargs={'data': {'num_records': 0}})
def test_import_dataset_mode_delete(self, mock_progress_ticker):
_self = MagicMock()
_self.mode = ImportMode.DELETE
with patch("builtins.open", mock_open(read_data="")) as m:
ImportClient.import_dataset(_self)
m.assert_called_once()
m.mock_calls[1] = call().write("[")
m.mock_calls[2] = call().write("]")
m.mock_calls[3] = call().close()
_self.merger.assert_not_called()
_self.import_rows.assert_not_called()
_self.entity_validator.assert_not_called()
_self.get_result_msg.assert_called_once()
def test_import_row_too_few_records(self, mock_Reader):
reader = MagicMock()
mock_Reader.return_value = reader
rows = set()
reader.read.return_value = rows
progress = MagicMock()
write = MagicMock()
_self = MagicMock()
_self.mode = ImportMode.FULL
_self.dataset = {}
ImportClient.import_rows(_self, write, progress)
_self.validator.result.assert_called_once_with()
self.assertEqual(len(_self.logger.info.call_args_list), 3)
self.assertEqual(len(_self.logger.error.call_args_list), 1)
def test_import_dataset_exception(self, mock_traceback, mock_ProgressTicker, mock_ContentsWriter):
_self = MagicMock()
_self.get_result_msg.return_value = 'res'
writer = MagicMock()
writer.side_effect = Exception('Boom')
mock_ContentsWriter.return_value.__enter__ = writer
res = ImportClient.import_dataset(_self)
self.assertEqual(res, 'res')
self.assertEqual(len(_self.logger.error.call_args_list), 2)
mock_traceback.format_exc.assert_called_once_with(limit=-5)
def handle_import_msg(msg):
dataset = extract_dataset_from_msg(msg)
msg['header'] |= {
'source': dataset['source']['name'],
'application': dataset['source']['application'],
'catalogue': dataset['catalogue'],
'entity': dataset['entity'],
}
logger.configure(msg, "IMPORT")
header = msg.get('header', {})
# Create a new import client and start the process
mode = ImportMode(header.get('mode', ImportMode.FULL.value))
import_client = ImportClient(dataset=dataset,
msg=msg,
mode=mode,
logger=logger)
return import_client.import_dataset()
def test_import_dataset(self, mock_ProgressTicker, mock_ContentsWriter):
_self = MagicMock()
_self.get_result_msg.return_value = 'res'
writer = MagicMock()
mock_ContentsWriter.return_value.__enter__.return_value = writer
filename = 'fname'
writer.filename = filename
writer.write = 'write'
progress = MagicMock()
mock_ProgressTicker.return_value.__enter__.return_value = progress
res = ImportClient.import_dataset(_self)
self.assertEqual(res, 'res')
mock_ProgressTicker.called_once()
_self.merger.prepare.assert_called_once_with(progress)
self.assertEqual(_self.filename, filename)
_self.import_rows.assert_called_once_with('write', progress)
_self.merger.finish.assert_called_once_with('write')
_self.entity_validator.result.assert_called_once()
def test_init(self):
logger = MagicMock()
self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger)
logger.info.assert_called()
class TestImportClient(TestCase):
import_client = None
def setUp(self):
self.mock_dataset = {
'source': {
'entity_id': fixtures.random_string(),
'application': fixtures.random_string(),
'name': fixtures.random_string(),
'type': 'file',
'config': {},
'query': fixtures.random_string(),
},
'version': 0.1,
'catalogue': fixtures.random_string(),
'entity': fixtures.random_string(),
'gob_mapping': {}
}
self.mock_msg = {
'header': {}
}
def test_init(self):
logger = MagicMock()
self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger)
logger.info.assert_called()
def test_publish(self):
logger = MagicMock()
self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger)
self.import_client.n_rows = 10
self.import_client.filename = "filename"
msg = self.import_client.get_result_msg()
self.assertEqual(msg['contents_ref'], 'filename')
self.assertEqual(msg['summary']['num_records'], 10)
self.assertEqual(msg['header']['version'], 0.1)
def test_publish_delete_mode(self):
logger = MagicMock()
self.import_client = ImportClient(self.mock_dataset, self.mock_msg, logger, ImportMode.DELETE)
self.import_client.n_rows = 0
self.import_client.filename = "filename"
app = self.mock_dataset['source']['application']
entity = self.mock_dataset['entity']
msg1 = f"Import dataset {entity} from {app} (mode = DELETE) started"
logger.info.assert_called_with(msg1)
self.import_client.get_result_msg()
msg2 = f"Import dataset {entity} from {app} completed. " \
"0 records imported, all known entities will be marked as deleted."
logger.info.assert_called_with(msg2, kwargs={'data': {'num_records': 0}})
@patch('gobimport.import_client.Reader')
def test_import_rows(self, mock_Reader):
mock_reader = MagicMock()
mock_Reader.return_value = mock_reader
rows = ((1, 2), (3, 4))
mock_reader.read.return_value = rows
progress = MagicMock()
write = MagicMock()
_self = MagicMock()
_self.logger = MagicMock()
_self.injector.inject = MagicMock()
_self.merger = MagicMock()
_self.converter = MagicMock()
entity = 'Entity'
_self.converter.convert.return_value = entity
_self.validator = MagicMock()
ImportClient.import_rows(_self, write, progress)
_self.logger.info.assert_called()
self.assertEqual(_self.injector.inject.call_args_list, [call(c) for c in rows])
self.assertEqual(_self.merger.merge.call_args_list, [call(c, write) for c in rows])
self.assertEqual(_self.converter.convert.call_args_list, [call(c) for c in rows])
self.assertEqual(_self.validator.validate.call_args_list, [call(entity) for c in rows])
self.assertEqual(write.call_args_list, [call(entity) for c in rows])
_self.validator.result.called_once_with()
self.assertEqual(len(_self.logger.info.call_args_list), 3)
@patch('gobimport.import_client.Reader')
def test_import_row_too_few_records(self, mock_Reader):
reader = MagicMock()
mock_Reader.return_value = reader
rows = set()
reader.read.return_value = rows
progress = MagicMock()
write = MagicMock()
_self = MagicMock()
_self.mode = ImportMode.FULL
_self.dataset = {}
ImportClient.import_rows(_self, write, progress)
_self.validator.result.assert_called_once_with()
self.assertEqual(len(_self.logger.info.call_args_list), 3)
self.assertEqual(len(_self.logger.error.call_args_list), 1)
@patch('gobimport.import_client.ContentsWriter')
@patch('gobimport.import_client.ProgressTicker')
def test_import_dataset(self, mock_ProgressTicker, mock_ContentsWriter):
_self = MagicMock()
_self.get_result_msg.return_value = 'res'
writer = MagicMock()
mock_ContentsWriter.return_value.__enter__.return_value = writer
filename = 'fname'
writer.filename = filename
writer.write = 'write'
progress = MagicMock()
mock_ProgressTicker.return_value.__enter__.return_value = progress
res = ImportClient.import_dataset(_self)
self.assertEqual(res, 'res')
mock_ProgressTicker.called_once()
_self.merger.prepare.assert_called_once_with(progress)
self.assertEqual(_self.filename, filename)
_self.import_rows.assert_called_once_with('write', progress)
_self.merger.finish.assert_called_once_with('write')
_self.entity_validator.result.assert_called_once()
@patch("gobimport.import_client.ProgressTicker")
def test_import_dataset_mode_delete(self, mock_progress_ticker):
_self = MagicMock()
_self.mode = ImportMode.DELETE
with patch("builtins.open", mock_open(read_data="")) as m:
ImportClient.import_dataset(_self)
m.assert_called_once()
m.mock_calls[1] = call().write("[")
m.mock_calls[2] = call().write("]")
m.mock_calls[3] = call().close()
_self.merger.assert_not_called()
_self.import_rows.assert_not_called()
_self.entity_validator.assert_not_called()
_self.get_result_msg.assert_called_once()
@patch('gobimport.import_client.ContentsWriter')
@patch('gobimport.import_client.ProgressTicker')
@patch('gobimport.import_client.traceback')
def test_import_dataset_exception(self, mock_traceback, mock_ProgressTicker, mock_ContentsWriter):
_self = MagicMock()
_self.get_result_msg.return_value = 'res'
writer = MagicMock()
writer.side_effect = Exception('Boom')
mock_ContentsWriter.return_value.__enter__ = writer
res = ImportClient.import_dataset(_self)
self.assertEqual(res, 'res')
self.assertEqual(len(_self.logger.error.call_args_list), 2)
mock_traceback.format_exc.assert_called_once_with(limit=-5)