def setUp(self):
self.migrations = GOBMigrations()
self.add_event = MockEvent('ADD', '0.1')
self.mock_migration = {
'target_version':
'0.2',
'conversions': [{
'action': 'rename',
'old_column': 'old',
'new_column': 'new'
}]
}
def database_to_gobevent(event) -> ImportEvent:
"""Reconstruct the original event out of the stored event
:param event: the database event
:return: a ADD, MODIFY, CONFIRM or DELETE event
"""
# Parse the json data of the event
if isinstance(event.contents, dict):
data = event.contents
else:
data = json.loads(event.contents)
# Get the model version to check if the event should be migrated to the correct version
model_version = GOBModel().get_collection(event.catalogue,
event.entity)['version']
if model_version != event.version:
# Event should be migrated to the correct GOBModel version
data = GOBMigrations().migrate_event_data(event, data, event.catalogue,
event.entity, model_version)
event_msg = {
"event": event.action,
"data": data,
}
msg_header = {
"process_id": None,
"source": event.source,
"application": event.application,
"id_column": data.get("id_column"),
"catalogue": event.catalogue,
"entity": event.entity,
"version": event.version,
"timestamp": event.timestamp,
}
# Construct the event out of the reconstructed event data
gob_event = GobEvent(event.tid, event_msg, MessageMetaData(msg_header))
# Store the id of the event in the gob_event
gob_event.id = event.eventid
return gob_event
def correct_version_numbers():
storage = GOBStorageHandler()
for catalog_name, collection in GOBMigrations()._migrations.items():
for collection_name, versions in collection.items():
print(f"{catalog_name} {collection_name}: Determine version boundaries in events")
current_version = '0.1'
change_eventids = [(current_version, 0)]
migration = versions[current_version]
while migration:
change_eventid = _get_change_eventid(catalog_name, collection_name, migration, storage)
target_version = migration['target_version']
if change_eventid is not None:
change_eventids.append((target_version, change_eventid))
migration = versions.get(target_version)
_update_version_numbers(catalog_name, collection_name, change_eventids, storage)
def test_initialize_model_once(self, mock_extract_migrations):
# Model has already been initialised, _extract_migrations should not be called again
migrations = GOBMigrations()
mock_extract_migrations.assert_not_called()
class TestMigrations(unittest.TestCase):
def setUp(self):
self.migrations = GOBMigrations()
self.add_event = MockEvent('ADD', '0.1')
self.mock_migration = {
'target_version': '0.2',
'conversions': [
{
'action': 'rename',
'old_column': 'old',
'new_column': 'new'
},
{
'action': 'delete',
'column': 'deleted_column'
},
{
'action': 'add',
'column': 'added_column',
'default': 'default value'
}
]
}
@patch('gobcore.model.migrations.GOBMigrations._extract_migrations')
def test_initialize_model_once(self, mock_extract_migrations):
# Model has already been initialised, _extract_migrations should not be called again
migrations = GOBMigrations()
mock_extract_migrations.assert_not_called()
def test_get_migration(self):
# Assert we get a list of relations for a collection
self.assertIsInstance(self.migrations._get_migration('brk', 'kadastralesecties', '0.1'), dict)
def test_get_migration_keyerror(self):
# Assert an empty list is returned if no migration is found
self.assertEqual(self.migrations._get_migration('catalog', 'collection', '0.1'), None)
def test_apply_migration_add(self):
# Test if a column has been renamed in event data
data = {
'entity': {
'old': 'value',
'deleted_column': 'some value',
}
}
expected_data = {
'entity': {
'new': 'value',
'added_column': 'default value',
}
}
result = self.migrations._apply_migration(self.add_event, data, self.mock_migration)
self.assertEqual(result, expected_data)
self.assertEqual(self.add_event.version, '0.2')
def test_apply_migration_modify(self):
# Test if a column has been renamed in event data
modify_event = MockEvent('MODIFY', '0.1')
data = {
'modifications': [
{
'key': 'old',
'old_value': 'test',
'new_value': 'modify',
},
{
'key': 'other_key',
'old_value': 'test',
'new_value': 'modify'
},
{
'key': 'deleted_column',
'old_value': 'test',
'new_value': 'modify',
}
]
}
expected_data = {
'modifications': [
{
'key': 'new',
'old_value': 'test',
'new_value': 'modify'
},
{
'key': 'other_key',
'old_value': 'test',
'new_value': 'modify'
},
]
}
result = self.migrations._apply_migration(modify_event, data, self.mock_migration)
self.assertEqual(result, expected_data)
self.assertEqual(modify_event.version, '0.2')
def test_apply_migration_not_implemented(self):
# Assert migration fails for notimplemented action
not_implemented_mock_migration = {
'target_version': '0.2',
'conversions': [
{
'action': 'non-existent',
}
]
}
data = {}
with self.assertRaises(NotImplementedError):
result = self.migrations._apply_migration(self.add_event, data, not_implemented_mock_migration)
@patch('gobcore.model.migrations.logger', MagicMock())
@patch('gobcore.model.migrations.GOBMigrations._get_migration')
def test_migrate_event_data(self, mock_get_migration):
mock_get_migration.return_value = self.mock_migration
data = {
'entity': {
'old': 'value',
}
}
expected_data = {
'entity': {
'new': 'value',
'added_column': 'default value',
}
}
result = self.migrations.migrate_event_data(self.add_event, data, 'catalog', 'collection', '0.2')
self.assertEqual(result, expected_data)
@patch('gobcore.model.migrations.logger', MagicMock())
@patch('gobcore.model.migrations.GOBMigrations._get_migration')
def test_migrate_event_data_multiple(self, mock_get_migration):
mock_migration2 = {
'target_version': '0.3',
'conversions': [
{
'action': 'rename',
'old_column': 'new',
'new_column': 'extra_new'
}
]
}
mock_get_migration.side_effect = [self.mock_migration, mock_migration2]
data = {
'entity': {
'old': 'value',
}
}
expected_data = {
'entity': {
'extra_new': 'value',
'added_column': 'default value',
}
}
result = self.migrations.migrate_event_data(self.add_event, data, 'catalog', 'collection', '0.3')
self.assertEqual(result, expected_data)
@patch('gobcore.model.migrations.logger')
@patch('gobcore.model.migrations.GOBMigrations._get_migration')
def test_migrate_event_data_missing_migration(self, mock_get_migration, mock_logger):
mock_get_migration.side_effect = [self.mock_migration, None]
data = {
'entity': {
'old': 'value',
}
}
with self.assertRaises(GOBException):
self.migrations.migrate_event_data(self.add_event, data, 'catalog', 'collection', '0.3')
mock_logger.assert_called()