def test_migrate_app_with_app_in_inconsistent_state(monkeypatch, mock_es_client):
"""
Test that migrate_app() resyncs an app in an inconsistent state.
This refers to an app with multiple read indices (i.e. an app with an interrupted migration).
In this case, a resync should be scheduled to attempt to complete the migration.
"""
migrate_model_task_mock = Mock()
monkeypatch.setattr('datahub.search.migrate.complete_model_migration', migrate_model_task_mock)
mock_client = mock_es_client.return_value
old_index = 'test-index-current-hash'
read_indices = ('test-index-current-hash', 'another-index')
current_hash = 'current-hash'
target_hash = 'current-hash'
mock_app = create_mock_search_app(
current_mapping_hash=current_hash,
target_mapping_hash=target_hash,
read_indices=read_indices,
write_index=old_index,
)
migrate_app(mock_app)
mock_app.es_model.create_index.assert_not_called()
mock_client.indices.update_aliases.assert_not_called()
migrate_model_task_mock.apply_async.assert_called_once_with(
args=(mock_app.name, target_hash),
)
def test_migrate_app_with_app_in_invalid_state(monkeypatch, mock_es_client):
"""
Test that migrate_app() raises an exception for apps in an invalid state.
This refers to an app that needs migrating, but the current write index is not one of
the read indices. This should never happen.
"""
migrate_model_task_mock = Mock()
monkeypatch.setattr('datahub.search.migrate.complete_model_migration', migrate_model_task_mock)
read_indices = ('read-index',)
write_index = 'write-index'
current_hash = 'current-hash'
target_hash = 'target-hash'
mock_app = create_mock_search_app(
current_mapping_hash=current_hash,
target_mapping_hash=target_hash,
read_indices=read_indices,
write_index=write_index,
)
with pytest.raises(DataHubException):
migrate_app(mock_app)
migrate_model_task_mock.apply_async.assert_not_called()
def test_complete_model_migration_aborts_when_already_in_progress(monkeypatch):
"""
Test that the complete_model_migration task aborts when the lock for the same search app is
already held.
"""
resync_after_migrate_mock = Mock()
monkeypatch.setattr('datahub.search.tasks.resync_after_migrate',
resync_after_migrate_mock)
mock_app = create_mock_search_app(
current_mapping_hash='current-hash',
target_mapping_hash='target-hash',
)
get_search_app_mock = Mock(return_value=mock_app)
monkeypatch.setattr('datahub.search.tasks.get_search_app',
get_search_app_mock)
# Have to mock rather than acquire the lock as locks are per connection (if the lock is
# already held by the current connection, the current connection can still acquire it again).
advisory_lock_mock = MagicMock()
advisory_lock_mock.return_value.__enter__.return_value = False
monkeypatch.setattr('datahub.search.tasks.advisory_lock',
advisory_lock_mock)
complete_model_migration.apply(args=('test-app', 'target-hash'))
# resync_after_migrate_mock should not have been called as the task should've exited instead
resync_after_migrate_mock.assert_not_called()
def test_complete_model_migration_with_mapping_hash_mismatch(monkeypatch):
"""
Test that the complete_model_migration task calls self.retry() when the target mapping hash is
not the expected one.
This is to catch cases where the complete_model_migration task is received by an old app
instance.
"""
resync_after_migrate_mock = Mock()
monkeypatch.setattr('datahub.search.tasks.resync_after_migrate', resync_after_migrate_mock)
mock_app = create_mock_search_app(
current_mapping_hash='current-hash',
target_mapping_hash='target-hash',
)
get_search_app_mock = Mock(return_value=mock_app)
monkeypatch.setattr('datahub.search.tasks.get_search_app', get_search_app_mock)
retry_mock = Mock(side_effect=MockRetryError())
monkeypatch.setattr(complete_model_migration, 'retry', retry_mock)
res = complete_model_migration.apply_async(args=('test-app', 'another-hash'))
with pytest.raises(MockRetryError):
res.get()
retry_mock.assert_called_once()
resync_after_migrate_mock.assert_not_called()
def test_sync_app_logic(monkeypatch):
"""Tests syncing an app to Elasticsearch during a mapping migration."""
bulk_mock = Mock()
monkeypatch.setattr('datahub.search.bulk_sync.bulk', bulk_mock)
search_app = create_mock_search_app(
current_mapping_hash='mapping-hash',
target_mapping_hash='mapping-hash',
read_indices=('index1', 'index2'),
write_index='index1',
queryset=MockQuerySet([Mock(id=1), Mock(id=2)]),
)
sync_app(search_app, batch_size=1000)
assert bulk_mock.call_args_list[0][1]['actions'] == [
{
'_index': 'index1',
'_id': 1,
'_type': 'test-type',
},
{
'_index': 'index1',
'_id': 2,
'_type': 'test-type',
},
]
assert bulk_mock.call_count == 1
def test_resync_in_invalid_state(self, monkeypatch, mock_es_client):
"""
Test that if the there is only a single read index, no aliases are updated and no indices
are deleted.
"""
sync_app_mock = Mock()
monkeypatch.setattr('datahub.search.migrate_utils.sync_app',
sync_app_mock)
get_aliases_for_index_mock = Mock(return_value=set())
monkeypatch.setattr(
'datahub.search.migrate_utils.get_aliases_for_index',
get_aliases_for_index_mock,
)
read_indices = {'index2', 'index3'}
write_index = 'index1'
mock_app = create_mock_search_app(
read_indices=read_indices,
write_index=write_index,
)
with pytest.raises(DataHubException):
resync_after_migrate(mock_app)
# The state is only checked once sync_app has run, as it's making sure nothing has
# changed while sync_app was running
sync_app_mock.assert_called_once_with(
mock_app,
post_batch_callback=delete_from_secondary_indices_callback,
)
def test_resync_with_single_read_index(self, monkeypatch, mock_es_client):
"""
Test that the function aborts if the there is only a single read index.
"""
sync_app_mock = Mock()
monkeypatch.setattr('datahub.search.migrate_utils.sync_app',
sync_app_mock)
get_aliases_for_index_mock = Mock(return_value=set())
monkeypatch.setattr(
'datahub.search.migrate_utils.get_aliases_for_index',
get_aliases_for_index_mock,
)
mock_client = mock_es_client.return_value
read_indices = {'index1'}
write_index = 'index1'
mock_app = create_mock_search_app(
read_indices=read_indices,
write_index=write_index,
)
resync_after_migrate(mock_app)
sync_app_mock.assert_not_called()
mock_client.indices.update_aliases.assert_not_called()
mock_client.indices.delete.assert_not_called()
def test_resync_with_deletion_error(self, monkeypatch, mock_es_client):
"""
Test that resync_after_migrate() raises an exception when there is an error deleting
documents.
"""
index_bulk_mock = Mock()
delete_bulk_mock = Mock(return_value=(True, ({
'delete': {
'status': 500
}
}, )))
monkeypatch.setattr('datahub.search.bulk_sync.bulk', index_bulk_mock)
monkeypatch.setattr('datahub.search.deletion.bulk', delete_bulk_mock)
get_aliases_for_index_mock = Mock(return_value=set())
monkeypatch.setattr(
'datahub.search.migrate_utils.get_aliases_for_index',
get_aliases_for_index_mock,
)
read_indices = {'index1', 'index2'}
write_index = 'index1'
mock_app = create_mock_search_app(
read_indices=read_indices,
write_index=write_index,
queryset=MockQuerySet([Mock(id=1), Mock(id=2)]),
)
with pytest.raises(DataHubException):
resync_after_migrate(mock_app)
def test_sync_app_with_overridden_batch_size(monkeypatch):
"""Tests syncing an app to OpenSearch with an overridden batch size."""
bulk_mock = Mock()
monkeypatch.setattr('datahub.search.bulk_sync.bulk', bulk_mock)
search_app = create_mock_search_app(queryset=MockQuerySet(
[Mock(id=1), Mock(id=2)]), )
sync_app(search_app, batch_size=1)
assert bulk_mock.call_count == 2
def test_sync_app_with_default_batch_size(monkeypatch):
"""Tests syncing an app to Elasticsearch with the default batch size."""
bulk_mock = Mock()
monkeypatch.setattr('datahub.search.bulk_sync.bulk', bulk_mock)
search_app = create_mock_search_app(queryset=MockQuerySet(
[Mock(id=1), Mock(id=2)]), )
sync_app(search_app)
assert bulk_mock.call_count == 1
def test_complete_model_migration(monkeypatch):
"""Test that the complete_model_migration task calls resync_after_migrate()."""
resync_after_migrate_mock = Mock()
monkeypatch.setattr('datahub.search.tasks.resync_after_migrate', resync_after_migrate_mock)
mock_app = create_mock_search_app(
current_mapping_hash='current-hash',
target_mapping_hash='target-hash',
)
get_search_app_mock = Mock(return_value=mock_app)
monkeypatch.setattr('datahub.search.tasks.get_search_app', get_search_app_mock)
complete_model_migration.apply_async(args=('test-app', 'target-hash'))
resync_after_migrate_mock.assert_called_once_with(mock_app)
def test_migrate_app_with_app_needing_migration(monkeypatch,
mock_opensearch_client):
"""Test that migrate_app() migrates an app needing migration."""
migrate_model_task_mock = Mock()
monkeypatch.setattr('datahub.search.migrate.complete_model_migration',
migrate_model_task_mock)
create_index_mock = Mock()
monkeypatch.setattr('datahub.search.migrate.create_index',
create_index_mock)
mock_client = mock_opensearch_client.return_value
old_index = 'test-index'
new_index = 'test-index-target-hash'
current_hash = 'current-hash'
target_hash = 'target-hash'
mock_app = create_mock_search_app(
current_mapping_hash=current_hash,
target_mapping_hash=target_hash,
write_index=old_index,
)
migrate_app(mock_app)
create_index_mock.assert_called_once_with(
new_index, mock_app.search_model._doc_type.mapping)
mock_client.indices.update_aliases.assert_called_once_with(body={
'actions': [
{
'add': {
'alias': 'test-read-alias',
'indices': [new_index],
},
},
{
'add': {
'alias': 'test-write-alias',
'indices': [new_index],
},
},
{
'remove': {
'alias': 'test-write-alias',
'indices': [old_index],
},
},
],
}, )
migrate_model_task_mock.apply_async.assert_called_once_with(
args=(mock_app.name, target_hash), )
def test_resync_with_old_index_referenced(self, monkeypatch,
mock_es_client):
"""
Test that if the old index is still referenced, resync_after_migrate() does not delete it.
"""
sync_app_mock = Mock()
monkeypatch.setattr('datahub.search.migrate_utils.sync_app',
sync_app_mock)
get_aliases_for_index_mock = Mock(return_value={'another-index'})
monkeypatch.setattr(
'datahub.search.migrate_utils.get_aliases_for_index',
get_aliases_for_index_mock,
)
mock_client = mock_es_client.return_value
read_indices = {'index1', 'index2'}
write_index = 'index1'
mock_app = create_mock_search_app(
read_indices=read_indices,
write_index=write_index,
)
resync_after_migrate(mock_app)
sync_app_mock.assert_called_once_with(
mock_app,
post_batch_callback=delete_from_secondary_indices_callback,
)
mock_client.indices.update_aliases.assert_called_once_with(body={
'actions': [
{
'remove': {
'alias': 'test-read-alias',
'indices': ['index2'],
},
},
],
}, )
mock_client.indices.delete.assert_not_called()
def test_migrate_app_with_app_not_needing_migration(monkeypatch, mock_es_client):
"""Test that migrate_app() migrates an app needing migration."""
migrate_model_task_mock = Mock()
monkeypatch.setattr('datahub.search.migrate.complete_model_migration', migrate_model_task_mock)
mock_client = mock_es_client.return_value
old_index = 'test-index-current-hash'
current_hash = 'current-hash'
target_hash = 'current-hash'
mock_app = create_mock_search_app(
current_mapping_hash=current_hash,
target_mapping_hash=target_hash,
write_index=old_index,
)
migrate_app(mock_app)
mock_app.es_model.create_index.assert_not_called()
mock_client.indices.update_aliases.assert_not_called()
migrate_model_task_mock.apply_async.assert_not_called()
def test_normal_resync(self, monkeypatch, mock_es_client):
"""
Test that resync_after_migrate() resyncs the app, updates the read alias and deletes the
old index.
"""
index_bulk_mock = Mock()
delete_bulk_mock = Mock(return_value=(True, ({
'delete': {
'status': 404
}
}, )))
monkeypatch.setattr('datahub.search.bulk_sync.bulk', index_bulk_mock)
monkeypatch.setattr('datahub.search.deletion.bulk', delete_bulk_mock)
get_aliases_for_index_mock = Mock(return_value=set())
monkeypatch.setattr(
'datahub.search.migrate_utils.get_aliases_for_index',
get_aliases_for_index_mock,
)
mock_client = mock_es_client.return_value
read_indices = {'index1', 'index2'}
write_index = 'index1'
mock_app = create_mock_search_app(
read_indices=read_indices,
write_index=write_index,
queryset=MockQuerySet([Mock(id=1), Mock(id=2)]),
)
resync_after_migrate(mock_app)
assert index_bulk_mock.call_count == 1
assert index_bulk_mock.call_args_list[0][1]['actions'] == [
{
'_index': 'index1',
'_id': 1,
'_type': 'test-type',
},
{
'_index': 'index1',
'_id': 2,
'_type': 'test-type',
},
]
assert delete_bulk_mock.call_count == 1
assert list(delete_bulk_mock.call_args_list[0][1]['actions']) == [
{
'_index': 'index2',
'_id': 1,
'_op_type': 'delete',
'_type': 'test-type',
},
{
'_index': 'index2',
'_id': 2,
'_op_type': 'delete',
'_type': 'test-type',
},
]
mock_client.indices.update_aliases.assert_called_once()
actions = mock_client.indices.update_aliases.call_args_list[0][1][
'body']['actions']
actions[0]['remove']['indices'] = sorted(
actions[0]['remove']['indices'])
mock_client.indices.update_aliases.assert_called_once_with(body={
'actions': [
{
'remove': {
'alias': 'test-read-alias',
'indices': ANY,
},
},
],
}, )
actions = mock_client.indices.update_aliases.call_args_list[0][1][
'body']['actions']
assert sorted(actions[0]['remove']['indices']) == ['index2']
assert mock_client.indices.delete.call_count == 1
mock_client.indices.delete.assert_any_call('index2')
