def test_delete_documents_with_errors(es_bulk, mock_es_client):
"""Test that if ES returns a non-404 error, DataHubException is raised."""
es_bulk.return_value = (
None,
[
{
'delete': {
'status': 404
}
},
{
'delete': {
'status': 500
}
},
],
)
index = 'test-index'
es_docs = [{'_type': 'model', '_id': 1}]
with pytest.raises(DataHubException) as excinfo:
delete_documents(index, es_docs)
assert excinfo.value.args == ((
'One or more errors during an Elasticsearch bulk deletion '
"operation: [{'delete': {'status': 500}}]"), )
def test_delete_documents(es_bulk, mock_es_client):
"""Test that delete_documents calls ES bulk to delete all documents."""
es_bulk.return_value = (None, [])
index = 'test-index'
es_docs = [
{'_type': 'model', '_id': 1},
{'_type': 'model', '_id': 2},
{'_type': 'model', '_id': 3},
]
delete_documents(index, es_docs)
assert es_bulk.call_count == 1
call_args, call_kwargs = es_bulk.call_args_list[0]
call_kwargs['actions'] = list(call_kwargs['actions']) # from generator to list
assert call_args == (mock_es_client.return_value,)
assert call_kwargs == {
'actions': [
{'_op_type': 'delete', '_index': index, **es_doc}
for es_doc in es_docs
],
'chunk_size': BULK_CHUNK_SIZE,
'request_timeout': BULK_DELETION_TIMEOUT_SECS,
'max_chunk_bytes': settings.ES_BULK_MAX_CHUNK_BYTES,
'raise_on_error': False,
}
def test_delete_documents_with_errors(opensearch_bulk, mock_opensearch_client):
"""Test that if OpenSearch returns a non-404 error, DataHubError is raised."""
opensearch_bulk.return_value = (
None,
[
{
'delete': {
'status': 404
}
},
{
'delete': {
'status': 500
}
},
],
)
index = 'test-index'
docs = [{'_id': 1}]
with pytest.raises(DataHubError) as excinfo:
delete_documents(index, docs)
assert excinfo.value.args == ((
"Errors during an OpenSearch bulk deletion operation: [{'delete': {'status': 500}}]"
), )
def delete_from_secondary_indices_callback(read_indices, write_index, actions):
"""
Callback for sync_app() and sync_objects() that deletes synced documents from any indices
that are currently being migrated from.
This is used to avoid multiple, differing copies of documents existing at the same time
whilst documents are being migrated from one index to another.
"""
remove_indices = read_indices - {write_index}
for index in remove_indices:
delete_documents(index, actions)