コード例 #1
0
def test_delete_documents_with_errors(es_bulk, mock_es_client):
    """Test that if ES returns a non-404 error, DataHubException is raised."""
    es_bulk.return_value = (
        None,
        [
            {
                'delete': {
                    'status': 404
                }
            },
            {
                'delete': {
                    'status': 500
                }
            },
        ],
    )

    index = 'test-index'
    es_docs = [{'_type': 'model', '_id': 1}]

    with pytest.raises(DataHubException) as excinfo:
        delete_documents(index, es_docs)

    assert excinfo.value.args == ((
        'One or more errors during an Elasticsearch bulk deletion '
        "operation: [{'delete': {'status': 500}}]"), )
コード例 #2
0
def test_delete_documents(es_bulk, mock_es_client):
    """Test that delete_documents calls ES bulk to delete all documents."""
    es_bulk.return_value = (None, [])

    index = 'test-index'
    es_docs = [
        {'_type': 'model', '_id': 1},
        {'_type': 'model', '_id': 2},
        {'_type': 'model', '_id': 3},
    ]
    delete_documents(index, es_docs)

    assert es_bulk.call_count == 1

    call_args, call_kwargs = es_bulk.call_args_list[0]
    call_kwargs['actions'] = list(call_kwargs['actions'])  # from generator to list
    assert call_args == (mock_es_client.return_value,)
    assert call_kwargs == {
        'actions': [
            {'_op_type': 'delete', '_index': index, **es_doc}
            for es_doc in es_docs
        ],
        'chunk_size': BULK_CHUNK_SIZE,
        'request_timeout': BULK_DELETION_TIMEOUT_SECS,
        'max_chunk_bytes': settings.ES_BULK_MAX_CHUNK_BYTES,
        'raise_on_error': False,
    }
コード例 #3
0
def test_delete_documents_with_errors(opensearch_bulk, mock_opensearch_client):
    """Test that if OpenSearch returns a non-404 error, DataHubError is raised."""
    opensearch_bulk.return_value = (
        None,
        [
            {
                'delete': {
                    'status': 404
                }
            },
            {
                'delete': {
                    'status': 500
                }
            },
        ],
    )

    index = 'test-index'
    docs = [{'_id': 1}]

    with pytest.raises(DataHubError) as excinfo:
        delete_documents(index, docs)

    assert excinfo.value.args == ((
        "Errors during an OpenSearch bulk deletion operation: [{'delete': {'status': 500}}]"
    ), )
コード例 #4
0
def delete_from_secondary_indices_callback(read_indices, write_index, actions):
    """
    Callback for sync_app() and sync_objects() that deletes synced documents from any indices
    that are currently being migrated from.

    This is used to avoid multiple, differing copies of documents existing at the same time
    whilst documents are being migrated from one index to another.
    """
    remove_indices = read_indices - {write_index}
    for index in remove_indices:
        delete_documents(index, actions)