def test_indexer_process(es_client):
mappings = TypeMappingRegistry()
mappings.register_type('page', {
'title': {'type': 'localized'},
})
index = "foo_bar-my_schema-en-page"
indexer = Indexer(
mappings, Queue(), hostname='foo.bar', es_client=es_client)
indexer.queue.put({
'action': 'index',
'schema': 'my-schema',
'type_name': 'page',
'id': 1,
'language': 'en',
'properties': {
'title': 'Go ahead and jump',
'es_public': True
}
})
assert indexer.process() == 1
assert indexer.process() == 0
es_client.indices.refresh(index=index)
search = es_client.search(index=index)
assert search['hits']['total'] == 1
assert search['hits']['hits'][0]['_id'] == '1'
assert search['hits']['hits'][0]['_source'] == {
'title': 'Go ahead and jump',
'es_public': True
}
assert search['hits']['hits'][0]['_type'] == 'page'
# check if the analyzer was applied correctly (stopword removal)
search = es_client.search(
index=index, body={'query': {'match': {'title': 'and'}}})
assert search['hits']['total'] == 0
search = es_client.search(
index=index, body={'query': {'match': {'title': 'go jump'}}})
assert search['hits']['total'] == 1
# delete the document again
indexer.queue.put({
'action': 'delete',
'schema': 'my-schema',
'type_name': 'page',
'id': 1
})
assert indexer.process() == 1
assert indexer.process() == 0
es_client.indices.refresh(index=index)
es_client.search(index=index)
assert search['hits']['total'] == 1
def test_elasticsearch_outage(es_client, es_url):
mappings = TypeMappingRegistry()
mappings.register_type('page', {
'title': {'type': 'localized'},
})
indexer = Indexer(
mappings, Queue(), hostname='foo.bar', es_client=es_client)
indexer.queue.put({
'action': 'index',
'schema': 'my-schema',
'type_name': 'page',
'id': 1,
'language': 'en',
'properties': {
'title': 'Foo',
'es_public': True
}
})
indexer.es_client.index = Mock(side_effect=TransportError)
for i in range(0, 2):
assert indexer.process() == 0
assert indexer.queue.empty()
assert indexer.failed_task is not None
indexer.queue.put({
'action': 'index',
'schema': 'my-schema',
'type_name': 'page',
'id': 2,
'language': 'en',
'properties': {
'title': 'Bar',
'es_public': True
}
})
for i in range(0, 2):
assert indexer.process() == 0
assert not indexer.queue.empty()
assert indexer.failed_task is not None
indexer.es_client = Elasticsearch(es_url)
indexer.es_client.indices.refresh(index='_all')
assert indexer.es_client.search(index='_all')['hits']['total'] == 0
assert indexer.process() == 2
assert indexer.failed_task is None
indexer.es_client.indices.refresh(index='_all')
assert indexer.es_client.search(index='_all')['hits']['total'] == 2
