def test_indexer_process(es_client):
mappings = TypeMappingRegistry()
mappings.register_type('page', {
'title': {'type': 'localized'},
})
index = "foo_bar-my_schema-en-page"
indexer = Indexer(
mappings, Queue(), hostname='foo.bar', es_client=es_client)
indexer.queue.put({
'action': 'index',
'schema': 'my-schema',
'type_name': 'page',
'id': 1,
'language': 'en',
'properties': {
'title': 'Go ahead and jump',
'es_public': True
}
})
assert indexer.process() == 1
assert indexer.process() == 0
es_client.indices.refresh(index=index)
search = es_client.search(index=index)
assert search['hits']['total'] == 1
assert search['hits']['hits'][0]['_id'] == '1'
assert search['hits']['hits'][0]['_source'] == {
'title': 'Go ahead and jump',
'es_public': True
}
assert search['hits']['hits'][0]['_type'] == 'page'
# check if the analyzer was applied correctly (stopword removal)
search = es_client.search(
index=index, body={'query': {'match': {'title': 'and'}}})
assert search['hits']['total'] == 0
search = es_client.search(
index=index, body={'query': {'match': {'title': 'go jump'}}})
assert search['hits']['total'] == 1
# delete the document again
indexer.queue.put({
'action': 'delete',
'schema': 'my-schema',
'type_name': 'page',
'id': 1
})
assert indexer.process() == 1
assert indexer.process() == 0
es_client.indices.refresh(index=index)
es_client.search(index=index)
assert search['hits']['total'] == 1
def test_orm_event_queue_overflow(capturelog):
capturelog.setLevel(logging.ERROR, logger='onegov.search')
class Tweet(Searchable):
def __init__(self, id):
self.id = id
@property
def es_suggestion(self):
return self.id
es_id = 'id'
es_type_name = 'tweet'
es_language = 'en'
es_public = True
es_properties = {}
mappings = TypeMappingRegistry()
mappings.register_type('tweet', {})
translator = ORMEventTranslator(mappings, max_queue_size=3)
translator.on_insert('foobar', Tweet(1))
translator.on_update('foobar', Tweet(2))
translator.on_delete('foobar', Tweet(3))
assert len(capturelog.records()) == 0
translator.on_insert('foobar', Tweet(4))
assert len(capturelog.records()) == 1
assert capturelog.records()[0].message == \
'The orm event translator queue is full!'
def test_elasticsearch_outage(es_client, es_url):
mappings = TypeMappingRegistry()
mappings.register_type('page', {
'title': {'type': 'localized'},
})
indexer = Indexer(
mappings, Queue(), hostname='foo.bar', es_client=es_client)
indexer.queue.put({
'action': 'index',
'schema': 'my-schema',
'type_name': 'page',
'id': 1,
'language': 'en',
'properties': {
'title': 'Foo',
'es_public': True
}
})
indexer.es_client.index = Mock(side_effect=TransportError)
for i in range(0, 2):
assert indexer.process() == 0
assert indexer.queue.empty()
assert indexer.failed_task is not None
indexer.queue.put({
'action': 'index',
'schema': 'my-schema',
'type_name': 'page',
'id': 2,
'language': 'en',
'properties': {
'title': 'Bar',
'es_public': True
}
})
for i in range(0, 2):
assert indexer.process() == 0
assert not indexer.queue.empty()
assert indexer.failed_task is not None
indexer.es_client = Elasticsearch(es_url)
indexer.es_client.indices.refresh(index='_all')
assert indexer.es_client.search(index='_all')['hits']['total'] == 0
assert indexer.process() == 2
assert indexer.failed_task is None
indexer.es_client.indices.refresh(index='_all')
assert indexer.es_client.search(index='_all')['hits']['total'] == 2
def test_orm_event_translator_delete():
class Page(Searchable):
def __init__(self, id):
self.id = id
es_id = 'id'
es_type_name = 'page'
mappings = TypeMappingRegistry()
mappings.register_type('page', {})
translator = ORMEventTranslator(mappings)
translator.on_delete('foobar', Page(123))
assert translator.queue.get() == {
'action': 'delete',
'schema': 'foobar',
'type_name': 'page',
'id': 123
}
assert translator.queue.empty()
def test_type_mapping_registry():
registry = TypeMappingRegistry()
registry.register_type('page', {
'title': {'type': 'string'}
})
registry.register_type('comment', {
'comment': {'type': 'string'}
})
assert set(t.name for t in registry) == {'page', 'comment'}
with pytest.raises(AssertionError):
registry.register_type('page', {})
def test_orm_event_translator_properties():
class Page(Searchable):
es_id = 'id'
es_type_name = 'page'
es_properties = {
'title': {'type': 'localized'},
'body': {'type': 'localized'},
'tags': {'type': 'string'},
'date': {'type': 'date'},
'published': {'type': 'boolean'},
'likes': {'type': 'long'}
}
def __init__(self, id, **kwargs):
self.id = id
self.language = kwargs.pop('language', 'en')
self.public = kwargs.pop('public', True)
for k, v in kwargs.items():
setattr(self, k, v)
@property
def es_language(self):
return self.language
@property
def es_public(self):
return self.public
@property
def es_suggest(self):
return self.title
mappings = TypeMappingRegistry()
mappings.register_type('page', Page.es_properties)
translator = ORMEventTranslator(mappings)
for on_event in (translator.on_insert, translator.on_update):
on_event('my-schema', Page(
id=1,
title='About',
body='We are Pied Piper',
tags=['aboutus', 'company'],
date=datetime(2015, 9, 11),
published=True,
likes=1000
))
assert translator.queue.get() == {
'action': 'index',
'schema': 'my-schema',
'type_name': 'page',
'id': 1,
'language': 'en',
'properties': {
'title': 'About',
'body': 'We are Pied Piper',
'tags': ['aboutus', 'company'],
'date': '2015-09-11T00:00:00',
'likes': 1000,
'published': True,
'es_public': True,
'es_public_categories': ['public'],
'es_suggestion': {
'input': ['About'],
'output': 'About'
}
}
}
assert translator.queue.empty()
