def test_init(self, mock_set):
obj = es.ES(source='Foo')
assert obj.index_name == mock_set.index_name
assert obj.doc_type == 'Foo'
assert obj.chunk_size == mock_set.asint()
obj = es.ES(source='Foo', index_name='a', chunk_size=2)
assert obj.index_name == 'a'
assert obj.doc_type == 'Foo'
assert obj.chunk_size == 2
def test_get_item_no_index_not_raise(self, mock_get):
obj = es.ES('Foo', 'foondex')
mock_get.side_effect = es.IndexNotFoundException()
try:
obj.get_item(name='foo', _raise_on_empty=False)
except JHTTPNotFound:
raise Exception('Unexpected error')
def test_get_by_ids_fields(self, mock_mget):
obj = es.ES('Foo', 'foondex')
documents = [{'_id': 1, '_type': 'Story'}]
mock_mget.return_value = {
'docs': [{
'_type': 'foo',
'_id': 1,
'_source': {
'_id': 1,
'_type': 'Story',
'name': 'bar'
},
}]
}
docs = obj.get_by_ids(documents, _limit=1, _fields=['name'])
mock_mget.assert_called_once_with(
body={'docs': [{
'_index': 'foondex',
'_type': 'Story',
'_id': 1
}]},
_source_include=['name', '_type'],
_source=True)
assert len(docs) == 1
assert hasattr(docs[0], '_id')
assert hasattr(docs[0], '_type')
assert docs[0].name == 'bar'
assert docs._nefertari_meta['total'] == 1
assert docs._nefertari_meta['start'] == 0
assert sorted(docs._nefertari_meta['fields']) == sorted(
['name', '_type'])
def test_get_item_not_found_not_raise(self, mock_get):
obj = es.ES('Foo', 'foondex')
mock_get.return_value = {}
try:
obj.get_item(name='foo', _raise_on_empty=False)
except JHTTPNotFound:
raise Exception('Unexpected error')
def test_get_by_ids(self, mock_mget):
obj = es.ES('Foo', 'foondex')
documents = [{'_id': 1, '_type': 'Story'}]
mock_mget.return_value = {
'docs': [{
'_type': 'Foo2',
'_id': 1,
'_source': {
'_id': 1,
'_type': 'Story',
'name': 'bar'
},
'fields': {
'name': 'bar'
}
}]
}
docs = obj.get_by_ids(documents, _page=0)
mock_mget.assert_called_once_with(
body={'docs': [{
'_index': 'foondex',
'_type': 'Story',
'_id': 1
}]})
assert len(docs) == 1
assert docs[0]._id == 1
assert docs[0].name == 'bar'
assert docs[0]._type == 'Foo2'
assert docs._nefertari_meta['total'] == 1
assert docs._nefertari_meta['start'] == 0
assert docs._nefertari_meta['fields'] == []
def test_build_search_params_with_body(self):
obj = es.ES('Foo', 'foondex')
params = obj.build_search_params({
'body': {
'query': {
'query_string': 'foo'
}
},
'_raw_terms': ' AND q:5',
'_limit': 10,
'_search_fields': 'a,b',
'_fields': ['a'],
'_sort': '+a,-b,c',
})
assert sorted(params.keys()) == sorted(
['body', 'doc_type', 'fields', 'from_', 'index', 'size', 'sort'])
assert params['body'] == {
'query': {
'query_string': {
'fields': ['b^1', 'a^2'],
'query': 'foo'
}
}
}
assert params['index'] == 'foondex'
assert params['doc_type'] == 'Foo'
assert params['fields'] == ['a']
assert params['sort'] == 'a:asc,b:desc,c:asc'
def test_get_collection_source(self, mock_search):
obj = es.ES('Foo', 'foondex')
mock_search.return_value = {
'hits': {
'hits': [{
'_source': {
'foo': 'bar',
'id': 1
},
'_score': 2,
'_type': 'Zoo'
}],
'total':
4,
},
'took': 2.8,
}
docs = obj.get_collection(body={'foo': 'bar'}, from_=0)
mock_search.assert_called_once_with(body={'foo': 'bar'}, from_=0)
assert len(docs) == 1
assert docs[0].id == 1
assert docs[0]._score == 2
assert docs[0].foo == 'bar'
assert docs[0]._type == 'Zoo'
assert docs._nefertari_meta['total'] == 4
assert docs._nefertari_meta['start'] == 0
assert docs._nefertari_meta['fields'] == ''
assert docs._nefertari_meta['took'] == 2.8
def test_get_by_ids_no_index_raise(self, mock_mget):
obj = es.ES('Foo', 'foondex')
documents = [{'_id': 1, '_type': 'Story'}]
mock_mget.side_effect = es.IndexNotFoundException()
with pytest.raises(JHTTPNotFound) as ex:
obj.get_by_ids(documents, _raise_on_empty=True)
assert 'resource not found (Index does not exist)' in str(ex.value)
def test_build_search_params_no_body_no_qs(self):
obj = es.ES('Foo', 'foondex')
params = obj.build_search_params({'_limit': 10})
assert sorted(params.keys()) == sorted(
['body', 'doc_type', 'from_', 'size', 'index'])
assert params['body'] == {'query': {'match_all': {}}}
assert params['index'] == 'foondex'
assert params['doc_type'] == 'Foo'
def test_get_collection_no_index_raise(self, mock_search):
obj = es.ES('Foo', 'foondex')
mock_search.side_effect = es.IndexNotFoundException()
with pytest.raises(JHTTPNotFound) as ex:
obj.get_collection(body={'foo': 'bar'},
_raise_on_empty=True,
from_=0)
assert 'resource not found (Index does not exist)' in str(ex.value)
def test_delete_single_obj(self, mock_bulk):
obj = es.ES('Foo', 'foondex')
obj.delete(ids=1)
mock_bulk.assert_called_once_with('delete', [{
'_pk': 1,
'_type': 'Foo'
}],
request=None)
def test_process_chunks_multiple(self):
obj = es.ES('Foo', 'foondex', chunk_size=3)
operation = Mock()
documents = [1, 2, 3, 4, 5]
obj.process_chunks(documents, operation)
operation.assert_has_calls([
call(documents_actions=[1, 2, 3]),
call(documents_actions=[4, 5]),
])
def test_get_by_ids_not_found_not_raise(self, mock_mget):
obj = es.ES('Foo', 'foondex')
documents = [{'_id': 1, '_type': 'Story'}]
mock_mget.return_value = {'docs': [{'_type': 'foo', '_id': 1}]}
try:
docs = obj.get_by_ids(documents, _raise_on_empty=False)
except JHTTPNotFound:
raise Exception('Unexpected error')
assert len(docs) == 0
def test_get_by_ids_no_index_not_raise(self, mock_mget):
obj = es.ES('Foo', 'foondex')
documents = [{'_id': 1, '_type': 'Story'}]
mock_mget.side_effect = es.IndexNotFoundException()
try:
docs = obj.get_by_ids(documents, _raise_on_empty=False)
except JHTTPNotFound:
raise Exception('Unexpected error')
assert len(docs) == 0
def test_get_item(self, mock_get):
obj = es.ES('Foo', 'foondex')
mock_get.return_value = {'foo': 'bar', 'id': 4, '_type': 'Story'}
story = obj.get_item(name='foo')
assert story.id == 4
assert story.foo == 'bar'
mock_get.assert_called_once_with(name='foo',
index='foondex',
doc_type='Foo')
def test_get_collection_no_index_not_raise(self, mock_search):
obj = es.ES('Foo', 'foondex')
mock_search.side_effect = es.IndexNotFoundException()
try:
docs = obj.get_collection(body={'foo': 'bar'},
_raise_on_empty=False,
from_=0)
except JHTTPNotFound:
raise Exception('Unexpected error')
assert len(docs) == 0
def test_do_count_no_index(self, mock_count):
obj = es.ES('Foo', 'foondex')
mock_count.side_effect = es.IndexNotFoundException()
val = obj.do_count({
'foo': 1,
'size': 2,
'from_': 0,
'sort': 'foo:asc'
})
assert val == 0
mock_count.assert_called_once_with(foo=1)
def test_do_count(self, mock_count):
obj = es.ES('Foo', 'foondex')
mock_count.return_value = {'count': 123}
val = obj.do_count({
'foo': 1,
'size': 2,
'from_': 0,
'sort': 'foo:asc'
})
assert val == 123
mock_count.assert_called_once_with(foo=1)
def test_get_collection_count_with_body(self, mock_count, mock_build):
obj = es.ES('Foo', 'foondex')
obj.get_collection(_count=True, foo=1, body={'foo': 'bar'})
mock_count.assert_called_once_with({
'body': {
'foo': 'bar'
},
'_count': True,
'foo': 1
})
assert not mock_build.called
def test_get_collection_count_with_body(self, mock_count):
obj = es.ES('Foo', 'foondex')
obj.get_collection(_count=True, foo=1, body={'foo': 'bar'})
mock_count.assert_called_once_with({
'body': {
'foo': 'bar'
},
'doc_type': 'Foo',
'from_': 0,
'size': 1,
'index': 'foondex'
})
def test_get_collection_not_found_raise(self, mock_search):
obj = es.ES('Foo', 'foondex')
mock_search.return_value = {
'hits': {
'hits': [],
'total': 4,
},
'took': 2.8,
}
with pytest.raises(JHTTPNotFound):
obj.get_collection(body={'foo': 'bar'},
_raise_on_empty=True,
from_=0)
def test_aggregation_nothing_returned(self, mock_search, mock_build):
mock_search.return_value = {}
mock_build.return_value = {
'size': 1,
'from_': 2,
'sort': 3,
'body': {
'query': 'query1'
}
}
obj = es.ES('Foo', 'foondex')
with pytest.raises(JHTTPNotFound) as ex:
obj.aggregate(_aggregations_params={'zoo': 5}, param1=6)
assert 'No aggregations returned from ES' in str(ex.value)
def test_get_collection_not_found_not_raise(self, mock_search):
obj = es.ES('Foo', 'foondex')
mock_search.return_value = {
'hits': {
'hits': [],
'total': 4,
},
'took': 2.8,
}
try:
docs = obj.get_collection(body={'foo': 'bar'},
_raise_on_empty=False,
from_=0)
except JHTTPNotFound:
raise Exception('Unexpected error')
assert len(docs) == 0
def test_aggregation_index_not_exists(self, mock_search, mock_build):
mock_search.side_effect = es.IndexNotFoundException()
mock_build.return_value = {
'size': 1,
'from_': 2,
'sort': 3,
'body': {
'query': 'query1'
}
}
obj = es.ES('Foo', 'foondex')
with pytest.raises(JHTTPNotFound) as ex:
obj.aggregate(_aggregations_params={'zoo': 5},
param1=6,
_raise_on_empty=True)
assert 'Aggregation failed: Index does not exist' in str(ex.value)
def test_prep_bulk_documents_no_type(self):
obj = es.ES('Foo', 'foondex')
docs = [
{
'_pk': 'story2'
},
]
prepared = obj.prep_bulk_documents('myaction', docs)
assert len(prepared) == 1
doc2 = prepared[0]
assert sorted(doc2.keys()) == sorted(
['_op_type', '_type', '_id', '_index', '_source'])
assert doc2['_source'] == {'_pk': 'story2'}
assert doc2['_op_type'] == 'myaction'
assert doc2['_index'] == 'foondex'
assert doc2['_type'] == 'Foo'
assert doc2['_id'] == 'story2'
def test_index_missing_documents(self, mock_mget, mock_bulk):
obj = es.ES('Foo', 'foondex')
documents = [
{
'_pk': 1,
'name': 'foo'
},
{
'_pk': 2,
'name': 'bar'
},
{
'_pk': 3,
'name': 'baz'
},
]
mock_mget.return_value = {
'docs': [
{
'_id': '1',
'name': 'foo',
'found': False
},
{
'_id': '2',
'name': 'bar',
'found': True
},
{
'_id': '3',
'name': 'baz'
},
]
}
obj.index_missing_documents(documents)
mock_mget.assert_called_once_with(index='foondex',
doc_type='Foo',
fields=['_id'],
body={'ids': [1, 2, 3]})
mock_bulk.assert_called_once_with('index', [{
'_pk': 1,
'name': 'foo'
}, {
'_pk': 3,
'name': 'baz'
}], None)
def test_bulk(self, mock_proc, mock_prep, mock_part):
obj = es.ES('Foo', 'foondex', chunk_size=1)
docs = [{
'_op_type': 'index',
'_id': 'story1',
'_source': {
'_type': 'Story',
'id': 'story1',
'timestamp': 1
}
}, {
'_op_type': 'index',
'_id': 'story2',
'_source': {
'_type': 'Story',
'id': 'story2',
'timestamp': 2
}
}]
mock_prep.return_value = docs
obj._bulk('index', docs)
mock_prep.assert_called_once_with('index', docs)
mock_part.assert_called_once_with(es._bulk_body, request=None)
mock_proc.assert_called_once_with(
documents=[{
'_id': 'story1',
'_op_type': 'index',
'_timestamp': 1,
'_source': {
'timestamp': 1,
'_type': 'Story',
'id': 'story1'
}
}, {
'_id': 'story2',
'_op_type': 'index',
'_timestamp': 2,
'_source': {
'timestamp': 2,
'_type': 'Story',
'id': 'story2'
}
}],
operation=mock_part(),
)
def test_index_missing_documents_no_index(self, mock_mget, mock_bulk):
obj = es.ES('Foo', 'foondex')
documents = [
{
'_pk': 1,
'name': 'foo'
},
]
mock_mget.side_effect = es.IndexNotFoundException()
obj.index_missing_documents(documents)
mock_mget.assert_called_once_with(index='foondex',
doc_type='Foo',
fields=['_id'],
body={'ids': [1]})
mock_bulk.assert_called_once_with('index', [{
'_pk': 1,
'name': 'foo'
}], None)
def test_build_search_params_search_fields(self):
obj = es.ES('Foo', 'foondex')
params = obj.build_search_params({
'foo': 1,
'_search_fields': 'a,b',
'_limit': 10
})
assert sorted(params.keys()) == sorted(
['body', 'doc_type', 'from_', 'size', 'index'])
assert params['body'] == {
'query': {
'query_string': {
'fields': ['b^1', 'a^2'],
'query': 'foo:1'
}
}
}
assert params['index'] == 'foondex'
assert params['doc_type'] == 'Foo'
def test_build_search_params_sort(self):
obj = es.ES('Foo', 'foondex')
params = obj.build_search_params({
'foo': 1,
'_sort': '+a,-b,c',
'_limit': 10
})
assert sorted(params.keys()) == sorted(
['body', 'doc_type', 'index', 'sort', 'from_', 'size'])
assert params['body'] == {
'query': {
'query_string': {
'query': 'foo:1'
}
}
}
assert params['index'] == 'foondex'
assert params['doc_type'] == 'Foo'
assert params['sort'] == 'a:asc,b:desc,c:asc'
