def test_do_update(self, mock_obj):
"""
Tests the update_index.do_update function
"""
engine = ElasticsearchMultilingualSearchEngine()
backend = mock.Mock()
engine._backend = backend
index = engine.get_unified_index()
qs = Document.objects.all()
start = 0
end = len(qs)
total = end - start
do_update(backend,
index,
qs,
start,
end,
total,
verbosity=1,
commit=False)
# The update method has been called
self.assertTrue(backend.update.called)
call_args_list = backend.update.call_args_list
# args, the queryset cannot be testet for equality.
self.assertEqual(call_args_list[0][0][0], index)
# kwargs
self.assertEqual(call_args_list[0][1], {'commit': False})
def test_query_method_modeltranslations(self):
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.setup()
# fill up the index
self.index_exists = True
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = Document.objects.all()
es.update(index, iterable)
time.sleep(1)
id = 'testproject.document.2'
i = 0
# use this document as a reference.
reference = Document.objects.get(id=2)
for language in es.languages:
# check all language indexes
index_name = es._index_name_for_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
self.assertEqual(self.count, count['count'])
# make sure the index has been created
while not es.conn.exists(index=index_name, id=id) and i < 5:
time.sleep(0.5)
i += 1
self.assertTrue(es.conn.exists(index=index_name, id=id))
# get the document with the above id from the correct index
doc = es.conn.get(index=index_name, id=id)
self.assertTrue(doc['found'])
self.assertEqual(doc['_type'], 'modelresult')
self.assertEqual(doc['_source']['docid'], reference.docid)
with translation.override(language):
self.assertIn(escape(reference.text), doc['_source']['text'])
with translation.override('en'):
# test the search queryset
sqs = SearchQuerySet()
# Django 1.5 doesn't clean up patches
self.assertFalse(isinstance(sqs.query.backend.conn, mock.Mock))
result = sqs.filter(content='United States')
# result might be empty the first time this is called. (Django < 1.8)
if len(result) == 0:
result = sqs.filter(content='United States')
self.assertEqual(2, len(result))
self.assertEqual('cyberpresse/2012/12/01/1564248', result[0].docid)
self.assertEqual('cyberpresse/2012/12/01/1564741', result[1].docid)
self.assertIn(escape(reference.text), result[1].text)
with translation.override('de'):
# test the search queryset
sqs = SearchQuerySet()
result = sqs.filter(content='Früherkennungstest')
self.assertEqual(1, len(result))
self.assertEqual('cyberpresse/2012/12/01/1564741', result[0].docid)
self.assertIn(escape(reference.text), result[0].text)
def test_query_method_modeltranslations(self):
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.setup()
# fill up the index
self.index_exists = True
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = Document.objects.all()
es.update(index, iterable)
time.sleep(1)
id = 'testproject.document.2'
i = 0
# use this document as a reference.
reference = Document.objects.get(id=2)
for language in es.languages:
# check all language indexes
index_name = es._index_name_for_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
self.assertEqual(self.count, count['count'])
# make sure the index has been created
while not es.conn.exists(index=index_name, doc_type='_all', id=id) and i < 5:
time.sleep(0.5)
i += 1
self.assertTrue(es.conn.exists(index=index_name, doc_type='_all', id=id))
# get the document with the above id from the correct index
doc = es.conn.get(index=index_name, id=id)
self.assertTrue(doc['found'])
self.assertEqual(doc['_type'], 'modelresult')
self.assertEqual(doc['_source']['docid'], reference.docid)
with translation.override(language):
self.assertIn(escape(reference.text), doc['_source']['text'])
with translation.override('en'):
# test the search queryset
sqs = SearchQuerySet()
# Django 1.5 doesn't clean up patches
self.assertFalse(isinstance(sqs.query.backend.conn, mock.Mock))
result = sqs.filter(content='United States')
# result might be empty the first time this is called. (Django < 1.8)
if len(result) == 0:
result = sqs.filter(content='United States')
self.assertEqual(2, len(result))
self.assertEqual('cyberpresse/2012/12/01/1564248', result[0].docid)
self.assertEqual('cyberpresse/2012/12/01/1564741', result[1].docid)
self.assertIn(escape(reference.text), result[1].text)
with translation.override('de'):
# test the search queryset
sqs = SearchQuerySet()
result = sqs.filter(content='Früherkennungstest')
self.assertEqual(1, len(result))
self.assertEqual('cyberpresse/2012/12/01/1564741', result[0].docid)
self.assertIn(escape(reference.text), result[0].text)
def test_live_query(self):
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.setup()
sqs = SearchQuerySet()
self.assertFalse(sqs.query.has_run())
self.assertIsInstance(sqs.query, ElasticsearchMultilingualSearchQuery)
all_results = sqs.all()
self.assertEqual(list(all_results), []) # execute a query
self.assertTrue(isinstance(all_results, SearchQuerySet))
def test_live_query(self):
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.setup()
sqs = SearchQuerySet()
self.assertFalse(sqs.query.has_run())
self.assertIsInstance(sqs.query, ElasticsearchMultilingualSearchQuery)
all_results = sqs.all()
self.assertEqual(list(all_results), []) # execute a query
self.assertTrue(isinstance(all_results, SearchQuerySet))
def test_hook_called(self):
documents = create_documents()
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.clear(models=None, commit=True)
self.assertTrue(signals.post_save.has_listeners(sender=Document))
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = Document.objects.all()
es.update(index, iterable)
time.sleep(1)
self.assertTrue(es.conn.indices.exists('testproject-en'))
count = es.conn.count(index='testproject-en')
self.assertEqual(0, count['count'])
# write a new entry in the database
documents[0].save()
reference = documents[0].object
self.assertIsNotNone(reference.id)
id = 'testproject.document.1'
time.sleep(0.5)
self.assertFalse(isinstance(es.conn, mock.Mock))
for language in es.languages:
index_name = es._index_name_for_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
print(count)
self.assertEqual(1, count['count'],
'Index %s contains the document' % index_name)
self.assertTrue(
es.conn.exists(index=index_name, doc_type='_all', id=id))
doc = es.conn.get(index=index_name, id=id)
self.assertTrue(doc['found'])
self.assertEqual(doc['_type'], 'modelresult')
self.assertEqual(doc['_source']['docid'], reference.docid)
with translation.override(language):
self.assertIn(escape(reference.text), doc['_source']['text'])
# delete the object. The object needs to be removed from the index.
reference.delete()
for language in es.languages:
index_name = es._index_name_for_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
self.assertEqual(0, count['count'])
def test_hook_called(self):
documents = create_documents()
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.clear(models=None, commit=True)
self.assertTrue(signals.post_save.has_listeners(sender=Document))
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = Document.objects.all()
es.update(index, iterable)
time.sleep(1)
self.assertTrue(es.conn.indices.exists('testproject-en'))
count = es.conn.count(index='testproject-en')
self.assertEqual(0, count['count'])
# write a new entry in the database
documents[0].save()
reference = documents[0].object
self.assertIsNotNone(reference.id)
id = 'testproject.document.1'
time.sleep(0.5)
self.assertFalse(isinstance(es.conn, mock.Mock))
for language in es.languages:
index_name = es._index_name_for_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
print(count)
self.assertEqual(1, count['count'], 'Index %s contains the document' % index_name)
self.assertTrue(es.conn.exists(index=index_name, doc_type='_all', id=id))
doc = es.conn.get(index=index_name, id=id)
self.assertTrue(doc['found'])
self.assertEqual(doc['_type'], 'modelresult')
self.assertEqual(doc['_source']['docid'], reference.docid)
with translation.override(language):
self.assertIn(escape(reference.text), doc['_source']['text'])
# delete the object. The object needs to be removed from the index.
reference.delete()
for language in es.languages:
index_name = es._index_name_for_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
self.assertEqual(0, count['count'])
def test_parler_modeltranslations_update(self):
"""
Test the update method on the multiligual backend for parler models.
"""
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
# the indexes don't exist yet.
for language in es.languages:
index_name = es._index_name_for_language(language)
self.assertFalse(es.conn.indices.exists(index_name))
es.setup()
self.index_exists = True
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = ParlerDocument.objects.prefetch_related(
'translations').all()
es.update(index, iterable)
time.sleep(1)
id = 'testproject.parlerdocument.1'
i = 0
# use this document as a reference.
reference = ParlerDocument.objects.get(id=1)
for language in es.languages:
# check all language indexes
index_name = es._index_name_for_language(language)
reference.set_current_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
self.assertEqual(3, count['count'])
# make sure the index has been created
while not es.conn.exists(index=index_name, doc_type='_all',
id=id) and i < 5:
time.sleep(0.5)
i += 1
self.assertTrue(
es.conn.exists(index=index_name, doc_type='_all', id=id))
# get the document with the above id from the correct index
doc = es.conn.get(index=index_name, id=id)
self.assertTrue(doc['found'])
self.assertEqual(doc['_type'], 'modelresult')
self.assertEqual(doc['_source']['docid'], reference.docid)
with translation.override(language):
self.assertIn(escape(reference.text), doc['_source']['text'])
def test_parler_modeltranslations_update(self):
"""
Test the update method on the multiligual backend for parler models.
"""
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
# the indexes don't exist yet.
for language in es.languages:
index_name = es._index_name_for_language(language)
self.assertFalse(es.conn.indices.exists(index_name))
es.setup()
self.index_exists = True
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = ParlerDocument.objects.prefetch_related('translations').all()
es.update(index, iterable)
time.sleep(1)
id = 'testproject.parlerdocument.1'
i = 0
# use this document as a reference.
reference = ParlerDocument.objects.get(id=1)
for language in es.languages:
# check all language indexes
index_name = es._index_name_for_language(language)
reference.set_current_language(language)
self.assertTrue(es.conn.indices.exists(index_name))
count = es.conn.count(index=index_name)
self.assertEqual(3, count['count'])
# make sure the index has been created
while not es.conn.exists(index=index_name, id=id) and i < 5:
time.sleep(0.5)
i += 1
self.assertTrue(es.conn.exists(index=index_name, id=id))
# get the document with the above id from the correct index
doc = es.conn.get(index=index_name, id=id)
self.assertTrue(doc['found'])
self.assertEqual(doc['_type'], 'modelresult')
self.assertEqual(doc['_source']['docid'], reference.docid)
with translation.override(language):
self.assertIn(escape(reference.text), doc['_source']['text'])
def test_do_update(self, mock_obj):
"""
Tests the update_index.do_update function
"""
engine = ElasticsearchMultilingualSearchEngine()
backend = mock.Mock()
engine._backend = backend
index = engine.get_unified_index()
qs = Document.objects.all()
start = 0
end = len(qs)
total = end - start
do_update(backend, index, qs, start, end, total, verbosity=1, commit=False)
# The update method has been called
self.assertTrue(backend.update.called)
call_args_list = backend.update.call_args_list
# args, the queryset cannot be testet for equality.
self.assertEqual(call_args_list[0][0][0], index)
# kwargs
self.assertEqual(call_args_list[0][1], {"commit": False})
def test_multilingual_update(self, mock_obj):
"""
Test the update method on the multilingual backend.
"""
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.setup()
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = Document.objects.all()
es.update(index, iterable)
es.conn.indices.refresh.assert_any_call(index='testproject-en')
es.conn.indices.refresh.assert_any_call(index='testproject-de')
es.conn.indices.refresh.assert_any_call(index='testproject-fr')
self.assertTrue(es.conn.bulk.called)
self.assertEqual(len(es.conn.bulk.call_args_list), len(settings.LANGUAGES))
for call_args in es.conn.bulk.call_args_list:
language = call_args[1]['index'][-2:]
content = call_args[0][0][1]
document = Document.objects.get(id=content['django_id'])
self.assertEqual(document.docid, content['docid'])
with translation.override(language):
self.assertIn(escape(document.text), content['text'])
def test_multilingual_update(self, mock_obj):
"""
Test the update method on the multilingual backend.
"""
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend("default", **Data.connection_options)
es.setup()
unified_index = engine.get_unified_index()
index = unified_index.get_index(Document)
iterable = Document.objects.all()
es.update(index, iterable)
es.conn.indices.refresh.assert_any_call(index="testproject-en")
es.conn.indices.refresh.assert_any_call(index="testproject-de")
es.conn.indices.refresh.assert_any_call(index="testproject-fr")
self.assertTrue(es.conn.bulk.called)
self.assertEqual(len(es.conn.bulk.call_args_list), len(settings.LANGUAGES))
for call_args in es.conn.bulk.call_args_list:
language = call_args[1]["index"][-2:]
content = call_args[0][0][1]
document = Document.objects.get(id=content["django_id"])
self.assertEqual(document.docid, content["docid"])
with translation.override(language):
self.assertIn(escape(document.text), content["text"])
def tearDown(self):
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.clear(models=None, commit=True)
self.sp.teardown()
time.sleep(1)
def tearDown(self):
if self.index_exists:
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.clear(commit=True)
time.sleep(1)
def tearDown(self):
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.clear(models=None, commit=True)
self.sp.teardown()
time.sleep(1)
def tearDown(self):
if self.index_exists:
engine = ElasticsearchMultilingualSearchEngine()
es = engine.backend('default', **Data.connection_options)
es.clear(commit=True)
time.sleep(1)
