def test_use_default_num_retries(self):
doc = {'id': 'doc', 'prop': 'val'}
exception = self._get_put_error(1, 0)
failing_put = test_utils.FailingFunction(
search.Index.put,
exception,
gae_search_services.DEFAULT_NUM_RETRIES,
)
add_docs_counter = test_utils.CallCounter(
gae_search_services.add_documents_to_index)
put_ctx = self.swap(search.Index, 'put', failing_put)
add_docs_ctx = self.swap(
gae_search_services,
'add_documents_to_index',
add_docs_counter)
assert_raises_ctx = self.assertRaises(
gae_search_services.SearchFailureError)
with put_ctx, add_docs_ctx, assert_raises_ctx as context_mgr:
gae_search_services.add_documents_to_index([doc], 'my_index')
self.assertEqual(context_mgr.exception.original_exception, exception)
self.assertEqual(
add_docs_counter.times_called,
gae_search_services.DEFAULT_NUM_RETRIES)
def test_search_with_custom_rank_and_language(self):
doc1 = {'id': 'doc1', 'k': 'abc def', 'rank': 3, 'language_code': 'en'}
doc2 = {'id': 'doc2', 'k': 'abc ghi', 'rank': 1, 'language_code': 'fr'}
doc3 = {'id': 'doc3', 'k': 'abc jkl', 'rank': 2, 'language_code': 'nl'}
gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index')
result = gae_search_services.search('k:abc', index='index')[0]
self.assertEqual(result, [doc1, doc3, doc2])
def test_put_error_with_transient_result(self):
docs = [{'id': 'doc1', 'prop': 'val1'},
{'id': 'doc2', 'prop': 'val2'},
{'id': 'doc3', 'prop': 'val3'}]
error = self._get_put_error(3, 1)
failing_put = test_utils.FailingFunction(
search.Index.put,
error,
4)
add_docs_counter = test_utils.CallCounter(
gae_search_services.add_documents_to_index)
put_ctx = self.swap(search.Index, 'put', failing_put)
add_docs_ctx = self.swap(
gae_search_services,
'add_documents_to_index',
add_docs_counter)
with put_ctx, add_docs_ctx:
gae_search_services.add_documents_to_index(docs, 'my_index', 5)
self.assertEqual(add_docs_counter.times_called, 5)
for i in xrange(1, 4):
result = search.Index('my_index').get('doc' + str(i))
self.assertEqual(result.field('prop').value, 'val' + str(i))
def test_add_document_with_existing_id_updates_it(self):
doc1 = {'id': 'doc', 'version': 1, 'rank': 10}
doc2 = {'id': 'doc', 'version': 2, 'rank': 20}
gae_search_services.add_documents_to_index([doc1], 'my_index')
index = search.Index('my_index')
self.assertEqual(index.get('doc').field('version').value, 1)
self.assertEqual(index.get('doc').rank, 10)
gae_search_services.add_documents_to_index([doc2], 'my_index')
self.assertEqual(index.get('doc').field('version').value, 2)
self.assertEqual(index.get('doc').rank, 20)
def test_search_using_multiple_sort_expressions(self):
doc1 = {'id': 'doc1', 'k1': 2, 'k2': 'abc ghi'}
doc2 = {'id': 'doc2', 'k1': 1, 'k2': 'abc def'}
doc3 = {'id': 'doc3', 'k1': 1, 'k2': 'abc jkl'}
gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index')
result = gae_search_services.search(
'k2:abc', 'index', sort='+k1 -k2')[0]
self.assertEqual(result[0].get('id'), 'doc3')
self.assertEqual(result[1].get('id'), 'doc2')
self.assertEqual(result[2].get('id'), 'doc1')
def test_validate_list_values(self):
doc1 = {'f': ['a', 'b', ['c', 'd']]}
doc2 = {'f': ['a', 'b', 3, set([4, 5, 6])]}
# The str() of list and set are passed in to ensure that we mention the
# type the user passed in, in our error message..
with self.assertRaisesRegexp(ValueError, str(list)):
gae_search_services.add_documents_to_index([doc1], 'my_index')
with self.assertRaisesRegexp(ValueError, str(set)):
gae_search_services.add_documents_to_index([doc2], 'my_index')
def test_search_using_single_sort_expression(self):
doc1 = {'id': 'doc1', 'k': 'abc ghi'}
doc2 = {'id': 'doc2', 'k': 'abc def'}
doc3 = {'id': 'doc3', 'k': 'abc jkl'}
gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index')
result, cursor = gae_search_services.search(
'k:abc', 'index', sort='+k')
self.assertEqual(result[0].get('id'), 'doc2')
self.assertEqual(result[1].get('id'), 'doc1')
self.assertEqual(result[2].get('id'), 'doc3')
result, cursor = gae_search_services.search(
'k:abc', 'index', sort='-k')
self.assertEqual(result[0].get('id'), 'doc3')
self.assertEqual(result[1].get('id'), 'doc1')
self.assertEqual(result[2].get('id'), 'doc2')
def test_insert_document_with_id(self):
date = datetime.date(year=2015, month=3, day=14)
datetime_value = datetime.datetime(
year=1991, month=6, day=20, hour=16, minute=30, second=14)
doc_id = 'abcdefghijklmnop'
doc = {
'id': doc_id,
'numberfield': 5,
'stringfield': 'abc',
'datefield': date,
'datetimefield': datetime_value
}
result = gae_search_services.add_documents_to_index([doc], 'my_index')
self.assertEqual(result, [doc_id])
result_doc = search.Index('my_index').get(doc_id)
self.assertEqual(result_doc.doc_id, doc_id)
self.assertEqual(result_doc.field('numberfield').value, 5)
self.assertEqual(result_doc.field('stringfield').value, 'abc')
# The search api returns date fields as datetime fields with
# time at midnight.
self.assertEqual(
result_doc.field('datefield').value,
datetime.datetime.combine(
date=date,
time=datetime.datetime.min.time()))
self.assertEqual(
result_doc.field('datetimefield').value, datetime_value)
def test_insert_multiple_with_id(self):
docs = [{'id': 'id%d' % n, 'name': 'doc%d' % n} for n in range(5)]
result = gae_search_services.add_documents_to_index(docs, 'my_index')
index = search.Index('my_index')
for ind in range(5):
retrieved_doc = index.get('id%d' % ind)
self.assertEqual(retrieved_doc.field('name').value, 'doc%d' % ind)
self.assertEqual(result[ind], 'id%d' % ind)
def test_use_custom_number_of_retries(self):
doc = {'id': 'doc', 'prop': 'val'}
exception = self._get_put_error(1, 0)
failing_put = test_utils.FailingFunction(
search.Index.put,
exception,
42)
add_docs_counter = test_utils.CallCounter(
gae_search_services.add_documents_to_index)
put_ctx = self.swap(search.Index, 'put', failing_put)
add_docs_ctx = self.swap(
gae_search_services, 'add_documents_to_index', add_docs_counter)
assert_raises_ctx = self.assertRaises(
gae_search_services.SearchFailureError)
with put_ctx, add_docs_ctx, assert_raises_ctx:
gae_search_services.add_documents_to_index([doc], 'my_index', 42)
self.assertEqual(add_docs_counter.times_called, 42)
def test_arguments_are_preserved_in_retries(self):
doc = {'id': 'doc', 'prop': 'val'}
exception = self._get_put_error(1, 0)
failing_put = test_utils.FailingFunction(
search.Index.put,
exception,
3
)
add_docs_counter = test_utils.CallCounter(
gae_search_services.add_documents_to_index)
put_ctx = self.swap(search.Index, 'put', failing_put)
add_docs_ctx = self.swap(
gae_search_services,
'add_documents_to_index',
add_docs_counter)
with put_ctx, add_docs_ctx:
gae_search_services.add_documents_to_index([doc], 'my_index', 4)
self.assertEqual(add_docs_counter.times_called, 4)
result = search.Index('my_index').get('doc')
self.assertEqual(result.field('prop').value, 'val')
def test_put_error_without_transient_result(self):
docs = [{'id': 'doc1', 'prop': 'val1'},
{'id': 'doc2', 'prop': 'val2'},
{'id': 'doc3', 'prop': 'val3'}]
error = self._get_put_error(3)
failing_put = test_utils.FailingFunction(search.Index.put, error, 1)
add_docs_counter = test_utils.CallCounter(
gae_search_services.add_documents_to_index)
add_docs_ctx = self.swap(
gae_search_services,
'add_documents_to_index',
add_docs_counter)
put_ctx = self.swap(search.Index, 'put', failing_put)
assert_raises_ctx = self.assertRaises(
gae_search_services.SearchFailureError)
with add_docs_ctx, put_ctx, assert_raises_ctx as e:
gae_search_services.add_documents_to_index(docs, 'my_index')
# assert that the method only gets called once, since the error is not
# transient.
self.assertEqual(add_docs_counter.times_called, 1)
self.assertEqual(e.exception.original_exception, error)
def test_default_rank_is_descending_date(self):
# Time is only saved with 1 second accuracy,
# so I'm putting a 1 second delay between puts.
dict1 = {'id': 'doc1', 'k': 'abc def'}
dict2 = {'id': 'doc2', 'k': 'abc ghi'}
dict3 = {'id': 'doc3', 'k': 'abc jkl'}
gae_search_services.add_documents_to_index([dict1], 'my_index')
time.sleep(1)
gae_search_services.add_documents_to_index([dict2], 'my_index')
time.sleep(1)
gae_search_services.add_documents_to_index([dict3], 'my_index')
result = gae_search_services.search(
'k:abc', index='my_index', ids_only=True)[0]
self.assertEqual(result, ['doc3', 'doc2', 'doc1'])
def test_disallow_unsuported_value_types(self):
with self.assertRaises(ValueError):
doc = {'abc': set('xyz')}
gae_search_services.add_documents_to_index(doc, 'my_index')
def test_insert_document_with_multi_valued_property(self):
doc = {'id': 'doc', 'prop': ['val1', 'val2', 'val3']}
gae_search_services.add_documents_to_index([doc], 'index')
resultdoc = search.Index('index').get('doc')
values = set([field.value for field in resultdoc['prop']])
self.assertEqual(values, set(['val1', 'val2', 'val3']))
def test_index_must_be_string(self):
index = search.Index('test')
# Check that the error message mentions the type the user passed in.
with self.assertRaisesRegexp(ValueError, str(type(index))):
gae_search_services.add_documents_to_index(
{'id': 'one', 'key': 'value'}, index)
def test_insert_document_without_id(self):
doc = {'abc': 'def'}
result = gae_search_services.add_documents_to_index([doc], 'my_index')
retrieved_doc = search.Index('my_index').get(result[0])
self.assertEqual(retrieved_doc.field('abc').value, 'def')
def test_add_document_with_rank(self):
doc = {'id': 'my_doc', 'field': 'value', 'rank': 42}
gae_search_services.add_documents_to_index([doc], 'my_index')
index = search.Index('my_index')
self.assertEqual(index.get('my_doc').rank, 42)
def test_disallow_unsuported_value_types(self):
with self.assertRaises(ValueError):
doc = {'abc': set('xyz')}
gae_search_services.add_documents_to_index(doc, 'my_index')
def test_insert_document_without_id(self):
doc = {'abc': 'def'}
result = gae_search_services.add_documents_to_index([doc], 'my_index')
retrieved_doc = search.Index('my_index').get(result[0])
self.assertEqual(retrieved_doc.field('abc').value, 'def')
def test_add_document_with_rank(self):
doc = {'id': 'my_doc', 'field': 'value', 'rank': 42}
gae_search_services.add_documents_to_index([doc], 'my_index')
index = search.Index('my_index')
self.assertEqual(index.get('my_doc').rank, 42)
def test_insert_document_with_multi_valued_property(self):
doc = {'id': 'doc', 'prop': ['val1', 'val2', 'val3']}
gae_search_services.add_documents_to_index([doc], 'index')
resultdoc = search.Index('index').get('doc')
values = set([field.value for field in resultdoc['prop']])
self.assertEqual(values, set(['val1', 'val2', 'val3']))
def test_disallow_unsuported_value_types(self):
with self.assertRaisesRegexp(
ValueError,
'document should be a dictionary, got <type \'unicode\'>'):
doc = {'abc': set('xyz')}
gae_search_services.add_documents_to_index(doc, 'my_index')
def test_index_must_be_string(self):
index = search.Index('test')
# Check that the error message mentions the type the user passed in.
with self.assertRaisesRegexp(ValueError, str(type(index))):
gae_search_services.add_documents_to_index(
{'id': 'one', 'key': 'value'}, index)
