def test_search_when_query_string_is_invalid(self): # The search result would be ([], None) if query strings contain "NOT" # or a string with backslashes. observed_log_messages = [] def mock_logging_function(msg, *_): observed_log_messages.append(msg) with self.swap(logging, 'exception', mock_logging_function): doc = { 'id': 'doc1', 'NOT': 'abc', 'rank': 3, 'language_code': 'en' } gae_search_services.add_documents_to_index([doc], 'index') result = gae_search_services.search('NOT:abc', 'my_index') self.assertEqual(result, ([], None)) result = gae_search_services.search(r'\k:abc', 'my_index') self.assertEqual(result, ([], None)) self.assertEqual(len(observed_log_messages), 2) self.assertEqual(observed_log_messages[0], ('Could not parse query string NOT:abc')) self.assertEqual(observed_log_messages[1], (r'Could not parse query string \k:abc'))
def test_clear_index(self): doc = {'id': 'doc1', 'k': 'abc def', 'rank': 3, 'language_code': 'en'} gae_search_services.add_documents_to_index([doc], 'index') result = gae_search_services.search('k:abc', 'index')[0] self.assertEqual(result, [doc]) gae_search_services.clear_index('index') result = gae_search_services.search('k:abc', 'index')[0] self.assertEqual(result, [])
def test_arguments_are_preserved_in_retries(self): for i in xrange(3): doc = search.Document(doc_id='doc%d' % i, fields=[ search.TextField('prop', 'val'), search.NumberField('index', i) ]) search.Index('my_index').put(doc) exception = search.TransientError('oops') failing_index_search = test_utils.FailingFunction( search.Index.search, exception, 3) search_counter = test_utils.CallCounter(gae_search_services.search) gae_search_ctx = self.swap( search.Index, 'search', failing_index_search) search_counter_ctx = self.swap( gae_search_services, 'search', search_counter) with gae_search_ctx, search_counter_ctx: result, cursor = gae_search_services.search( 'prop:val', 'my_index', sort='-index', limit=2, ids_only=True, retries=4) failing_index_search2 = test_utils.FailingFunction( search.Index.search, exception, 3) search_counter2 = test_utils.CallCounter(gae_search_services.search) gae_search_ctx2 = self.swap( search.Index, 'search', failing_index_search2) search_counter_ctx2 = self.swap( gae_search_services, 'search', search_counter2) with gae_search_ctx2, search_counter_ctx2: result2, cursor = gae_search_services.search( 'prop:val', 'my_index', sort='-index', limit=2, cursor=cursor, ids_only=True, retries=4) self.assertEqual(search_counter.times_called, 4) self.assertEqual(result, ['doc2', 'doc1']) # also check that the cursor is preserved self.assertEqual(search_counter2.times_called, 4) self.assertEqual(result2, ['doc0'])
def test_raise_error_when_sort_starts_with_invalid_character(self): doc = {'id': 'doc1', 'k': 'abc def', 'rank': 3, 'language_code': 'en'} gae_search_services.add_documents_to_index([doc], 'index') # Fields in the sort expression need to start with '+' or '-' # to indicate sort direction. If no such indicator is there, it will # raise ValueError. sort_expression = 'invalid_sort_symbol' with self.assertRaisesRegexp( ValueError, r'Fields in the sort expression must start with "\+"' ' or "-" to indicate sort direction. The field %s has no such ' 'indicator in expression "%s".' % (sort_expression, sort_expression)): gae_search_services.search('k:abc', 'index', sort=sort_expression)
def test_search_using_single_sort_expression(self): doc1 = {'id': 'doc1', 'k': 'abc ghi'} doc2 = {'id': 'doc2', 'k': 'abc def'} doc3 = {'id': 'doc3', 'k': 'abc jkl'} gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index') result = gae_search_services.search('k:abc', 'index', sort='+k')[0] self.assertEqual(result[0].get('id'), 'doc2') self.assertEqual(result[1].get('id'), 'doc1') self.assertEqual(result[2].get('id'), 'doc3') result = gae_search_services.search('k:abc', 'index', sort='-k')[0] self.assertEqual(result[0].get('id'), 'doc3') self.assertEqual(result[1].get('id'), 'doc1') self.assertEqual(result[2].get('id'), 'doc2')
def test_use_offset(self): doc1 = search.Document( doc_id='doc1', language='en', rank=1, fields=[search.TextField(name='category', value='abc def ghi')]) doc2 = search.Document( doc_id='doc2', language='en', rank=1, fields=[search.TextField(name='category', value='abc jkl mno')]) doc3 = search.Document( doc_id='doc3', language='en', rank=1, fields=[search.TextField(name='category', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) result1, result1_offset = gae_search_services.search('', 'my_index', ['abc'], [], size=2) result2, _ = gae_search_services.search('', 'my_index', ['abc'], [], offset=result1_offset) self.assertEqual(len(result1), 2) self.assertEqual(len(result2), 1) dict1 = { 'id': 'doc1', 'category': 'abc def ghi', 'language_code': 'en', 'rank': 1 } self.assertIn(dict1, result1 + result2) dict2 = { 'id': 'doc2', 'category': 'abc jkl mno', 'language_code': 'en', 'rank': 1 } self.assertIn(dict2, result1 + result2) dict3 = { 'id': 'doc3', 'category': 'abc jkl ghi', 'language_code': 'en', 'rank': 1 } self.assertIn(dict3, result1 + result2)
def test_use_custom_number_of_retries(self): exception = search.TransientError('oops') failing_index_search = test_utils.FailingFunction( search.Index.search, exception, 3) search_counter = test_utils.CallCounter(gae_search_services.search) index_ctx = self.swap(search.Index, 'search', failing_index_search) search_counter_ctx = self.swap(gae_search_services, 'search', search_counter) assert_raises_ctx = self.assertRaises( gae_search_services.SearchFailureError) with index_ctx, search_counter_ctx, assert_raises_ctx: gae_search_services.search('query', 'my_index', retries=3) self.assertEqual(search_counter.times_called, 3)
def test_search_with_custom_rank_and_language(self): doc1 = {'id': 'doc1', 'k': 'abc def', 'rank': 3, 'language_code': 'en'} doc2 = {'id': 'doc2', 'k': 'abc ghi', 'rank': 1, 'language_code': 'fr'} doc3 = {'id': 'doc3', 'k': 'abc jkl', 'rank': 2, 'language_code': 'nl'} gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index') result = gae_search_services.search('k:abc', 'index')[0] self.assertEqual(result, [doc1, doc3, doc2])
def test_search_with_custom_rank_and_language(self): doc1 = {'id': 'doc1', 'k': 'abc def', 'rank': 3, 'language_code': 'en'} doc2 = {'id': 'doc2', 'k': 'abc ghi', 'rank': 1, 'language_code': 'fr'} doc3 = {'id': 'doc3', 'k': 'abc jkl', 'rank': 2, 'language_code': 'nl'} gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index') result = gae_search_services.search('k:abc', index='index')[0] self.assertEqual(result, [doc1, doc3, doc2])
def test_use_cursor(self): doc1 = search.Document( doc_id='doc1', language='en', rank=1, fields=[search.TextField(name='k', value='abc def ghi')]) doc2 = search.Document( doc_id='doc2', language='en', rank=1, fields=[search.TextField(name='k', value='abc jkl mno')]) doc3 = search.Document( doc_id='doc3', language='en', rank=1, fields=[search.TextField(name='k', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) result1, cursor = gae_search_services.search('k:abc', 'my_index', limit=2) result2, cursor = gae_search_services.search('k:abc', 'my_index', cursor=cursor) self.assertEqual(len(result1), 2) self.assertEqual(len(result2), 1) dict1 = { 'id': 'doc1', 'k': 'abc def ghi', 'language_code': 'en', 'rank': 1 } self.assertIn(dict1, result1 + result2) dict2 = { 'id': 'doc2', 'k': 'abc jkl mno', 'language_code': 'en', 'rank': 1 } self.assertIn(dict2, result1 + result2) dict3 = { 'id': 'doc3', 'k': 'abc jkl ghi', 'language_code': 'en', 'rank': 1 } self.assertIn(dict3, result1 + result2)
def test_use_custom_number_of_retries(self): exception = search.TransientError('oops') failing_index_search = test_utils.FailingFunction( search.Index.search, exception, 3) search_counter = test_utils.CallCounter(gae_search_services.search) index_ctx = self.swap(search.Index, 'search', failing_index_search) search_counter_ctx = self.swap( gae_search_services, 'search', search_counter) assert_raises_ctx = self.assertRaises( gae_search_services.SearchFailureError) with index_ctx, search_counter_ctx, assert_raises_ctx: gae_search_services.search('query', 'my_index', retries=3) self.assertEqual(search_counter.times_called, 3)
def test_respect_limit(self): doc1 = search.Document(doc_id='doc1', fields=[ search.TextField(name='k', value='abc def ghi')]) doc2 = search.Document(doc_id='doc2', fields=[ search.TextField(name='k', value='abc jkl mno')]) doc3 = search.Document(doc_id='doc3', fields=[ search.TextField(name='k', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) result = gae_search_services.search('k:abc', 'my_index', limit=2)[0] self.assertEqual(len(result), 2)
def test_cursor_is_none_if_no_more_results(self): doc1 = search.Document(doc_id='doc1', fields=[ search.TextField(name='k', value='abc def ghi')]) doc2 = search.Document(doc_id='doc2', fields=[ search.TextField(name='k', value='abc jkl mno')]) doc3 = search.Document(doc_id='doc3', fields=[ search.TextField(name='k', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) cursor = gae_search_services.search('k:abc', 'my_index')[1] self.assertIsNone(cursor)
def test_use_default_num_retries(self): exception = search.TransientError('oops') failing_index_search = test_utils.FailingFunction( search.Index.search, exception, 1) search_counter = test_utils.CallCounter(gae_search_services.search) search_ctx = self.swap(search.Index, 'search', failing_index_search) search_counter_ctx = self.swap(gae_search_services, 'search', search_counter) assert_raises_ctx = self.assertRaisesRegexp( gae_search_services.SearchFailureError, '<class \'google.appengine.api.search.search.TransientError\'>: ' 'oops') with search_ctx, search_counter_ctx, assert_raises_ctx as context_mgr: gae_search_services.search('query', 'my_index') self.assertEqual(context_mgr.exception.original_exception, exception) self.assertEqual(search_counter.times_called, 1)
def test_search_using_multiple_sort_expressions(self): doc1 = {'id': 'doc1', 'k1': 2, 'k2': 'abc ghi'} doc2 = {'id': 'doc2', 'k1': 1, 'k2': 'abc def'} doc3 = {'id': 'doc3', 'k1': 1, 'k2': 'abc jkl'} gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index') result, cursor = gae_search_services.search('k2:abc', 'index', sort='+k1 -k2') self.assertEqual(result[0].get('id'), 'doc3') self.assertEqual(result[1].get('id'), 'doc2') self.assertEqual(result[2].get('id'), 'doc1')
def test_use_default_num_retries(self): exception = search.TransientError('oops') failing_index_search = test_utils.FailingFunction( search.Index.search, exception, gae_search_services.DEFAULT_NUM_RETRIES) search_counter = test_utils.CallCounter(gae_search_services.search) search_ctx = self.swap(search.Index, 'search', failing_index_search) search_counter_ctx = self.swap(gae_search_services, 'search', search_counter) assert_raises_ctx = self.assertRaises( gae_search_services.SearchFailureError) with search_ctx, search_counter_ctx, assert_raises_ctx as context_mgr: gae_search_services.search('query', 'my_index') self.assertEqual(context_mgr.exception.original_exception, exception) self.assertEqual(search_counter.times_called, gae_search_services.DEFAULT_NUM_RETRIES)
def test_search_using_multiple_sort_expressions(self): doc1 = {'id': 'doc1', 'k1': 2, 'k2': 'abc ghi'} doc2 = {'id': 'doc2', 'k1': 1, 'k2': 'abc def'} doc3 = {'id': 'doc3', 'k1': 1, 'k2': 'abc jkl'} gae_search_services.add_documents_to_index([doc1, doc2, doc3], 'index') result, cursor = gae_search_services.search( 'k2:abc', 'index', sort='+k1 -k2') self.assertEqual(result[0].get('id'), 'doc3') self.assertEqual(result[1].get('id'), 'doc2') self.assertEqual(result[2].get('id'), 'doc1')
def test_use_default_num_retries(self): exception = search.TransientError('oops') failing_index_search = test_utils.FailingFunction( search.Index.search, exception, gae_search_services.DEFAULT_NUM_RETRIES) search_counter = test_utils.CallCounter(gae_search_services.search) search_ctx = self.swap(search.Index, 'search', failing_index_search) search_counter_ctx = self.swap( gae_search_services, 'search', search_counter) assert_raises_ctx = self.assertRaises( gae_search_services.SearchFailureError) with search_ctx, search_counter_ctx, assert_raises_ctx as context_mgr: gae_search_services.search('query', 'my_index') self.assertEqual(context_mgr.exception.original_exception, exception) self.assertEqual( search_counter.times_called, gae_search_services.DEFAULT_NUM_RETRIES)
def test_offset_is_none_if_no_more_results(self): doc1 = search.Document( doc_id='doc1', fields=[search.TextField(name='category', value='abc def ghi')]) doc2 = search.Document( doc_id='doc2', fields=[search.TextField(name='category', value='abc jkl mno')]) doc3 = search.Document( doc_id='doc3', fields=[search.TextField(name='category', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) offset = gae_search_services.search('', 'my_index', ['abc'], [])[1] self.assertIsNone(offset)
def test_default_rank_is_descending_date(self): # Time is only saved with 1 second accuracy, # so I'm putting a 1 second delay between puts. dict1 = {'id': 'doc1', 'k': 'abc def'} dict2 = {'id': 'doc2', 'k': 'abc ghi'} dict3 = {'id': 'doc3', 'k': 'abc jkl'} gae_search_services.add_documents_to_index([dict1], 'my_index') time.sleep(1) gae_search_services.add_documents_to_index([dict2], 'my_index') time.sleep(1) gae_search_services.add_documents_to_index([dict3], 'my_index') result = gae_search_services.search('k:abc', 'my_index', ids_only=True)[0] self.assertEqual(result, ['doc3', 'doc2', 'doc1'])
def test_ids_only(self): doc1 = search.Document(doc_id='doc1', fields=[ search.TextField(name='k', value='abc def ghi')]) doc2 = search.Document(doc_id='doc2', fields=[ search.TextField(name='k', value='abc jkl mno')]) doc3 = search.Document(doc_id='doc3', fields=[ search.TextField(name='k', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) result = gae_search_services.search( 'k:abc', 'my_index', ids_only=True)[0] self.assertIn('doc1', result) self.assertIn('doc2', result) self.assertIn('doc3', result)
def test_default_rank_is_descending_date(self): # Time is only saved with 1 second accuracy, # so I'm putting a 1 second delay between puts. dict1 = {'id': 'doc1', 'k': 'abc def'} dict2 = {'id': 'doc2', 'k': 'abc ghi'} dict3 = {'id': 'doc3', 'k': 'abc jkl'} gae_search_services.add_documents_to_index([dict1], 'my_index') time.sleep(1) gae_search_services.add_documents_to_index([dict2], 'my_index') time.sleep(1) gae_search_services.add_documents_to_index([dict3], 'my_index') result = gae_search_services.search( 'k:abc', index='my_index', ids_only=True)[0] self.assertEqual(result, ['doc3', 'doc2', 'doc1'])
def test_use_cursor(self): doc1 = search.Document(doc_id='doc1', language='en', rank=1, fields=[ search.TextField(name='k', value='abc def ghi')]) doc2 = search.Document(doc_id='doc2', language='en', rank=1, fields=[ search.TextField(name='k', value='abc jkl mno')]) doc3 = search.Document(doc_id='doc3', language='en', rank=1, fields=[ search.TextField(name='k', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) result1, cursor = gae_search_services.search( 'k:abc', 'my_index', limit=2) result2, cursor = gae_search_services.search( 'k:abc', 'my_index', cursor=cursor) self.assertEqual(len(result1), 2) self.assertEqual(len(result2), 1) dict1 = {'id': 'doc1', 'k': 'abc def ghi', 'language_code': 'en', 'rank': 1} self.assertIn(dict1, result1 + result2) dict2 = {'id': 'doc2', 'k': 'abc jkl mno', 'language_code': 'en', 'rank': 1} self.assertIn(dict2, result1 + result2) dict3 = {'id': 'doc3', 'k': 'abc jkl ghi', 'language_code': 'en', 'rank': 1} self.assertIn(dict3, result1 + result2)
def test_respect_search_query(self): doc1 = search.Document(doc_id='doc1', rank=1, language='en', fields=[ search.TextField(name='k', value='abc def ghi')]) doc2 = search.Document(doc_id='doc2', rank=1, language='en', fields=[ search.TextField(name='k', value='abc jkl mno')]) doc3 = search.Document(doc_id='doc3', rank=1, language='en', fields=[ search.TextField(name='k', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) result = gae_search_services.search('k:jkl', 'my_index')[0] self.assertNotIn({ 'id': 'doc1', 'k': 'abc def ghi', 'language_code': 'en', 'rank': 1 }, result) self.assertIn({ 'id': 'doc2', 'k': 'abc jkl mno', 'language_code': 'en', 'rank': 1 }, result) self.assertIn({ 'id': 'doc3', 'k': 'abc jkl ghi', 'language_code': 'en', 'rank': 1 }, result)
def test_search_all_documents(self): doc1 = search.Document(doc_id='doc1', language='en', rank=1, fields=[ search.TextField(name='k', value='abc def ghi')]) doc2 = search.Document(doc_id='doc2', language='en', rank=2, fields=[ search.TextField(name='k', value='abc jkl mno')]) doc3 = search.Document(doc_id='doc3', language='en', rank=3, fields=[ search.TextField(name='k', value='abc jkl ghi')]) index = search.Index('my_index') index.put([doc1, doc2, doc3]) result, cursor = gae_search_services.search('k:abc', 'my_index') self.assertIn({ 'id': 'doc1', 'k': 'abc def ghi', 'rank': 1, 'language_code': 'en' }, result) self.assertIn({ 'id': 'doc2', 'k': 'abc jkl mno', 'rank': 2, 'language_code': 'en' }, result) self.assertIn({ 'id': 'doc3', 'k': 'abc jkl ghi', 'rank': 3, 'language_code': 'en' }, result)