def search(self, query_txt, role=None):
"""Performs a search and adds view_url attributes to the results."""
results = None
if self.config.external_search_backends:
results = external_search.search(
self.repo, TextQuery(query_txt), MAX_RESULTS,
self.config.external_search_backends)
# External search backends are not always complete. Fall back to the
# original search when they fail or return no results.
if not results:
if config.get('enable_fulltext_search'):
results = full_text_search.search(self.repo,
query_txt, MAX_RESULTS)
else:
results = indexing.search(self.repo,
TextQuery(query_txt), MAX_RESULTS, role=self.params.role)
for result in results:
result.view_url = self.get_url('/view',
id=result.record_id,
role=self.params.role,
query=self.params.query,
given_name=self.params.given_name,
family_name=self.params.family_name)
result.latest_note_status = get_person_status_text(result)
if result.is_clone():
result.provider_name = result.get_original_domain()
sanitize_urls(result)
return results
def search(self, query_txt):
"""Performs a search and adds view_url attributes to the results."""
results = None
if self.config.external_search_backends:
results = external_search.search(
self.repo, TextQuery(query_txt), MAX_RESULTS,
self.config.external_search_backends)
# External search backends are not always complete. Fall back to the
# original search when they fail or return no results.
if not results:
if config.get('enable_fulltext_search'):
results = full_text_search.search(self.repo, query_txt,
MAX_RESULTS)
else:
results = indexing.search(self.repo, TextQuery(query_txt),
MAX_RESULTS)
for result in results:
result.view_url = self.get_url('/view',
id=result.record_id,
role=self.params.role,
query=self.params.query,
given_name=self.params.given_name,
family_name=self.params.family_name)
result.latest_note_status = get_person_status_text(result)
if result.is_clone():
result.provider_name = result.get_original_domain()
result.should_show_inline_photo = (self.should_show_inline_photo(
result.photo_url))
sanitize_urls(result)
return results
def test_search_recover_from_bad_response(self):
good_response = MockUrlFetchResponse(200, {
'name_entries': [{'person_record_id': 'test/1'}],
'all_entries': [],
})
bad_response = MockUrlFetchResponse(500, '')
urlfetch.fetch('http://backend1/?q=mori', deadline=IsSeconds(0.9))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndReturn(bad_response)
urlfetch.fetch('http://backend2/?q=mori', deadline=IsSeconds(0.8))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndReturn(bad_response)
urlfetch.fetch('http://backend3/?q=mori', deadline=IsSeconds(0.6))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndReturn(good_response)
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100,
['http://backend1/?q=%s', 'http://backend2/?q=%s',
'http://backend3/?q=%s'])
self.assertEquals(1, len(results))
self.assertEquals('test/1', results[0].record_id)
self.assertEquals(['Bad status code: 500', 'Bad status code: 500'],
self.mock_logging_handler.messages['info'])
self.mox.VerifyAll()
def test_search_remove_non_name_matches(self):
response = MockUrlFetchResponse(200, {
'name_entries': [],
'all_entries': [
{'person_record_id': 'test/1'},
{'person_record_id': 'test/2'},
{'person_record_id': 'test/3'},
{'person_record_id': 'test/4'},
{'person_record_id': 'test/5'},
],
})
urlfetch.fetch('http://backend/?q=mori',
deadline=IsSeconds(0.9)).AndReturn(response)
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100,
['http://backend/?q=%s'])
self.assertEquals(3, len(results))
self.assertEquals('test/1', results[0].record_id)
self.assertEquals('test/3', results[1].record_id)
self.assertEquals('test/4', results[2].record_id)
self.assertTrue(results[0].is_address_match)
self.assertTrue(results[1].is_address_match)
self.assertTrue(results[2].is_address_match)
self.mox.VerifyAll()
def test_search_retry_time_out(self):
good_response = MockUrlFetchResponse(200, {
'name_entries': [{
'person_record_id': 'test/1'
}],
'all_entries': [],
})
deactivation_message_html = 'de<i>acti</i>vated'
bad_response = MockUrlFetchResponse(500, '')
urlfetch.fetch('http://backend1/?q=mori', deadline=IsSeconds(0.9))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndReturn(bad_response)
urlfetch.fetch('http://backend2/?q=mori', deadline=IsSeconds(0.8))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.75))\
.AndRaise(urlfetch_errors.Error('bad'))
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100, [
'http://backend1/?q=%s', 'http://backend2/?q=%s',
'http://backend3/?q=%s'
])
self.assertEquals(None, results)
self.assertEquals([
'Bad status code: 500', 'Failed to fetch: bad',
'Fetch retry timed out.'
], self.mock_logging_handler.messages['info'])
self.mox.VerifyAll()
def search(self, query_name, query_location=None):
"""Get results for a query.
Args:
query_name: A name to query for (string).
query_location: A location to query for (optional, string).
"""
text_query = TextQuery(
'%s %s' %
(query_name, query_location) if query_location else query_name)
results = None
if self._external_search_backends:
results = external_search.search(self._repo, text_query,
self._max_results,
self._external_search_backends)
# External search backends are not always complete. Fall back to the
# original search when they fail or return no results.
if not results:
query_dict = {'name': query_name}
if query_location:
query_dict['location'] = query_location
if self._enable_fulltext_search:
results = full_text_search.search(self._repo, query_dict,
self._max_results)
else:
results = indexing.search(self._repo, text_query,
self._max_results)
return results
def test_search_recover_from_fetch_failure(self):
good_response = MockUrlFetchResponse(200, {
'name_entries': [{
'person_record_id': 'test/1'
}],
'all_entries': [],
})
urlfetch.fetch('http://backend1/?q=mori', deadline=IsSeconds(0.9))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndRaise(urlfetch_errors.Error('bad'))
urlfetch.fetch('http://backend2/?q=mori', deadline=IsSeconds(0.8))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndRaise(urlfetch_errors.Error('bad'))
urlfetch.fetch('http://backend3/?q=mori', deadline=IsSeconds(0.6))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndReturn(good_response)
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100, [
'http://backend1/?q=%s', 'http://backend2/?q=%s',
'http://backend3/?q=%s'
])
self.assertEquals(1, len(results))
self.assertEquals('test/1', results[0].record_id)
self.assertEquals(['Failed to fetch: bad', 'Failed to fetch: bad'],
self.mock_logging_handler.messages['info'])
self.mox.VerifyAll()
def test_search_remove_non_name_matches(self):
response = MockUrlFetchResponse(
200, {
'name_entries': [],
'all_entries': [
{
'person_record_id': 'test/1'
},
{
'person_record_id': 'test/2'
},
{
'person_record_id': 'test/3'
},
{
'person_record_id': 'test/4'
},
{
'person_record_id': 'test/5'
},
],
})
urlfetch.fetch('http://backend/?q=mori',
deadline=IsSeconds(0.9)).AndReturn(response)
self.mox.ReplayAll()
results = external_search.search('japan', text_query.TextQuery('mori'),
100, ['http://backend/?q=%s'])
self.assertEquals(3, len(results))
self.assertEquals('test/1', results[0].record_id)
self.assertEquals('test/3', results[1].record_id)
self.assertEquals('test/4', results[2].record_id)
self.assertTrue(results[0].is_address_match)
self.assertTrue(results[1].is_address_match)
self.assertTrue(results[2].is_address_match)
self.mox.VerifyAll()
def get(self):
if self.config.search_auth_key_required and not (
self.auth and self.auth.search_permission):
self.info(403,
message='Missing or invalid authorization key',
style='plain')
return
pfif_version = self.params.version
# Retrieve parameters and do some sanity checks on them.
record_id = self.request.get('id')
query_string = self.request.get('q')
max_results = min(self.params.max_results or 100, HARD_MAX_RESULTS)
results = []
if record_id:
# Search by record ID (always returns just 1 result or nothing).
person = model.Person.get(self.repo, record_id)
if person:
results = [person]
elif query_string:
# Search by query words.
if self.config.external_search_backends:
query = TextQuery(query_string)
results = external_search.search(
self.repo, query, max_results,
self.config.external_search_backends)
# External search backends are not always complete. Fall back to
# the original search when they fail or return no results.
if not results:
if config.get('enable_fulltext_search'):
results = full_text_search.search(self.repo, query_string,
max_results)
else:
results = indexing.search(self.repo,
TextQuery(query_string),
max_results)
else:
self.info(400,
message='Neither id nor q parameter specified',
style='plain')
records = [pfif_version.person_to_dict(result) for result in results]
utils.optionally_filter_sensitive_fields(records, self.auth)
# Define the function to retrieve notes for a person.
def get_notes_for_person(person):
notes = model.Note.get_by_person_record_id(
self.repo, person['person_record_id'])
notes = [note for note in notes if not note.hidden]
records = map(pfif_version.note_to_dict, notes)
utils.optionally_filter_sensitive_fields(records, self.auth)
return records
self.response.headers[
'Content-Type'] = 'application/xml; charset=utf-8'
pfif_version.write_file(self.response.out, records,
get_notes_for_person)
utils.log_api_action(self, ApiActionLog.SEARCH, len(records))
def get(self):
if self.config.search_auth_key_required and not (
self.auth and self.auth.search_permission):
self.info(
403,
message='Missing or invalid authorization key',
style='plain')
return
pfif_version = self.params.version
# Retrieve parameters and do some sanity checks on them.
record_id = self.request.get('id')
query_string = self.request.get('q')
max_results = min(self.params.max_results or 100, HARD_MAX_RESULTS)
results = []
if record_id:
# Search by record ID (always returns just 1 result or nothing).
person = model.Person.get(self.repo, record_id)
if person:
results = [person]
elif query_string:
# Search by query words.
if self.config.external_search_backends:
query = TextQuery(query_string)
results = external_search.search(self.repo, query, max_results,
self.config.external_search_backends)
# External search backends are not always complete. Fall back to
# the original search when they fail or return no results.
if not results:
if config.get('enable_fulltext_search'):
results = full_text_search.search(
self.repo, query_string, max_results)
else:
results = indexing.search(
self.repo, TextQuery(query_string), max_results)
else:
self.info(
400,
message='Neither id nor q parameter specified',
style='plain')
records = [pfif_version.person_to_dict(result) for result in results]
utils.optionally_filter_sensitive_fields(records, self.auth)
# Define the function to retrieve notes for a person.
def get_notes_for_person(person):
notes = model.Note.get_by_person_record_id(
self.repo, person['person_record_id'])
notes = [note for note in notes if not note.hidden]
records = map(pfif_version.note_to_dict, notes)
utils.optionally_filter_sensitive_fields(records, self.auth)
return records
self.response.headers['Content-Type'] = 'application/xml; charset=utf-8'
pfif_version.write_file(
self.response.out, records, get_notes_for_person)
utils.log_api_action(self, ApiActionLog.SEARCH, len(records))
def test_search_broken_content(self):
response = MockUrlFetchResponse(200, '')
response.content = 'broken'
urlfetch.fetch('http://backend/?q=mori',
deadline=IsSeconds(0.9)).AndReturn(response)
self.mox.ReplayAll()
results = external_search.search('japan', text_query.TextQuery('mori'),
100, ['http://backend/?q=%s'])
self.assertEquals(None, results)
self.assertEquals(['Fetched content is broken.'],
self.mock_logging_handler.messages['warning'])
self.mox.VerifyAll()
def test_search_broken_content(self):
response = MockUrlFetchResponse(200, '')
response.content = 'broken'
urlfetch.fetch('http://backend/?q=mori',
deadline=IsSeconds(0.9)).AndReturn(response)
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100,
['http://backend/?q=%s'])
self.assertEquals(None, results)
self.assertEquals(['Fetched content is broken.'],
self.mock_logging_handler.messages['warning'])
self.mox.VerifyAll()
def test_search_shuffle_backends(self):
bad_response = MockUrlFetchResponse(500, '')
urlfetch.fetch('http://backend1/?q=mori', deadline=IsSeconds(0.9))\
.InAnyOrder().AndReturn(bad_response)
urlfetch.fetch('http://backend2/?q=mori', deadline=IsSeconds(0.9))\
.InAnyOrder().AndReturn(bad_response)
urlfetch.fetch('http://backend3/?q=mori', deadline=IsSeconds(0.9))\
.InAnyOrder().AndReturn(bad_response)
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100,
['http://backend1/?q=%s', 'http://backend2/?q=%s',
'http://backend3/?q=%s'])
self.assertEquals(None, results)
self.mox.VerifyAll()
def test_search_shuffle_backends(self):
bad_response = MockUrlFetchResponse(500, '')
urlfetch.fetch('http://backend1/?q=mori', deadline=IsSeconds(0.9))\
.InAnyOrder().AndReturn(bad_response)
urlfetch.fetch('http://backend2/?q=mori', deadline=IsSeconds(0.9))\
.InAnyOrder().AndReturn(bad_response)
urlfetch.fetch('http://backend3/?q=mori', deadline=IsSeconds(0.9))\
.InAnyOrder().AndReturn(bad_response)
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100, [
'http://backend1/?q=%s', 'http://backend2/?q=%s',
'http://backend3/?q=%s'
])
self.assertEquals(None, results)
self.mox.VerifyAll()
def search(self, query_dict):
"""
Performs a search and adds view_url attributes to the results.
Args:
query_dict: A list contains two queries: Name query and Location query
"""
query_txt = " ".join(query_dict.values())
results = None
if self.config.external_search_backends:
results = external_search.search(
self.repo, TextQuery(query_txt), MAX_RESULTS,
self.config.external_search_backends)
# External search backends are not always complete. Fall back to the
# original search when they fail or return no results.
if not results:
if config.get('enable_fulltext_search'):
results = full_text_search.search(self.repo,
query_dict, MAX_RESULTS)
else:
results = indexing.search(self.repo,
TextQuery(query_txt), MAX_RESULTS)
query_name = self.get_query_value()
for result in results:
result.view_url = self.get_url('/view',
id=result.record_id,
role=self.params.role,
query_name=query_name,
query_location=
self.params.query_location,
given_name=self.params.given_name,
family_name=self.params.family_name)
result.latest_note_status = get_person_status_text(result)
if result.is_clone():
result.provider_name = result.get_original_domain()
result.should_show_inline_photo = (
self.should_show_inline_photo(result.photo_url))
if result.should_show_inline_photo and result.photo:
# Only use a thumbnail URL if the photo was uploaded; we don't
# have thumbnails for other photos.
result.thumbnail_url = self.get_thumbnail_url(result.photo_url)
sanitize_urls(result)
return results
def test_search_retry_time_out(self):
good_response = MockUrlFetchResponse(200, {
'name_entries': [{'person_record_id': 'test/1'}],
'all_entries': [],
})
deactivation_message_html='de<i>acti</i>vated'
bad_response = MockUrlFetchResponse(500, '')
urlfetch.fetch('http://backend1/?q=mori', deadline=IsSeconds(0.9))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.2))\
.AndReturn(bad_response)
urlfetch.fetch('http://backend2/?q=mori', deadline=IsSeconds(0.8))\
.WithSideEffects(lambda url, deadline: self.advance_seconds(0.75))\
.AndRaise(urlfetch_errors.Error('bad'))
self.mox.ReplayAll()
results = external_search.search(
'japan', text_query.TextQuery('mori'), 100,
['http://backend1/?q=%s', 'http://backend2/?q=%s',
'http://backend3/?q=%s'])
self.assertEquals(None, results)
self.assertEquals(['Bad status code: 500', 'Failed to fetch: bad',
'Fetch retry timed out.'],
self.mock_logging_handler.messages['info'])
self.mox.VerifyAll()
