def search(self, query_dict):
"""
Performs a search and adds view_url attributes to the results.
Args:
query_dict: A list contains two queries: Name query and Location query
"""
searcher = Searcher(self.repo, config.get('enable_fulltext_search'),
MAX_RESULTS)
results = searcher.search(query_dict['name'],
query_dict.get('location'))
query_name = self.get_query_value()
for result in results:
result.view_url = self.get_url(
'/view',
id=result.record_id,
role=self.params.role,
query_name=query_name,
query_location=self.params.query_location,
given_name=self.params.given_name,
family_name=self.params.family_name)
result.latest_note_status = get_person_status_text(result)
if result.is_clone():
result.provider_name = result.get_original_domain()
result.should_show_inline_photo = (self.should_show_inline_photo(
result.photo_url))
if result.should_show_inline_photo and result.photo:
# Only use a thumbnail URL if the photo was uploaded; we don't
# have thumbnails for other photos.
result.thumbnail_url = self.get_thumbnail_url(result.photo_url)
sanitize_urls(result)
return results
def search(self, query_dict):
"""
Performs a search and adds view_url attributes to the results.
Args:
query_dict: A list contains two queries: Name query and Location query
"""
searcher = Searcher(
self.repo, config.get('enable_fulltext_search'), MAX_RESULTS)
results = searcher.search(
query_dict['name'],
query_dict.get('location'))
query_name = self.get_query_value()
for result in results:
result.view_url = self.get_url('/view',
id=result.record_id,
role=self.params.role,
query_name=query_name,
query_location=
self.params.query_location,
given_name=self.params.given_name,
family_name=self.params.family_name)
result.latest_note_status = get_person_status_text(result)
if result.is_clone():
result.provider_name = result.get_original_domain()
result.should_show_inline_photo = (
self.should_show_inline_photo(result.photo_url))
if result.should_show_inline_photo and result.photo:
# Only use a thumbnail URL if the photo was uploaded; we don't
# have thumbnails for other photos.
result.thumbnail_url = self.get_thumbnail_url(result.photo_url)
sanitize_urls(result)
return results
def get(self):
searcher = Searcher(
self.repo, config.get('enable_fulltext_search'),
Results.MAX_RESULTS)
results = searcher.search(
self.params.query_name or self.params.query)
self._return_json([self._result_to_dict(r) for r in results])
def get(self):
searcher = Searcher(
self.repo, self.config.external_search_backends,
config.get('enable_fulltext_search'), Results.MAX_RESULTS)
results = searcher.search(
self.params.query_name or self.params.query)
self._return_json([self._result_to_dict(r) for r in results])
def get(self):
args = anime_search_parser.parse_args()
searcher = Searcher("persist")
entry_title = args['query']
entry_date = datetime.utcnow()
results = searcher.search(entry_title, entry_date)[0:10]
searcher.cleanup()
return results
def post(self, owner_id, ds_name): args = parser.parse_args() query = args['query'] ranker = args['ranker'] num_results = args['num_results'] params = args['params'] owner = User.objects(id=owner_id).first() path = cfg["anno_dataset_base_path"] + str(owner.gitlab_id) searcher = Searcher(ds_name, path) documents = jsonify(searcher.search(query, ranker, params, num_results)) return make_response(documents)
def test_full_text_search_results(self):
"""Use full_text_search.search results when enabled."""
with mock.patch('full_text_search.search') as full_text_search_mock:
full_text_search_mock.return_value = SearcherTests.FULLTEXT_RETURN_VALUE
searcher = Searcher(SearcherTests.REPO_NAME,
enable_fulltext_search=True,
max_results=SearcherTests.MAX_RESULTS)
assert (
searcher.search('matt') == SearcherTests.FULLTEXT_RETURN_VALUE)
assert len(full_text_search_mock.call_args_list) == 1
call_args, _ = full_text_search_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1] == {'name': 'matt'}
assert call_args[2] == SearcherTests.MAX_RESULTS
def get(self):
if self.config.search_auth_key_required and not (
self.auth and self.auth.search_permission):
self.info(
403,
message='Missing or invalid authorization key',
style='plain')
return
pfif_version = self.params.version
# Retrieve parameters and do some sanity checks on them.
record_id = self.request.get('id')
query_string = self.request.get('q')
max_results = min(self.params.max_results or 100, HARD_MAX_RESULTS)
results = []
if record_id:
# Search by record ID (always returns just 1 result or nothing).
person = model.Person.get(self.repo, record_id)
if person:
results = [person]
elif query_string:
searcher = Searcher(
self.repo, self.config.external_search_backends,
config.get('enable_fulltext_search'), max_results)
results = searcher.search(query_string)
else:
self.info(
400,
message='Neither id nor q parameter specified',
style='plain')
records = [pfif_version.person_to_dict(result) for result in results]
utils.optionally_filter_sensitive_fields(records, self.auth)
# Define the function to retrieve notes for a person.
def get_notes_for_person(person):
notes = model.Note.get_by_person_record_id(
self.repo, person['person_record_id'])
notes = [note for note in notes if not note.hidden]
records = map(pfif_version.note_to_dict, notes)
utils.optionally_filter_sensitive_fields(records, self.auth)
return records
self.response.headers['Content-Type'] = 'application/xml; charset=utf-8'
pfif_version.write_file(
self.response.out, records, get_notes_for_person)
utils.log_api_action(self, ApiActionLog.SEARCH, len(records))
def get(self):
if self.config.search_auth_key_required and not (
self.auth and self.auth.search_permission):
self.info(
403,
message='Missing or invalid authorization key',
style='plain')
return
pfif_version = self.params.version
# Retrieve parameters and do some sanity checks on them.
record_id = self.request.get('id')
query_string = self.request.get('q')
max_results = min(self.params.max_results or 100, HARD_MAX_RESULTS)
results = []
if record_id:
# Search by record ID (always returns just 1 result or nothing).
person = model.Person.get(self.repo, record_id)
if person:
results = [person]
elif query_string:
searcher = Searcher(
self.repo, config.get('enable_fulltext_search'), max_results)
results = searcher.search(query_string)
else:
self.info(
400,
message='Neither id nor q parameter specified',
style='plain')
records = [pfif_version.person_to_dict(result) for result in results]
utils.optionally_filter_sensitive_fields(records, self.auth)
# Define the function to retrieve notes for a person.
def get_notes_for_person(person):
notes = model.Note.get_by_person_record_id(
self.repo, person['person_record_id'])
notes = [note for note in notes if not note.hidden]
records = map(pfif_version.note_to_dict, notes)
utils.optionally_filter_sensitive_fields(records, self.auth)
return records
self.response.headers['Content-Type'] = 'application/xml; charset=utf-8'
pfif_version.write_file(
self.response.out, records, get_notes_for_person)
utils.log_api_action(self, ApiActionLog.SEARCH, len(records))
def test_full_text_search_results(self):
"""Use full_text_search.search results when enabled."""
with mock.patch('full_text_search.search') as full_text_search_mock:
full_text_search_mock.return_value = (
SearcherTests.FULLTEXT_RETURN_VALUE)
searcher = Searcher(
SearcherTests.REPO_NAME,
enable_fulltext_search=True,
max_results=SearcherTests.MAX_RESULTS)
assert (
searcher.search('matt') == SearcherTests.FULLTEXT_RETURN_VALUE)
assert len(full_text_search_mock.call_args_list) == 1
call_args, _ = full_text_search_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1] == {'name': 'matt'}
assert call_args[2] == SearcherTests.MAX_RESULTS
def test_full_text_search_results_with_location(self):
"""Use full_text_search.search results when enabled, including location.
"""
with patch('full_text_search.search') as full_text_search_mock, \
patch('indexing.search') as indexing_mock:
full_text_search_mock.return_value = SearcherTests.FULLTEXT_RETURN_VALUE
searcher = Searcher(SearcherTests.REPO_NAME,
enable_fulltext_search=True,
max_results=SearcherTests.MAX_RESULTS)
assert (searcher.search(
'matt', 'schenectady') == SearcherTests.FULLTEXT_RETURN_VALUE)
assert len(full_text_search_mock.call_args_list) == 1
call_args, _ = full_text_search_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1] == {'name': 'matt', 'location': 'schenectady'}
assert call_args[2] == SearcherTests.MAX_RESULTS
def search(assignment, dataset_name, queries, ranker, params, num_results):
author = User.objects(email=current_user.email).first()
path = os.path.join(current_app.root_path, 'data', author.name)
searcher = Searcher(dataset_name, path)
for query in queries:
results = searcher.search(query, ranker, params,
num_results)['results']
for result in results:
doc_path = str(
os.path.join(path, result['path'].encode('utf8')[2:]))
doc_score = result['score']
document = Document.objects(path=doc_path).first()
q = Query.objects(content=query).first()
Score(result=doc_score,
assignment=assignment,
query=q,
document=document).save()
def test_indexing_results_with_loc(self):
"""Fall back to indexing.search results, including a location value.
When full-text search is disabled, fall back to indexing.search.
"""
with mock.patch('indexing.search') as indexing_mock:
indexing_mock.return_value = SearcherTests.INDEXING_RETURN_VALUE
searcher = Searcher(SearcherTests.REPO_NAME,
enable_fulltext_search=False,
max_results=SearcherTests.MAX_RESULTS)
assert (searcher.search(
'matt', 'schenectady') == SearcherTests.INDEXING_RETURN_VALUE)
assert len(indexing_mock.call_args_list) == 1
call_args, _ = indexing_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1].query == 'matt schenectady'
assert call_args[2] == SearcherTests.MAX_RESULTS
def test_indexing_results_with_loc(self):
"""Fall back to indexing.search results, including a location value.
When full-text search is disabled, fall back to indexing.search.
"""
with mock.patch('indexing.search') as indexing_mock:
indexing_mock.return_value = SearcherTests.INDEXING_RETURN_VALUE
searcher = Searcher(
SearcherTests.REPO_NAME,
enable_fulltext_search=False,
max_results=SearcherTests.MAX_RESULTS)
assert (searcher.search(
'matt', 'schenectady') == SearcherTests.INDEXING_RETURN_VALUE)
assert len(indexing_mock.call_args_list) == 1
call_args, _ = indexing_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1].query == 'matt schenectady'
assert call_args[2] == SearcherTests.MAX_RESULTS
def test_indexing_results(self):
"""Fall back to indexing.search results.
When external search backends don't return any results and full-text
search is disabled, fall back to indexing.search.
"""
with patch('external_search.search') as external_search_mock, \
patch('full_text_search.search') as full_text_search_mock, \
patch('indexing.search') as indexing_mock:
external_search_mock.return_value = []
indexing_mock.return_value = SearcherTests.INDEXING_RETURN_VALUE
searcher = Searcher(SearcherTests.REPO_NAME,
external_search_backends='any value',
enable_fulltext_search=False,
max_results=SearcherTests.MAX_RESULTS)
assert (
searcher.search('matt') == SearcherTests.INDEXING_RETURN_VALUE)
assert len(indexing_mock.call_args_list) == 1
call_args, _ = indexing_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1].query == 'matt'
assert call_args[2] == SearcherTests.MAX_RESULTS
def test_full_text_search_results_with_location(self):
"""Fall back to full_text_search.search results, including location.
When external search backends don't return any results and full-text
search is enabled, fall back to full_text_search.search.
"""
with patch('external_search.search') as external_search_mock, \
patch('full_text_search.search') as full_text_search_mock, \
patch('indexing.search') as indexing_mock:
external_search_mock.return_value = []
full_text_search_mock.return_value = SearcherTests.FULLTEXT_RETURN_VALUE
searcher = Searcher(SearcherTests.REPO_NAME,
external_search_backends='any value',
enable_fulltext_search=True,
max_results=SearcherTests.MAX_RESULTS)
assert (searcher.search(
'matt', 'schenectady') == SearcherTests.FULLTEXT_RETURN_VALUE)
assert len(full_text_search_mock.call_args_list) == 1
call_args, _ = full_text_search_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1] == {'name': 'matt', 'location': 'schenectady'}
assert call_args[2] == SearcherTests.MAX_RESULTS
def test_external_search_backend_results(self):
"""Return external search backend results when available."""
with patch('external_search.search') as external_search_mock, \
patch('full_text_search.search') as full_text_search_mock, \
patch('indexing.search') as indexing_mock:
full_text_search_mock.return_value = []
indexing_mock.return_value = []
external_search_mock.return_value = (
SearcherTests.EXTERNAL_SEARCH_RETURN_VALUE)
external_search_backends_value = 'abc external search'
searcher = Searcher(
SearcherTests.REPO_NAME,
external_search_backends=external_search_backends_value,
enable_fulltext_search=False,
max_results=SearcherTests.MAX_RESULTS)
assert (searcher.search('matt') ==
SearcherTests.EXTERNAL_SEARCH_RETURN_VALUE)
assert len(external_search_mock.call_args_list) == 1
call_args, _ = external_search_mock.call_args_list[0]
assert call_args[0] == SearcherTests.REPO_NAME
assert call_args[1].query == 'matt'
assert call_args[2] == SearcherTests.MAX_RESULTS
assert call_args[3] == external_search_backends_value
searcher = Searcher()
name = raw_input("Name? ")
print "[INFO]: name: {0}".format(name.encode("utf-8"))
if name == "all":
media = searcher.db["randomc"].find()
else:
search_terms = [re.escape(name)]
search_terms_str = "|".join(search_terms)
regex = re.compile(ur"{0}".format(search_terms_str), re.IGNORECASE)
media = list(searcher.db["randomc"].find({"titles": {"$regex": regex, "$options": "-i"}}))
for medium in media:
print medium["titles"][0]
results = searcher.search(medium["titles"][0], medium["start_date"])[0:10]
results = searcher.reduce(results)
if len(results) > 0:
persister.db["media"].update(
{"titles": {"$in": [medium["titles"][0]]}, "request_url": medium["request_url"]},
{"$set": dict(results[0])},
upsert=True,
)
else:
print "[ERROR] {0} has no search results".format(medium["titles"][0].encode("utf-8"))
searcher.cleanup()
persister.cleanup()