def read_all(self, identity, fields, type, cache=True, **kwargs):
"""Search for records matching the querystring."""
cache_key = type + "_" + "-".join(fields)
results = current_cache.get(cache_key)
es_query = Q("match_all")
if not results:
# If not found, NoResultFound is raised (caught by the resource).
vocabulary_type = VocabularyType.query.filter_by(id=type).one()
vocab_id_filter = Q('term', type__id=vocabulary_type.id)
results = self._read_many(identity,
es_query,
fields,
extra_filter=vocab_id_filter,
**kwargs)
if cache:
# ES DSL Response is not pickable.
# If saved in cache serialization wont work with to_dict()
current_cache.set(cache_key, results.to_dict())
else:
search = self.create_search(
identity=identity,
record_cls=self.record_cls,
search_opts=self.config.search,
permission_action='search',
).query(es_query)
results = Response(search, results)
return self.result_list(self, identity, results)
def test_render_people():
hits = Response(
Search(), {
'hits': {
'hits': [
{
'_type': 'authors',
'_source': {
'control_number': 1,
'name': {
'preferred_name': 'preferred_name',
},
},
},
],
'total':
1,
},
}).hits
expected = ([
[
"<a href='/authors/1'>preferred_name</a>",
],
], 1)
result = render_people(hits)
assert expected == result
def test_render_contributions_handles_unicode():
hits = Response(
Search(), {
'hits': {
'hits': [
{
'_type': 'hep',
'_source': {
'control_number':
1427573,
'titles': [
{
'title':
u'Storage Ring Based EDM Search — Achievements and Goals'
},
],
},
},
],
'total':
1,
},
}).hits
expected = ([
[
u"<a href='/literature/1427573'>Storage Ring Based EDM Search — Achievements and Goals</a>",
u'\n\n',
'',
0,
],
], 1)
result = render_contributions(hits)
assert expected == result
def read_all(self, identity, fields, cache=True, **kwargs):
"""Search for records matching the querystring."""
cache_key = "-".join(fields)
results = current_cache.get(cache_key)
es_query = Q("match_all")
if not results:
results = self._read_many(
identity, es_query, fields, **kwargs)
if cache:
# ES DSL Response is not pickable.
# If saved in cache serialization wont work with to_dict()
current_cache.set(cache_key, results.to_dict())
else:
search = self.create_search(
identity=identity,
record_cls=self.record_cls,
search_opts=self.config.search,
permission_action='search',
).query(es_query)
results = Response(search, results)
return self.result_list(self, identity, results)
async def execute(self, ignore_cache=False, raise_on_error=True):
"""
Execute the multi search request and return a list of search results.
"""
if ignore_cache or not hasattr(self, '_response'):
es = get_connection(self._using)
responses = await es.msearch(index=self._index,
body=self.to_dict(),
**self._params)
out = []
for s, r in zip(self._searches, responses['responses']):
if r.get('error', False):
if raise_on_error:
raise TransportError('N/A', r['error']['type'],
r['error'])
r = None
else:
r = Response(s, r)
out.append(r)
self._response = out
return self._response
def test_render_contributions():
hits = Response(
Search(), {
'hits': {
'hits': [
{
'_type': 'hep',
'_source': {
'citation_count':
1,
'control_number':
1,
'publication_info': [
{
'journal_title': 'first-journal_title'
},
],
'titles': [
{
'title': 'first-title'
},
],
},
},
{
'_type': 'hep',
'_source': {
'control_number': 2,
'titles': [
{
'title': 'second-title'
},
],
},
},
],
'total':
2,
},
}).hits
expected = ([
[
"<a href='/literature/1'>first-title</a>",
u'\n\n',
'first-journal_title',
1,
],
[
"<a href='/literature/2'>second-title</a>",
u'\n\n',
'',
0,
],
], 2)
result = render_contributions(hits)
assert expected == result
def _query_suggestions(response: Response):
"""
Get suggestions on a misspelt query
"""
obj_suggestion = response.to_dict()['suggest']
if not obj_suggestion['get_suggestion']:
suggestion = None
else:
get_suggestion = obj_suggestion['get_suggestion'][0]
suggestions = get_suggestion['options']
if not suggestions:
suggestion = None
else:
suggestion = suggestions[0]['text']
return suggestion
def test_render_conferences(request_context):
hits = Response(
Search(), {
'hits': {
'hits': [
{
'_type': 'conferences',
'_source': {
'addresses': [
{
'original_address': 'original_address'
},
],
'control_number':
1,
'titles': [
{
'title': 'title'
},
],
},
},
{
'_type': 'conferences',
'_source': {
'control_number': 2,
},
},
],
'total':
2,
},
}).hits
expected = ([
[
'<a href="/conferences/1">title</a>',
'original_address',
'',
u' ',
],
], 1)
result = render_conferences(2, hits)
assert expected == result
def test_render_conferences_handles_unicode(request_context):
hits = Response(
Search(), {
'hits': {
'hits': [
{
'_type': 'conference',
'_source': {
'addresses': [
{
'original_address': 'Paris, France'
},
],
'control_number':
1351301,
'titles': [
{
'title': u'Théorie de Cordes en France'
},
],
},
},
],
'total':
1,
},
}).hits
expected = ([
[
u'<a href="/conferences/1351301">Théorie de Cordes en France</a>',
'Paris, France',
'',
u' ',
],
], 1)
result = render_conferences(1, hits)
assert expected == result
def fake_es_all_but_first_none(*args):
return [Response(Search(), response) for response in get_json_file('es_all_but_first_none.json')]
def test_es_search_format_response_multiple_resources(self) -> None:
mock_es_dsl_search = Search()
mock_es_dsl_responses = [
Response(mock_es_dsl_search, r) for r in RESPONSE_2
]
print(mock_es_dsl_responses)
formatted_response = self.es_proxy._format_response(
page_index=0,
results_per_page=10,
responses=mock_es_dsl_responses,
resource_types=[Resource.TABLE, Resource.USER, Resource.FEATURE])
expected = SearchResponse(
msg='Success',
page_index=0,
results_per_page=10,
results={
"table": {
"results": [{
"key":
"mock_db://mock_cluster.mock_schema/mock_table_1",
"description":
"mock table description",
"badges": ["pii", "beta"],
"tag": ["mock_tag_1", "mock_tag_2", "mock_tag_3"],
"schema":
"mock_schema",
"table":
"mock_table_1",
"column": ["mock_col_1", "mock_col_2", "mock_col_3"],
"database":
"mock_db",
"cluster":
"mock_cluster",
"search_score":
804.52716,
"resource_type":
"table"
}, {
"key":
"mock_db://mock_cluster.mock_schema/mock_table_2",
"description":
"mock table description",
"badges": [],
"tag": ["mock_tag_4", "mock_tag_5", "mock_tag_6"],
"schema":
"mock_schema",
"table":
"mock_table_2",
"column": ["mock_col_1", "mock_col_2", "mock_col_3"],
"database":
"mock_db",
"cluster":
"mock_cluster",
"search_score":
9.104584,
"resource_type":
"table"
}],
"total_results":
2
},
"user": {
"results": [{
"full_name": "Allison Suarez Miranda",
"first_name": "Allison",
"last_name": "Suarez Miranda",
"email": "*****@*****.**",
"search_score": 61.40606,
"resource_type": "user"
}],
"total_results":
1
},
"feature": {
"results": [{
"key": "none/feature_1/1",
"feature_group": "fg_2",
"feature_name": "feature_1",
"description": "mock feature description",
"entity": None,
"status": "active",
"version": 1,
"availability": None,
"tags": [],
"badges": [],
"search_score": 62.66787,
"resource_type": "feature"
}, {
"key": "fg_2/feature_2/1",
"feature_group": "fg_2",
"feature_name": "feature_2",
"description": "mock feature description",
"entity": None,
"status": "active",
"version": 1,
"availability": None,
"tags": [],
"badges": [],
"search_score": 62.66787,
"resource_type": "feature"
}, {
"key": "fg_3/feature_3/2",
"feature_group": "fg_3",
"feature_name": "feature_3",
"description": "mock feature description",
"entity": None,
"status": "active",
"version": 2,
"availability": None,
"tags": [],
"badges": ["pii"],
"search_score": 62.66787,
"resource_type": "feature"
}],
"total_results":
3
}
},
status_code=200)
self.assertEqual(formatted_response, expected)
def fake_es_multiple_queries(*args):
return [Response(Search(), response) for response in get_json_file('es_response.json')] + \
[Response(Search(), response) for response in get_json_file('es_percentile.json')]
def fake_es_multiple_terms(*args):
return [Response(Search(), response) for response in get_json_file('es_multiple_terms.json')]
def fake_es_percentile(*args):
return [Response(Search(), response) for response in get_json_file('es_percentile.json')]
def fake_es_filters_aggregation(*args):
return [Response(Search(), response) for response in get_json_file('es_filters_aggregation.json')]
def empty_es_response(*args):
return [Response(Search(), response) for response in get_json_file('es_empty_response.json')]
def fake_es_one_datapoint(*args):
return [Response(Search(), response) for response in get_json_file('es_one_datapoint.json')]
def fake_es_only_none(*args):
return [Response(Search(), response) for response in get_json_file('es_only_none.json')]
def test_es_search_format_response_1_resource(self) -> None:
mock_es_dsl_search = Search()
mock_es_dsl_responses = [
Response(mock_es_dsl_search, r) for r in RESPONSE_1
]
formatted_response = self.es_proxy._format_response(
page_index=0,
results_per_page=10,
responses=mock_es_dsl_responses,
resource_types=[Resource.TABLE, Resource.USER])
expected = SearchResponse(
msg='Success',
page_index=0,
results_per_page=10,
results={
"table": {
"results": [{
"key":
"mock_db://mock_cluster.mock_schema/mock_table_1",
"description":
"mock table description",
"badges": ["pii", "beta"],
"tag": ["mock_tag_1", "mock_tag_2", "mock_tag_3"],
"schema":
"mock_schema",
"table":
"mock_table_1",
"column": ["mock_col_1", "mock_col_2", "mock_col_3"],
"database":
"mock_db",
"cluster":
"mock_cluster",
"search_score":
804.52716,
"resource_type":
"table"
}, {
"key":
"mock_db://mock_cluster.mock_schema/mock_table_2",
"description":
"mock table description",
"badges": [],
"tag": ["mock_tag_4", "mock_tag_5", "mock_tag_6"],
"schema":
"mock_schema",
"table":
"mock_table_2",
"column": ["mock_col_1", "mock_col_2", "mock_col_3"],
"database":
"mock_db",
"cluster":
"mock_cluster",
"search_score":
9.104584,
"resource_type":
"table"
}],
"total_results":
2
},
"user": {
"results": [],
"total_results": 0
}
},
status_code=200)
self.assertEqual(formatted_response, expected)