def _ex_search(self, limit=None, info=False, retries=3):
"""Execute a search and return the results, up to the ``SEARCH_LIMIT``.
Uses the query currently in this SearchHelper.
Arguments:
limit (int): Maximum number of entries to return. **Default**: ``10`` for basic
queries, and ``10000`` for advanced.
info (bool): If ``False``, search will return a list of the results.
If ``True``, search will return a tuple containing the results list
and other information about the query.
**Default:** ``False``.
retries (int): The number of times to retry a Search query if it fails.
**Default:** 3.
Returns:
If ``info`` is ``False``, *list*: The search results.
If ``info`` is ``True``, *tuple*: The search results,
and a dictionary of query information.
"""
# Make sure there is query information present
if not self.initialized:
raise ValueError('No query has been set.')
# Create Search-ready query
if limit is not None:
self.__query["limit"] = limit
query = _validate_query(self.__query)
tries = 0
errors = []
while True:
# Try searching until success or `retries` number of failures
# Raise exception after `retries` failures
try:
search_res = self.__search_client.post_search(
self.index, query)
except globus_sdk.SearchAPIError as e:
if tries >= retries:
raise
else:
errors.append(repr(e))
except Exception as e:
if tries >= retries:
raise
else:
errors.append(repr(e))
else:
break
tries += 1
# Remove the wrapping on each entry from Globus search
res = mdf_toolbox.gmeta_pop(search_res, info=info)
# Add more information to output if requested
if info:
# Add everything from the query itself
info_dict = mdf_toolbox.dict_merge(res[1], query)
# But rename "q" to "query" for clarity
info_dict["query"] = info_dict.pop("q")
# Add other useful/interesting parameters
info_dict["index_uuid"] = self.index
info_dict["retries"] = tries
info_dict["errors"] = errors
# Remake tuple because tuples don't suport assignment
res = (res[0], info_dict)
return res
def test_gmeta_pop():
class TestResponse():
status_code = 200
headers = {"Content-Type": "json"}
data = {
'@datatype':
'GSearchResult',
'@version':
'2016-11-09',
'count':
11,
'gmeta': [{
'@datatype':
'GMetaResult',
'@version':
'2016-11-09',
'content': [{
'mdf': {
'links': {
'landing_page':
'https://data.materialsdatafacility.org/test/test_fetch.txt',
'txt': {
"globus_endpoint":
"82f1b5c6-6e9b-11e5-ba47-22000b92c6ec",
"http_host":
"https://data.materialsdatafacility.org",
"path": "/test/test_fetch.txt"
}
}
}
}, {
'mdf': {
'links': {
'landing_page':
'https://data.materialsdatafacility.org/test/test_fetch.txt',
'txt': {
"globus_endpoint":
"82f1b5c6-6e9b-11e5-ba47-22000b92c6ec",
"http_host":
"https://data.materialsdatafacility.org",
"path": "/test/test_fetch.txt"
}
}
}
}],
'subject':
'https://data.materialsdatafacility.org/test/test_fetch.txt',
}],
'offset':
0,
'total':
22
}
text = json.dumps(data)
def json(self):
return self.data
ghttp = globus_sdk.GlobusHTTPResponse(TestResponse())
popped = mdf_toolbox.gmeta_pop(ghttp)
assert popped == [{
'mdf': {
'links': {
'landing_page':
'https://data.materialsdatafacility.org/test/test_fetch.txt',
'txt': {
'globus_endpoint': '82f1b5c6-6e9b-11e5-ba47-22000b92c6ec',
'http_host': 'https://data.materialsdatafacility.org',
'path': '/test/test_fetch.txt'
}
}
}
}, {
'mdf': {
'links': {
'landing_page':
'https://data.materialsdatafacility.org/test/test_fetch.txt',
'txt': {
'globus_endpoint': '82f1b5c6-6e9b-11e5-ba47-22000b92c6ec',
'http_host': 'https://data.materialsdatafacility.org',
'path': '/test/test_fetch.txt'
}
}
}
}]
info_pop = mdf_toolbox.gmeta_pop(ghttp, info=True)
assert info_pop == (popped, {'total_query_matches': 22})
# String loading
str_gmeta = json.dumps({
"gmeta": [{
"content": [{
"test1": "test1"
}, {
"test2": "test2"
}]
}, {
"content": [{
"test3": "test3"
}, {
"test4": "test4"
}]
}]
})
assert mdf_toolbox.gmeta_pop(str_gmeta) == [{
"test1": "test1"
}, {
"test2": "test2"
}, {
"test3": "test3"
}, {
"test4": "test4"
}]
# Error on bad data
with pytest.raises(TypeError):
mdf_toolbox.gmeta_pop(1)