def _set_version(self):
"""Set self.database_version from an /info query"""
base_url = self.database[1].base_url
if base_url not in self.__cached_versions:
# Retrieve and cache version
response = perform_optimade_query(
base_url=self.database[1].base_url, endpoint="/info")
msg, _ = handle_errors(response)
if msg:
raise QueryError(msg)
if "meta" not in response:
raise QueryError(
f"'meta' field not found in /info endpoint for base URL: {base_url}"
)
if "api_version" not in response["meta"]:
raise QueryError(
f"'api_version' field not found in 'meta' for base URL: {base_url}"
)
version = response["meta"]["api_version"]
if version.startswith("v"):
version = version[1:]
self.__cached_versions[base_url] = version
LOGGER.debug(
"Cached version %r for base URL: %r",
self.__cached_versions[base_url],
base_url,
)
self.database_version = self.__cached_versions[base_url]
def retrieve_data(self, _):
"""Perform query and retrieve data"""
self.offset = 0
self.number = 1
try:
# Freeze and disable list of structures in dropdown widget
# We don't want changes leading to weird things happening prior to the query ending
self.freeze()
# Reset the error or status message
if self.error_or_status_messages.value:
self.error_or_status_messages.value = ""
# Update button text and icon
self.query_button.description = "Querying ... "
self.query_button.icon = "cog"
self.query_button.tooltip = "Please wait ..."
# Query database
response = self._query()
msg, _ = handle_errors(response)
if msg:
self.error_or_status_messages.value = msg
raise QueryError(msg)
# Update list of structures in dropdown widget
self._update_structures(response["data"])
# Update pageing
if self._data_available is None:
self._data_available = response.get("meta", {}).get(
"data_available", None)
data_returned = response.get("meta",
{}).get("data_returned",
len(response.get("data", [])))
self.structure_page_chooser.set_pagination_data(
data_returned=data_returned,
data_available=self._data_available,
links_to_page=response.get("links", {}),
reset_cache=True,
)
except QueryError:
self.structure_drop.reset()
self.structure_page_chooser.reset()
raise
except Exception as exc:
self.structure_drop.reset()
self.structure_page_chooser.reset()
raise QueryError(
f"Bad stuff happened: {traceback.format_exc()}") from exc
finally:
self.query_button.description = "Search"
self.query_button.icon = "search"
self.query_button.tooltip = "Search"
self.unfreeze()
def _get_more_results(self, change):
"""Query for more results according to pageing"""
if not self.__perform_query:
self.__perform_query = True
LOGGER.debug(
"NOT going to perform query with change: name=%s value=%s",
change["name"],
change["new"],
)
return
pageing: Union[int, str] = change["new"]
LOGGER.debug(
"Updating results with pageing change: name=%s value=%s",
change["name"],
pageing,
)
if change["name"] == "page_offset":
self.offset = pageing
pageing = None
elif change["name"] == "page_number":
self.number = pageing
pageing = None
else:
# It is needed to update page_offset, but we do not wish to query again
with self.hold_trait_notifications():
self.__perform_query = False
self.structure_page_chooser.update_offset()
try:
# Freeze and disable list of structures in dropdown widget
# We don't want changes leading to weird things happening prior to the query ending
self.freeze()
# Update button text and icon
self.query_button.description = "Updating ... "
self.query_button.icon = "cog"
self.query_button.tooltip = "Please wait ..."
# Query database
response = self._query(pageing)
msg, _ = handle_errors(response)
if msg:
self.error_or_status_messages.value = msg
return
# Update list of structures in dropdown widget
self._update_structures(response["data"])
# Update pageing
self.structure_page_chooser.set_pagination_data(
links_to_page=response.get("links", {}), )
finally:
self.query_button.description = "Search"
self.query_button.icon = "search"
self.query_button.tooltip = "Search"
self.unfreeze()
def _set_intslider_ranges(self):
"""Update IntRangeSlider ranges according to chosen database
Query database to retrieve ranges.
Cache ranges in self.__cached_ranges.
"""
defaults = {
"nsites": {
"min": 0,
"max": 10000
},
"nelements": {
"min": 0,
"max": len(CHEMICAL_SYMBOLS)
},
}
db_base_url = self.database[1].base_url
if db_base_url not in self.__cached_ranges:
self.__cached_ranges[db_base_url] = {}
sortable_fields = check_entry_properties(
base_url=db_base_url,
entry_endpoint="structures",
properties=["nsites", "nelements"],
checks=["sort"],
)
for response_field in sortable_fields:
if response_field in self.__cached_ranges[db_base_url]:
# Use cached value(s)
continue
page_limit = 1
new_range = {}
for extremum, sort in [
("min", response_field),
("max", f"-{response_field}"),
]:
query_params = {
"base_url": db_base_url,
"page_limit": page_limit,
"response_fields": response_field,
"sort": sort,
}
LOGGER.debug(
"Querying %s to get %s of %s.\nParameters: %r",
self.database[0],
extremum,
response_field,
query_params,
)
response = perform_optimade_query(**query_params)
msg, _ = handle_errors(response)
if msg:
raise QueryError(msg)
if not response.get("meta", {}).get("data_available", 0):
new_range[extremum] = defaults[response_field][extremum]
else:
new_range[extremum] = (response.get("data", [{}])[0].get(
"attributes", {}).get(response_field, None))
# Cache new values
LOGGER.debug(
"Caching newly found range values for %s\nValue: %r",
db_base_url,
{response_field: new_range},
)
self.__cached_ranges[db_base_url].update(
{response_field: new_range})
if not self.__cached_ranges[db_base_url]:
LOGGER.debug("No values found for %s, storing default values.",
db_base_url)
self.__cached_ranges[db_base_url].update({
"nsites": {
"min": 0,
"max": 10000
},
"nelements": {
"min": 0,
"max": len(CHEMICAL_SYMBOLS)
},
})
# Set widget's new extrema
LOGGER.debug(
"Updating range extrema for %s\nValues: %r",
db_base_url,
self.__cached_ranges[db_base_url],
)
self.filters.update_range_filters(self.__cached_ranges[db_base_url])
def _query( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
self,
link: str = None,
exclude_ids: List[str] = None) -> Tuple[List[dict], dict, int,
int]:
"""Query helper function"""
# If a complete link is provided, use it straight up
if link is not None:
try:
if exclude_ids:
filter_value = " AND ".join(
[f'NOT id="{id_}"' for id_ in exclude_ids])
parsed_url = urllib.parse.urlparse(link)
queries = urllib.parse.parse_qs(parsed_url.query)
# Since parse_qs wraps all values in a list,
# this extracts the values from the list(s).
queries = {key: value[0] for key, value in queries.items()}
if "filter" in queries:
queries[
"filter"] = f"( {queries['filter']} ) AND ( {filter_value} )"
else:
queries["filter"] = filter_value
parsed_query = urllib.parse.urlencode(queries)
link = (
f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
f"?{parsed_query}")
link = ordered_query_url(link)
response = SESSION.get(link, timeout=TIMEOUT_SECONDS)
if response.from_cache:
LOGGER.debug("Request to %s was taken from cache !", link)
response = response.json()
except (
requests.exceptions.ConnectTimeout,
requests.exceptions.ConnectionError,
requests.exceptions.ReadTimeout,
) as exc:
response = {
"errors": {
"msg": "CLIENT: Connection error or timeout.",
"url": link,
"Exception": repr(exc),
}
}
except json.JSONDecodeError as exc:
response = {
"errors": {
"msg": "CLIENT: Could not decode response to JSON.",
"url": link,
"Exception": repr(exc),
}
}
else:
filter_ = '( link_type="child" OR type="child" )'
if exclude_ids:
filter_ += (
" AND ( " +
" AND ".join([f'NOT id="{id_}"'
for id_ in exclude_ids]) + " )")
response = perform_optimade_query(
filter=filter_,
base_url=self.provider.base_url,
endpoint="/links",
page_limit=self.child_db_limit,
page_offset=self.offset,
page_number=self.number,
)
msg, http_errors = handle_errors(response)
if msg:
if 404 in http_errors:
# If /links not found move on
pass
else:
self.error_or_status_messages.value = msg
raise QueryError(msg=msg, remove_target=True)
# Check implementation API version
msg = validate_api_version(response.get("meta",
{}).get("api_version", ""),
raise_on_fail=False)
if msg:
self.error_or_status_messages.value = (
f"{msg}<br>The provider has been removed.")
raise QueryError(msg=msg, remove_target=True)
LOGGER.debug(
"Manually remove `exclude_ids` if filters are not supported")
child_db_data = {
impl.get("id", "N/A"): impl
for impl in response.get("data", [])
}
if exclude_ids:
for links_id in exclude_ids:
if links_id in list(child_db_data.keys()):
child_db_data.pop(links_id)
LOGGER.debug("child_db_data after popping: %r", child_db_data)
response["data"] = list(child_db_data.values())
if "meta" in response:
if "data_available" in response["meta"]:
old_data_available = response["meta"].get(
"data_available", 0)
if len(response["data"]) > old_data_available:
LOGGER.debug("raising OptimadeClientError")
raise OptimadeClientError(
f"Reported data_available ({old_data_available}) is smaller than "
f"curated list of responses ({len(response['data'])}).",
)
response["meta"]["data_available"] = len(response["data"])
else:
raise OptimadeClientError(
"'meta' not found in response. Bad response")
LOGGER.debug(
"Attempt for %r (in /links): Found implementations (names+base_url only):\n%s",
self.provider.name,
[
f"(id: {name}; base_url: {base_url}) " for name, base_url in [(
impl.get("id", "N/A"),
impl.get("attributes", {}).get("base_url", "N/A"),
) for impl in response.get("data", [])]
],
)
# Return all implementations of link_type "child"
implementations = [
implementation for implementation in response.get("data", [])
if (implementation.get("attributes", {}).get("link_type", "") ==
"child" or implementation.get("type", "") == "child")
]
LOGGER.debug(
"After curating for implementations which are of 'link_type' = 'child' or 'type' == "
"'child' (old style):\n%s",
[
f"(id: {name}; base_url: {base_url}) " for name, base_url in [(
impl.get("id", "N/A"),
impl.get("attributes", {}).get("base_url", "N/A"),
) for impl in implementations]
],
)
# Get links, data_returned, and data_available
links = response.get("links", {})
data_returned = response.get("meta", {}).get("data_returned",
len(implementations))
if data_returned > 0 and not implementations:
# Most probably dealing with pre-v1.0.0-rc.2 implementations
data_returned = 0
data_available = response.get("meta", {}).get("data_available", 0)
return implementations, links, data_returned, data_available