def _on_database_select(self, _):
"""Load chosen database"""
self.structure_drop.reset()
if (self.database[1] is None
or getattr(self.database[1], "base_url", None) is None):
self.query_button.tooltip = "Search - No database chosen"
self.freeze()
else:
self.offset = 0
self.number = 1
self.structure_page_chooser.silent_reset()
try:
self.freeze()
self.query_button.description = "Updating ..."
self.query_button.icon = "cog"
self.query_button.tooltip = "Updating filters ..."
self._set_intslider_ranges()
self._set_version()
except Exception as exc: # pylint: disable=broad-except
LOGGER.error(
"Exception raised during setting IntSliderRanges: %s",
exc.with_traceback(),
)
finally:
self.query_button.description = "Search"
self.query_button.icon = "search"
self.query_button.tooltip = "Search"
self.unfreeze()
def _set_version(self):
"""Set self.database_version from an /info query"""
base_url = self.database[1].base_url
if base_url not in self.__cached_versions:
# Retrieve and cache version
response = perform_optimade_query(
base_url=self.database[1].base_url, endpoint="/info")
msg, _ = handle_errors(response)
if msg:
raise QueryError(msg)
if "meta" not in response:
raise QueryError(
f"'meta' field not found in /info endpoint for base URL: {base_url}"
)
if "api_version" not in response["meta"]:
raise QueryError(
f"'api_version' field not found in 'meta' for base URL: {base_url}"
)
version = response["meta"]["api_version"]
if version.startswith("v"):
version = version[1:]
self.__cached_versions[base_url] = version
LOGGER.debug(
"Cached version %r for base URL: %r",
self.__cached_versions[base_url],
base_url,
)
self.database_version = self.__cached_versions[base_url]
def _on_database_change(self, change):
"""Update database summary, since self.database has been changed"""
LOGGER.debug("Database changed in summary. New value: %r",
change["new"])
if not change["new"] or change["new"] is None:
self.database_summary.value = ""
else:
self._update_database()
def _on_provider_change(self, change: dict):
"""Update provider summary, since self.provider has been changed"""
LOGGER.debug("Provider changed in summary. New value: %r",
change["new"])
self.database_summary.value = ""
if not change["new"] or change["new"] is None:
self.provider_summary.value = ""
else:
self._update_provider()
def __init__(self, *args: Tuple[Any]):
LOGGER.error(
"%s raised.\nError message: %s\nAbout this exception: %s",
args[0].__class__.__name__ if args
and isinstance(args[0], Exception) else self.__class__.__name__,
str(args[0]) if args else "",
args[0].__doc__
if args and isinstance(args[0], Exception) else self.__doc__,
)
super().__init__(*args)
def __init__(self, *args: Tuple[Any]):
LOGGER.warning(
"%s warned.\nWarning message: %s\nAbout this warning: %s",
args[0].__class__.__name__ if args
and isinstance(args[0], Exception) else self.__class__.__name__,
str(args[0]) if args else "",
args[0].__doc__
if args and isinstance(args[0], Exception) else self.__doc__,
)
super().__init__(*args)
def _get_file(filename: str) -> Union[str, bytes]:
"""Read and return file"""
path = Path(filename).resolve()
LOGGER.debug("Trying image file path: %s", str(path))
if path.exists() and path.is_file():
with open(path, "rb") as file_handle:
res = file_handle.read()
return res
LOGGER.debug("File %s either does not exist or is not a file", str(path))
return ""
def _query(self, link: str = None) -> dict:
"""Query helper function"""
# If a complete link is provided, use it straight up
if link is not None:
try:
response = requests.get(link, timeout=TIMEOUT_SECONDS).json()
except (
requests.exceptions.ConnectTimeout,
requests.exceptions.ConnectionError,
) as exc:
response = {
"errors": {
"msg": "CLIENT: Connection error or timeout.",
"url": link,
"Exception": repr(exc),
}
}
except JSONDecodeError as exc:
response = {
"errors": {
"msg": "CLIENT: Could not decode response to JSON.",
"url": link,
"Exception": repr(exc),
}
}
return response
# Avoid structures with null positions and with assemblies.
add_to_filter = 'NOT structure_features HAS ANY "assemblies"'
if not self._uses_new_structure_features():
add_to_filter += ',"unknown_positions"'
optimade_filter = self.filters.collect_value()
optimade_filter = ("( {} ) AND ( {} )".format(optimade_filter,
add_to_filter)
if optimade_filter else add_to_filter)
LOGGER.debug("Querying with filter: %s", optimade_filter)
# OPTIMADE queries
queries = {
"base_url": self.database[1].base_url,
"filter": optimade_filter,
"page_limit": self.page_limit,
"page_offset": self.offset,
"page_number": self.number,
}
LOGGER.debug(
"Parameters (excluding filter) sent to query util func: %s",
{key: value
for key, value in queries.items() if key != "filter"},
)
return perform_optimade_query(**queries)
def operator_and_integer(field: str, value: str) -> str:
"""Handle operator for values with integers and a possible operator prefixed"""
LOGGER.debug(
"Parsing input with operator_and_integer. <field: %r>, <value: %r>",
field,
value,
)
match_operator = re.findall(r"[<>]?=?", value)
match_no_operator = re.findall(r"^\s*[0-9]+", value)
LOGGER.debug(
"Finding all operators (or none):\nmatch_operator: %r\nmatch_no_operator: %r",
match_operator,
match_no_operator,
)
if match_operator and any(match_operator):
match_operator = [_ for _ in match_operator if _]
if len(match_operator) != 1:
raise ParserError(
"Multiple values given with operators.",
field,
value,
extras=("match_operator", match_operator),
)
number = re.findall(r"[0-9]+", value)[0]
operator = match_operator[0].replace(r"\s*", "")
return f"{operator}{number}"
if match_no_operator and any(match_no_operator):
match_no_operator = [_ for _ in match_no_operator if _]
if len(match_no_operator) != 1:
raise ParserError(
"Multiple values given, must be an integer, "
"either with or without an operator prefixed.",
field,
value,
extras=("match_no_operator", match_no_operator),
)
result = match_no_operator[0].replace(r"\s*", "")
return f"={result}"
raise ParserError(
"Not proper input. Should be, e.g., '>=3' or '5'",
field,
value,
extras=[
("match_operator", match_operator),
("match_no_operator", match_no_operator),
],
)
def ranged_int(field: str, value: Tuple[int, int]) -> str:
"""Turn IntRangeSlider widget value into OPTIMADE filter string"""
LOGGER.debug("ranged_int: Received value %r for field %r", value,
field)
low, high = value
if low == high:
# Exactly N of property
res = f"={low}"
else:
# Range of property
res = [f">={low}", f"<={high}"]
LOGGER.debug("ranged_int: Concluded the response is %r", res)
return res
def _uses_new_structure_features(self) -> bool:
"""Check whether self.database_version is >= v1.0.0-rc.2"""
critical_version = SemanticVersion("1.0.0-rc.2")
version = SemanticVersion(self.database_version)
LOGGER.debug("Semantic version: %r", version)
if version.base_version > critical_version.base_version:
return True
if version.base_version == critical_version.base_version:
if version.prerelease:
return version.prerelease >= critical_version.prerelease
# Version is bigger than critical version and is not a pre-release
return True
# Major.Minor.Patch is lower than critical version
return False
def _toggle_debug_logging(change: dict):
"""Set logging level depending on toggle button"""
if change["new"]:
# Set logging level DEBUG
WIDGET_HANDLER.setLevel(logging.DEBUG)
LOGGER.info("Set log output in widget to level DEBUG")
LOGGER.debug("This should now be shown")
else:
# Set logging level to INFO
WIDGET_HANDLER.setLevel(logging.INFO)
LOGGER.info("Set log output in widget to level INFO")
LOGGER.debug("This should now NOT be shown")
def update_ranged_inputs(self, change: dict):
"""Update ranged inputs' min/max values"""
ranges = change["new"]
if not ranges or ranges is None:
return
for field, config in ranges.items():
if field not in self.query_fields:
raise ParserError(
field=field,
value="N/A",
extras=[
("config", config),
("self.query_fields.keys", self.query_fields.keys()),
],
msg=
"Provided field is unknown. Can not update range for unknown field.",
)
widget = self.query_fields[field].input_widget
cached_value: Tuple[int, int] = widget.value
for attr in ("min", "max"):
if attr in config:
try:
new_value = int(config[attr])
except (TypeError, ValueError) as exc:
raise ParserError(
field=field,
value=cached_value,
extras=[("attr", attr),
("config[attr]", config[attr])],
msg=
f"Could not cast config[attr] to int. Exception: {exc!s}",
)
LOGGER.debug(
"Setting %s for %s to %d.\nWidget immediately before: %r",
attr,
field,
new_value,
widget,
)
# Since "min" is always set first, to be able to set "min" to a valid value,
# "max" is first set to the new "min" value + 1 IF the new "min" value is
# larger than the current "max" value, otherwise there is no reason,
# and it may indeed lead to invalid attribute setting, if this is done.
# For "max", coming last, this should then be fine, as the new "min" and "max"
# values should never be an invalid pair.
if attr == "min" and new_value > cached_value[1]:
widget.max = new_value + 1
setattr(widget, attr, new_value)
LOGGER.debug("Updated widget %r:\n%r", attr, widget)
widget.value = (widget.min, widget.max)
LOGGER.debug("Final state, updated widget:\n%r", widget)
def _update_child_dbs(
data: List[dict]) -> Tuple[List[str], List[LinksResource]]:
"""Update child DB dropdown from response data"""
child_dbs = []
exclude_dbs = []
for entry in data:
child_db = update_old_links_resources(entry)
if child_db is None:
continue
attributes = child_db.attributes
# Skip if not a 'child' link_type database
if attributes.link_type != LinkType.CHILD:
LOGGER.debug(
"Skip %s: Links resource not a %r link_type, instead: %r",
attributes.name,
LinkType.CHILD,
attributes.link_type,
)
continue
# Skip if there is no base_url
if attributes.base_url is None:
LOGGER.debug(
"Skip %s: Base URL found to be None for child DB: %r",
attributes.name,
child_db,
)
exclude_dbs.append(child_db.id)
continue
versioned_base_url = get_versioned_base_url(attributes.base_url)
if versioned_base_url:
attributes.base_url = versioned_base_url
else:
# Not a valid/supported child DB: skip
LOGGER.debug(
"Skip %s: Could not determine versioned base URL for child DB: %r",
attributes.name,
child_db,
)
exclude_dbs.append(child_db.id)
continue
child_dbs.append((attributes.name, attributes))
return exclude_dbs, child_dbs
def _get_more_child_dbs(self, change):
"""Query for more child DBs according to page_offset"""
if self.providers.value is None:
# This may be called if a provider is suddenly removed (bad provider)
return
pageing: Union[int, str] = change["new"]
LOGGER.debug(
"Detected change in page_chooser's .page_offset, .page_number, or .page_link: %r",
pageing,
)
if change["name"] == "page_offset":
LOGGER.debug(
"Got offset %d to retrieve more child DBs from %r",
pageing,
self.providers.value,
)
self.offset = pageing
pageing = None
elif change["name"] == "page_number":
LOGGER.debug(
"Got number %d to retrieve more child DBs from %r",
pageing,
self.providers.value,
)
self.number = pageing
pageing = None
else:
LOGGER.debug(
"Got link %r to retrieve more child DBs from %r",
pageing,
self.providers.value,
)
# It is needed to update page_offset, but we do not wish to query again
with self.hold_trait_notifications():
self.__perform_query = False
self.page_chooser.update_offset()
if not self.__perform_query:
self.__perform_query = True
LOGGER.debug("Will not perform query with pageing: %r", pageing)
return
try:
# Freeze and disable both dropdown widgets
# We don't want changes leading to weird things happening prior to the query ending
self.freeze()
# Query index meta-database
LOGGER.debug("Querying for more child DBs using pageing: %r",
pageing)
child_dbs, links, _, _ = self._query(pageing)
data_returned = self.page_chooser.data_returned
while True:
# Update list of structures in dropdown widget
exclude_child_dbs, final_child_dbs = self._update_child_dbs(
child_dbs)
data_returned -= len(exclude_child_dbs)
if exclude_child_dbs and data_returned:
child_dbs, links, data_returned, _ = self._query(
link=pageing, exclude_ids=exclude_child_dbs)
else:
break
self._set_child_dbs(final_child_dbs)
# Update pageing
self.page_chooser.set_pagination_data(data_returned=data_returned,
links_to_page=links)
except QueryError as exc:
LOGGER.debug(
"Trying to retrieve more child DBs (new page). QueryError caught: %r",
exc,
)
if exc.remove_target:
LOGGER.debug(
"Remove target: %r. Will remove target at %r: %r",
exc.remove_target,
self.providers.index,
self.providers.value,
)
self.providers.options = self._remove_current_dropdown_option(
self.providers)
self.reset()
else:
LOGGER.debug(
"Remove target: %r. Will NOT remove target at %r: %r",
exc.remove_target,
self.providers.index,
self.providers.value,
)
self.child_dbs.options = self.INITIAL_CHILD_DBS
self.child_dbs.disabled = True
else:
self.unfreeze()
def _set_intslider_ranges(self):
"""Update IntRangeSlider ranges according to chosen database
Query database to retrieve ranges.
Cache ranges in self.__cached_ranges.
"""
defaults = {
"nsites": {
"min": 0,
"max": 10000
},
"nelements": {
"min": 0,
"max": len(CHEMICAL_SYMBOLS)
},
}
db_base_url = self.database[1].base_url
if db_base_url not in self.__cached_ranges:
self.__cached_ranges[db_base_url] = {}
sortable_fields = check_entry_properties(
base_url=db_base_url,
entry_endpoint="structures",
properties=["nsites", "nelements"],
checks=["sort"],
)
for response_field in sortable_fields:
if response_field in self.__cached_ranges[db_base_url]:
# Use cached value(s)
continue
page_limit = 1
new_range = {}
for extremum, sort in [
("min", response_field),
("max", f"-{response_field}"),
]:
query_params = {
"base_url": db_base_url,
"page_limit": page_limit,
"response_fields": response_field,
"sort": sort,
}
LOGGER.debug(
"Querying %s to get %s of %s.\nParameters: %r",
self.database[0],
extremum,
response_field,
query_params,
)
response = perform_optimade_query(**query_params)
msg, _ = handle_errors(response)
if msg:
raise QueryError(msg)
if not response.get("meta", {}).get("data_available", 0):
new_range[extremum] = defaults[response_field][extremum]
else:
new_range[extremum] = (response.get("data", [{}])[0].get(
"attributes", {}).get(response_field, None))
# Cache new values
LOGGER.debug(
"Caching newly found range values for %s\nValue: %r",
db_base_url,
{response_field: new_range},
)
self.__cached_ranges[db_base_url].update(
{response_field: new_range})
if not self.__cached_ranges[db_base_url]:
LOGGER.debug("No values found for %s, storing default values.",
db_base_url)
self.__cached_ranges[db_base_url].update({
"nsites": {
"min": 0,
"max": 10000
},
"nelements": {
"min": 0,
"max": len(CHEMICAL_SYMBOLS)
},
})
# Set widget's new extrema
LOGGER.debug(
"Updating range extrema for %s\nValues: %r",
db_base_url,
self.__cached_ranges[db_base_url],
)
self.filters.update_range_filters(self.__cached_ranges[db_base_url])
def _initialize_child_dbs(self):
"""New provider chosen; initialize child DB dropdown"""
self.offset = 0
self.number = 1
try:
# Freeze and disable list of structures in dropdown widget
# We don't want changes leading to weird things happening prior to the query ending
self.freeze()
# Reset the error or status message
if self.error_or_status_messages.value:
self.error_or_status_messages.value = ""
if self.provider.base_url in self.__cached_child_dbs:
cache = self.__cached_child_dbs[self.provider.base_url]
LOGGER.debug(
"Initializing child DBs for %s. Using cached info:\n%r",
self.provider.name,
cache,
)
self._set_child_dbs(cache["child_dbs"])
data_returned = cache["data_returned"]
data_available = cache["data_available"]
links = cache["links"]
else:
LOGGER.debug("Initializing child DBs for %s.",
self.provider.name)
# Query database and get child_dbs
child_dbs, links, data_returned, data_available = self._query()
while True:
# Update list of structures in dropdown widget
exclude_child_dbs, final_child_dbs = self._update_child_dbs(
child_dbs)
LOGGER.debug("Exclude child DBs: %r", exclude_child_dbs)
data_returned -= len(exclude_child_dbs)
if exclude_child_dbs and data_returned:
child_dbs, links, data_returned, _ = self._query(
exclude_ids=exclude_child_dbs)
else:
break
self._set_child_dbs(final_child_dbs)
# Cache initial child_dbs and related information
self.__cached_child_dbs[self.provider.base_url] = {
"child_dbs": final_child_dbs,
"data_returned": data_returned,
"data_available": data_available,
"links": links,
}
LOGGER.debug(
"Found the following, which has now been cached:\n%r",
self.__cached_child_dbs[self.provider.base_url],
)
# Update pageing
self.page_chooser.set_pagination_data(
data_returned=data_returned,
data_available=data_available,
links_to_page=links,
reset_cache=True,
)
except QueryError as exc:
LOGGER.debug(
"Trying to initalize child DBs. QueryError caught: %r", exc)
if exc.remove_target:
LOGGER.debug(
"Remove target: %r. Will remove target at %r: %r",
exc.remove_target,
self.providers.index,
self.providers.value,
)
self.providers.options = self._remove_current_dropdown_option(
self.providers)
self.reset()
else:
LOGGER.debug(
"Remove target: %r. Will NOT remove target at %r: %r",
exc.remove_target,
self.providers.index,
self.providers.value,
)
self.child_dbs.options = self.INITIAL_CHILD_DBS
self.child_dbs.disabled = True
else:
self.unfreeze()
def _query( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
self,
link: str = None,
exclude_ids: List[str] = None) -> Tuple[List[dict], dict, int,
int]:
"""Query helper function"""
# If a complete link is provided, use it straight up
if link is not None:
try:
if exclude_ids:
filter_value = " AND ".join(
[f"id!={id_}" for id_ in exclude_ids])
parsed_url = urllib.parse.urlparse(link)
queries = urllib.parse.parse_qs(parsed_url.query)
# Since parse_qs wraps all values in a list,
# this extracts the values from the list(s).
queries = {key: value[0] for key, value in queries.items()}
if "filter" in queries:
queries[
"filter"] = f"( {queries['filter']} ) AND ( {filter_value} )"
else:
queries["filter"] = filter_value
parsed_query = urllib.parse.urlencode(queries)
link = (
f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
f"?{parsed_query}")
response = requests.get(link, timeout=TIMEOUT_SECONDS).json()
except (
requests.exceptions.ConnectTimeout,
requests.exceptions.ConnectionError,
) as exc:
response = {
"errors": {
"msg": "CLIENT: Connection error or timeout.",
"url": link,
"Exception": repr(exc),
}
}
except json.JSONDecodeError as exc:
response = {
"errors": {
"msg": "CLIENT: Could not decode response to JSON.",
"url": link,
"Exception": repr(exc),
}
}
else:
filter_ = "link_type=child OR type=child"
if exclude_ids:
filter_ += (" AND ( " +
" AND ".join([f"id!={id_}"
for id_ in exclude_ids]) + " )")
response = perform_optimade_query(
filter=filter_,
base_url=self.provider.base_url,
endpoint="/links",
page_limit=self.child_db_limit,
page_offset=self.offset,
page_number=self.number,
)
msg, http_errors = handle_errors(response)
if msg:
if 404 in http_errors:
# If /links not found move on
pass
else:
self.error_or_status_messages.value = msg
raise QueryError(msg=msg, remove_target=True)
# Check implementation API version
msg = validate_api_version(response.get("meta",
{}).get("api_version", ""),
raise_on_fail=False)
if msg:
self.error_or_status_messages.value = (
f"{msg}<br>The provider has been removed.")
raise QueryError(msg=msg, remove_target=True)
LOGGER.debug(
"Manually remove `exclude_ids` if filters are not supported")
child_db_data = {
impl.get("id", "N/A"): impl
for impl in response.get("data", [])
}
if exclude_ids:
for links_id in exclude_ids:
if links_id in list(child_db_data.keys()):
child_db_data.pop(links_id)
LOGGER.debug("child_db_data after popping: %r", child_db_data)
response["data"] = list(child_db_data.values())
if "meta" in response:
if "data_available" in response["meta"]:
old_data_available = response["meta"].get(
"data_available", 0)
if len(response["data"]) > old_data_available:
LOGGER.debug("raising OptimadeClientError")
raise OptimadeClientError(
f"Reported data_available ({old_data_available}) is smaller than "
f"curated list of responses ({len(response['data'])}).",
)
response["meta"]["data_available"] = len(response["data"])
else:
raise OptimadeClientError(
"'meta' not found in response. Bad response")
LOGGER.debug(
"Attempt for %r (in /links): Found implementations (names+base_url only):\n%s",
self.provider.name,
[
f"(id: {name}; base_url: {base_url}) " for name, base_url in [(
impl.get("id", "N/A"),
impl.get("attributes", {}).get("base_url", "N/A"),
) for impl in response.get("data", [])]
],
)
# Return all implementations of link_type "child"
implementations = [
implementation for implementation in response.get("data", [])
if (implementation.get("attributes", {}).get("link_type", "") ==
"child" or implementation.get("type", "") == "child")
]
LOGGER.debug(
"After curating for implementations which are of 'link_type' = 'child' or 'type' == "
"'child' (old style):\n%s",
[
f"(id: {name}; base_url: {base_url}) " for name, base_url in [(
impl.get("id", "N/A"),
impl.get("attributes", {}).get("base_url", "N/A"),
) for impl in implementations]
],
)
# Get links, data_returned, and data_available
links = response.get("links", {})
data_returned = response.get("meta", {}).get("data_returned",
len(implementations))
if data_returned > 0 and not implementations:
# Most probably dealing with pre-v1.0.0-rc.2 implementations
data_returned = 0
data_available = response.get("meta", {}).get("data_available", 0)
return implementations, links, data_returned, data_available