def _prune_answers_to_achieve_curie_to_curie_query(
kg: QGOrganizedKnowledgeGraph, output_qnode_key: str,
qg: QueryGraph) -> QGOrganizedKnowledgeGraph:
"""
This is a way of hacking around BTE's limitation where it can only do (node with curie)-->(non-specific node)
kinds of queries. We do the non-specific query, and then use this function to remove all of the answer nodes
that do not correspond to the curie we wanted for the 'output' node.
"""
# Remove 'output' nodes in the KG that aren't actually the ones we were looking for
output_qnode = qg.nodes[output_qnode_key]
qedge_key = next(qedge_key for qedge_key in qg.edges)
qedge = qg.edges[qedge_key]
desired_output_curies = set(
eu.convert_string_or_list_to_list(output_qnode.id))
all_output_node_keys = set(kg.nodes_by_qg_id[output_qnode_key])
output_node_keys_to_remove = all_output_node_keys.difference(
desired_output_curies)
for node_key in output_node_keys_to_remove:
kg.nodes_by_qg_id[output_qnode_key].pop(node_key)
# And remove any edges that used them
edge_keys_to_remove = set()
for edge_key, edge in kg.edges_by_qg_id[qedge_key].items():
if edge.object in output_node_keys_to_remove: # Edge object always contains output node ID for BTE
edge_keys_to_remove.add(edge_key)
for edge_key in edge_keys_to_remove:
kg.edges_by_qg_id[qedge_key].pop(edge_key)
return kg
def _send_query_to_kp(self, query_graph: QueryGraph,
log: ARAXResponse) -> Dict[str, any]:
# Send query to their API (stripping down qnode/qedges to only the properties they like)
stripped_qnodes = []
for qnode_key, qnode in query_graph.nodes.items():
stripped_qnode = {'id': qnode_key, 'type': qnode.category}
if qnode.id:
stripped_qnode['curie'] = qnode.id
stripped_qnodes.append(stripped_qnode)
qedge_key = next(qedge_key for qedge_key in
query_graph.edges) # Our query graph is single-edge
qedge = query_graph.edges[qedge_key]
stripped_qedge = {
'id': qedge_key,
'source_id': qedge.subject,
'target_id': qedge.object,
'type': list(self.accepted_edge_types)[0]
}
source_stripped_qnode = next(qnode for qnode in stripped_qnodes
if qnode['id'] == qedge.subject)
input_curies = eu.convert_string_or_list_to_list(
source_stripped_qnode['curie'])
combined_response = dict()
for input_curie in input_curies: # Until we have batch querying, ping them one-by-one for each input curie
log.debug(
f"Sending {qedge_key} query to {self.kp_name} for {input_curie}"
)
source_stripped_qnode['curie'] = input_curie
kp_response = requests.post(self.kp_query_endpoint,
json={
'message': {
'query_graph': {
'nodes': stripped_qnodes,
'edges': [stripped_qedge]
}
}
},
headers={'accept': 'application/json'})
if kp_response.status_code != 200:
log.warning(
f"{self.kp_name} KP API returned response of {kp_response.status_code}"
)
else:
kp_response_json = kp_response.json()
if kp_response_json.get('results'):
if not combined_response:
combined_response = kp_response_json
else:
combined_response['knowledge_graph'][
'nodes'] += kp_response_json['knowledge_graph'][
'nodes']
combined_response['knowledge_graph'][
'edges'] += kp_response_json['knowledge_graph'][
'edges']
combined_response['results'] += kp_response_json[
'results']
return combined_response
def _convert_kg1_node_to_swagger_node(
self, neo4j_node: Dict[str, any]) -> Tuple[str, Node]:
swagger_node = Node()
swagger_node_key = neo4j_node.get('id')
swagger_node.name = neo4j_node.get('name')
node_category = neo4j_node.get('category')
swagger_node.category = eu.convert_string_or_list_to_list(
node_category)
other_properties = ["symbol", "description", "uri"]
swagger_node.attributes = self._create_swagger_attributes(
other_properties, neo4j_node)
return swagger_node_key, swagger_node
def _build_kg_to_qg_id_dict(
results: Dict[str, any]) -> Dict[str, Dict[str, List[str]]]:
kg_to_qg_ids = {'nodes': dict(), 'edges': dict()}
for node_binding in results['node_bindings']:
node_key = node_binding['kg_id']
qnode_key = node_binding['qg_id']
kg_to_qg_ids['nodes'][node_key] = qnode_key
for edge_binding in results['edge_bindings']:
edge_keys = eu.convert_string_or_list_to_list(
edge_binding['kg_id'])
qedge_keys = edge_binding['qg_id']
for kg_id in edge_keys:
kg_to_qg_ids['edges'][kg_id] = qedge_keys
return kg_to_qg_ids
def _convert_kg2_node_to_swagger_node(
self, neo4j_node: Dict[str, any]) -> Tuple[str, Node]:
swagger_node = Node()
swagger_node_key = neo4j_node.get('id')
swagger_node.name = neo4j_node.get('name')
node_category = neo4j_node.get('category_label')
swagger_node.category = eu.convert_string_or_list_to_list(
node_category)
# Add all additional properties on KG2 nodes as swagger Attribute objects
other_properties = [
"full_name", "description", "iri", "publications", "synonym",
"category", "provided_by", "deprecated", "update_date"
]
swagger_node.attributes = self._create_swagger_attributes(
other_properties, neo4j_node)
return swagger_node_key, swagger_node
def _pre_process_query_graph(self, query_graph: QueryGraph,
log: ARAXResponse) -> QueryGraph:
for qnode_key, qnode in query_graph.nodes.items():
# Convert node types to preferred format and verify we can do this query
formatted_qnode_categories = {
self.node_category_overrides_for_kp.get(
qnode_category, qnode_category)
for qnode_category in eu.convert_string_or_list_to_list(
qnode.category)
}
accepted_qnode_categories = formatted_qnode_categories.intersection(
self.accepted_node_categories)
if not accepted_qnode_categories:
log.error(
f"{self.kp_name} can only be used for queries involving {self.accepted_node_categories} "
f"and QNode {qnode_key} has category '{qnode.category}'",
error_code="UnsupportedQueryForKP")
return query_graph
else:
qnode.category = list(accepted_qnode_categories)[0]
# Convert curies to equivalent curies accepted by the KP (depending on qnode type)
if qnode.id:
equivalent_curies = eu.get_curie_synonyms(qnode.id, log)
desired_curies = [
curie for curie in equivalent_curies if curie.startswith(
f"{self.kp_preferred_prefixes[qnode.category]}:")
]
if desired_curies:
qnode.id = desired_curies if len(
desired_curies) > 1 else desired_curies[0]
log.debug(
f"Converted qnode {qnode_key} curie to {qnode.id}")
else:
log.warning(
f"Could not convert qnode {qnode_key} curie(s) to preferred prefix ({self.kp_preferred_prefixes[qnode.category]})"
)
return query_graph
def _validate_and_pre_process_input(qg: QueryGraph,
valid_bte_inputs_dict: Dict[str,
Set[str]],
enforce_directionality: bool,
use_synonyms: bool,
log: ARAXResponse) -> Tuple[str, str]:
# Make sure we have a valid one-hop query graph
if len(qg.edges) != 1 or len(qg.nodes) != 2:
log.error(
f"BTE can only accept one-hop query graphs (your QG has {len(qg.nodes)} nodes and "
f"{len(qg.edges)} edges)",
error_code="InvalidQueryGraph")
return "", ""
qedge_key = next(qedge_key for qedge_key in qg.edges)
qedge = qg.edges[qedge_key]
# Make sure at least one of our qnodes has a curie
qnodes_with_curies = [
qnode_key for qnode_key, qnode in qg.nodes.items() if qnode.id
]
if not qnodes_with_curies:
log.error(
f"Neither qnode for qedge {qedge_key} has a curie specified. BTE requires that at least one of "
f"them has a curie. Your query graph is: {qg.to_dict()}",
error_code="UnsupportedQueryForKP")
return "", ""
# Figure out which query node is input vs. output
if enforce_directionality:
input_qnode_key = qedge.subject
output_qnode_key = qedge.object
else:
input_qnode_key = next(qnode_key
for qnode_key, qnode in qg.nodes.items()
if qnode.id)
output_qnode_key = list(
set(qg.nodes).difference({input_qnode_key}))[0]
log.warning(
f"BTE cannot do bidirectional queries; the query for this edge will be directed, going: "
f"{input_qnode_key}-->{output_qnode_key}")
input_qnode = qg.nodes[input_qnode_key]
output_qnode = qg.nodes[output_qnode_key]
# Make sure predicate is allowed
if qedge.predicate not in valid_bte_inputs_dict[
'predicates'] and qedge.predicate is not None:
log.error(
f"BTE does not accept predicate '{qedge.predicate}'. Valid options are "
f"{valid_bte_inputs_dict['predicates']}",
error_code="InvalidInput")
return "", ""
# Process qnode types (convert to preferred format, make sure allowed)
input_qnode.category = [
eu.convert_string_to_pascal_case(node_category) for node_category
in eu.convert_string_or_list_to_list(input_qnode.category)
]
output_qnode.category = [
eu.convert_string_to_pascal_case(node_category) for node_category
in eu.convert_string_or_list_to_list(output_qnode.category)
]
qnodes_missing_type = [
qnode_key for qnode_key in [input_qnode_key, output_qnode_key]
if not qg.nodes[qnode_key].category
]
if qnodes_missing_type:
log.error(
f"BTE requires every query node to have a category. QNode(s) missing a category: "
f"{', '.join(qnodes_missing_type)}",
error_code="InvalidInput")
return "", ""
invalid_qnode_categories = [
node_category for qnode in [input_qnode, output_qnode]
for node_category in qnode.category
if node_category not in valid_bte_inputs_dict['node_categories']
]
if invalid_qnode_categories:
log.error(
f"BTE does not accept QNode category(s): {', '.join(invalid_qnode_categories)}. Valid options are "
f"{valid_bte_inputs_dict['node_categories']}",
error_code="InvalidInput")
return "", ""
# Sub in curie synonyms as appropriate
if use_synonyms:
qnodes_with_curies = [
qnode for qnode in [input_qnode, output_qnode] if qnode.id
]
for qnode in qnodes_with_curies:
synonymized_curies = eu.get_curie_synonyms(qnode.id, log)
qnode.id = synonymized_curies
# Make sure our input node curies are in list form and use prefixes BTE prefers
input_curie_list = eu.convert_string_or_list_to_list(input_qnode.id)
input_qnode.id = [
eu.convert_curie_to_bte_format(curie) for curie in input_curie_list
]
return input_qnode_key, output_qnode_key
def _add_answers_to_kg(self, answer_kg: QGOrganizedKnowledgeGraph,
reasoner_std_response: Dict[str, any],
input_qnode_key: str, output_qnode_key: str,
qedge_key: str,
log: ARAXResponse) -> QGOrganizedKnowledgeGraph:
kg_to_qg_ids_dict = self._build_kg_to_qg_id_dict(
reasoner_std_response['results'])
if reasoner_std_response['knowledge_graph']['edges']:
remapped_node_keys = dict()
log.debug(
f"Got results back from BTE for this query "
f"({len(reasoner_std_response['knowledge_graph']['edges'])} edges)"
)
for node in reasoner_std_response['knowledge_graph']['nodes']:
swagger_node = Node()
bte_node_key = node.get('id')
swagger_node.name = node.get('name')
swagger_node.category = eu.convert_string_or_list_to_list(
eu.convert_string_to_snake_case(node.get('type')))
# Map the returned BTE qg_ids back to the original qnode_keys in our query graph
bte_qg_id = kg_to_qg_ids_dict['nodes'].get(bte_node_key)
if bte_qg_id == "n0":
qnode_key = input_qnode_key
elif bte_qg_id == "n1":
qnode_key = output_qnode_key
else:
log.error("Could not map BTE qg_id to ARAX qnode_key",
error_code="UnknownQGID")
return answer_kg
# Find and use the preferred equivalent identifier for this node (if it's an output node)
if qnode_key == output_qnode_key:
if bte_node_key in remapped_node_keys:
swagger_node_key = remapped_node_keys.get(bte_node_key)
else:
equivalent_curies = [
f"{prefix}:{eu.get_curie_local_id(local_id)}"
for prefix, local_ids in node.get(
'equivalent_identifiers').items()
for local_id in local_ids
]
swagger_node_key = self._get_best_equivalent_bte_curie(
equivalent_curies, swagger_node.category[0])
remapped_node_keys[bte_node_key] = swagger_node_key
else:
swagger_node_key = bte_node_key
answer_kg.add_node(swagger_node_key, swagger_node, qnode_key)
for edge in reasoner_std_response['knowledge_graph']['edges']:
swagger_edge = Edge()
swagger_edge_key = edge.get("id")
swagger_edge.predicate = edge.get('type')
swagger_edge.subject = remapped_node_keys.get(
edge.get('source_id'), edge.get('source_id'))
swagger_edge.object = remapped_node_keys.get(
edge.get('target_id'), edge.get('target_id'))
swagger_edge.attributes = [
Attribute(name="provided_by",
value=edge.get('edge_source'),
type=eu.get_attribute_type("provided_by")),
Attribute(name="is_defined_by",
value="BTE",
type=eu.get_attribute_type("is_defined_by"))
]
# Map the returned BTE qg_id back to the original qedge_key in our query graph
bte_qg_id = kg_to_qg_ids_dict['edges'].get(swagger_edge_key)
if bte_qg_id != "e1":
log.error("Could not map BTE qg_id to ARAX qedge_key",
error_code="UnknownQGID")
return answer_kg
answer_kg.add_edge(swagger_edge_key, swagger_edge, qedge_key)
return answer_kg
def _convert_one_hop_query_graph_to_cypher_query(
self, qg: QueryGraph, enforce_directionality: bool, kg_name: str,
log: ARAXResponse) -> str:
qedge_key = next(qedge_key for qedge_key in qg.edges)
qedge = qg.edges[qedge_key]
log.debug(f"Generating cypher for edge {qedge_key} query graph")
try:
# Build the match clause
source_qnode_key = qedge.subject
target_qnode_key = qedge.object
qedge_cypher = self._get_cypher_for_query_edge(
qedge_key, qg, enforce_directionality)
source_qnode_cypher = self._get_cypher_for_query_node(
source_qnode_key, qg, kg_name)
target_qnode_cypher = self._get_cypher_for_query_node(
target_qnode_key, qg, kg_name)
match_clause = f"MATCH {source_qnode_cypher}{qedge_cypher}{target_qnode_cypher}"
# Build the where clause
where_fragments = []
for qnode_key in [source_qnode_key, target_qnode_key]:
qnode = qg.nodes[qnode_key]
if qnode.id and isinstance(qnode.id,
list) and len(qnode.id) > 1:
where_fragments.append(f"{qnode_key}.id in {qnode.id}")
if qnode.category:
if kg_name == "KG2c":
qnode_categories = eu.convert_string_or_list_to_list(
qnode.category)
category_fragments = [
f"'{qnode_category}' in {qnode_key}.types"
for qnode_category in qnode_categories
]
joined_category_fragments = " OR ".join(
category_fragments)
category_where_clause = joined_category_fragments if len(
category_fragments
) < 2 else f"({joined_category_fragments})"
where_fragments.append(category_where_clause)
elif isinstance(qnode.category, list):
if kg_name == "KG2":
node_category_property = "category_label"
else:
node_category_property = "category"
where_fragments.append(
f"{qnode_key}.{node_category_property} in {qnode.category}"
)
if where_fragments:
where_clause = f"WHERE {' AND '.join(where_fragments)}"
else:
where_clause = ""
# Build the with clause
source_qnode_col_name = f"nodes_{source_qnode_key}"
target_qnode_col_name = f"nodes_{target_qnode_key}"
qedge_col_name = f"edges_{qedge_key}"
# This line grabs the edge's ID and a record of which of its nodes correspond to which qnode ID
extra_edge_properties = "{.*, " + f"id:ID({qedge_key}), {source_qnode_key}:{source_qnode_key}.id, {target_qnode_key}:{target_qnode_key}.id" + "}"
with_clause = f"WITH collect(distinct {source_qnode_key}) as {source_qnode_col_name}, " \
f"collect(distinct {target_qnode_key}) as {target_qnode_col_name}, " \
f"collect(distinct {qedge_key}{extra_edge_properties}) as {qedge_col_name}"
# Build the return clause
return_clause = f"RETURN {source_qnode_col_name}, {target_qnode_col_name}, {qedge_col_name}"
cypher_query = f"{match_clause} {where_clause} {with_clause} {return_clause}"
return cypher_query
except Exception:
tb = traceback.format_exc()
error_type, error, _ = sys.exc_info()
log.error(f"Problem generating cypher for query. {tb}",
error_code=error_type.__name__)
return ""