def get_preview():
"""Get a preview of query
Returns
-------
json
resultsPreview: Preview of the query results
headerPreview: Header of the results table
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
try:
# If public datasets and queries are protected, dont return anything to unlogged users
if "user" not in session and current_app.iniconfig.getboolean("askomics", "protect_public"):
preview = []
header = []
else:
data = request.get_json()
query = SparqlQuery(current_app, session, data["graphState"])
query.build_query_from_json(preview=True, for_editor=False)
endpoints = query.endpoints
federated = query.federated
header = query.selects
preview = []
if query.graphs:
query_launcher = SparqlQueryLauncher(current_app, session, get_result_query=True, federated=federated, endpoints=endpoints)
header, preview = query_launcher.process_query(query.sparql)
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'resultsPreview': [],
'headerPreview': [],
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({
'resultsPreview': preview,
'headerPreview': header,
'error': False,
'errorMessage': ''
})
def init():
"""Get the default sparql query
Returns
-------
json
"""
try:
# Disk space
files_utils = FilesUtils(current_app, session)
disk_space = files_utils.get_size_occupied_by_user(
) if "user" in session else None
# Get graphs and endpoints
query = SparqlQuery(current_app, session)
graphs, endpoints = query.get_graphs_and_endpoints(all_selected=True)
# Default query
default_query = query.prefix_query(query.get_default_query())
console_enabled = can_access(session['user'])
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
"error": True,
"errorMessage": str(e),
"defaultQuery": "",
"graphs": {},
"endpoints": {},
"diskSpace": None,
"console_enabled": False
}), 500
return jsonify({
"error": False,
"errorMessage": "",
"defaultQuery": default_query,
"graphs": graphs,
"endpoints": endpoints,
"diskSpace": disk_space,
"console_enabled": console_enabled
})
def update_base_url(self, graph, old_url, new_url):
"""Update base url for a graph
Parameters
----------
graph : string
Graph to update
old_url : string
Old base_url
new_url : string
New base_url
Returns
-------
list
List of graphs
"""
query_launcher = SparqlQueryLauncher(self.app, self.session)
query_builder = SparqlQuery(self.app, self.session)
query = """
WITH <{0}>
DELETE{{
?s ?p ?o
}}
INSERT{{
?s2 ?p2 ?o2
}}
WHERE {{
?s ?p ?o
FILTER(REGEX(?s, '{1}', 'i') || REGEX(?p, '{1}', 'i') || REGEX(?o, '{1}', 'i')) .
BIND(IF (isURI(?o), URI(REPLACE(STR(?o), '{1}', '{2}')), ?o) AS ?o2) .
BIND(IF (isURI(?s), URI(REPLACE(STR(?s), '{1}', '{2}')), ?s) AS ?s2) .
BIND(IF (isURI(?p), URI(REPLACE(STR(?p), '{1}', '{2}')), ?p) AS ?p2) .
}}
""".format(graph, old_url, new_url)
header, data = query_launcher.process_query(
query_builder.prefix_query(query))
def get_data(uri):
"""Get information about uri
Returns
-------
json
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
try:
query = SparqlQuery(current_app, session)
graphs, endpoints = query.get_graphs_and_endpoints(all_selected=True)
endpoints = [val['uri'] for val in endpoints.values()]
data = []
# If the user do not have access to any endpoint (no viewable graph), skip
if endpoints:
uri = urllib.parse.quote(uri)
base_uri = current_app.iniconfig.get('triplestore',
'namespace_data')
full_uri = "<%s%s>" % (base_uri, uri)
data = query.get_uri_parameters(full_uri, endpoints)
except Exception as e:
current_app.logger.error(str(e).replace('\\n', '\n'))
traceback.print_exc(file=sys.stdout)
return jsonify({
'error': True,
'errorMessage': str(e).replace('\\n', '\n'),
'data': []
}), 500
return jsonify({'data': data, 'error': False, 'errorMessage': ""})
def get_sparql_query():
"""Get sparql query of result for the query editor
Returns
-------
json
query: the sparql query
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
try:
files_utils = FilesUtils(current_app, session)
disk_space = files_utils.get_size_occupied_by_user(
) if "user" in session else None
file_id = request.get_json()["fileId"]
result_info = {"id": file_id}
result = Result(current_app, session, result_info)
query = SparqlQuery(current_app, session)
sparql = result.get_sparql_query()
# get graphs and endpoints used in the query
graphs = result.graphs
endpoints = result.endpoints
# Get all graphs and endpoint, and mark as selected the used one
graphs, endpoints = query.get_graphs_and_endpoints(
selected_graphs=graphs, selected_endpoints=endpoints)
# Build sparql query from json if needed
if sparql is None:
query.json = result.get_graph_state()
query.build_query_from_json(for_editor=True)
sparql = query.sparql
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'query': {},
'graphs': [],
'endpoints': [],
'diskSpace': 0,
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({
'query': sparql,
'graphs': graphs,
'endpoints': endpoints,
'diskSpace': disk_space,
'error': False,
'errorMessage': ''
})
def query():
"""Perform a sparql query
Returns
-------
json
query results
"""
q = request.get_json()['query']
graphs = request.get_json()['graphs']
endpoints = request.get_json()['endpoints']
local_endpoint_f = current_app.iniconfig.get('triplestore', 'endpoint')
try:
local_endpoint_f = current_app.iniconfig.get('federation', 'local_endpoint')
except Exception:
pass
# No graph selected in local TS
if not graphs and local_endpoint_f in endpoints:
return jsonify({
'error': True,
'errorMessage': "No graph selected in local triplestore",
'header': [],
'data': []
}), 500
# No endpoint selected
if not endpoints:
return jsonify({
'error': True,
'errorMessage': "No endpoint selected",
'header': [],
'data': []
}), 500
try:
query = SparqlQuery(current_app, session)
query.set_graphs_and_endpoints(graphs=graphs, endpoints=endpoints)
federated = query.is_federated()
replace_froms = query.replace_froms()
sparql = query.format_query(q, replace_froms=replace_froms, federated=federated)
# header, data = query_launcher.process_query(query)
header = query.selects
data = []
if query.graphs or query.endpoints:
query_launcher = SparqlQueryLauncher(current_app, session, get_result_query=True, federated=federated, endpoints=endpoints)
header, data = query_launcher.process_query(sparql)
except Exception as e:
current_app.logger.error(str(e).replace('\\n', '\n'))
traceback.print_exc(file=sys.stdout)
return jsonify({
'error': True,
'errorMessage': str(e).replace('\\n', '\n'),
'header': [],
'data': []
}), 500
return jsonify({
'header': header,
'data': data
})
def save_query():
"""Perform a sparql query
Returns
-------
json
query results
"""
q = request.get_json()['query']
graphs = request.get_json()['graphs']
endpoints = request.get_json()['endpoints']
local_endpoint_f = current_app.iniconfig.get('triplestore', 'endpoint')
try:
local_endpoint_f = current_app.iniconfig.get('federation', 'local_endpoint')
except Exception:
pass
# No graph selected in local TS
if not graphs and local_endpoint_f in endpoints:
return jsonify({
'error': True,
'errorMessage': "No graph selected in local triplestore",
'task_id': None
}), 500
# No endpoint selected
if not endpoints:
return jsonify({
'error': True,
'errorMessage': "No endpoint selected",
'task_id': None
}), 500
try:
files_utils = FilesUtils(current_app, session)
disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None
if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
return jsonify({
'error': True,
'errorMessage': "Exceeded quota",
'task_id': None
}), 500
# Is query federated?
query = SparqlQuery(current_app, session)
query.set_graphs_and_endpoints(graphs=graphs, endpoints=endpoints)
federated = query.is_federated()
replace_froms = query.replace_froms()
formatted_query = query.format_query(q, limit=None, replace_froms=replace_froms, federated=federated)
info = {
"sparql_query": q, # Store the non formatted query in db
"graphs": graphs,
"endpoints": endpoints,
"federated": federated,
"celery_id": None
}
result = Result(current_app, session, info)
info["id"] = result.save_in_db()
# execute the formatted query
info["sparql_query"] = formatted_query
session_dict = {"user": session["user"]}
task = current_app.celery.send_task("sparql_query", (session_dict, info))
result.update_celery(task.id)
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'error': True,
'errorMessage': str(e),
'task_id': None
}), 500
return jsonify({
'error': False,
'errorMessage': '',
'task_id': task.id
})
def get_graph_and_sparql_query():
"""Get query (graphState or Sparql)
Returns
-------
json
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
try:
data = request.get_json()
if not (data and data.get("fileId")):
return jsonify({
'graphState': {},
'sparqlQuery': "",
'graphs': [],
'endpoints': [],
'diskSpace': 0,
'error': True,
'errorMessage': "Missing fileId parameter"
}), 400
file_id = data["fileId"]
result_info = {"id": file_id}
result = Result(current_app, session, result_info)
if not result:
return jsonify({
'graphState': {},
'sparqlQuery':
"",
'graphs': [],
'endpoints': [],
'diskSpace':
0,
'error':
True,
'errorMessage':
"You do not have access to this query"
}), 401
# Get graph state and sparql query
graph_state = result.get_graph_state(formated=True)
sparql_query = result.get_sparql_query()
# Get disk space
files_utils = FilesUtils(current_app, session)
disk_space = files_utils.get_size_occupied_by_user(
) if "user" in session else None
# Get graphs and endpoints
graphs = result.graphs
endpoints = result.endpoints
# Get all graphs and endpoint, and mark as selected the used one
query = SparqlQuery(current_app, session)
graphs, endpoints = query.get_graphs_and_endpoints(
selected_graphs=graphs, selected_endpoints=endpoints)
console_enabled = can_access(session['user'])
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'graphState': {},
'sparqlQuery': "",
'graphs': [],
'endpoints': [],
'diskSpace': 0,
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({
'graphState': graph_state,
'sparqlQuery': sparql_query,
'graphs': graphs,
'endpoints': endpoints,
'diskSpace': disk_space,
'error': False,
'errorMessage': '',
'console_enabled': console_enabled
})
def get_startpoints(self):
"""Get public and user startpoints
Returns
-------
list
Startpoints
"""
filter_user = ""
if self.logged_user():
filter_user = "******".format(self.session["user"]["username"])
query_launcher = SparqlQueryLauncher(self.app, self.session)
query_builder = SparqlQuery(self.app, self.session)
query = '''
SELECT DISTINCT ?endpoint ?graph ?entity ?entity_label ?creator ?public
WHERE {{
?graph askomics:public ?public .
?graph dc:creator ?creator .
GRAPH ?graph {{
?graph prov:atLocation ?endpoint .
?entity a askomics:entity .
?entity a askomics:startPoint .
?entity rdfs:label ?entity_label .
}}
FILTER (
?public = <true>{}
)
}}
'''.format(filter_user)
header, data = query_launcher.process_query(query_builder.prefix_query(query))
startpoints = []
entities = []
for result in data:
if result["endpoint"] == self.settings.get("triplestore", "endpoint"):
endpoint_name = "local"
else:
try:
endpoint_name = tld.get_fld(result["endpoint"]).split('.')[0]
except Exception:
endpoint_name = urlparse(result["endpoint"]).netloc
if result["entity"] not in entities:
# new entity
entities.append(result['entity'])
startpoint = {
"entity": result["entity"],
"entity_label": result["entity_label"],
"graphs": [{
"uri": result["graph"],
"public": result["public"],
"creator": result["creator"]
}],
"endpoints": [{"url": result["endpoint"], "name": endpoint_name}],
"public": self.str_to_bool(result["public"]),
"private": not self.str_to_bool(result["public"])
}
startpoints.append(startpoint)
else:
# update existing entity
index = entities.index(result['entity'])
graph = {
"uri": result["graph"],
"public": result["public"],
"creator": result["creator"]
}
startpoints[index]["graphs"].append(graph)
startpoints[index]["endpoints"].append({"url": result["endpoint"], "name": endpoint_name})
if self.str_to_bool(result["public"]):
startpoints[index]["public"] = True
else:
startpoints[index]["private"] = True
return startpoints
def get_abstraction_relations(self):
"""Get user abstraction relations from the triplestore
Returns
-------
list
Relations
"""
filter_user = ""
if self.logged_user():
filter_user = "******".format(self.session["user"]["username"])
query_launcher = SparqlQueryLauncher(self.app, self.session)
query_builder = SparqlQuery(self.app, self.session)
query = '''
SELECT DISTINCT ?graph ?entity_uri ?entity_faldo ?entity_label ?node_type ?attribute_uri ?attribute_faldo ?attribute_label ?attribute_range ?property_uri ?property_faldo ?property_label ?range_uri ?category_value_uri ?category_value_label
WHERE {{
# Graphs
?graph askomics:public ?public .
?graph dc:creator ?creator .
GRAPH ?graph {{
# Property (relations and categories)
?property_uri a owl:ObjectProperty .
?property_uri a askomics:AskomicsRelation .
?property_uri rdfs:label ?property_label .
?property_uri rdfs:range ?range_uri .
}}
# Relation of entity (or motherclass of entity)
{{
?property_uri rdfs:domain ?mother .
?entity_uri rdfs:subClassOf ?mother .
}} UNION {{
?property_uri rdfs:domain ?entity_uri .
}}
FILTER (
?public = <true>{}
)
}}
'''.format(filter_user)
header, data = query_launcher.process_query(query_builder.prefix_query(query))
relations_list = []
relations = []
for result in data:
# Relation
if "property_uri" in result:
rel_tpl = (result["property_uri"], result["entity_uri"], result["range_uri"])
if rel_tpl not in relations_list:
relations_list.append(rel_tpl)
relation = {
"uri": result["property_uri"],
"label": result["property_label"],
"graphs": [result["graph"], ],
"source": result["entity_uri"],
"target": result["range_uri"]
}
relations.append(relation)
else:
# if graph is diff, append it
index_relation = relations_list.index(rel_tpl)
if result["graph"] not in relations[index_relation]["graphs"]:
relations[index_relation]["graphs"].append(result["graph"])
return relations
def get_abstraction_attributes(self):
"""Get user abstraction attributes from the triplestore
Returns
-------
list
AskOmics attributes
"""
filter_user = ""
if self.logged_user():
filter_user = "******".format(self.session["user"]["username"])
litterals = (
"http://www.w3.org/2001/XMLSchema#string",
"http://www.w3.org/2001/XMLSchema#decimal",
"http://www.w3.org/2001/XMLSchema#boolean"
)
query_launcher = SparqlQueryLauncher(self.app, self.session)
query_builder = SparqlQuery(self.app, self.session)
query = '''
SELECT DISTINCT ?graph ?entity_uri ?attribute_uri ?attribute_type ?attribute_faldo ?attribute_label ?attribute_range ?category_value_uri ?category_value_label
WHERE {{
# Graphs
?graph askomics:public ?public .
?graph dc:creator ?creator .
GRAPH ?graph {{
?attribute_uri a ?attribute_type .
VALUES ?attribute_type {{ owl:DatatypeProperty askomics:AskomicsCategory }}
?attribute_uri rdfs:label ?attribute_label .
?attribute_uri rdfs:range ?attribute_range .
# Faldo
OPTIONAL {{
?attribute_uri a ?attribute_faldo .
VALUES ?attribute_faldo {{ askomics:faldoStart askomics:faldoEnd askomics:faldoStrand askomics:faldoReference }}
}}
# Categories (DK)
OPTIONAL {{
?attribute_range askomics:category ?category_value_uri .
?category_value_uri rdfs:label ?category_value_label .
}}
}}
# Attribute of entity (or motherclass of entity)
{{
?attribute_uri rdfs:domain ?mother .
?entity_uri rdfs:subClassOf ?mother .
}} UNION {{
?attribute_uri rdfs:domain ?entity_uri .
}}
FILTER (
?public = <true>{}
)
}}
'''.format(filter_user)
header, data = query_launcher.process_query(query_builder.prefix_query(query))
attributes_list = []
attributes = []
for result in data:
# Attributes
if "attribute_uri" in result and "attribute_label" in result and result["attribute_type"] != "{}AskomicsCategory".format(self.settings.get("triplestore", "namespace_internal")) and result["attribute_range"] in litterals:
attr_tpl = (result["attribute_uri"], result["entity_uri"])
if attr_tpl not in attributes_list:
attributes_list.append(attr_tpl)
attribute = {
"uri": result["attribute_uri"],
"label": result["attribute_label"],
"graphs": [result["graph"], ],
"entityUri": result["entity_uri"],
"type": result["attribute_range"],
"faldo": result["attribute_faldo"] if "attribute_faldo" in result else None,
"categories": []
}
attributes.append(attribute)
else:
# if graph is different, store it
index_attribute = attributes_list.index(attr_tpl)
if result["graph"] not in attributes[index_attribute]["graphs"]:
attributes[index_attribute]["graphs"].append(result["graph"])
index_attribute = attributes_list.index(attr_tpl)
# Categories
if "attribute_uri" in result and result["attribute_type"] == "{}AskomicsCategory".format(self.settings.get("triplestore", "namespace_internal")) and "category_value_uri" in result:
attr_tpl = (result["attribute_uri"], result["entity_uri"])
if attr_tpl not in attributes_list:
attributes_list.append(attr_tpl)
attribute = {
"uri": result["attribute_uri"],
"label": result["attribute_label"],
"graphs": [result["graph"], ],
"entityUri": result["entity_uri"],
"type": result["attribute_type"],
"faldo": result["attribute_faldo"] if "attribute_faldo" in result else None,
"categories": [{
"uri": result["category_value_uri"],
"label": result["category_value_label"]
}]
}
attributes.append(attribute)
else:
# if graph diff, store it
index_attribute = attributes_list.index(attr_tpl)
if result["graph"] not in attributes[index_attribute]["graphs"]:
attributes[index_attribute]["graphs"].append(result["graph"])
# Store value if new
value = {
"uri": result["category_value_uri"],
"label": result["category_value_label"]
}
if value not in attributes[index_attribute]["categories"]:
attributes[index_attribute]["categories"].append(value)
return attributes
def get_abstraction_entities(self):
"""Get abstraction entities
Returns
-------
list
List of entities available
"""
filter_user = ""
if self.logged_user():
filter_user = "******".format(self.session["user"]["username"])
query_launcher = SparqlQueryLauncher(self.app, self.session)
query_builder = SparqlQuery(self.app, self.session)
query = '''
SELECT DISTINCT ?endpoint ?graph ?entity_uri ?entity_type ?entity_faldo ?entity_label ?have_no_label
WHERE {{
?graph askomics:public ?public .
?graph dc:creator ?creator .
GRAPH ?graph {{
?graph prov:atLocation ?endpoint .
?entity_uri a ?entity_type .
VALUES ?entity_type {{ askomics:entity askomics:bnode }} .
# Faldo
OPTIONAL {{
?entity_uri a ?entity_faldo .
VALUES ?entity_faldo {{ askomics:faldo }} .
}}
# Label
OPTIONAL {{ ?entity_uri rdfs:label ?entity_label . }}
OPTIONAL {{ ?entity_uri askomics:instancesHaveNoLabels ?have_no_label . }}
}}
FILTER (
?public = <true>{}
)
}}
'''.format(filter_user)
header, data = query_launcher.process_query(query_builder.prefix_query(query))
entities_list = [] # list of entity uri
entities = [] # list of entity dict
for result in data:
if result["entity_uri"] not in entities_list:
# New entity
entities_list.append(result["entity_uri"])
# Uri, graph and label
label = "" if "entity_label" not in result else result["entity_label"]
entity_type = "bnode" if result["entity_type"] == "{}bnode".format(self.settings.get("triplestore", "namespace_internal")) else "node"
entity = {
"uri": result["entity_uri"],
"type": entity_type,
"label": label,
"instancesHaveLabels": True if "have_no_label" not in result else False if result["have_no_label"] == "1" else True,
"faldo": True if "entity_faldo" in result else False,
"endpoints": [result["endpoint"]],
"graphs": [result["graph"]],
}
entities.append(entity)
else:
# if graph is different, store it
index_entity = entities_list.index(result['entity_uri'])
if result["graph"] not in entities[index_entity]["graphs"]:
entities[index_entity]["graphs"].append(result["graph"])
# If endpoint is different, store it
if result["endpoint"] not in entities[index_entity]["endpoints"]:
entities[index_entity]["endpoints"].append(result["endpoint"])
return entities
def save_result():
"""Save a query in filesystem and db, using a celery task
Returns
-------
json
result_id: result id
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
try:
files_utils = FilesUtils(current_app, session)
disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None
if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
return jsonify({
'error': True,
'errorMessage': "Exceeded quota",
'result_id': None
}), 400
# Get query and endpoints and graphs of the query
data = request.get_json()
if not (data and data.get("graphState")):
return jsonify({
'result_id': None,
'error': True,
'errorMessage': "Missing graphState parameter"
}), 400
query = SparqlQuery(current_app, session, data["graphState"], get_graphs=False)
query.build_query_from_json(preview=False, for_editor=False)
federated = query.is_federated()
info = {
"graph_state": data["graphState"],
"query": query.sparql,
"graphs": query.graphs,
"endpoints": query.endpoints,
"federated": federated,
"celery_id": None,
"selects": query.selects,
}
result = Result(current_app, session, info)
info["id"] = result.save_in_db()
session_dict = {"user": session["user"]}
task = current_app.celery.send_task("query", (session_dict, info))
result.update_celery(task.id)
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'error': True,
'errorMessage': str(e),
'result_id': None
}), 500
return jsonify({
'error': False,
'errorMessage': '',
'result_id': info["id"]
})