def insert_metadatas(self, accessL):
"""
Insert the metadatas into the parent graph
"""
self.log.debug('--- insert_metadatas ---')
sqb = SparqlQueryBuilder(self.settings, self.session)
query_laucher = QueryLauncher(self.settings, self.session)
valAcces = 'public' if accessL else 'private'
ttl = '<' + self.graph + '> prov:generatedAtTime "' + self.timestamp + '"^^xsd:dateTime .\n'
ttl += '<' + self.graph + '> dc:creator "' + self.session[
'username'] + '" .\n'
ttl += '<' + self.graph + '> :accessLevel "' + valAcces + '" .\n'
ttl += '<' + self.graph + '> foaf:Group "' + self.session[
'group'] + '" .\n'
ttl += '<' + self.graph + '> prov:wasDerivedFrom "' + self.name + '" .\n'
ttl += '<' + self.graph + '> dc:hasVersion "' + get_distribution(
'Askomics').version + '" .\n'
ttl += '<' + self.graph + '> prov:describesService "' + os.uname(
)[1] + '" .\n'
if self.is_defined("askomics.endpoint"):
ttl += '<' + self.graph + '> prov:atLocation "' + self.get_param(
"askomics.endpoint") + '" .\n'
else:
raise ValueError("askomics.endpoint does not exit.")
sparql_header = sqb.header_sparql_config('')
query_laucher.insert_data(ttl, self.graph, sparql_header)
def empty(self):
#empty database
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
namedGraphs = self.list_named_graphs()
for graph in namedGraphs:
ql.execute_query(sqb.get_delete_query_string(graph).query)
def build_sparql_query_from_json(self,
fromgraphs,
variates,
constraintes_relations,
limit,
send_request_to_tps=True):
"""
Build a sparql query from JSON constraints
"""
select = ' '.join(variates)
sqb = SparqlQueryBuilder(self.settings, self.session)
query_launcher = QueryLauncher(self.settings, self.session)
query = self.build_recursive_block('', constraintes_relations)
# if limit != None and limit > 0:
# query += ' LIMIT ' + str(limit)
if send_request_to_tps:
results = query_launcher.process_query(
sqb.custom_query(fromgraphs, select, query).query)
else:
results = []
return results, sqb.custom_query(fromgraphs, select, query).query
def deleteMoState(self, urimo):
self.log.debug(' ***** Delete module ' + urimo + ' on TPS ***** ')
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
ql.execute_query(
sqb.prepare_query("""
DELETE WHERE { GRAPH <""" + self.graph_modules + """> { <""" + urimo +
"""> ?p ?o } }
""").query)
def build_sparql_query_from_json(self, variates, constraintesRelations,
limit, sendRequestToTPS):
self.log.debug("variates")
self.log.debug(variates)
self.log.debug("constraintesRelations")
self.log.debug(constraintesRelations)
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
res = ql.execute_query(sqb.get_list_named_graphs().query)
namedGraphs = []
#for indexResult in range(len(res['results']['bindings'])):
# namedGraphs.append(res['results']['bindings'][indexResult]['g']['value'])
req = ""
req += "SELECT DISTINCT " + ' '.join(variates) + "\n"
#TODO OFI: External Service do not work and, anyway, graphes have to be selectionned by the user in the UI
#
#for graph in namedGraphs:
# req += "FROM "+ "<"+graph+ ">"+"\n"
req += "WHERE \n"
req += self.buildRecursiveBlock('', constraintesRelations)
if limit != None and limit > 0:
req += " LIMIT " + str(limit)
sqb = SparqlQueryBuilder(self.settings, self.session)
prefixes = sqb.header_sparql_config(req)
query = prefixes + req
results = {}
if sendRequestToTPS:
ql = QueryLauncher(self.settings, self.session)
results = ql.process_query(query)
else:
# add comment inside query to inform user
query = "# endpoint = " + self.get_param(
"askomics.endpoint") + "\n" + query
return results, query
def build_sparql_query_from_json2(self, variates, constraintes_relations,
limit, send_request_to_TPS):
"""
build a sparql query from json
"""
self.log.debug("variates")
self.log.debug(variates)
self.log.debug("constraintes_relations")
self.log.debug(constraintes_relations)
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
req = ""
req += "SELECT DISTINCT " + ' '.join(variates) + "\n"
#TODO OFI: External Service do not work and, anyway, graphes have to be selectionned by the user in the UI
#
#for graph in namedGraphs:
# req += "FROM "+ "<"+graph+ ">"+"\n"
req += "WHERE \n"
req += self.build_recursive_block('', constraintes_relations)
if limit != None and limit > 0:
req += " LIMIT " + str(limit)
sqb = SparqlQueryBuilder(self.settings, self.session)
prefixes = sqb.header_sparql_config(req)
query = prefixes + req
results = {}
if send_request_to_TPS:
ql = QueryLauncher(self.settings, self.session)
results = ql.process_query(query)
else:
# add comment inside query to inform user
query = "# endpoint = " + self.get_param(
"askomics.endpoint") + "\n" + query
return results, query
def list_named_graphs(self):
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
res = ql.execute_query(sqb.get_list_named_graphs().query)
namedGraphs = []
for indexResult in range(len(res['results']['bindings'])):
namedGraphs.append(
res['results']['bindings'][indexResult]['g']['value'])
return namedGraphs
def moStateOnTPS(self):
'''
check if module files state is saved on the TPS.
if not all modules files are saved with the unchecked status !
'''
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
results = ql.process_query(
sqb.build_query_on_the_fly(
{
'select':
'?uri ?module ?comment ?version ?graph ?state',
'from': [self.graph_modules],
'query':
'{\n' + '?uri rdfs:label ?module .\n' +
'?uri rdfs:comment ?comment .\n' +
'?uri :module_version ?version .\n' +
'?uri :module_state ?state .\n' +
'OPTIONAL { ?uri :module_graph ?graph . } \n' + '}\n'
}, True).query)
self.log.debug(' ***** module on TPS ***** ')
listMoOnTps = {}
for result in results:
result['checked'] = (result['state'] == "ok")
result['wait'] = (result['state'] == "wait")
listMoOnTps[result['module']] = 0
self.log.debug('module : ' + result['module'])
#=======================*************** A ENLEVER *********=======================================
#pour debugger
#if result['wait'] :
# result['wait'] = False
#==============================================================
self.log.debug(' ***** check Available Modules ***** ')
requestAgain = False
for mo in self.moduleFiles:
self.log.debug(" --> module " + mo)
if mo not in listMoOnTps:
self.log.debug(" --====== > new module < =======")
self.importMoSate(self.moduleFiles[mo], 'off')
requestAgain = True
if requestAgain:
return False
return results
def delete_graph(self):
"""
Delete triples from a list of graph
"""
self.log.debug("=== DELETE SELECTED GRAPHS ===")
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
graphs = self.request.json_body['namedGraphs']
for graph in graphs:
self.log.debug("--- DELETE GRAPH : %s", graph)
ql.execute_query(sqb.get_drop_named_graph(graph).query)
#delete metadatas
ql.execute_query(sqb.get_delete_metadatas_of_graph(graph).query)
def get_metadatas(self):
"""
Create metadatas and insert them into AskOmics main graph.
"""
self.log.debug("====== INSERT METADATAS ======")
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
ttlMetadatas = "<" + self.metadatas['graphName'] + "> " + "prov:generatedAtTime " + '"' + self.metadatas['loadDate'] + '"^^xsd:dateTime .\n'
ttlMetadatas += "<" + self.metadatas['graphName'] + "> " + "dc:creator " + '"' + self.metadatas['username'] + '"^^xsd:string .\n'
ttlMetadatas += "<" + self.metadatas['graphName'] + "> " + "prov:wasDerivedFrom " + '"' + self.metadatas['fileName'] + '"^^xsd:string .\n'
ttlMetadatas += "<" + self.metadatas['graphName'] + "> " + "dc:hasVersion " + '"' + self.metadatas['version'] + '"^^xsd:string .\n'
ttlMetadatas += "<" + self.metadatas['graphName'] + "> " + "prov:describesService " + '"' + self.metadatas['server'] + '"^^xsd:string .'
sparqlHeader = sqb.header_sparql_config("")
ql.insert_data(ttlMetadatas, self.get_param("askomics.graph"), sparqlHeader)
def positionable_attr(self):
"""
Return the positionable attributes in common between two positionable entity
"""
#FIXEME: Rewrite this ugly method
body = self.request.json_body
data = {}
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
# Check if the two entity are positionable
positionable1 = ql.process_query(
sqb.get_if_positionable(body['node']).query)
positionable2 = ql.process_query(
sqb.get_if_positionable(body['node']).query)
if positionable1 == 0 or positionable2 == 0:
data['error'] = 'not positionable nodes'
return data
results = ql.process_query(
sqb.get_common_positionable_attributes(body['node'],
body['second_node']).query)
self.log.debug(results)
data['results'] = {}
list_pos_attr = []
for elem in results:
if elem['pos_attr'] not in list_pos_attr:
list_pos_attr.append(elem['pos_attr'].replace(
"http://www.semanticweb.org/irisa/ontologies/2016/1/igepp-ontology#",
""))
for elem in list_pos_attr:
data['results'][elem] = False not in [
bool(int(p['status'])) for p in results if p['pos_attr'] ==
"http://www.semanticweb.org/irisa/ontologies/2016/1/igepp-ontology#"
+ elem
]
return data
def test_statistics(self):
request = testing.DummyRequest()
sqb = SparqlQueryBuilder(self.settings, request.session)
ql = QueryLauncher(self.settings, request.session)
sqb.get_statistics_number_of_triples()
sqb.get_statistics_number_of_entities()
sqb.get_statistics_distinct_classes()
sqb.get_statistics_list_classes()
sqb.get_statistics_nb_instances_by_classe()
sqb.get_statistics_by_startpoint()
sqb.get_list_named_graphs()
res = ql.execute_query(sqb.get_list_named_graphs().query)
for indexResult in range(len(res['results']['bindings'])):
sqb.get_delete_query_string(
res['results']['bindings'][indexResult]['g']['value'])
sqb.get_metadatas(
res['results']['bindings'][indexResult]['g']['value'])
def get_list_named_graphs(self):
"""
Return a list with all the named graphs.
"""
self.log.debug("=== LIST OF NAMED GRAPHS ===")
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
res = ql.execute_query(sqb.get_list_named_graphs().query)
namedGraphs = []
for indexResult in range(len(res['results']['bindings'])):
namedGraphs.append(
res['results']['bindings'][indexResult]['g']['value'])
return namedGraphs
def load_data_from_file(self, fp, urlbase):
"""
Load a locally created ttl file in the triplestore using http (with load_data(url)) or with the filename for Fuseki (with fuseki_load_data(fp.name)).
:param fp: a file handle for the file to load
:param urlbase:the base URL of current askomics instance. It is used to let triple stores access some askomics temporary ttl files using http.
:return: a dictionnary with information on the success or failure of the operation
"""
if not fp.closed:
fp.flush() # This is required as otherwise, data might not be really written to the file before being sent to triplestore
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
graphName = "urn:sparql:" + self.name + '_' + self.timestamp
self.metadatas['graphName'] = graphName
ttlNamedGraph = "<" + graphName + "> " + "rdfg:subGraphOf" + " <" + self.get_param("askomics.graph") + "> ."
sparqlHeader = sqb.header_sparql_config("")
ql.insert_data(ttlNamedGraph, self.get_param("askomics.graph"), sparqlHeader)
url = urlbase+"/ttl/"+os.path.basename(fp.name)
self.log.debug(url)
data = {}
try:
if self.is_defined("askomics.file_upload_url"):
queryResults = ql.upload_data(fp.name, graphName)
self.metadatas['server'] = queryResults.headers['Server']
self.metadatas['loadDate'] = self.timestamp
else:
queryResults = ql.load_data(url, graphName)
self.metadatas['server'] = queryResults.info()['server']
self.metadatas['loadDate'] = self.timestamp
data['status'] = 'ok'
except Exception as e:
self._format_exception(e, data=data)
finally:
if self.settings["askomics.debug"]:
data['url'] = url
else:
os.remove(fp.name) # Everything ok, remove temp file
self.get_metadatas()
return data
def existing_relations(self):
"""
Fetch from triplestore the existing relations if entities of the same name exist
:return: a List of relation names
:rtype: List
"""
self.log.debug("existing_relations")
existing_relations = []
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
sparql_template = self.get_template_sparql(self.ASKOMICS_get_class_info_from_abstraction_queryFile)
query = sqb.load_from_file(sparql_template, {"nodeClass": self.headers[0]}).query
results = ql.process_query(query)
return existing_relations
def load_data_from_file(self, fp, urlbase):
"""
Load a locally created ttl file in the triplestore using http (with load_data(url)) or with the filename for Fuseki (with fuseki_load_data(fp.name)).
:param fp: a file handle for the file to load
:param urlbase:the base URL of current askomics instance. It is used to let triple stores access some askomics temporary ttl files using http.
:return: a dictionnary with information on the success or failure of the operation
"""
if not fp.closed:
fp.flush(
) # This is required as otherwise, data might not be really written to the file before being sent to triplestore
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
if self.is_defined('askomics.load_url'):
urlbase = self.settings['askomics.load_url']
url = urlbase + "/ttl/" + self.session[
'username'] + '/' + os.path.basename(fp.name)
self.log.debug(url)
data = {}
data["status"] = "ok"
try:
if self.is_defined("askomics.file_upload_url"):
queryResults = ql.upload_data(fp.name, self.graph)
else:
queryResults = ql.load_data(url, self.graph)
except Exception as e:
self.log.error(self._format_exception(e))
raise e
finally:
if self.settings["askomics.debug"]:
data['url'] = url
else:
self.log.debug("3source file : load_data_from_file delete =>" +
fp.name)
os.remove(fp.name) # Everything ok, remove temp file
return data
def importMoSate(self, mo, state):
'''
Import in the TPS all triplet necessary to defined an askomics module
'''
rdf = ":" + self.escape['entity'](
mo['module']) + " rdfs:label " + self.escape['text'](
mo['module']) + ";\n"
rdf += " rdfs:comment " + self.escape['text'](mo['comment']) + ";\n"
rdf += " :module_version " + self.escape['text'](mo['version']) + ";\n"
rdf += " :module_state " + self.escape['text'](state) + ""
if (state == 'ok'):
rdf += ";\n :module_graph " + '<' + mo['graph'] + '>.\n'
else:
rdf += ".\n"
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
sh = sqb.header_sparql_config('')
ql.insert_data(rdf, self.graph_modules, sh)
def empty_database(self):
"""
Delete all triples in the triplestore
"""
data = {}
self.log.debug("=== DELETE ALL TRIPLES ===")
try:
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
namedGraphs = self.get_list_named_graphs()
namedGraphs.append(ql.get_param("askomics.graph"))
for graph in namedGraphs:
ql.execute_query(sqb.get_delete_query_string(graph).query)
except Exception as e:
traceback.print_exc(file=sys.stdout)
data['error'] = traceback.format_exc(limit=8) + "\n\n\n" + str(e)
self.log.error(str(e))
return data
def get_start_points(self):
"""
Get the possible starting points for your graph.
:return: List of starting points
:rtype: Node list
"""
self.log.debug(
" =========== TripleStoreExplorer:get_start_points ===========")
nodes = []
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
sparql_template = self.get_template_sparql(self.ASKOMICS_initial_query)
query = sqb.load_from_file(sparql_template, {}).query
results = ql.process_query(query)
for result in results:
uri = result["nodeUri"]
label = result["nodeLabel"]
nodes.append({'uri': uri, 'label': label})
return nodes
def statistics(self):
""" Get information about triplet store """
self.log.debug("== STATS ==")
data = {}
pm = ParamManager(self.settings, self.request.session)
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
tse = TripleStoreExplorer(self.settings, self.request.session)
results = ql.process_query(
sqb.get_statistics_number_of_triples().query)
resultsGraphs = ql.process_query(
sqb.get_statistics_number_of_triples_AskOmics_graphs().query)
data["ntriples"] = int(results[0]["no"]) + int(resultsGraphs[0]["no"])
results = ql.process_query(
sqb.get_statistics_number_of_entities().query)
data["nentities"] = results[0]["no"]
results = ql.process_query(sqb.get_statistics_distinct_classes().query)
data["nclasses"] = results[0]["no"]
# Get the number of graphs
results = ql.process_query(sqb.get_statistics_number_of_graphs().query)
data["ngraphs"] = results[0]["no"]
self.log.debug("=== LIST OF METADATAS ")
# Get the list of named graphs
namedGraphsResults = ql.execute_query(
sqb.get_list_named_graphs().query)
namedGraphsMetadatas = {}
# Get a dictionnary containing the metadatas for each graph
for indexResult in range(len(
namedGraphsResults['results']['bindings'])):
metadatasResults = ql.execute_query(
sqb.get_metadatas(namedGraphsResults['results']['bindings']
[indexResult]['g']['value']).query)
metadatas = {}
for indexMetadatasResults in range(
len(metadatasResults['results']['bindings'])):
if metadatasResults['results']['bindings'][
indexMetadatasResults]['p'][
'value'] == "http://www.w3.org/ns/prov#generatedAtTime":
metadatas['loadDate'] = metadatasResults['results'][
'bindings'][indexMetadatasResults]['o']['value']
if metadatasResults['results']['bindings'][
indexMetadatasResults]['p'][
'value'] == "http://purl.org/dc/elements/1.1/creator":
metadatas['username'] = metadatasResults['results'][
'bindings'][indexMetadatasResults]['o']['value']
if metadatasResults['results']['bindings'][
indexMetadatasResults]['p'][
'value'] == "http://purl.org/dc/elements/1.1/hasVersion":
metadatas['version'] = metadatasResults['results'][
'bindings'][indexMetadatasResults]['o']['value']
if metadatasResults['results']['bindings'][
indexMetadatasResults]['p'][
'value'] == "http://www.w3.org/ns/prov#describesService":
metadatas['server'] = metadatasResults['results'][
'bindings'][indexMetadatasResults]['o']['value']
if metadatasResults['results']['bindings'][
indexMetadatasResults]['p'][
'value'] == "http://www.w3.org/ns/prov#wasDerivedFrom":
metadatas['filename'] = metadatasResults['results'][
'bindings'][indexMetadatasResults]['o']['value']
namedGraphsMetadatas[namedGraphsResults['results']['bindings']
[indexResult]['g']['value']] = metadatas
data['metadata'] = namedGraphsMetadatas
# Get the list of classes
res_list_classes = ql.process_query(
sqb.get_statistics_list_classes().query)
data["class"] = {}
for obj in res_list_classes:
print(obj['class'])
class_name = pm.remove_prefix(obj['class'])
print(class_name)
data["class"][class_name] = {}
# Get the number of instances by class
res_nb_instances = ql.process_query(
sqb.get_statistics_nb_instances_by_classe().query)
for obj in res_nb_instances:
if 'class' in obj:
print(data['class'])
class_name = pm.remove_prefix(obj['class'])
data["class"][class_name]["count"] = obj['count']
return data
def drop_graph(self, graph):
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
ql.execute_query(sqb.get_drop_named_graph(graph).query)
def test_generateAbstractAskomicsRDF(self):
import os
from askomics.libaskomics.rdfdb.SparqlQueryBuilder import SparqlQueryBuilder
from askomics.libaskomics.rdfdb.QueryLauncher import QueryLauncher
m = ModulesManager(self.settings, self.request.session)
sqb = SparqlQueryBuilder(self.settings, self.request.session)
ql = QueryLauncher(self.settings, self.request.session)
sh = sqb.header_sparql_config('')
rdf = """
<http://bidon/relationTest> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#ObjectProperty> ;
<http://www.w3.org/2000/01/rdf-schema#label> "relationBidon" ;
<http://www.w3.org/2000/01/rdf-schema#domain> <http://bidon/Type1> ;
<http://www.w3.org/2000/01/rdf-schema#range> <http://bidon/Type2>.
<http://bidon/Type1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class>.
<http://bidon/Type2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class>.
"""
ql.insert_data(rdf, "urn:test:askomics", sh)
m.generateAbstractAskomicsRDF("urn:test:askomics")
rdf = """
<http://bidon/relationTest> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#ObjectProperty> ;
<http://www.w3.org/2000/01/rdf-schema#label> "relationBidon" ;
<http://www.w3.org/2000/01/rdf-schema#domain> <http://bidon/Type1> ;
<http://www.w3.org/2000/01/rdf-schema#range> <http://bidon/Type2>.
<http://bidon/Type1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class>.
<http://bidon/Type2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class>.
<http://bidon/Type1> <http://www.w3.org/2000/01/rdf-schema#label> "Type1".
<http://bidon/Type2> <http://www.w3.org/2000/01/rdf-schema#label> "Type2".
<http://bidon/Attribute1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#DatatypeProperty> ;
<http://www.w3.org/2000/01/rdf-schema#label> "Attribute1";
<http://www.w3.org/2000/01/rdf-schema#domain> <http://bidon/Type1> ;
<http://www.w3.org/2000/01/rdf-schema#range> <http://www.w3.org/2001/XMLSchema#int>.
"""
ql.insert_data(rdf, "urn:test:askomics2", sh)
m.generateAbstractAskomicsRDF("urn:test:askomics2")
rdf = """
<http://bidon/relationTest> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#ObjectProperty> ;
<http://www.w3.org/2000/01/rdf-schema#label> "relationBidon" ;
<http://www.w3.org/2000/01/rdf-schema#domain> <http=bidon=Type1> ;
<http://www.w3.org/2000/01/rdf-schema#range> <http=bidon=Type2>.
<http=bidon=Type1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class>.
<http=bidon=Type2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class>.
<http=bidon=Type1> <http://www.w3.org/2000/01/rdf-schema#label> "Type1".
<http=bidon=Type2> <http://www.w3.org/2000/01/rdf-schema#label> "Type2".
<http://bidon/Attribute1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#DatatypeProperty> ;
<http://www.w3.org/2000/01/rdf-schema#label> "Attribute1";
<http://www.w3.org/2000/01/rdf-schema#domain> <http=bidon=Type1> ;
<http://www.w3.org/2000/01/rdf-schema#range> <http://www.w3.org/2001/XMLSchema#int>.
"""
ql.insert_data(rdf, "urn:test:askomics3", sh)
m.generateAbstractAskomicsRDF("urn:test:askomics3")
def build_sparql_query_from_json(self,listEndpoints, typeEndpoints, fromgraphs, variates, constraintes_relations,limit, send_request_to_tps=True):
"""
Build a sparql query from JSON constraints
"""
if len(typeEndpoints) != len(listEndpoints):
self.log.warn("listEndpoints:"+str(listEndpoints))
self.log.warn("typeEndpoints:"+str(typeEndpoints))
raise ValueError("Devel error. Different size for List Endpoints and List type Endpoints. ")
select = ' '.join(variates)
sqb = SparqlQueryBuilder(self.settings, self.session)
query = self.build_recursive_block('', constraintes_relations)
# if limit != None and limit > 0:
# query += ' LIMIT ' + str(limit)
self.log.debug("============ build_sparql_query_from_json ========")
self.log.debug("type_endpoints:"+str(typeEndpoints))
self.log.debug("endpoints:"+str(listEndpoints))
self.log.debug("graphs"+str(fromgraphs))
extreq = False
typeQuery = ''
if send_request_to_tps:
if len(listEndpoints) == 0:
#raise ValueError("None endpoint are defined fo the current SPARLQ query !")
query_launcher = QueryLauncher(self.settings, self.session)
elif len(listEndpoints)==1:
self.log.debug("============ QueryLauncher ========")
endpoint = ''
type_endpoint='askomics'
if len(listEndpoints) == 1 :
endpoint = listEndpoints[0]
if typeEndpoints[0] != 'askomics':
extreq = True
query_launcher = QueryLauncher(self.settings, self.session,name = endpoint, endpoint = endpoint)
else:
self.log.debug("============ FederationQueryLauncher ========")
typeQuery = '(Federation)'
lE = []
iCount = 0
for i in range(0, len(listEndpoints)):
iCount+=1
end = {}
end['name'] = "endpoint"+str(iCount)
end['endpoint'] = listEndpoints[i]
end['askomics'] = (typeEndpoints[i] == 'askomics')
end['auth'] = 'Basic'
end['username'] = None
end['password'] = None
lE.append(end)
query_launcher = FederationQueryLauncher(self.settings, self.session,lE)
req = sqb.custom_query(fromgraphs, select, query,externalrequest=extreq).query
results = query_launcher.process_query(req)
else:
results = []
return results, sqb.custom_query(fromgraphs, select, query).query,typeQuery
def manageModules(self, host_url, urimodule, namemodule, active):
'''
activate/desactivate module
'''
self.log.debug(
" --======================> manageModules <========================--- "
)
self.log.debug(" uri:" + urimodule)
self.log.debug(" namemodule:" + namemodule)
self.log.debug(" active:" + str(active))
listMo = self.getListModules()
mo = None
for i in listMo:
if i["uri"] == urimodule:
mo = i
break
if mo == None:
raise ValueError("Can not find Mo on TPS !")
##########################################################################################
if mo['state'] == 'wait':
self.log.debug(
" ****************** WAIT MODE **************** :" +
urimodule)
return
self.log.debug(" delete MO state :" + urimodule)
self.deleteMoState(urimodule)
self.log.debug(" insert new MO state :" + urimodule)
self.importMoSate(mo, "wait")
ql = QueryLauncher(self.settings, self.session)
if active:
try:
self.importRDF(mo, namemodule, host_url)
#loading owl file
if 'owl' in self.moduleFiles[namemodule] and self.moduleFiles[
namemodule]['owl'].strip() != '':
ql.load_data(self.moduleFiles[namemodule]['owl'],
mo['graph'])
except Exception as e:
self.log.error('failed: ' + str(e))
self.log.debug(" delete MO state :" + urimodule)
self.deleteMoState(urimodule)
self.log.debug(" insert new MO state :" + urimodule)
self.importMoSate(mo, "off")
raise e
self.log.debug(" delete MO state :" + urimodule)
self.deleteMoState(urimodule)
self.log.debug(" insert new MO state :" + urimodule)
self.importMoSate(mo, "ok")
##########################################################################################
# manage owl if dos not exist in the MO file
if 'rdf' not in self.moduleFiles[namemodule]:
self.moduleFiles[namemodule]['rdf'] = []
if len(self.moduleFiles[namemodule]['rdf']) <= 0:
self.moduleFiles[namemodule][
'rdf'] = self.generateAbstractAskomicsRDF(mo['graph'])
self.importRDF(mo, namemodule, host_url, mo['graph'])
self.saveMo(namemodule)
else:
if 'graph' in mo:
sqb = SparqlQueryBuilder(self.settings, self.session)
ql.execute_query(sqb.get_drop_named_graph(mo['graph']).query)
ql.execute_query(
sqb.get_delete_metadatas_of_graph(mo['graph']).query)
self.log.debug(" delete MO state :" + urimodule)
self.deleteMoState(urimodule)
self.log.debug(" insert new MO state :" + urimodule)
self.importMoSate(mo, "off")
def generateAbstractAskomicsRDF(self, graph):
'''
'''
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
results = ql.process_query(
sqb.build_query_on_the_fly(
{
'select':
'?entityDom ?entityDomLab ?relation ?entityRan ?entityRanLab',
'query':
'{\n' + 'GRAPH ?g { \n' +
'?relation a owl:ObjectProperty.\n' +
'?relation rdfs:domain ?entityDom.\n' +
'?entityDom a owl:Class .\n' +
'OPTIONAL { ?entityDom rdfs:label ?entityDomLab }.\n' +
'?relation rdfs:range ?entityRan .\n' +
'?entityRan a owl:Class .\n' +
'OPTIONAL { ?entityRan rdfs:label ?entityRanLab }.\n' +
'FILTER ( isIRI(?entityDom)).\n ' +
'FILTER ( isIRI(?entityRan)).\n ' + '}\n' +
'VALUES ?g {<' + graph + '>}'
'}\n'
}, True).query)
entities = {}
attributes = {}
label = {}
for r in results:
if r['entityDom'] not in entities:
entities[r['entityDom']] = {}
if r['entityRan'] not in entities:
entities[r['entityRan']] = {}
entities[r['entityDom']][r['relation']] = r['entityRan']
if ('entityDomLab' in r) and (r['entityDom'] not in label):
if r['entityDomLab'] != '':
label[r['entityDom']] = r['entityDomLab']
if ('entityRanLab' in r) and (r['entityRan'] not in label):
if r['entityRan'] != '':
label[r['entityRan']] = r['entityRanLab']
if len(entities) > 0:
values = ""
for ent in entities:
values += '<' + ent + '> '
results = ql.process_query(
sqb.build_query_on_the_fly(
{
'select':
'?entity ?attribute ?basetype',
'query':
'{\n' + 'GRAPH ?g { \n' +
'?attribute a owl:DatatypeProperty.\n' +
'?attribute rdfs:domain ?entity.\n' +
'?entity a owl:Class .\n' +
'?attribute rdfs:range ?basetype .\n' +
'FILTER ( isIRI(?basetype)).\n ' + 'VALUES ?entity {' +
values + '}.\n ' + '}\n' + 'VALUES ?g {<' + graph +
'>}'
'}\n'
}, True).query)
for r in results:
if r['entity'] not in attributes:
attributes[r['entity']] = {}
attributes[r['entity']][r['attribute']] = r['basetype']
rdftab = []
rdftab.append("@prefix displaySetting: <" +
self.ASKOMICS_prefix['displaySetting'] + ">.")
rdftab.append("@prefix rdfs: <" + self.ASKOMICS_prefix['rdfs'] + ">.")
for ent in entities:
rdftab.append("<" + ent +
"> displaySetting:entity \"true\"^^xsd:boolean.")
if ent not in label:
label = ent
idxList = [label.rfind('#'), label.rfind('/')]
idx = -1
for i in idxList:
if i > idx:
idx = i
if idx >= 0:
label = self.escape['text'](
self.reverse_prefix(label[:idx + 1]) + ':' +
label[idx + 1:])
else:
label = self.escape['text'](label)
rdftab.append("<" + ent + "> rdfs:label " + label +
"^^xsd:string.")
if len(entities[ent]) > 0:
rdftab.append(
"<" + ent +
"> displaySetting:startPoint \"true\"^^xsd:boolean.")
if ent in attributes:
for at in attributes[ent]:
rdftab.append(
"<" + at +
"> displaySetting:attribute \"true\"^^xsd:boolean.")
return rdftab
def getUserAbstraction(self):
"""
Get the user abstraction (relation and entity as subject and object)
:return:
:rtype:
"""
data = {}
listEntities = {}
self.log.debug(
" =========== TripleStoreExplorer:getUserAbstraction ===========")
nodes_startpoint = self.get_start_points()
# add start node at first
for node in nodes_startpoint:
listEntities[node['uri']] = 0
sqb = SparqlQueryBuilder(self.settings, self.session)
ql = QueryLauncher(self.settings, self.session)
sparql_template = self.get_template_sparql(
self.ASKOMICS_abstractionRelationUser)
query = sqb.load_from_file(sparql_template, {
'OwlProperty': 'owl:ObjectProperty'
}).query
results = ql.process_query(query)
data['relations'] = results
for elt in results:
if not elt['object'] in listEntities:
listEntities[elt['object']] = 0
if not elt['subject'] in listEntities:
listEntities[elt['subject']] = 0
#sparql_template = self.get_template_sparql(self.ASKOMICS_abstractionRelationUser)
#query = sqb.load_from_file(sparql_template, { 'OwlProperty' : 'owl:SymmetricProperty'}).query
#results = ql.process_query(query)
#data['relationsSym'] = results
#for elt in results:
# if not elt['object'] in listEntities:
# listEntities[elt['object']]=0
# if not elt['subject'] in listEntities:
# listEntities[elt['subject']]=0
filterEntities = ' '.join(["<" + s + ">" for s in listEntities.keys()])
sparql_template = self.get_template_sparql(
self.ASKOMICS_abstractionEntityUser)
query = sqb.load_from_file(sparql_template, {
"entities": filterEntities
}).query
results = ql.process_query(query)
data['entities'] = results
sparql_template = self.get_template_sparql(
self.ASKOMICS_abstractionAttributesEntityUser)
query = sqb.load_from_file(sparql_template, {
"entities": filterEntities
}).query
results = ql.process_query(query)
data['attributes'] = results
sparql_template = self.get_template_sparql(
self.ASKOMICS_abstractionCategoriesEntityUser)
query = sqb.load_from_file(sparql_template, {
"entities": filterEntities
}).query
results = ql.process_query(query)
data['categories'] = results
sparql_template = self.get_template_sparql(
self.ASKOMICS_abstractionPositionableEntityUser)
query = sqb.load_from_file(sparql_template, {}).query
results = ql.process_query(query)
data['positionable'] = results
return data
def test_print_ids(self):
request = testing.DummyRequest()
sqb = SparqlQueryBuilder(self.settings, request.session)
def test_load_from_file(self):
request = testing.DummyRequest()
sqb = SparqlQueryBuilder(self.settings, request.session)
temp = self.get_template1()
sqb.load_from_file(temp)
def test_prepare_query(self):
request = testing.DummyRequest()
sqb = SparqlQueryBuilder(self.settings, request.session)
temp = self.get_template1()
sqb.prepare_query(temp)
def persist(self, urlbase, public):
"""
Store the current source file in the triple store
:param urlbase: the base URL of current askomics instance. It is used to let triple stores access some askomics temporary ttl files using http.
:return: a dictionnary with information on the success or failure of the operation
:rtype: Dict
"""
self.insert_metadatas(public)
content_ttl = self.get_turtle()
ql = QueryLauncher(self.settings, self.session)
# use insert data instead of load sparql procedure when the dataset is small
total_triple_count = 0
chunk_count = 1
chunk = ""
pathttl = self.get_rdf_user_directory()
method = 'load'
if self.get_param("askomics.upload_user_data_method"):
method = self.get_param("askomics.upload_user_data_method")
if method == 'load':
fp = None
triple_count = 0
for triple in content_ttl:
chunk += triple + '\n'
triple_count += 1
# with open('/tmp/DEBUGTTL' + str(triple_count), 'w') as debug_file:
# debug_file.write(chunk)
if triple_count > int(self.settings[
'askomics.max_content_size_to_update_database']):
# Temp file must be accessed by http so we place it in askomics/ttl/ dir
fp = tempfile.NamedTemporaryFile(dir=pathttl,
prefix="tmp_" +
self.alphanum_name,
suffix=".ttl",
mode="w",
delete=False)
# We have reached the maximum chunk size, load it and then we will start a new chunk
self.log.debug("Loading ttl chunk %s file %s" %
(chunk_count, fp.name))
header_ttl = self.get_turtle_template(chunk)
fp.write(header_ttl + '\n')
fp.write(chunk)
fp.close()
data = self.load_data_from_file(fp, urlbase)
if data['status'] == 'failed':
return data
chunk = ""
total_triple_count += triple_count
triple_count = 0
chunk_count += 1
# Load the last chunk
if triple_count > 0:
self.log.debug("Loading ttl chunk %s (last)" % (chunk_count))
fp = tempfile.NamedTemporaryFile(dir=pathttl,
prefix="tmp_" +
self.alphanum_name,
suffix=".ttl",
mode="w",
delete=False)
header_ttl = self.get_turtle_template(chunk)
fp.write(header_ttl + '\n')
fp.write(chunk)
fp.close()
data = self.load_data_from_file(fp, urlbase)
if data['status'] == 'failed':
return data
total_triple_count += triple_count
# Data is inserted, now insert the abstraction
# We get the abstraction now as we need first to parse the whole file to have category_values
abstraction_ttl = self.get_abstraction()
domain_knowledge_ttl = self.get_domain_knowledge()
header_ttl = self.get_turtle_template(abstraction_ttl + "\n" +
domain_knowledge_ttl)
fp = tempfile.NamedTemporaryFile(dir=pathttl,
prefix="tmp_" +
self.alphanum_name,
suffix=".ttl",
mode="w",
delete=False)
fp.write(header_ttl + '\n')
fp.write(abstraction_ttl + '\n')
fp.write(domain_knowledge_ttl + '\n')
self.log.debug("Loading ttl abstraction file %s" % (fp.name))
fp.close()
data = self.load_data_from_file(fp, urlbase)
if data['status'] == 'failed':
return data
data['total_triple_count'] = total_triple_count
else:
sqb = SparqlQueryBuilder(self.settings, self.session)
triple_count = 0
chunk = ""
for triple in content_ttl:
chunk += triple + '\n'
triple_count += 1
if triple_count > int(
self.settings[
'askomics.max_content_size_to_update_database']
) / 10: # FIXME the limit is much lower than for load
# We have reached the maximum chunk size, load it and then we will start a new chunk
self.log.debug("Inserting ttl chunk %s" % (chunk_count))
try:
header_ttl = sqb.header_sparql_config(chunk)
queryResults = ql.insert_data(chunk, self.graph,
header_ttl)
except Exception as e:
return self._format_exception(e)
chunk = ""
total_triple_count += triple_count
triple_count = 0
chunk_count += 1
# Load the last chunk
if triple_count > 0:
self.log.debug("Inserting ttl chunk %s (last)" % (chunk_count))
try:
header_ttl = sqb.header_sparql_config(chunk)
queryResults = ql.insert_data(chunk, self.graph,
header_ttl)
except Exception as e:
return self._format_exception(e)
total_triple_count += triple_count
# Data is inserted, now insert the abstraction
# We get the abstraction now as we need first to parse the whole file to have category_values
abstraction_ttl = self.get_abstraction()
domain_knowledge_ttl = self.get_domain_knowledge()
chunk += abstraction_ttl + '\n'
chunk += domain_knowledge_ttl + '\n'
self.log.debug("Inserting ttl abstraction")
try:
header_ttl = sqb.header_sparql_config(chunk)
ql.insert_data(chunk, self.graph, header_ttl)
except Exception as e:
return self._format_exception(e)
self.metadatas['graphName'] = self.graph
sparqlHeader = sqb.header_sparql_config("")
data = {}
if 'server' in queryResults.info():
self.metadatas['server'] = queryResults.info()['server']
else:
self.metadatas['server'] = 'unknown'
data['status'] = 'ok'
data['total_triple_count'] = total_triple_count
data['expected_lines_number'] = self.get_number_of_lines()
return data