def get_graph_of_user(self, username): """get all graph of a user Parameters ---------- username : string Username Returns ------- list List of graphs """ query_launcher = SparqlQueryLauncher(self.app, self.session) query_builder = SparqlQuery(self.app, self.session) query = """ SELECT DISTINCT ?graph WHERE {{ ?graph dc:creator <{}> . }} """.format(username) header, data = query_launcher.process_query(query_builder.prefix_query(query)) graphs = [] for result in data: graphs.append(result["graph"]) return graphs
def load_graph(self, rdf_graph, tmp_file_name): """Load a rdflib graph into the triplestore Write rdf to a tmp file, and send the url to this file to the triplestore with a LOAD request Parameters ---------- rdf_graph : Graph rdf graph to load tmp_file_name : string Path to a tmp file """ sparql = SparqlQueryLauncher(self.app, self.session) temp_file_path = '{}/{}'.format(self.ttl_dir, tmp_file_name) encoding = 'utf-8' if self.serialization_format != 'nt' else None rdf_graph.serialize(format=self.serialization_format, encoding=encoding, destination=temp_file_path) # Load the chunk sparql.load_data(tmp_file_name, self.file_graph, self.host_url) # Remove tmp file if not self.settings.getboolean('askomics', 'debug_ttl'): os.remove(temp_file_path)
def toggle_public(self, graph, public): """Change public status of data into the triplestore Parameters ---------- graph : string Graph to update public status public : string true or false (string) """ query = ''' WITH GRAPH <{graph}> DELETE {{ <{graph}> :public ?public . }} INSERT {{ <{graph}> :public <{public}> . }} WHERE {{ <{graph}> :public ?public . }} '''.format(graph=graph, public=public) query_launcher = SparqlQueryLauncher(self.app, self.session) query_launcher.execute_query(self.prefix_query(query))
def get_all_graphs(self): """get all graph of a user Returns ------- list List of graphs """ query_launcher = SparqlQueryLauncher(self.app, self.session) query_builder = SparqlQuery(self.app, self.session) query = """ SELECT DISTINCT ?graph WHERE {{ ?graph dc:creator ?user . }} """ header, data = query_launcher.process_query( query_builder.prefix_query(query)) graphs = [] for result in data: graphs.append(result["graph"]) return graphs
def set_graphs_and_endpoints(self, entities=None, graphs=None, endpoints=None): """Get all public and private graphs containing the given entities Parameters ---------- entities : list, optional list of entity uri """ substrlst = [] filter_entity_string = '' if entities: for entity in entities: substrlst.append("?entity_uri = <{}>".format(entity)) filter_entity_string = 'FILTER (' + ' || '.join(substrlst) + ')' filter_public_string = 'FILTER (?public = <true>)' if 'user' in self.session: filter_public_string = 'FILTER (?public = <true> || ?creator = <{}>)'.format( self.session["user"]["username"]) query = ''' SELECT DISTINCT ?graph ?endpoint WHERE {{ ?graph :public ?public . ?graph dc:creator ?creator . GRAPH ?graph {{ ?graph prov:atLocation ?endpoint . ?entity_uri a :entity . }} {} {} }} '''.format(filter_public_string, filter_entity_string) query_launcher = SparqlQueryLauncher(self.app, self.session) header, results = query_launcher.process_query( self.prefix_query(query)) self.graphs = [] self.endpoints = [] for res in results: if not graphs or res["graph"] in graphs: self.graphs.append(res["graph"]) # If local triplestore url is not accessible by federetad query engine if res["endpoint"] == self.settings.get( 'triplestore', 'endpoint') and self.local_endpoint_f is not None: endpoint = self.local_endpoint_f else: endpoint = res["endpoint"] if not endpoints or endpoint in endpoints: self.endpoints.append(endpoint) self.endpoints = Utils.unique(self.endpoints) self.federated = len(self.endpoints) > 1
def query(self, session, info): """Save the query results in filesystem and db Parameters ---------- session : dict AskOmics session graph_state : dict JSON graph state Returns ------- dict error: True if error, else False errorMessage: the error message of error, else an empty string """ try: info["celery_id"] = self.request.id result = Result(app, session, info, force_no_db=True) # Save job in database database result.set_celery_id(self.request.id) result.update_db_status("started", update_celery=True, update_date=True) # launch query query = SparqlQuery(app, session, info["graph_state"]) query.build_query_from_json(for_editor=False) headers = query.selects results = [] if query.graphs: query_launcher = SparqlQueryLauncher(app, session, get_result_query=True, federated=query.federated, endpoints=query.endpoints) headers, results = query_launcher.process_query(query.sparql, isql_api=True) # write result to a file file_size = result.save_result_in_file(headers, results) # Update database status result.update_db_status("success", size=file_size) except Exception as e: traceback.print_exc(file=sys.stdout) trace = traceback.format_exc() result.update_db_status("error", error=True, error_message=str(e), traceback=trace) result.rollback() raise e return {'error': True, 'errorMessage': str(e)} return {'error': False, 'errorMessage': ''}
def create_result(self, has_form=False): """Create a result entry in db Returns ------- dict Result info """ # Query: transcript concerned by DE and included in QTL if has_form: with open("tests/data/graphState_simple_query_form.json", "r") as file: file_content = file.read() else: with open("tests/data/graphState_simple_query.json", "r") as file: file_content = file.read() json_query = json.loads(file_content) # Get query and endpoints and graphs of the query query = SparqlQuery(self.app, self.session, json_query) query_launcher = SparqlQueryLauncher(self.app, self.session) query.build_query_from_json(preview=False, for_editor=False) info = { "graph_state": json_query, "query": query.sparql, "celery_id": '00000000-0000-0000-0000-000000000000', "graphs": query.graphs, "endpoints": query.endpoints } # Save job in database database result = Result(self.app, self.session, info) result.save_in_db() # Execute query and write result to file headers, results = query_launcher.process_query(query.sparql) file_size = result.save_result_in_file(headers, results) # Update database status result.update_db_status("success", size=file_size) return { "id": result.id, "path": result.file_path, "start": result.start, "end": result.end, "size": file_size }
def get_preview(): """Get a preview of query Returns ------- json resultsPreview: Preview of the query results headerPreview: Header of the results table error: True if error, else False errorMessage: the error message of error, else an empty string """ try: # If public datasets and queries are protected, dont return anything to unlogged users if "user" not in session and current_app.iniconfig.getboolean("askomics", "protect_public"): preview = [] header = [] else: data = request.get_json() if not (data and data.get("graphState")): return jsonify({ 'resultsPreview': [], 'headerPreview': [], 'error': True, 'errorMessage': "Missing graphState parameter" }), 400 query = SparqlQuery(current_app, session, data["graphState"], get_graphs=False) query.build_query_from_json(preview=True, for_editor=False) endpoints = query.endpoints federated = query.federated header = query.selects preview = [] if query.graphs: query_launcher = SparqlQueryLauncher(current_app, session, get_result_query=True, federated=federated, endpoints=endpoints) header, preview = query_launcher.process_query(query.sparql) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'resultsPreview': [], 'headerPreview': [], 'error': True, 'errorMessage': str(e) }), 500 return jsonify({ 'resultsPreview': preview, 'headerPreview': header, 'error': False, 'errorMessage': '' })
def set_triples_number(self): """Set graph triples number by requesting the triplestore""" query = """ SELECT count(*) AS ?count FROM <{}> WHERE {{ ?s ?p ?o . }} """.format(self.file_graph) sparql = SparqlQueryLauncher(self.app, self.session) result = sparql.process_query(query) self.ntriples = result[1][0]["count"]
def sparql_query(self, session, info): """Save the sparql query results in filesystem and db Parameters ---------- session : dict AskOmics session info : dict sparql query Returns ------- dict error: True if error, else False errorMessage: the error message of error, else an empty string """ try: info["celery"] = self.request.id result = Result(app, session, info, force_no_db=True) # Save job in db result.set_celery_id(self.request.id) result.update_db_status("started", update_celery=True) query_launcher = SparqlQueryLauncher(app, session, get_result_query=True, federated=info["federated"], endpoints=info["endpoints"]) header, data = query_launcher.process_query(info["sparql_query"], isql_api=True) # Write results in file file_size = result.save_result_in_file(header, data) # Update database status result.update_db_status("success", size=file_size) except Exception as e: traceback.print_exc(file=sys.stdout) trace = traceback.format_exc() result.update_db_status("error", error=True, error_message=str(e), traceback=trace) raise e return {'error': True, 'errorMessage': str(e)} return {'error': False, 'errorMessage': ''}
def clean_triplestore(self): """remove all test graph in the triplestore""" query = ''' SELECT ?graph WHERE {{ GRAPH ?graph {{ ?s ?p ?o . }} FILTER (strStarts(str(?graph), "{}")) }} '''.format(self.get_config("triplestore", "default_graph")) query_launcher = SparqlQueryLauncher(self.app, {}) header, data = query_launcher.process_query(query) for result in data: query_launcher.drop_dataset(result["graph"])
def integrate(self, public=False): """Integrate the file into the triplestore Parameters ---------- public : bool, optional Integrate in private or public graph """ sparql = SparqlQueryLauncher(self.app, self.session) tse = TriplestoreExplorer(self.app, self.session) self.public = public method = self.settings.get('triplestore', 'upload_method') # Load file into a RDF graph self.graph_chunk.parse(self.path, format=self.type_dict[self.type]) # get metadata self.set_metadata() # Remove metadata from data self.delete_metadata_location() # insert metadata sparql.insert_data(self.graph_metadata, self.file_graph, metadata=True) if method == "load": # write rdf into a tmpfile and load it temp_file_name = 'tmp_{}_{}.{}'.format(Utils.get_random_string(5), self.name, self.rdf_extention) # Try to load data. if failure, wait 5 sec and retry 5 time Utils.redo_if_failure(self.log, 5, 5, self.load_graph, self.graph_chunk, temp_file_name) else: # Insert # Try to insert data. if failure, wait 5 sec and retry 5 time Utils.redo_if_failure(self.log, 5, 5, sparql.insert_data, self.graph_chunk, self.file_graph) # Remove chached abstraction tse.uncache_abstraction(public=self.public) self.set_triples_number()
def get_preview(): """Get a preview of query Returns ------- json resultsPreview: Preview of the query results headerPreview: Header of the results table error: True if error, else False errorMessage: the error message of error, else an empty string """ try: data = request.get_json() query_builder = SparqlQueryBuilder(current_app, session) query = query_builder.build_query_from_json(data["graphState"], preview=True, for_editor=False) endpoints = query_builder.endpoints federated = query_builder.federated header = query_builder.selects preview = [] if query_builder.graphs: query_launcher = SparqlQueryLauncher(current_app, session, get_result_query=True, federated=federated, endpoints=endpoints) header, preview = query_launcher.process_query(query) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'resultsPreview': [], 'headerPreview': [], 'error': True, 'errorMessage': str(e) }), 500 return jsonify({ 'resultsPreview': preview, 'headerPreview': header, 'error': False, 'errorMessage': '' })
def delete_datasets(self): """delete the datasets from the database and the triplestore""" sparql = SparqlQueryLauncher(self.app, self.session) for dataset in self.datasets: # Delete from triplestore if dataset.graph_name: Utils.redo_if_failure(self.log, 3, 1, sparql.drop_dataset, dataset.graph_name) # Delete from db dataset.delete_from_db()
def update_base_url(self, graph, old_url, new_url): """Update base url for a graph Parameters ---------- graph : string Graph to update old_url : string Old base_url new_url : string New base_url Returns ------- list List of graphs """ query_launcher = SparqlQueryLauncher(self.app, self.session) query_builder = SparqlQuery(self.app, self.session) query = """ WITH <{0}> DELETE{{ ?s ?p ?o }} INSERT{{ ?s2 ?p2 ?o2 }} WHERE {{ ?s ?p ?o FILTER(REGEX(?s, '{1}', 'i') || REGEX(?p, '{1}', 'i') || REGEX(?o, '{1}', 'i')) . BIND(IF (isURI(?o), URI(REPLACE(STR(?o), '{1}', '{2}')), ?o) AS ?o2) . BIND(IF (isURI(?s), URI(REPLACE(STR(?s), '{1}', '{2}')), ?s) AS ?s2) . BIND(IF (isURI(?p), URI(REPLACE(STR(?p), '{1}', '{2}')), ?p) AS ?p2) . }} """.format(graph, old_url, new_url) header, data = query_launcher.process_query( query_builder.prefix_query(query))
def delete_datasets(self): """delete the datasets from the database and the triplestore""" sparql = SparqlQueryLauncher(self.app, self.session) tse = TriplestoreExplorer(self.app, self.session) for dataset in self.datasets: # Delete from triplestore if dataset.graph_name: Utils.redo_if_failure(self.log, 3, 1, sparql.drop_dataset, dataset.graph_name) # Delete from db dataset.delete_from_db() # Uncache abstraction tse.uncache_abstraction(public=dataset.public)
def delete_user_rdf(self, username): """Delete a user rdf graphs Delete in DB, TS and filesystem Parameters ---------- username : string Username to delete """ tse = TriplestoreExplorer(self.app, self.session) query_launcher = SparqlQueryLauncher(self.app, self.session) graphs = tse.get_graph_of_user(username) for graph in graphs: Utils.redo_if_failure(self.log, 3, 1, query_launcher.drop_dataset, graph)
def integrate(self, public=False): """Integrate the file into the triplestore Parameters ---------- public : bool, optional Integrate in private or public graph """ sparql = SparqlQueryLauncher(self.app, self.session) self.public = public method = self.settings.get('triplestore', 'upload_method') # insert metadata sparql.insert_data(self.get_metadata(), self.file_graph, metadata=True) if method == "load": # cp file into ttl dir tmp_file_name = 'tmp_{}_{}.ttl'.format( Utils.get_random_string(5), self.name, ) temp_file_path = '{}/{}'.format(self.ttl_dir, tmp_file_name) copyfile(self.path, temp_file_path) # Load the chunk sparql.load_data(tmp_file_name, self.file_graph, self.host_url) # Remove tmp file if not self.settings.getboolean('askomics', 'debug_ttl'): os.remove(temp_file_path) else: with open(self.path) as ttl_file: ttl_content = ttl_file.read() sparql.insert_ttl_string(ttl_content, self.user_graph) self.set_triples_number()
def get_abstraction_relations(self): """Get user abstraction relations from the triplestore Returns ------- list Relations """ filter_user = "" if self.logged_user(): filter_user = "******".format( self.session["user"]["username"]) query_launcher = SparqlQueryLauncher(self.app, self.session) query_builder = SparqlQueryBuilder(self.app, self.session) query = ''' SELECT DISTINCT ?graph ?entity_uri ?entity_faldo ?entity_label ?node_type ?attribute_uri ?attribute_faldo ?attribute_label ?attribute_range ?property_uri ?property_faldo ?property_label ?range_uri ?category_value_uri ?category_value_label WHERE {{ # Graphs ?graph :public ?public . ?graph dc:creator ?creator . GRAPH ?graph {{ # Property (relations and categories) ?property_uri a owl:ObjectProperty . ?property_uri a :AskomicsRelation . ?property_uri rdfs:label ?property_label . ?property_uri rdfs:domain ?entity_uri . ?property_uri rdfs:range ?range_uri . }} FILTER ( ?public = <true>{} ) }} '''.format(filter_user) header, data = query_launcher.process_query( query_builder.prefix_query(query)) relations_list = [] relations = [] for result in data: # Relation if "property_uri" in result: rel_tpl = (result["property_uri"], result["entity_uri"], result["range_uri"]) if rel_tpl not in relations_list: relations_list.append(rel_tpl) relation = { "uri": result["property_uri"], "label": result["property_label"], "graphs": [ result["graph"], ], "source": result["entity_uri"], "target": result["range_uri"] } relations.append(relation) else: # if graph is diff, append it index_relation = relations_list.index(rel_tpl) if result["graph"] not in relations[index_relation][ "graphs"]: relations[index_relation]["graphs"].append( result["graph"]) return relations
def get_startpoints(self): """Get public and user startpoints Returns ------- list Startpoints """ filter_user = "" if self.logged_user(): filter_user = "******".format( self.session["user"]["username"]) query_launcher = SparqlQueryLauncher(self.app, self.session) query_builder = SparqlQueryBuilder(self.app, self.session) query = ''' SELECT DISTINCT ?endpoint ?graph ?entity ?entity_label ?creator ?public WHERE {{ ?graph :public ?public . ?graph dc:creator ?creator . GRAPH ?graph {{ ?graph prov:atLocation ?endpoint . ?entity a :entity . ?entity a :startPoint . ?entity rdfs:label ?entity_label . }} FILTER ( ?public = <true>{} ) }} '''.format(filter_user) header, data = query_launcher.process_query( query_builder.prefix_query(query)) startpoints = [] entities = [] for result in data: if result["endpoint"] == self.settings.get("triplestore", "endpoint"): endpoint_name = "local" else: try: endpoint_name = tld.get_fld( result["endpoint"]).split('.')[0] except Exception: endpoint_name = urlparse(result["endpoint"]).netloc if result["entity"] not in entities: # new entity entities.append(result['entity']) startpoint = { "entity": result["entity"], "entity_label": result["entity_label"], "graphs": [{ "uri": result["graph"], "public": result["public"], "creator": result["creator"] }], "endpoints": [{ "url": result["endpoint"], "name": endpoint_name }], "public": self.str_to_bool(result["public"]), "private": not self.str_to_bool(result["public"]) } startpoints.append(startpoint) else: # update existing entity index = entities.index(result['entity']) graph = { "uri": result["graph"], "public": result["public"], "creator": result["creator"] } startpoints[index]["graphs"].append(graph) startpoints[index]["endpoints"].append({ "url": result["endpoint"], "name": endpoint_name }) if self.str_to_bool(result["public"]): startpoints[index]["public"] = True else: startpoints[index]["private"] = True return startpoints
def get_abstraction_attributes(self): """Get user abstraction attributes from the triplestore Returns ------- list AskOmics attributes """ filter_user = "" if self.logged_user(): filter_user = "******".format( self.session["user"]["username"]) litterals = ("http://www.w3.org/2001/XMLSchema#string", "http://www.w3.org/2001/XMLSchema#decimal") query_launcher = SparqlQueryLauncher(self.app, self.session) query_builder = SparqlQueryBuilder(self.app, self.session) query = ''' SELECT DISTINCT ?graph ?entity_uri ?attribute_uri ?attribute_type ?attribute_faldo ?attribute_label ?attribute_range ?category_value_uri ?category_value_label WHERE {{ # Graphs ?graph :public ?public . ?graph dc:creator ?creator . GRAPH ?graph {{ ?attribute_uri a ?attribute_type . VALUES ?attribute_type {{ owl:DatatypeProperty :AskomicsCategory }} ?attribute_uri rdfs:label ?attribute_label . ?attribute_uri rdfs:domain ?entity_uri . ?attribute_uri rdfs:range ?attribute_range . # Faldo OPTIONAL {{ ?attribute_uri a ?attribute_faldo . VALUES ?attribute_faldo {{ askomics:faldoStart askomics:faldoEnd askomics:faldoStrand askomics:faldoReference }} }} # Categories (DK) OPTIONAL {{ ?attribute_range askomics:category ?category_value_uri . ?category_value_uri rdfs:label ?category_value_label . }} }} FILTER ( ?public = <true>{} ) }} '''.format(filter_user) header, data = query_launcher.process_query( query_builder.prefix_query(query)) attributes_list = [] attributes = [] for result in data: # Attributes if "attribute_uri" in result and "attribute_label" in result and result[ "attribute_type"] != "{}AskomicsCategory".format( self.settings.get("triplestore", "prefix") ) and result["attribute_range"] in litterals: attr_tpl = (result["attribute_uri"], result["entity_uri"]) if attr_tpl not in attributes_list: attributes_list.append(attr_tpl) attribute = { "uri": result["attribute_uri"], "label": result["attribute_label"], "graphs": [ result["graph"], ], "entityUri": result["entity_uri"], "type": result["attribute_range"], "faldo": result["attribute_faldo"] if "attribute_faldo" in result else None, "categories": [] } attributes.append(attribute) else: # if graph is different, store it index_attribute = attributes_list.index(attr_tpl) if result["graph"] not in attributes[index_attribute][ "graphs"]: attributes[index_attribute]["graphs"].append( result["graph"]) index_attribute = attributes_list.index(attr_tpl) # Categories if "attribute_uri" in result and result[ "attribute_type"] == "{}AskomicsCategory".format( self.settings.get("triplestore", "prefix")): attr_tpl = (result["attribute_uri"], result["entity_uri"]) if attr_tpl not in attributes_list: attributes_list.append(attr_tpl) attribute = { "uri": result["attribute_uri"], "label": result["attribute_label"], "graphs": [ result["graph"], ], "entityUri": result["entity_uri"], "type": result["attribute_type"], "faldo": result["attribute_faldo"] if "attribute_faldo" in result else None, "categories": [{ "uri": result["category_value_uri"], "label": result["category_value_label"] }] } attributes.append(attribute) else: # if graph diff, store it index_attribute = attributes_list.index(attr_tpl) if result["graph"] not in attributes[index_attribute][ "graphs"]: attributes[index_attribute]["graphs"].append( result["graph"]) # Store value if new value = { "uri": result["category_value_uri"], "label": result["category_value_label"] } if value not in attributes[index_attribute]["categories"]: attributes[index_attribute]["categories"].append(value) return attributes
def get_abstraction_entities(self): """Get abstraction entities Returns ------- list List of entities available """ filter_user = "" if self.logged_user(): filter_user = "******".format( self.session["user"]["username"]) query_launcher = SparqlQueryLauncher(self.app, self.session) query_builder = SparqlQueryBuilder(self.app, self.session) query = ''' SELECT DISTINCT ?endpoint ?graph ?entity_uri ?entity_type ?entity_faldo ?entity_label ?have_no_label WHERE {{ ?graph :public ?public . ?graph dc:creator ?creator . GRAPH ?graph {{ ?graph prov:atLocation ?endpoint . ?entity_uri a ?entity_type . VALUES ?entity_type {{ :entity :bnode }} . # Faldo OPTIONAL {{ ?entity_uri a ?entity_faldo . VALUES ?entity_faldo {{ :faldo }} . }} # Label OPTIONAL {{ ?entity_uri rdfs:label ?entity_label . }} OPTIONAL {{ ?entity_uri :instancesHaveNoLabels ?have_no_label . }} }} FILTER ( ?public = <true>{} ) }} '''.format(filter_user) header, data = query_launcher.process_query( query_builder.prefix_query(query)) entities_list = [] # list of entity uri entities = [] # list of entity dict for result in data: if result["entity_uri"] not in entities_list: # New entity entities_list.append(result["entity_uri"]) # Uri, graph and label label = "" if "entity_label" not in result else result[ "entity_label"] entity_type = "bnode" if result[ "entity_type"] == "{}bnode".format( self.settings.get("triplestore", "prefix")) else "node" entity = { "uri": result["entity_uri"], "type": entity_type, "label": label, "instancesHaveLabels": True if "have_no_label" not in result else False if result["have_no_label"] == "1" else True, "faldo": True if "entity_faldo" in result else False, "endpoints": [result["endpoint"]], "graphs": [result["graph"]], } entities.append(entity) else: # if graph is different, store it index_entity = entities_list.index(result['entity_uri']) if result["graph"] not in entities[index_entity]["graphs"]: entities[index_entity]["graphs"].append(result["graph"]) # If endpoint is different, store it if result["endpoint"] not in entities[index_entity][ "endpoints"]: entities[index_entity]["endpoints"].append( result["endpoint"]) return entities
def integrate(self, dataset_id=None): """Integrate the file into the triplestore""" sparql = SparqlQueryLauncher(self.app, self.session) # insert metadata sparql.insert_data(self.get_metadata(), self.file_graph, metadata=True) content_generator = self.generate_rdf_content() # Insert content chunk_number = 0 for _ in content_generator: if self.graph_chunk.ntriple >= self.max_chunk_size: if self.graph_chunk.percent and dataset_id: self.update_percent_in_db(self.graph_chunk.percent, dataset_id) if self.method == 'load': # write rdf into a tmpfile and load it temp_file_name = 'tmp_{}_{}_chunk_{}.{}'.format( Utils.get_random_string(5), self.name, chunk_number, self.rdf_extention) # Try to load data. if failure, wait 5 sec and retry 5 time Utils.redo_if_failure(self.log, 5, 5, self.load_graph, self.graph_chunk, temp_file_name) else: # Insert # Try to insert data. if failure, wait 5 sec and retry 5 time Utils.redo_if_failure(self.log, 5, 5, sparql.insert_data, self.graph_chunk, self.file_graph) chunk_number += 1 self.graph_chunk = RdfGraph(self.app, self.session) # Load the last chunk if self.graph_chunk.percent and dataset_id: self.update_percent_in_db(100, dataset_id) if self.method == 'load': temp_file_name = 'tmp_{}_{}_chunk_{}.{}'.format( Utils.get_random_string(5), self.name, chunk_number, self.rdf_extention) # Try to load data. if failure, wait 5 sec and retry 5 time Utils.redo_if_failure(self.log, 5, 5, self.load_graph, self.graph_chunk, temp_file_name) else: # Insert # Try to insert data. if failure, wait 5 sec and retry 5 time Utils.redo_if_failure(self.log, 5, 5, sparql.insert_data, self.graph_chunk, self.file_graph) # Content is inserted, now insert abstraction and domain_knowledge self.set_rdf_abstraction_domain_knowledge() if self.method == 'load': temp_file_name = 'tmp_{}_{}_abstraction_domain_knowledge.{}'.format( Utils.get_random_string(5), self.name, self.rdf_extention) self.load_graph(self.graph_abstraction_dk, temp_file_name) else: # Insert sparql.insert_data(self.graph_abstraction_dk, self.file_graph) self.set_triples_number()
def rollback(self): """Drop the dataset from the triplestore in case of error""" sparql = SparqlQueryLauncher(self.app, self.session) sparql.drop_dataset(self.file_graph)
def query(): """Perform a sparql query Returns ------- json query results """ q = request.get_json()['query'] graphs = request.get_json()['graphs'] endpoints = request.get_json()['endpoints'] local_endpoint_f = current_app.iniconfig.get('triplestore', 'endpoint') try: local_endpoint_f = current_app.iniconfig.get('federation', 'local_endpoint') except Exception: pass # No graph selected in local TS if not graphs and local_endpoint_f in endpoints: return jsonify({ 'error': True, 'errorMessage': "No graph selected in local triplestore", 'header': [], 'data': [] }), 500 # No endpoint selected if not endpoints: return jsonify({ 'error': True, 'errorMessage': "No endpoint selected", 'header': [], 'data': [] }), 500 try: query_builder = SparqlQueryBuilder(current_app, session) query_builder.set_graphs_and_endpoints(graphs=graphs, endpoints=endpoints) federated = query_builder.is_federated() replace_froms = query_builder.replace_froms() query = query_builder.format_query(q, replace_froms=replace_froms, federated=federated) # header, data = query_launcher.process_query(query) header = query_builder.selects data = [] if query_builder.graphs or query_builder.endpoints: query_launcher = SparqlQueryLauncher(current_app, session, get_result_query=True, federated=federated, endpoints=endpoints) header, data = query_launcher.process_query(query) except Exception as e: current_app.logger.error(str(e).replace('\\n', '\n')) traceback.print_exc(file=sys.stdout) return jsonify({ 'error': True, 'errorMessage': str(e).replace('\\n', '\n'), 'header': [], 'data': [] }), 500 return jsonify({'header': header, 'data': data})
def query(): """Perform a sparql query Returns ------- json query results """ if not can_access(session['user']): return jsonify({"error": True, "errorMessage": "Admin required"}), 401 data = request.get_json() if not (data and data.get("query")): return jsonify({ 'error': True, 'errorMessage': "Missing query parameter", 'header': [], 'data': [] }), 400 q = data['query'] graphs = data.get('graphs', []) endpoints = data.get('endpoints', []) local_endpoint_f = current_app.iniconfig.get('triplestore', 'endpoint') try: local_endpoint_f = current_app.iniconfig.get('federation', 'local_endpoint') except Exception: pass # No graph selected in local TS if not graphs and local_endpoint_f in endpoints: return jsonify({ 'error': True, 'errorMessage': "No graph selected in local triplestore", 'header': [], 'data': [] }), 400 # No endpoint selected if not endpoints: return jsonify({ 'error': True, 'errorMessage': "No endpoint selected", 'header': [], 'data': [] }), 400 try: query = SparqlQuery(current_app, session, get_graphs=False) query.set_graphs_and_endpoints(graphs=graphs, endpoints=endpoints) federated = query.is_federated() replace_froms = query.replace_froms() sparql = query.format_query(q, replace_froms=replace_froms, federated=federated) # header, data = query_launcher.process_query(query) header = query.selects data = [] if query.graphs or query.endpoints: query_launcher = SparqlQueryLauncher(current_app, session, get_result_query=True, federated=federated, endpoints=endpoints) header, data = query_launcher.process_query(sparql) except Exception as e: current_app.logger.error(str(e).replace('\\n', '\n')) traceback.print_exc(file=sys.stdout) return jsonify({ 'error': True, 'errorMessage': str(e).replace('\\n', '\n'), 'header': [], 'data': [] }), 500 return jsonify({'header': header, 'data': data})