def query_database(query_type: str, query: str, database: str) -> (bool, dict): """ Process a SELECT or UPDATE query :param query_type: A string = 'select' or 'update' :param query: The text of a SPARQL query :param database: The database (name) to be queried :return: True if successful; False otherwise Query results (if the query_type is 'select'); An empty dictionary otherwise """ logging.info( f'Querying database, {database}, using {query_type}, with query, {query}' ) if query_type != 'select' and query_type != 'update': capture_error(f'Invalid query_type {query_type} for query_db', True) return False, dict() try: conn = stardog.Connection(database, **sd_conn_details) if query_type == 'select': # Select query, which will return results, if successful query_results = conn.select( query, content_type='application/sparql-results+json') if query_results['results']['bindings']: return True, query_results else: return True, dict() else: # Update query; No results (either success or failure) conn.update(query) return True, dict() except Exception as e: capture_error( f'Database ({database}) query exception for {query}: {str(e)}', True) return False, dict()
def getTerms(input_term): with stardog.Admin(**conn_details) as admin: with stardog.Connection('capability', **conn_details) as conn: # get all equivalent calsses query # No. 1 : get all parents query_for_parents = """ SELECT ?entity where {{ ?src {0}* <http://stardog.com/{1}> . FILTER(?src != <http://stardog.com/ManufacturingCapabilities>) BIND(?src AS ?entity) . }} """.format(PARENT_OF, input_term) result_for_parents = conn.select(query_for_parents, reasoning=True) # No. 2 : get all requires query_for_requires = """ SELECT ?entity where {{ <http://stardog.com/{1}> {0}* ?tgt . FILTER(?tgt != <http://stardog.com/ManufacturingCapabilities>) BIND(?tgt AS ?entity) . }} """.format(REQUIRES, input_term) result_for_requires = conn.select(query_for_requires, reasoning=True) stardog_responses = [result_for_parents, result_for_requires] result_list = set() for response in stardog_responses: for result in response['results']['bindings']: temp_out = result[next(iter(result))]['value'] temp_out = temp_out.rsplit("/", 1)[-1] result_list.add(temp_out) return result_list
def download_ontology(self, save_as, progress_callback=None): try: assert self.conn_details if progress_callback: progress_callback("establishing connection ...") dbname = self.conn_details['dbname'] with stardog.Connection(dbname, **self.open_conn_details) as conn: if progress_callback: progress_callback("connection OK") # conn.export(content_type='application/rdf+xml') # to get RDF/XML contents = conn.export() # запись в файл ext = '.ttl' if not save_as.endswith(ext): save_as += ext if progress_callback: progress_callback("writing to file: " + save_as[-50:]) with open(save_as, 'wb') as f: f.write(contents) if progress_callback: progress_callback("dropping database if set so...") self.stardog_drop_db_if_set() if progress_callback: progress_callback("finished!") except Exception as e: print(e) return str(e)
def stardog_create_db_if_set(self): # conn_details = { # 'endpoint': self.server_url_var.get(), # 'username': '******', # 'password': '******', # 'schemafile': self.schema_file_var.get(), # 'dbname': self.server_dbname_var.get(), # 'createdb': self.server_db_create_var.get(), # 'dropdb': self.server_db_drop_var.get(), # } assert self.conn_details if self.conn_details['createdb']: dbname = self.conn_details['dbname'] schema_file = stardog.content.File(self.conn_details['schemafile']) print('Probably schema_file OK: ', self.conn_details['schemafile']) with stardog.Admin(**self.open_conn_details) as admin: # check if database already exist and drop it try: self.stardog_drop_db_if_set(force=True) except: pass db = admin.new_database(dbname) db = None # forget pointer # init schema with stardog.Connection(dbname, **self.open_conn_details) as conn: conn.begin() conn.add(schema_file) conn.commit()
def get_query(self, spqrql, prefix): conn = stardog.Connection( DATASOURCE_YML_CONFIG_FILE["database"]["stardog"]["db"], **STARDOG_CONN_DETAILS) results = conn.select(spqrql, content_type='application/sparql-results+xml') return results
def runQuery(self, query): results = {} if self.timeout == 0: return query, results try: with stardog.Connection(self.database, **conn_details) as conn: output = conn.select(query) results = output.get("results").get("bindings") logging.info("Results: " + str(len(results))) except Exception as inst: logging.error(inst) return query, results
def post(self): if not request.json: return jsonify({'status': 'could not get requested data', 'statuscode': 409}) else: query = buildQueryFromJSON(request.json) # Anfrage an Stardog with stardog.Connection(dbName, **conn_details) as conn: conn.begin() print(query) results = conn.select(query) print(results) return results
def append_rule(message): print("new rule received") rule_obj = json.loads(message) rule = rule_obj['semantic_ruleminer'] with open("rule_tmp.ttl", "w") as text_file: text_file.write(rule) for i in range(0, DATABASES): with stardog.Connection('db'+str(i), **conn_details) as conn: conn.begin() conn.add(stardog.content.File('rule_tmp.ttl')) conn.commit()
def stardog_eg_csv(ark): conn_details = { 'endpoint': 'http://stardog.uvadcos.io', 'username': '******', 'password': '******' } with stardog.Connection('db', **conn_details) as conn: conn.begin() #results = conn.select('select * { ?a ?p ?o }') results = conn.paths("PATHS START ?x=<" + ark + "> END ?y VIA ?p", content_type='text/csv') with open(root_dir + '/star/test.csv', 'wb') as f: f.write(results) return
def create_named_graph(meta, id): with open(root_dir + '/star/meta.json', 'w') as f: json.dump(meta, f) conn_details = { 'endpoint': 'http://stardog.uvadcos.io', 'username': '******', 'password': '******' } with stardog.Connection('db', **conn_details) as conn: conn.begin() conn.add(stardog.content.File(root_dir + "/star/meta.json"), graph_uri='http://ors.uvadcos/' + id) conn.commit() # cmd = 'stardog data add --named-graph http://ors.uvadcos.io/' + id + ' -f JSONLD test "/star/meta.json"' # test = os.system(cmd) # warnings.warn('Creating named graph returned: ' + str(test)) return
def add_remove_data(op_type: str, triples: str, database: str, graph: str = '') -> bool: """ Add or remove data to/from the database/store :param op_type: A string = 'add' or 'remove' :param triples: A string with the triples to be inserted/removed :param database: The database name :param graph: An optional named graph in which to insert/remove the triples :return: True if successful; False otherwise """ logging.info( f'Data {"added to" if op_type == "add" else "removed from"} {database}' f'{" and graph, " if graph else ""}{graph}') if op_type != 'add' and op_type != 'remove': capture_error(f'Invalid op_type {op_type} for add_remove_graph', True) return False try: conn = stardog.Connection(database, **sd_conn_details) conn.begin() if op_type == 'add': # Add to the database if graph: conn.add(stardog.content.Raw(triples, 'text/turtle'), graph_uri=graph) else: conn.add(stardog.content.Raw(triples, 'text/turtle')) else: # Remove from the database if graph: conn.remove(stardog.content.Raw(triples, 'text/turtle'), graph_uri=graph) else: conn.remove(stardog.content.Raw(triples, 'text/turtle')) conn.commit() return True except Exception as e: capture_error(f'Database ({op_type}) exception: {str(e)}', True) return False
def create_delete_database(op_type: str, database: str) -> str: """ Create or delete a database. If created, add the DNA ontologies. :param op_type: A string = 'create' or 'delete' :param database: The database name :return: Empty string if successful or the details of an exception """ logging.info(f'Database {database} being {op_type}d') if op_type != 'create' and op_type != 'delete': capture_error(f'Invalid op_type {op_type} for create_delete_db', True) return '' try: admin = stardog.Admin(**sd_conn_details) if op_type == 'create': # Create database admin.new_database( database, { 'search.enabled': True, 'edge.properties': True, 'reasoning': True, 'reasoning.punning.enabled': True, 'query.timeout': '20m' }) # Load ontologies to the newly created database conn = stardog.Connection(database, **sd_conn_details) conn.begin() logging.info(f'Loading DNA ontologies to {database}') _load_directory_to_database(ontol_path, conn) _load_directory_to_database(f'{ontol_path}domain-context/', conn) conn.commit() else: # Delete database database_obj = admin.database(database) database_obj.drop() return '' except Exception as e: return f'Database ({op_type}) exception: {str(e)}'
def create_dataset_database(self, database_name, dataset_path): connection_details = { 'endpoint': self.endpoint, 'username': self.username, 'password': self.password } with stardog.Admin(**connection_details) as admin: if database_name in [db.name for db in admin.databases()]: admin.database(database_name).drop() db = admin.new_database(database_name) with stardog.Connection(database_name, **connection_details) as conn: conn.begin() conn.add(stardog.content.File(str(dataset_path))) conn.commit() if conn.size(exact=True) <= 0: admin.database(database_name).drop() raise StardogException('No triples loaded!') return database_name
def upload_triples(self, triples, progress_callback=None): try: assert self.conn_details if progress_callback: progress_callback("establishing connection ...") self.stardog_create_db_if_set() prefix_str = """ BASE <http://vstu.ru/poas/se/c_schema_2020-01#> PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> """ f = lambda s: make_namespace_prefix( s, default_prefix=':', known_prefixes={'rdf'}) dbname = self.conn_details['dbname'] with stardog.Connection(dbname, **self.open_conn_details) as conn: if progress_callback: progress_callback("connection OK") for i, trpl in enumerate(triples): # ensure prefixes OK trpl = tuple(f(a) for a in trpl) q = triple_to_sparql_insert(trpl, prefix_str) print('Send: ', trpl, end='') # отправка запросов SPARQL UPDATE results = conn.update(q, reasoning=False) print(' done!') if progress_callback: progress_callback("%d/%d done" % ((i + 1), len(triples))) if progress_callback: progress_callback("100% finished!") except Exception as e: print(e) return str(e)
def reason_group(message_group, db_i, box, roomtype): try: with stardog.Connection('db'+str(db_i), **conn_details) as conn: task.reason(conn, message_group, db_i, box, roomtype) except Exception as e: print(e)
#session = requests.Session() #session.auth = (SD_USERNAME,SD_PASSWORD) SD_URL = os.environ.get('STARDOG_URL','http://stardog.uvadcos.io') SD_USERNAME = os.environ.get('STARDOG_USERNAME') SD_PASSWORD = os.environ.get('STARDOG_PASSWORD') HOST_URL = os.environ.get('HOST_URL','') ORS_URL = os.environ.get("ORS_URL","ors.uvadco.io/") EVI_PREFIX = 'evi:' conn_details = { 'endpoint': SD_URL, 'username': SD_USERNAME, 'password': SD_PASSWORD } conn = stardog.Connection('ors', **conn_details) class EverythingConverter(PathConverter): regex = '.*?' def mint_eg_id(eg): ''' Mints Id for newly created evidence graph ''' r = requests.post(ORS_URL + "shoulder/ark:99999",data = json.dumps(eg)) if 'created' in r.json(): return HOST_URL + 'evidence/' + r.json()['created'] def add_eg_to_og_id(ark,eg_id):
time.sleep(30) manager = Manager() db_pool = manager.list([True for x in range(0, DATABASES)]) # create database pool for i in range(0, DATABASES): try: with stardog.Admin(**conn_details) as admin: admin.database('db'+str(i)).drop() except: print("no database to drop") print('db'+str(i)) with stardog.Admin(**conn_details) as admin: admin.new_database('db'+str(i), {'index.type': 'Memory'}) with stardog.Connection('db'+str(i), **conn_details) as conn: conn.begin() conn.add(stardog.content.File('rule.ttl')) conn.commit() message_queue = {} proccesses = [] counter = 0 start = time.time() for message in consumer: counter += 1 plain_message = message.value if counter%100==0: print("processed: 100 messages in ", time.time()-start) start = time.time()
# Далее должен быть цикл загрузки всех триплетов ... import stardog conn_details = { 'endpoint': 'http://localhost:5820', 'username': '******', 'password': '******' } with stardog.Admin(**conn_details) as admin: with stardog.Connection('с_owl', **conn_details) as conn: for trpl in triples: q = triple_to_sparql_insert(trpl, prefix_str) print('Go...', end='') # как называется метод для отправки запросов SPARQL UPDATE, я не в курсе: проверь. Но не `select` - точно. results = conn.update(q, reasoning=False) print(' done!') # pprint(results) contents = str(conn.export()) contents = contents[2:] contents = contents[:-1] contents = contents.replace('\\n', '\n')
def update(self, query: str): with stardog.Connection(self.__database, **self.__credentials) as conn: conn.update(query)
def select(self, query: str, reasoning: bool = False): with stardog.Connection(self.__database, **self.__credentials) as conn: results = conn.select(query, reasoning=reasoning) return results["results"]["bindings"]
class ConnectionFactory(): ''' A class used to represent and manage a Stardog database connection Attributes ---------- connectionPool : dict A dict with three items used to connect to stardog. endpoint : str the host where stardog is running. Default = 'http://localhost:5820' username : str the username used to log in to the base. Default = 'admin' password : str the password dused to log in to the base. Default = 'admin' ''' connectionPool = { 'endpoint': 'http://192.168.56.104:5820', 'username': '******', 'password': '******' } connectionPoolUfes = { 'endpoint': 'http://200.137.66.31:5820', 'username': '******', 'password': '******' } databaseName = 'riodoce' connection = stardog.Connection(databaseName, **connectionPool) if (connection): print(str(connection)) else: print('null') def __init__(self): pass def __listAllProps(self, prd): return self.connection.select( 'PREFIX prd: <' + Prefixes.PREFIX + '>' + ' SELECT distinct ?prop WHERE{?prop rdf:type prd:' + prd + '}') def listProperty(self): return self.__listAllProps('Measurable_Property') def listParty(self): return self.__listAllProps('Agent_Party') def listRiver(self): return self.__listAllProps('River') def getFilteredData(self, allFilters): finalQuery = '' filterPropertiesSelected = '' print('all filters ' + allFilters) finalQuery = ( 'PREFIX prd: <' + Prefixes.PREFIX + '>' + 'SELECT ?author ?property ?value ?unity ?sample' + '?sampleDate ?lat ?long ?monitoringFacility ?locale' + ' WHERE {' + ' ?measurement rdf:type prd:Measurement.' + ' ?measurement prd:measure_unit ?unity.' + ' ?measurement prd:measures ?property.' + ' ?measurement prd:val ?value.' + ' ?measurement prd:uses ?sample.' + ' ?measurement prd:isDoneBy ?author.' + filterPropertiesSelected + ' ?sample rdf:type prd:Sampling.' + ' ?sample prd:date ?sampleDate.' + ' ?monitoringFacility rdf:type prd:Monitoring_Facility.' + ' ?monitoringFacility prd:performs ?sample.' + ' ?monitoringFacility prd:locates ?locale.' + ' ?locale rdf:type prd:Geographic_Point.' + ' ?locale prd:latitude ?lat.' + ' ?locale prd:longitude ?long' + allFilters + '.}LIMIT 10') return self.connection.select(finalQuery) def measurement(self, allFilters): finalQuery = '' filterPropertiesSelected = '' print('all filters ' + allFilters) finalQuery = ( 'PREFIX prd: <' + Prefixes.PREFIX + '>' + 'SELECT ?author ?property ?value ?unity ?sample' + '?sampleDate ?lat ?long ?monitoringFacility ?locale' + ' WHERE {' + ' ?measurement rdf:type prd:Measurement.' + ' ?measurement prd:measure_unit ?unity.' + ' ?measurement prd:measures ?property.' + ' ?measurement prd:val ?value.' + ' ?measurement prd:uses ?sample.' + ' ?measurement prd:isDoneBy ?author.' + filterPropertiesSelected + ' ?sample rdf:type prd:Sampling.' + ' ?sample prd:date ?sampleDate.' + ' ?monitoringFacility rdf:type prd:Monitoring_Facility.' + ' ?monitoringFacility prd:performs ?sample.' + ' ?monitoringFacility prd:locates ?locale.' + ' ?locale rdf:type prd:Geographic_Point.' + ' ?locale prd:latitude ?lat.' + ' ?locale prd:longitude ?long' + allFilters + '.}LIMIT 10') return self.connection.select(finalQuery)