def test_one_spurious_semicolons_bnode(): sparql = """ PREFIX : <http://example.org/> CONSTRUCT { [ :b :c ; :d :e ; ] } WHERE {} """ expected = Graph() expected.addN(t + (expected,) for t in [ (BNode("a"), NS.b, NS.c), (BNode("a"), NS.d, NS.e), ]) got = Graph().query(sparql).graph assert isomorphic(got, expected), got.serialize(format="turtle")
def test_one_spurious_semicolons_bnode(): sparql = """ PREFIX : <http://example.org/> CONSTRUCT { [ :b :c ; :d :e ; ] } WHERE {} """ expected = Graph() expected.addN(t + (expected, ) for t in [ (BNode("a"), NS.b, NS.c), (BNode("a"), NS.d, NS.e), ]) got = Graph().query(sparql).graph assert isomorphic(got, expected), got.serialize(format="turtle")
def test_two_spurious_semicolons_no_period(): sparql = """ PREFIX : <http://example.org/> CONSTRUCT { :a :b :c ; :d :e ; ; } WHERE {} """ expected = Graph() expected.addN(t + (expected,) for t in [ (NS.a, NS.b, NS.c), (NS.a, NS.d, NS.e), ]) got = Graph().query(sparql).graph assert isomorphic(got, expected), got.serialize(format="turtle")
def _serialize_with_rdflib(rdflib_format, graph, bindings, base_uri): "Common implementation of all rdflib-based serialize functions." assert isinstance(rdflib_format, str) assert isinstance(graph, Graph) # copy in another graph to prevent polluting the original graph namespaces # TODO LATER find an efficient way to serialize a graph with custom NSs ser = Graph() ser.addN( (s, p, o, ser) for s, p, o in graph ) for prefix, nsuri in bindings.items(): ser.bind(prefix, nsuri) if base_uri is not None and not isinstance(base_uri, URIRef): base_uri = coerce_to_uri(base_uri) # We use yield to prevent the serialization to happen if a 304 is returned yield ser.serialize(None, format=rdflib_format, base=base_uri)
def _serialize_with_rdflib(rdflib_format, graph, bindings, base_uri): "Common implementation of all rdflib-based serialize functions." assert isinstance(rdflib_format, str) assert isinstance(graph, Graph) # copy in another graph to prevent polluting the original graph namespaces # TODO LATER find an efficient way to serialize a graph with custom NSs ser = Graph() ser.addN((s, p, o, ser) for s, p, o in graph) for prefix, nsuri in bindings.items(): ser.bind(prefix, nsuri) if base_uri is not None and not isinstance(base_uri, URIRef): base_uri = coerce_to_uri(base_uri) # We use yield to prevent the serialization to happen if a 304 is returned yield ser.serialize(None, format=rdflib_format, base=base_uri)
def test_two_spurious_semicolons_no_period(): sparql = """ PREFIX : <http://example.org/> CONSTRUCT { :a :b :c ; :d :e ; ; } WHERE {} """ expected = Graph() expected.addN(t + (expected, ) for t in [ (NS.a, NS.b, NS.c), (NS.a, NS.d, NS.e), ]) got = Graph().query(sparql).graph assert isomorphic(got, expected), got.serialize(format="turtle")
def add_list_to_graph(graph: Graph, spo_list: RDFList) -> None: """ 将一个SPO三元组的列表一次性地加入 graph 中 :param graph: 需要加入RDF的graph :param spo_list: SPO三元组的列表 :return: None Examples -------- >>> g = Graph() >>> spo_list = [ ... (URIRef('http://www.ifa.com#Firm/tencent'), URIRef('http://www.ifa.com#hasName'), Literal('腾讯', datatype=XSD.string)), ... (URIRef('http://www.ifa.com#Firm/tencent'), URIRef('http://www.ifa.com#hasApp'), URIRef('http://www.ifa.com#App/wechat')) ... ] >>> add_list_to_graph(g, spo_list) """ quads = [(s, p, o, graph) for s, p, o in spo_list] graph.addN(quads)
def test_pathological(): """ This test did not raise an exception, but generated a graph completely different from the expected one. """ sparql = """ PREFIX : <http://example.org/> CONSTRUCT { :a :b :c ; ; :d :e ; ; :f :g ; } WHERE {} """ expected = Graph() expected.addN(t + (expected,) for t in [ (NS.a, NS.b, NS.c), (NS.a, NS.d, NS.e), (NS.a, NS.f, NS.g), ]) got = Graph().query(sparql).graph assert isomorphic(got, expected), got.serialize(format="turtle")
def test_pathological(): """ This test did not raise an exception, but generated a graph completely different from the expected one. """ sparql = """ PREFIX : <http://example.org/> CONSTRUCT { :a :b :c ; ; :d :e ; ; :f :g ; } WHERE {} """ expected = Graph() expected.addN(t + (expected, ) for t in [ (NS.a, NS.b, NS.c), (NS.a, NS.d, NS.e), (NS.a, NS.f, NS.g), ]) got = Graph().query(sparql).graph assert isomorphic(got, expected), got.serialize(format="turtle")
def add_dict_to_graph(graph: Graph, s: Union[URIRef, BNode], po_dict: Dict[URIRef, Identifier]) -> None: """ 将一个P-O的字典一次性地加入 graph 中 :param graph: 需要加入RDF的graph :param s: subject的URIRef :param po_dict: predicate-object 组成的字典类型 :return: None Examples -------- >>> from rdflib import XSD, Graph, Literal, URIRef >>> >>> g = Graph() >>> po_dict = { ... URIRef('http://www.ifa.com#hasName'): Literal('腾讯', datatype=XSD.string), ... URIRef('http://www.ifa.com#hasApp'): URIRef('http://www.ifa.com#App/wechat') ... } >>> s = URIRef('http://www.ifa.com#Firm/tencent') >>> add_dict_to_graph(g, s, po_dict) """ quads = [(s, p, o, graph) for p, o in po_dict.items()] graph.addN(quads)
class ToRDFConverter: def __init__(self, atdm, metadata): super().__init__() self.graph = Graph() self.metadata = metadata self.atdm = atdm self.mode = CONST_STANDARD_MODE RDFGraphUtils.add_default_bindings(self.graph) RDFGraphUtils.add_bindings_from_metadata(self.graph, metadata) def convert(self, mode=CONST_STANDARD_MODE, format=None): self.mode = mode main_node = BNode() if mode == CONST_STANDARD_MODE: self.graph.add((main_node, RDF.type, CSVW.TableGroup)) self._add_file_metadata(self.metadata, main_node) # TODO: 4.5 non-core annotations for table_metadata, table_data in zip(self.metadata['tables'], self.atdm['tables']): self._parse_table(main_node, table_metadata, table_data) return self.graph if format is None else self.graph.serialize( format=format).decode() def _parse_table(self, main_node, table_metadata, table_data): table_node = URIRef( table_metadata['@id']) if '@id' in table_metadata else BNode() property_url = table_metadata['url'] + '#' if 'propertyUrl' in table_metadata['tableSchema']: property_url = table_metadata['tableSchema']['propertyUrl'] if self.mode == CONST_STANDARD_MODE: self.graph.add((main_node, CSVW.table, table_node)) self.graph.add((table_node, RDF.type, CSVW.Table)) self.graph.add( (table_node, CSVW.url, URIRef(table_metadata['url']))) self._add_file_metadata(table_metadata, table_node) for index, atdm_row in enumerate(table_data['rows']): if self.mode == CONST_STANDARD_MODE: row_node = BNode() self._parse_generic_row(atdm_row, table_node, table_metadata, property_url, row_node, table_data) else: dummy = BNode() row_node = BNode() self._parse_row(atdm_row, row_node, dummy, table_metadata, property_url, table_data) self.graph.remove((dummy, CSVW.describes, row_node)) self.parse_virtual_columns(row_node, atdm_row, table_metadata) def parse_virtual_columns(self, row_node, atdm_row, table_metadata): for virtual_column in table_metadata['tableSchema']['columns']: if 'virtual' not in virtual_column or virtual_column[ 'virtual'] is False: continue subject = URIRef( UriTemplateUtils.insert_value(virtual_column['aboutUrl'], atdm_row, '', table_metadata['url'])) predicate = Namespaces.get_term(virtual_column['propertyUrl']) obj = UriTemplateUtils.insert_value(virtual_column['valueUrl'], atdm_row, '', table_metadata['url']) obj = CommonProperties.expand_property_if_possible(obj) self.graph.add((subject, predicate, URIRef(obj))) if self.mode == CONST_STANDARD_MODE: self.graph.add((row_node, CSVW.describes, subject)) def _add_file_metadata(self, metadata, node): language = JSONLDUtils.language(self.metadata['@context']) for key, value in metadata.items(): if CommonProperties.is_common_property(key) or key == 'notes': triples = CommonProperties.property_to_triples( (key, metadata[key]), node, language) self.graph.addN(triple + (self.graph, ) for triple in triples) def _parse_generic_row(self, atdm_row, table_node, metadata, property_url, row_node, atdm_table): self.graph.add((table_node, CSVW.row, row_node)) self.graph.add((row_node, RDF.type, CSVW.Row)) self.graph.add((row_node, CSVW.rownum, Literal(atdm_row['number'], datatype=XSD.integer))) self.graph.add((row_node, CSVW.url, URIRef(atdm_row['@id']))) if 'rowTitles' in metadata['tableSchema']: col_names = metadata['tableSchema']['rowTitles'] for col_name in col_names: col_value = atdm_row['cells'][col_name][0] self.graph.add((row_node, CSVW.title, Literal(col_value))) values_node = BNode() self._parse_row(atdm_row, values_node, row_node, metadata, property_url, atdm_table) def _parse_row(self, atdm_row, values_node, row_node, metadata, property_url, atdm_Table): if not all( map(lambda column: 'aboutUrl' in column, metadata['tableSchema']['columns'])): self.graph.add((row_node, CSVW.describes, values_node)) self._parse_row_data(atdm_row, values_node, metadata, property_url, row_node, atdm_Table) def _parse_row_data(self, atdm_row, subject, table_metadata, property_url, row_node, atdm_table): top_level_property_url = property_url atdm_columns = atdm_table['columns'] for index, entry in enumerate(atdm_row['cells'].items()): col_name, values = entry for col_metadata in atdm_columns: if col_metadata['name'] == col_name: break if col_metadata.get('suppressOutput', False): continue property_url = col_metadata.get('propertyUrl', top_level_property_url) if 'aboutUrl' in col_metadata: subject = UriTemplateUtils.insert_value_rdf( col_metadata['aboutUrl'], atdm_row, col_name, table_metadata['url']) if self.mode == CONST_STANDARD_MODE: self.graph.add((row_node, CSVW.describes, subject)) property_namespace = PropertyUrlUtils.create_namespace( property_url, table_metadata['url']) predicate = self._predicate_node(property_namespace, property_url, col_name) self._parse_cell_values(values, col_metadata, subject, predicate) def _parse_cell_values(self, values, col_metadata, subject, predicate): """ Parses single cell value, values if 'separator' is present""" if 'ordered' in col_metadata and col_metadata[ 'ordered'] is True and len(values) > 1: next_item = BNode() rdf_list = next_item values_count = len(values) for i in range(values_count): item = next_item self.graph.add((item, RDF.first, Literal(values[i]))) if i != values_count - 1: next_item = BNode() self.graph.add((item, RDF.rest, next_item)) Collection(self.graph, rdf_list) self.graph.add((subject, predicate, rdf_list)) else: for value in values: object_node = self._object_node(value, col_metadata) self.graph.add((subject, predicate, object_node)) @staticmethod def _object_node(value, col_metadata): if 'valueUrl' in col_metadata: return ValueUrlUtils.create_uri_ref(value, col_metadata['valueUrl']) else: lang = col_metadata.get('lang') if not datatypeutils.is_compatible_with_datatype( value, col_metadata.get('datatype')): datatype = None else: datatype = OntologyUtils.type(col_metadata) lang = None if datatype is not None else lang return Literal(value, lang=lang, datatype=datatype) @staticmethod def _normalize_to_uri(string): string = quote(string, safe='') string = string.replace('-', '%2D') return string @staticmethod def _predicate_node(namespace, property_url, col_name): col_name = ToRDFConverter._normalize_to_uri(col_name) if '{' in property_url: return namespace.term(col_name) else: if CommonProperties.is_common_property(property_url): return URIRef( CommonProperties.expand_property_if_possible(property_url)) else: if property_url.endswith('#'): return URIRef(property_url + col_name) else: return URIRef(namespace)
def get_state(self, parameters=None): """I override `~rdfrest.cores.ICore.get_state`:meth: I support parameter "prop" to enrich the Base description with additional information. I consider an empty dict as equivalent to no dict. """ state = super(Base, self).get_state(parameters) if not parameters: return state enriched_state = Graph() enriched_state += state whole = ConjunctiveGraph(self.service.store) initNs = { '': KTBS, 'rdfs': RDFS, 'skos': SKOS } initBindings = { 'base': self.uri } for prop in parameters['prop']: if prop == 'comment': enriched_state.addN( (s, RDFS.comment, o, enriched_state) for s, o, _ in whole.query(''' SELECT ?s ?o $base # selected solely to please Virtuoso { GRAPH $base { $base :contains ?s } GRAPH ?s { ?s rdfs:comment ?o } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'hasMethod': enriched_state.addN( (s, KTBS.hasMethod, o, enriched_state) for s, o, _ in whole.query(''' SELECT ?t ?m $base # selected solely to please Virtuoso { GRAPH $base { $base :contains ?t } GRAPH ?t { ?t :hasMethod ?m } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'hasModel': enriched_state.addN( (s, KTBS.hasModel, o, enriched_state) for s, o, _ in whole.query(''' SELECT ?t ?m $base # selected solely to please Virtuoso { GRAPH $base { $base :contains ?t } GRAPH ?t { ?t :hasModel ?m } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'hasSource': enriched_state.addN( (s, KTBS.hasSource, o, enriched_state) for s, o, _ in whole.query(''' SELECT ?t1 ?t2 $base # selected solely to please Virtuoso { GRAPH $base { $base :contains ?t1 } GRAPH ?t1 { ?t1 :hasSource ?t2 } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'label': enriched_state.addN( (s, p, o, enriched_state) for s, p, o, _ in whole.query(''' SELECT ?s ?p ?o $base # selected solely to please Virtuoso { VALUES ?p { rdfs:label skos:prefLabel } GRAPH $base { $base :contains ?s } GRAPH ?s { ?s ?p ?o } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'obselCount': enriched_state.addN( (s, KTBS.hasObselCount, o, enriched_state) for s, o, _ in whole.query(''' SELECT ?t (COUNT(?obs) as ?c) (SAMPLE($base) as ?sample_base) # selected solely to please Virtuoso { VALUES ?tt { :StoredTrace :ComputedTrace } GRAPH $base { $base :contains ?t. ?t a ?tt } OPTIONAL { ?obs :hasTrace ?t } } GROUP BY ?t ''', initNs=initNs, initBindings=initBindings) ) else: pass # ignoring unrecognized properties # should we signal them instead (diagnosis?) return enriched_state
class OntologyDatabase: """A front-end of an ontology database. This class provides "safe" access to most of the standard operations provided by the rdflib.Graph class. The "safeness" of the methods lies in catching exceptions and reconnecting shall the connection to the database "die" for whatever reason. Additionally, this class implements the SQLAlchemy store for the triples """ def __init__(self, config=None, create=None): """Create ontology database API with SQLAlchemy store. Parameters ---------- config : [str, knowl.DBConfig], optional The path to a configuration file or the configuration object. By default None, which results in a configuration with default parameters (see knowl.DBConfig). create : bool, optional Whether or not the tables for the ontology (triplestore) should be initalized. Set to True if you are creating a new database, by default None. As per SQLAlchemy documentation, the creation operation is idempotent. Thus, could be left at True, unless you specifically do not want to create a new database if one does not exist. """ # initialize database config self.__config = DBConfig.factory(config) self.__username = None self.__password = None self.__create = create self.__store_type = self.config["store"] # configure database identifier (ontology IRI/base URL) self.__identifier = self.config.baseURL if self.store_type == "alchemy": self.__store = SQLAlchemy(identifier=self.identifier) self._graph = Graph(self.__store, identifier=self.identifier) elif self.store_type == "fuseki": self.__query_endpoint = f'http://{self.config["host"]}:{self.config["port"]}/{self.config["database"]}' self.__update_endpoint = f'http://{self.config["host"]}:{self.config["port"]}/{self.config["database"]}/update' self.__store = SPARQLUpdateStore( queryEndpoint=self.__query_endpoint + '/sparql', update_endpoint=self.__update_endpoint, context_aware=True, postAsEncoded=False, node_to_sparql=my_bnode_ext) self.__query_endpoint += '/query' self.__store.method = 'POST' else: raise Exception(f"Unknown store type {self.store_type}!") def setup(self, create=False, username: str = None, password: str = None): """Sets-up a new database connection. Call this to initialize access to the database. Parameters ---------- create : bool, optional Whether the tables should be created (idempotent). Only set to True if creating a new database, by default False. Setting the object property self.create to anything but None will override this value! username : str, optional Database access credentials. Only set this if you didn't set it before (e.g. in the config file), by default None password : str, optional Database access credentials. Only set this if you didn't set it before (e.g. in the config file), by default None """ if self.store_type == "alchemy": if self.__create is not None: create = self.__create self._graph.open(self.config.getDB_URI( self.__username if username is None else username, self.__password if password is None else password), create=create) elif self.store_type == "fuseki": print( f"Query endpoint: {self.__query_endpoint}\nUpdate endpoint: {self.__update_endpoint}\nIndentifier: {self.identifier}" ) self.__store.open((self.__query_endpoint, self.__update_endpoint)) self._graph = Graph(self.__store, identifier=self.identifier) for ns, uri in self.config.namespaces.items(): self._graph.bind(ns.lower(), uri) def closelink(self): """Closes the database connection. """ try: self._graph.close() except Exception as e: print("Exception in Closing", e) def destroy(self, confirmation: str = None): """Destroys the store for the Ontology This will erase/destroy the database (triplestore) used to store the data. Be very careful when calling this function. Parameters ---------- confirmation : str, optional [description], by default None """ if confirmation == "I know what I am doing": self._graph.destroy(self.identifier) else: raise ValueError( "Destroying the DB attempted but failed - wrong confirmation string!" ) def setCredentials(self, username: str = None, password: str = None): """Set access credentials for the database server containing the triplestore. Parameters ---------- username : str, optional The username, by default None password : str, optional The password. Warning, this will be visible in the DB URL! By default None """ self.__username = username self.__password = password @interact_with_db def mergeFileIntoDB(self, filepath: str): """Merge an existing ontology file into the current database. This could be used to populate a new ontology from an existing one stored as a file. The ontology is automatically merged and stored in the triplestore database server after calling this function. Parameters ---------- filepath : str Path to the file containing the ontology. See RDFLib documentation, specifically, the function Graph.parse for supported formats. """ tmpGraph = Graph() tmpGraph.parse(filepath) self._graph += tmpGraph @property def config(self): return self.__config @property def identifier(self): return self.__identifier @property def store_type(self): return self.__store_type @interact_with_db def bind(self, prefix, namespace, override=True): self._graph.bind(prefix.lower(), namespace, override) @interact_with_db def query(self, *args, **kwargs) -> Generator: return self._graph.query(*args, **kwargs) @interact_with_db def update(self, *args, **kwargs) -> Generator: return self._graph.update(*args, **kwargs) @interact_with_db def add(self, triple: tuple): """Adds a triple (s, p, o) into the database. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- triple : tuple (s, p, o) triple """ self._graph.add(triple) @interact_with_db def addN(self, triples: list): """Adds n-triples into the database. This method is faster than adding individual triples one-by-one using the "add" method. This function also automatically adds the current graph as the context (unlike the original method from rdflib). This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- triples : list list of (s, p, o) triples to be added into the database """ # automatically add self.graph as context if not specified directly quads = [t + (self._graph, ) for t in triples if len(t) == 3] self._graph.addN(quads) @interact_with_db def remove(self, triple: tuple): """Remove the specified triple or triples from the database. Not all fields of the triple needs to be specified. Omitted parts shall be replaced with "None" value. In such case, all triples matching the provided variables are removed. E.g., (someEntity, None, None) will remove containing "someEntity" as subject. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- triple : tuple (s, p, o) triple """ self._graph.remove(triple) @interact_with_db def triples(self, triple: tuple): """Returns a generator for triples matching the provided pattern. The pattern/template triple can contain concrete values or None where the item shall be matched to anything. E.g., (None, RDF.type, None) will return all triples containing RDF.type as the predicate. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- triple : tuple (s, p, o) triple Returns ------- generator generator of matching triples """ return self._graph.triples(triple) @interact_with_db def subjects(self, predicate: Identifier = None, object: Identifier = None): """Returns a (list of) subject(s) matching the values provided as predicate and object. Similarly to triples, wildcard items can be replaced with None. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- predicate : Identifier, optional p, by default None object : Identifier, optional o, by default None Returns ------- generator Subjects matching the query """ return self._graph.subjects(predicate, object) @interact_with_db def subject_objects(self, predicate: Identifier = None): """Returns subjects and objects matching the value provided as predicate. See "subjects" and "triples" methods for more info. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- predicate : Identifier, optional p, by default None Returns ------- generator The subjects and objects matching where predicate is set to the provided value """ return self._graph.subject_objects(predicate) @interact_with_db def subject_predicates(self, object: Identifier = None): """Returns subjects and predicates matching the value provided as object. See "subjects" and "triples" methods for more info. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- object : Identifier, optional o, by default None Returns ------- generator The subjects and predicates matching the query """ return self._graph.subject_predicates(object) @interact_with_db def objects(self, subject: Identifier = None, predicate: Identifier = None): """Returns (a list of) object(s) matching the query. See "subjects" and "triples" methods for more info. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- subject : Identifier, optional s, by default None predicate : Identifier, optional p, by default None Returns ------- generator The objects matching the query """ return self._graph.objects(subject, predicate) @interact_with_db def predicates(self, subject: Identifier = None, object: Identifier = None): """Returns (a list of) predicate(s) matching the query. See "subjects" and "triples" methods for more info. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- subject : Identifier, optional s, by default None object : Identifier, optional o, by default None Returns ------- generator The predicates matching the query. """ return self._graph.predicates(subject, object) @interact_with_db def predicate_objects(self, subject: Identifier = None): """Returns predicates and objects where the subject matches the value specified in the function parameter. See "subjects" and "triples" methods for more info. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- subject : Identifier, optional s, by default None Returns ------- generator The predicates and objects matching the query """ return self._graph.predicate_objects(subject) @interact_with_db def transitive_subjects(self, predicate: Identifier, object: Identifier): """This function transitively generates subjects for the object, using only the value specified as predicate as the property. I.e., it "walks backwards" using only the predicate. E.g., transitive_subjects(parentOf, entity) will generate all ancestor of the object "entity". See "subjects" and "triples" methods for more info. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- predicate : Identifier p object : Identifier o Returns ------- generator Generator of subjects matching the query. """ return self._graph.transitive_subjects(predicate, object) @interact_with_db def transitive_objects(self, subject: Identifier, property: Identifier): """This function generates objects for the subject using only the property. It is the revers of "transitive_subjects". I.e., it "walks forwards" in the ontology, using only the property/predicate. E.g., transitive_objects(entity, hasComponent) will generate all objects that are part of the entity (i.e., all the components of the entity). See "subjects" and "triples" methods for more info. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- subject : Identifier s property : Identifier p Returns ------- generator Objects matchting the query """ return self._graph.transitive_objects(subject, property) @interact_with_db def set(self, triple: set): """Convenience function for "set" operations in the database. Values set by this function are first removed and than assigned, ensuring there is only one record for the specified subject + property. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- triple : set (s, p, o) triple """ self._graph.set(triple) @interact_with_db def value(self, subject: Identifier = None, predicate: Identifier = RDF.value, object: Identifier = None, default=None, any=True): """Complementery function for the "set" method. It expects that there is only one value matching the subject + predicate combination. Error is risen otherwise! This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. Parameters ---------- subject : Identifier, optional s, by default None predicate : Identifier, optional p, by default RDF.value object : Identifier, optional o, by default None default : any, optional Default value to be returned if it is not specified in the database, by default None any : bool, optional No idea, see rdflib documentation, by default True Returns ------- any The expected value """ return self._graph.value(subject, predicate, object, default, any) @interact_with_db def compute_qname(self, uri): return self._graph.compute_qname(uri) @interact_with_db def __getitem__(self, item): """Convenience function. Allows queries/triples to be specified via the "object[index]" notation. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. """ return self._graph.__getitem__(item) @interact_with_db def __len__(self): """Allows the use of the len(container) function to return the number of entries in the database This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. """ return len(self._graph) @interact_with_db def __contains__(self, item): """Allows the use of "item in container" notation to be used to test if database contains entries matching the query. The item shall be an (s, p, o) triple, obeying the standard contrains. This function is only a "safe" re-implementation of the original rdflib graph function. See rdflib.Graph documentation for more information. """ return item in self._graph @property def graph(self): # REMOVE: temporal debugging property, shall not be present at release return self._graph
class Indexer(): """ Index the KG in either a sparql engine or in memory. This is required for rule learning """ def __init__(self, store='remote', endpoint=None, identifier=None, graph=None, batch_size=100, remove_invalid_ids=True): self.remove_invalid_ids = remove_invalid_ids self.batch_size = batch_size self.store = 'SPARQLUpdateStore' if store == 'remote' or store == 'SPARQLUpdateStore' else 'default' self.endpoint = endpoint self.identifier = identifier self.graph = graph def index_triples(self, triples_source: TriplesSource, prefix='', safe_urls=False, drop_old=False): if drop_old: logger.info("Drop %s " % self.identifier) self.drop() if self.store != 'SPARQLUpdateStore' and not self.graph: self.graph = Graph(store=self.store, identifier=self.identifier) # print(self.graph.store) # if self.store == 'SPARQLUpdateStore': # self.graph.open(self.endpoint) # self._index(triples_source, prefix, safe_urls) self._index_np(triples_source) # , prefix, safe_urls) return self.graph def _index_np(self, triples_source, prefix='', safe_urls=False): logger.info("Start indexing " + triples_source.get_name()) data = triples_source.as_numpy_array() data_size = triples_source.size() number_splits = math.ceil(data_size / self.batch_size) logger.info("data size %i" % data_size) logger.info("chunks %i" % number_splits) # ch=0 chunks = np.array_split(data, number_splits) for chunk in tqdm(chunks): if self.store == 'SPARQLUpdateStore': self.insert_sparql(chunk) else: self.insert_memory(chunk) logger.info("Done indexing " + triples_source.get_name()) def drop(self): if self.store == 'SPARQLUpdateStore': if self.graph_exists(): return self._drop_sparql() else: self.graph = Graph(store=self.store, identifier=self.identifier) return True return True def insert_memory(self, triples): chunk_context = [(URIRef(s), URIRef(p), URIRef(o), self.graph) for s, p, o in triples] self.graph.addN(chunk_context) return True def insert_sparql(self, triples): triples_filtered = filter( lambda a: data_formating.valid_id_triple(a), triples) if self.remove_invalid_ids else triples query = 'INSERT DATA into <%s> {%s}' % (self.identifier, '\n'.join( map(data_formating.sparql_repr, triples_filtered))) # print(query) sparql = SPARQLWrapper(self.endpoint) sparql.setMethod(POST) sparql.setReturnFormat(JSON) sparql.setQuery(query) results = sparql.query().convert() return results def graph_exists(self): if self.store == 'SPARQLUpdateStore': query = 'ASK WHERE { GRAPH <%s> { ?s ?p ?o } }' % self.identifier sparql = SPARQLWrapper(self.endpoint) sparql.setReturnFormat(JSON) sparql.setQuery(query) results = sparql.query().convert() return results['boolean'] else: return False def _drop_sparql(self): query = 'DROP SILENT GRAPH <%s>' % self.identifier sparql = SPARQLWrapper(self.endpoint) sparql.setMethod(POST) sparql.setReturnFormat(JSON) sparql.setQuery(query) results = sparql.query().convert() # print(results) result = results['results']['bindings'][0]['callret-0']['value'] if 'triples were removed' in result: return True elif 'nothing to do' in result: return False raise Exception( 'Problem Dropping the graph using: %s Message from sparql : \"%s\"' % (query, result))
def get_state(self, parameters=None): """I override `~rdfrest.cores.ICore.get_state`:meth: I support parameter "prop" to enrich the Base description with additional information. I consider an empty dict as equivalent to no dict. """ state = super(Base, self).get_state(parameters) if not parameters: return state enriched_state = Graph() enriched_state += state whole = ConjunctiveGraph(self.service.store) initNs = { '': KTBS, 'rdfs': RDFS, 'skos': SKOS } initBindings = { 'base': self.uri } for prop in parameters['prop']: if prop == 'comment': enriched_state.addN( (s, RDFS.comment, o, enriched_state) for s, o in whole.query(''' SELECT ?s ?o { GRAPH ?base { ?base :contains ?s } GRAPH ?t { ?s rdfs:comment ?o } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'hasModel': enriched_state.addN( (s, KTBS.hasModel, o, enriched_state) for s, o in whole.query(''' SELECT ?t ?m { GRAPH ?base { ?base :contains ?t } GRAPH ?t { ?t :hasModel ?m } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'hasSource': enriched_state.addN( (s, KTBS.hasSource, o, enriched_state) for s, o in whole.query(''' SELECT ?t1 ?t2 { GRAPH ?base { ?base :contains ?t1 } GRAPH ?t1 { ?t1 :hasSource ?t2 } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'label': enriched_state.addN( (s, p, o, enriched_state) for s, p, o in whole.query(''' SELECT ?s ?p ?o { GRAPH ?base { ?base :contains ?s } GRAPH ?s { ?s ?p ?o } VALUES ?p { rdfs:label skos:prefLabel } } ''', initNs=initNs, initBindings=initBindings) ) elif prop == 'obselCount': enriched_state.addN( (s, KTBS.hasObselCount, o, enriched_state) for s, o in whole.query(''' SELECT ?t (COUNT(?obs) as ?c) { GRAPH ?base { ?base :contains ?t. ?t a ?tt } OPTIONAL { ?obs :hasTrace ?t } VALUES ?tt { :StoredTrace :ComputedTrace } } GROUP BY ?t ''', initNs=initNs, initBindings=initBindings) ) else: pass # ignoring unrecognized properties # should we signal them instead (diagnosis?) return enriched_state
def get_state(self, parameters=None): """I override `~rdfrest.cores.ICore.get_state`:meth: I support parameter "prop" to enrich the Base description with additional information. I consider an empty dict as equivalent to no dict. """ state = super(Base, self).get_state(parameters) if not parameters: return state enriched_state = Graph() enriched_state += state whole = ConjunctiveGraph(self.service.store) initNs = {"": KTBS, "rdfs": RDFS, "skos": SKOS} initBindings = {"base": self.uri} for prop in parameters["prop"]: if prop == "comment": enriched_state.addN( (s, RDFS.comment, o, enriched_state) for s, o in whole.query( """ SELECT ?s ?o { GRAPH ?base { ?base :contains ?s } GRAPH ?t { ?s rdfs:comment ?o } } """, initNs=initNs, initBindings=initBindings, ) ) elif prop == "hasModel": enriched_state.addN( (s, KTBS.hasModel, o, enriched_state) for s, o in whole.query( """ SELECT ?t ?m { GRAPH ?base { ?base :contains ?t } GRAPH ?t { ?t :hasModel ?m } } """, initNs=initNs, initBindings=initBindings, ) ) elif prop == "hasSource": enriched_state.addN( (s, KTBS.hasSource, o, enriched_state) for s, o in whole.query( """ SELECT ?t1 ?t2 { GRAPH ?base { ?base :contains ?t1 } GRAPH ?t1 { ?t1 :hasSource ?t2 } } """, initNs=initNs, initBindings=initBindings, ) ) elif prop == "label": enriched_state.addN( (s, p, o, enriched_state) for s, p, o in whole.query( """ SELECT ?s ?p ?o { GRAPH ?base { ?base :contains ?s } GRAPH ?s { ?s ?p ?o } VALUES ?p { rdfs:label skos:prefLabel } } """, initNs=initNs, initBindings=initBindings, ) ) elif prop == "obselCount": enriched_state.addN( (s, KTBS.hasObselCount, o, enriched_state) for s, o in whole.query( """ SELECT ?t (COUNT(?obs) as ?c) { GRAPH ?base { ?base :contains ?t. ?t a ?tt } OPTIONAL { ?obs :hasTrace ?t } VALUES ?tt { :StoredTrace :ComputedTrace } } GROUP BY ?t """, initNs=initNs, initBindings=initBindings, ) ) else: pass # ignoring unrecognized properties # should we signal them instead (diagnosis?) return enriched_state
class GraphTestCase(unittest.TestCase): storetest = True identifier = URIRef("rdflib_test") michel = URIRef(u"michel") tarek = URIRef(u"tarek") bob = URIRef(u"bob") likes = URIRef(u"likes") hates = URIRef(u"hates") pizza = URIRef(u"pizza") cheese = URIRef(u"cheese") namespace_dc = "http://purl.org/dc/elements/1.1/" namespace_dct = "http://purl.org/dc/terms/" namespace_saws = "http://purl.org/saws/ontology#" def setUp(self, uri="sqlite://", storename=None): store = plugin.get(storename, Store)(identifier=self.identifier) self.graph = Graph(store, identifier=self.identifier) self.graph.open(uri, create=True) def tearDown(self, uri="sqlite://"): self.graph.destroy(uri) self.graph.close() def addStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese self.graph.add((tarek, likes, pizza)) self.graph.add((tarek, likes, cheese)) self.graph.add((michel, likes, pizza)) self.graph.add((michel, likes, cheese)) self.graph.add((bob, likes, cheese)) self.graph.add((bob, hates, pizza)) self.graph.add((bob, hates, michel)) # gasp! self.graph.commit() def removeStuff(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese self.graph.remove((tarek, likes, pizza)) self.graph.remove((tarek, likes, cheese)) self.graph.remove((michel, likes, pizza)) self.graph.remove((michel, likes, cheese)) self.graph.remove((bob, likes, cheese)) self.graph.remove((bob, hates, pizza)) self.graph.remove((bob, hates, michel)) # gasp! def testAdd(self): self.addStuff() def testRemove(self): self.addStuff() self.removeStuff() def testTriples(self): tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes hates = self.hates pizza = self.pizza cheese = self.cheese asserte = self.assertEqual triples = self.graph.triples Any = None self.addStuff() # unbound subjects asserte(len(list(triples((Any, likes, pizza)))), 2) asserte(len(list(triples((Any, hates, pizza)))), 1) asserte(len(list(triples((Any, likes, cheese)))), 3) asserte(len(list(triples((Any, hates, cheese)))), 0) # unbound objects asserte(len(list(triples((michel, likes, Any)))), 2) asserte(len(list(triples((tarek, likes, Any)))), 2) asserte(len(list(triples((bob, hates, Any)))), 2) asserte(len(list(triples((bob, likes, Any)))), 1) # unbound predicates asserte(len(list(triples((michel, Any, cheese)))), 1) asserte(len(list(triples((tarek, Any, cheese)))), 1) asserte(len(list(triples((bob, Any, pizza)))), 1) asserte(len(list(triples((bob, Any, michel)))), 1) # unbound subject, objects asserte(len(list(triples((Any, hates, Any)))), 2) asserte(len(list(triples((Any, likes, Any)))), 5) # unbound predicates, objects asserte(len(list(triples((michel, Any, Any)))), 2) asserte(len(list(triples((bob, Any, Any)))), 3) asserte(len(list(triples((tarek, Any, Any)))), 2) # unbound subjects, predicates asserte(len(list(triples((Any, Any, pizza)))), 3) asserte(len(list(triples((Any, Any, cheese)))), 3) asserte(len(list(triples((Any, Any, michel)))), 1) # all unbound asserte(len(list(triples((Any, Any, Any)))), 7) self.removeStuff() asserte(len(list(triples((Any, Any, Any)))), 0) def testConnected(self): graph = self.graph self.addStuff() self.assertEqual(True, graph.connected()) jeroen = URIRef("jeroen") unconnected = URIRef("unconnected") graph.add((jeroen, self.likes, unconnected)) self.assertEqual(False, graph.connected()) def testSub(self): g1 = Graph() g2 = Graph() tarek = self.tarek # michel = self.michel bob = self.bob likes = self.likes # hates = self.hates pizza = self.pizza cheese = self.cheese g1.add((tarek, likes, pizza)) g1.add((bob, likes, cheese)) g2.add((bob, likes, cheese)) g3 = g1 - g2 self.assertEqual(len(g3), 1) self.assertEqual((tarek, likes, pizza) in g3, True) self.assertEqual((tarek, likes, cheese) in g3, False) self.assertEqual((bob, likes, cheese) in g3, False) g1 -= g2 self.assertEqual(len(g1), 1) self.assertEqual((tarek, likes, pizza) in g1, True) self.assertEqual((tarek, likes, cheese) in g1, False) self.assertEqual((bob, likes, cheese) in g1, False) def testGraphAdd(self): g1 = Graph() g2 = Graph() tarek = self.tarek # michel = self.michel bob = self.bob likes = self.likes # hates = self.hates pizza = self.pizza cheese = self.cheese g1.add((tarek, likes, pizza)) g2.add((bob, likes, cheese)) g3 = g1 + g2 self.assertEqual(len(g3), 2) self.assertEqual((tarek, likes, pizza) in g3, True) self.assertEqual((tarek, likes, cheese) in g3, False) self.assertEqual((bob, likes, cheese) in g3, True) g1 += g2 self.assertEqual(len(g1), 2) self.assertEqual((tarek, likes, pizza) in g1, True) self.assertEqual((tarek, likes, cheese) in g1, False) self.assertEqual((bob, likes, cheese) in g1, True) def testGraphIntersection(self): g1 = Graph() g2 = Graph() tarek = self.tarek michel = self.michel bob = self.bob likes = self.likes # hates = self.hates pizza = self.pizza cheese = self.cheese g1.add((tarek, likes, pizza)) g1.add((michel, likes, cheese)) g2.add((bob, likes, cheese)) g2.add((michel, likes, cheese)) g3 = g1 * g2 self.assertEqual(len(g3), 1) self.assertEqual((tarek, likes, pizza) in g3, False) self.assertEqual((tarek, likes, cheese) in g3, False) self.assertEqual((bob, likes, cheese) in g3, False) self.assertEqual((michel, likes, cheese) in g3, True) g1 *= g2 self.assertEqual(len(g1), 1) self.assertEqual((tarek, likes, pizza) in g1, False) self.assertEqual((tarek, likes, cheese) in g1, False) self.assertEqual((bob, likes, cheese) in g1, False) self.assertEqual((michel, likes, cheese) in g1, True) def testStoreLiterals(self): bob = self.bob says = URIRef(u"says") hello = Literal(u"hello", lang="en") konichiwa = Literal(u"こんにちは", lang="ja") something = Literal(u"something") self.graph.add((bob, says, hello)) self.graph.add((bob, says, konichiwa)) self.graph.add((bob, says, something)) self.graph.commit() objs = list(self.graph.objects(subject=bob, predicate=says)) for o in objs: if o.value == u"hello": self.assertEqual(o.language, "en") elif o.value == u"こんにちは": self.assertEqual(o.language, "ja") elif o.value == u"something": self.assertEqual(o.language, None) else: self.fail() self.assertEqual(len(list(objs)), 3) def testStoreLiteralsXml(self): bob = self.bob says = URIRef(u"http://www.rdflib.net/terms/says") objects = [ Literal(u"I'm the one", lang="en"), Literal(u"こんにちは", lang="ja"), Literal(u"les garçons à Noël reçoivent des œufs", lang="fr") ] testdoc = (PY3 and bytes(xmltestdocXml, "UTF-8")) or xmltestdocXml self.graph.parse(StringInputSource(testdoc), formal="xml") objs = list(self.graph) self.assertEqual(len(objs), 3) for o in objs: self.assertEqual(o[0], bob) self.assertEqual(o[1], says) self.assertIn(o[2], objects) def testStoreLiteralsXmlQuote(self): bob = self.bob says = URIRef(u"http://www.rdflib.net/terms/says") imtheone = Literal(u"I'm the one", lang="en") testdoc = (PY3 and bytes(xmltestdocXmlQuote, "UTF-8")) or xmltestdocXmlQuote self.graph.parse(StringInputSource(testdoc), formal="xml") objs = list(self.graph) self.assertEqual(len(objs), 1) o = objs[0] self.assertEqual(o, (bob, says, imtheone)) def testBindNamespace(self): """ Check that bound namespaced with prefix (including empty ones) are correctly kept """ self.graph.bind("", self.namespace_dc) self.graph.bind("dct", self.namespace_dct) self.assertEqual(self.graph.qname(self.namespace_dct + u"title"), "dct:title", "Prefixed namespace should be stored and retrieved") self.assertEqual( self.graph.qname(self.namespace_dc + u"title"), "title", "Empty prefixes for namespace should be stored and retrieved") self.assertEqual( self.graph.qname(self.namespace_saws + u"title"), "ns1:title", "Unknown prefixes for namespace should be transformed to nsX") def testTriplesChoices(self): likes = self.likes pizza = self.pizza cheese = self.cheese tarek = self.tarek michel = self.michel bob = self.bob self.addStuff() trips = self.graph.triples_choices((None, likes, [pizza, cheese])) self.assertEqual( set(trips), set([(tarek, likes, pizza), (tarek, likes, pizza), (tarek, likes, cheese), (michel, likes, pizza), (michel, likes, cheese), (bob, likes, cheese)])) def test_type_add(self): trip = (URIRef('http://example.org#type-add'), RDF.type, URIRef('http://example.org/cra')) self.graph.add(trip) self.graph.add(trip) # No exception raised def test_type_addn(self): quad = (URIRef('http://example.org#type-addn'), RDF.type, URIRef('http://example.org/cra'), self.graph) self.graph.addN([quad, quad]) # No exception raised def test_add(self): trip = (URIRef('http://example.org#add'), URIRef('http://example.org/blah'), URIRef('http://example.org/cra')) self.graph.add(trip) self.graph.add(trip) # No exception raised def test_addn(self): quad = (URIRef('http://example.org#addn'), URIRef('http://example.org/blah'), URIRef('http://example.org/cra'), self.graph) self.graph.addN([quad, quad]) # No exception raised def test_namespace_change_prefix_binding(self): nm = self.graph.namespace_manager nm.bind('change_binding', URIRef('http://example.org/change-binding-1#'), replace=True) nm.bind('change_binding', URIRef('http://example.org/change-binding-2#'), replace=True) self.assertIn( ('change_binding', URIRef('http://example.org/change-binding-2#')), list(nm.namespaces())) def test_namespace_rebind_prefix(self): nm = self.graph.namespace_manager nm.bind('rebind', URIRef('http://example.org/rebind#')) nm.bind('rebind', URIRef('http://example.org/rebind#')) def test_add_duplicate_length(self): ''' Test that adding duplicate triples doesn't increase the length of the graph ''' trip = (URIRef('http://example.org#add'), URIRef('http://example.org/blah'), URIRef('http://example.org/cra')) self.graph.add(trip) self.graph.add(trip) self.assertEqual(len(self.graph), 1)