def test_one_spurious_semicolons_bnode():
sparql = """
PREFIX : <http://example.org/>
CONSTRUCT {
[ :b :c ; :d :e ; ]
} WHERE {}
"""
expected = Graph()
expected.addN(t + (expected,) for t in [
(BNode("a"), NS.b, NS.c),
(BNode("a"), NS.d, NS.e),
])
got = Graph().query(sparql).graph
assert isomorphic(got, expected), got.serialize(format="turtle")
def test_one_spurious_semicolons_bnode():
sparql = """
PREFIX : <http://example.org/>
CONSTRUCT {
[ :b :c ; :d :e ; ]
} WHERE {}
"""
expected = Graph()
expected.addN(t + (expected, ) for t in [
(BNode("a"), NS.b, NS.c),
(BNode("a"), NS.d, NS.e),
])
got = Graph().query(sparql).graph
assert isomorphic(got, expected), got.serialize(format="turtle")
def test_two_spurious_semicolons_no_period():
sparql = """
PREFIX : <http://example.org/>
CONSTRUCT {
:a :b :c ; :d :e ; ;
} WHERE {}
"""
expected = Graph()
expected.addN(t + (expected,) for t in [
(NS.a, NS.b, NS.c),
(NS.a, NS.d, NS.e),
])
got = Graph().query(sparql).graph
assert isomorphic(got, expected), got.serialize(format="turtle")
def _serialize_with_rdflib(rdflib_format, graph, bindings, base_uri):
"Common implementation of all rdflib-based serialize functions."
assert isinstance(rdflib_format, str)
assert isinstance(graph, Graph)
# copy in another graph to prevent polluting the original graph namespaces
# TODO LATER find an efficient way to serialize a graph with custom NSs
ser = Graph()
ser.addN( (s, p, o, ser) for s, p, o in graph )
for prefix, nsuri in bindings.items():
ser.bind(prefix, nsuri)
if base_uri is not None and not isinstance(base_uri, URIRef):
base_uri = coerce_to_uri(base_uri)
# We use yield to prevent the serialization to happen if a 304 is returned
yield ser.serialize(None, format=rdflib_format, base=base_uri)
def _serialize_with_rdflib(rdflib_format, graph, bindings, base_uri):
"Common implementation of all rdflib-based serialize functions."
assert isinstance(rdflib_format, str)
assert isinstance(graph, Graph)
# copy in another graph to prevent polluting the original graph namespaces
# TODO LATER find an efficient way to serialize a graph with custom NSs
ser = Graph()
ser.addN((s, p, o, ser) for s, p, o in graph)
for prefix, nsuri in bindings.items():
ser.bind(prefix, nsuri)
if base_uri is not None and not isinstance(base_uri, URIRef):
base_uri = coerce_to_uri(base_uri)
# We use yield to prevent the serialization to happen if a 304 is returned
yield ser.serialize(None, format=rdflib_format, base=base_uri)
def test_two_spurious_semicolons_no_period():
sparql = """
PREFIX : <http://example.org/>
CONSTRUCT {
:a :b :c ; :d :e ; ;
} WHERE {}
"""
expected = Graph()
expected.addN(t + (expected, ) for t in [
(NS.a, NS.b, NS.c),
(NS.a, NS.d, NS.e),
])
got = Graph().query(sparql).graph
assert isomorphic(got, expected), got.serialize(format="turtle")
def add_list_to_graph(graph: Graph, spo_list: RDFList) -> None:
"""
将一个SPO三元组的列表一次性地加入 graph 中
:param graph: 需要加入RDF的graph
:param spo_list: SPO三元组的列表
:return: None
Examples
--------
>>> g = Graph()
>>> spo_list = [
... (URIRef('http://www.ifa.com#Firm/tencent'), URIRef('http://www.ifa.com#hasName'), Literal('腾讯', datatype=XSD.string)),
... (URIRef('http://www.ifa.com#Firm/tencent'), URIRef('http://www.ifa.com#hasApp'), URIRef('http://www.ifa.com#App/wechat'))
... ]
>>> add_list_to_graph(g, spo_list)
"""
quads = [(s, p, o, graph) for s, p, o in spo_list]
graph.addN(quads)
def test_pathological():
"""
This test did not raise an exception,
but generated a graph completely different from the expected one.
"""
sparql = """
PREFIX : <http://example.org/>
CONSTRUCT {
:a :b :c ; ;
:d :e ; ;
:f :g ;
} WHERE {}
"""
expected = Graph()
expected.addN(t + (expected,) for t in [
(NS.a, NS.b, NS.c),
(NS.a, NS.d, NS.e),
(NS.a, NS.f, NS.g),
])
got = Graph().query(sparql).graph
assert isomorphic(got, expected), got.serialize(format="turtle")
def test_pathological():
"""
This test did not raise an exception,
but generated a graph completely different from the expected one.
"""
sparql = """
PREFIX : <http://example.org/>
CONSTRUCT {
:a :b :c ; ;
:d :e ; ;
:f :g ;
} WHERE {}
"""
expected = Graph()
expected.addN(t + (expected, ) for t in [
(NS.a, NS.b, NS.c),
(NS.a, NS.d, NS.e),
(NS.a, NS.f, NS.g),
])
got = Graph().query(sparql).graph
assert isomorphic(got, expected), got.serialize(format="turtle")
def add_dict_to_graph(graph: Graph, s: Union[URIRef, BNode],
po_dict: Dict[URIRef, Identifier]) -> None:
"""
将一个P-O的字典一次性地加入 graph 中
:param graph: 需要加入RDF的graph
:param s: subject的URIRef
:param po_dict: predicate-object 组成的字典类型
:return: None
Examples
--------
>>> from rdflib import XSD, Graph, Literal, URIRef
>>>
>>> g = Graph()
>>> po_dict = {
... URIRef('http://www.ifa.com#hasName'): Literal('腾讯', datatype=XSD.string),
... URIRef('http://www.ifa.com#hasApp'): URIRef('http://www.ifa.com#App/wechat')
... }
>>> s = URIRef('http://www.ifa.com#Firm/tencent')
>>> add_dict_to_graph(g, s, po_dict)
"""
quads = [(s, p, o, graph) for p, o in po_dict.items()]
graph.addN(quads)
class ToRDFConverter:
def __init__(self, atdm, metadata):
super().__init__()
self.graph = Graph()
self.metadata = metadata
self.atdm = atdm
self.mode = CONST_STANDARD_MODE
RDFGraphUtils.add_default_bindings(self.graph)
RDFGraphUtils.add_bindings_from_metadata(self.graph, metadata)
def convert(self, mode=CONST_STANDARD_MODE, format=None):
self.mode = mode
main_node = BNode()
if mode == CONST_STANDARD_MODE:
self.graph.add((main_node, RDF.type, CSVW.TableGroup))
self._add_file_metadata(self.metadata, main_node)
# TODO: 4.5 non-core annotations
for table_metadata, table_data in zip(self.metadata['tables'],
self.atdm['tables']):
self._parse_table(main_node, table_metadata, table_data)
return self.graph if format is None else self.graph.serialize(
format=format).decode()
def _parse_table(self, main_node, table_metadata, table_data):
table_node = URIRef(
table_metadata['@id']) if '@id' in table_metadata else BNode()
property_url = table_metadata['url'] + '#'
if 'propertyUrl' in table_metadata['tableSchema']:
property_url = table_metadata['tableSchema']['propertyUrl']
if self.mode == CONST_STANDARD_MODE:
self.graph.add((main_node, CSVW.table, table_node))
self.graph.add((table_node, RDF.type, CSVW.Table))
self.graph.add(
(table_node, CSVW.url, URIRef(table_metadata['url'])))
self._add_file_metadata(table_metadata, table_node)
for index, atdm_row in enumerate(table_data['rows']):
if self.mode == CONST_STANDARD_MODE:
row_node = BNode()
self._parse_generic_row(atdm_row, table_node, table_metadata,
property_url, row_node, table_data)
else:
dummy = BNode()
row_node = BNode()
self._parse_row(atdm_row, row_node, dummy, table_metadata,
property_url, table_data)
self.graph.remove((dummy, CSVW.describes, row_node))
self.parse_virtual_columns(row_node, atdm_row, table_metadata)
def parse_virtual_columns(self, row_node, atdm_row, table_metadata):
for virtual_column in table_metadata['tableSchema']['columns']:
if 'virtual' not in virtual_column or virtual_column[
'virtual'] is False:
continue
subject = URIRef(
UriTemplateUtils.insert_value(virtual_column['aboutUrl'],
atdm_row, '',
table_metadata['url']))
predicate = Namespaces.get_term(virtual_column['propertyUrl'])
obj = UriTemplateUtils.insert_value(virtual_column['valueUrl'],
atdm_row, '',
table_metadata['url'])
obj = CommonProperties.expand_property_if_possible(obj)
self.graph.add((subject, predicate, URIRef(obj)))
if self.mode == CONST_STANDARD_MODE:
self.graph.add((row_node, CSVW.describes, subject))
def _add_file_metadata(self, metadata, node):
language = JSONLDUtils.language(self.metadata['@context'])
for key, value in metadata.items():
if CommonProperties.is_common_property(key) or key == 'notes':
triples = CommonProperties.property_to_triples(
(key, metadata[key]), node, language)
self.graph.addN(triple + (self.graph, ) for triple in triples)
def _parse_generic_row(self, atdm_row, table_node, metadata, property_url,
row_node, atdm_table):
self.graph.add((table_node, CSVW.row, row_node))
self.graph.add((row_node, RDF.type, CSVW.Row))
self.graph.add((row_node, CSVW.rownum,
Literal(atdm_row['number'], datatype=XSD.integer)))
self.graph.add((row_node, CSVW.url, URIRef(atdm_row['@id'])))
if 'rowTitles' in metadata['tableSchema']:
col_names = metadata['tableSchema']['rowTitles']
for col_name in col_names:
col_value = atdm_row['cells'][col_name][0]
self.graph.add((row_node, CSVW.title, Literal(col_value)))
values_node = BNode()
self._parse_row(atdm_row, values_node, row_node, metadata,
property_url, atdm_table)
def _parse_row(self, atdm_row, values_node, row_node, metadata,
property_url, atdm_Table):
if not all(
map(lambda column: 'aboutUrl' in column,
metadata['tableSchema']['columns'])):
self.graph.add((row_node, CSVW.describes, values_node))
self._parse_row_data(atdm_row, values_node, metadata, property_url,
row_node, atdm_Table)
def _parse_row_data(self, atdm_row, subject, table_metadata, property_url,
row_node, atdm_table):
top_level_property_url = property_url
atdm_columns = atdm_table['columns']
for index, entry in enumerate(atdm_row['cells'].items()):
col_name, values = entry
for col_metadata in atdm_columns:
if col_metadata['name'] == col_name:
break
if col_metadata.get('suppressOutput', False):
continue
property_url = col_metadata.get('propertyUrl',
top_level_property_url)
if 'aboutUrl' in col_metadata:
subject = UriTemplateUtils.insert_value_rdf(
col_metadata['aboutUrl'], atdm_row, col_name,
table_metadata['url'])
if self.mode == CONST_STANDARD_MODE:
self.graph.add((row_node, CSVW.describes, subject))
property_namespace = PropertyUrlUtils.create_namespace(
property_url, table_metadata['url'])
predicate = self._predicate_node(property_namespace, property_url,
col_name)
self._parse_cell_values(values, col_metadata, subject, predicate)
def _parse_cell_values(self, values, col_metadata, subject, predicate):
""" Parses single cell value, values if 'separator' is present"""
if 'ordered' in col_metadata and col_metadata[
'ordered'] is True and len(values) > 1:
next_item = BNode()
rdf_list = next_item
values_count = len(values)
for i in range(values_count):
item = next_item
self.graph.add((item, RDF.first, Literal(values[i])))
if i != values_count - 1:
next_item = BNode()
self.graph.add((item, RDF.rest, next_item))
Collection(self.graph, rdf_list)
self.graph.add((subject, predicate, rdf_list))
else:
for value in values:
object_node = self._object_node(value, col_metadata)
self.graph.add((subject, predicate, object_node))
@staticmethod
def _object_node(value, col_metadata):
if 'valueUrl' in col_metadata:
return ValueUrlUtils.create_uri_ref(value,
col_metadata['valueUrl'])
else:
lang = col_metadata.get('lang')
if not datatypeutils.is_compatible_with_datatype(
value, col_metadata.get('datatype')):
datatype = None
else:
datatype = OntologyUtils.type(col_metadata)
lang = None if datatype is not None else lang
return Literal(value, lang=lang, datatype=datatype)
@staticmethod
def _normalize_to_uri(string):
string = quote(string, safe='')
string = string.replace('-', '%2D')
return string
@staticmethod
def _predicate_node(namespace, property_url, col_name):
col_name = ToRDFConverter._normalize_to_uri(col_name)
if '{' in property_url:
return namespace.term(col_name)
else:
if CommonProperties.is_common_property(property_url):
return URIRef(
CommonProperties.expand_property_if_possible(property_url))
else:
if property_url.endswith('#'):
return URIRef(property_url + col_name)
else:
return URIRef(namespace)
def get_state(self, parameters=None):
"""I override `~rdfrest.cores.ICore.get_state`:meth:
I support parameter "prop" to enrich the Base description with additional information.
I consider an empty dict as equivalent to no dict.
"""
state = super(Base, self).get_state(parameters)
if not parameters:
return state
enriched_state = Graph()
enriched_state += state
whole = ConjunctiveGraph(self.service.store)
initNs = { '': KTBS, 'rdfs': RDFS, 'skos': SKOS }
initBindings = { 'base': self.uri }
for prop in parameters['prop']:
if prop == 'comment':
enriched_state.addN(
(s, RDFS.comment, o, enriched_state)
for s, o, _ in whole.query('''
SELECT ?s ?o
$base # selected solely to please Virtuoso
{
GRAPH $base { $base :contains ?s }
GRAPH ?s { ?s rdfs:comment ?o }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'hasMethod':
enriched_state.addN(
(s, KTBS.hasMethod, o, enriched_state)
for s, o, _ in whole.query('''
SELECT ?t ?m
$base # selected solely to please Virtuoso
{
GRAPH $base { $base :contains ?t }
GRAPH ?t { ?t :hasMethod ?m }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'hasModel':
enriched_state.addN(
(s, KTBS.hasModel, o, enriched_state)
for s, o, _ in whole.query('''
SELECT ?t ?m
$base # selected solely to please Virtuoso
{
GRAPH $base { $base :contains ?t }
GRAPH ?t { ?t :hasModel ?m }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'hasSource':
enriched_state.addN(
(s, KTBS.hasSource, o, enriched_state)
for s, o, _ in whole.query('''
SELECT ?t1 ?t2
$base # selected solely to please Virtuoso
{
GRAPH $base { $base :contains ?t1 }
GRAPH ?t1 { ?t1 :hasSource ?t2 }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'label':
enriched_state.addN(
(s, p, o, enriched_state)
for s, p, o, _ in whole.query('''
SELECT ?s ?p ?o
$base # selected solely to please Virtuoso
{
VALUES ?p { rdfs:label skos:prefLabel }
GRAPH $base { $base :contains ?s }
GRAPH ?s { ?s ?p ?o }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'obselCount':
enriched_state.addN(
(s, KTBS.hasObselCount, o, enriched_state)
for s, o, _ in whole.query('''
SELECT ?t (COUNT(?obs) as ?c)
(SAMPLE($base) as ?sample_base) # selected solely to please Virtuoso
{
VALUES ?tt { :StoredTrace :ComputedTrace }
GRAPH $base { $base :contains ?t. ?t a ?tt }
OPTIONAL { ?obs :hasTrace ?t }
} GROUP BY ?t
''', initNs=initNs, initBindings=initBindings)
)
else:
pass # ignoring unrecognized properties
# should we signal them instead (diagnosis?)
return enriched_state
class OntologyDatabase:
"""A front-end of an ontology database. This class provides "safe" access to most of the standard
operations provided by the rdflib.Graph class. The "safeness" of the methods lies in catching
exceptions and reconnecting shall the connection to the database "die" for whatever reason.
Additionally, this class implements the SQLAlchemy store for the triples
"""
def __init__(self, config=None, create=None):
"""Create ontology database API with SQLAlchemy store.
Parameters
----------
config : [str, knowl.DBConfig], optional
The path to a configuration file or the configuration object. By default None,
which results in a configuration with default parameters (see knowl.DBConfig).
create : bool, optional
Whether or not the tables for the ontology (triplestore) should be initalized.
Set to True if you are creating a new database, by default None.
As per SQLAlchemy documentation, the creation operation is idempotent. Thus,
could be left at True, unless you specifically do not want to create a new database
if one does not exist.
"""
# initialize database config
self.__config = DBConfig.factory(config)
self.__username = None
self.__password = None
self.__create = create
self.__store_type = self.config["store"]
# configure database identifier (ontology IRI/base URL)
self.__identifier = self.config.baseURL
if self.store_type == "alchemy":
self.__store = SQLAlchemy(identifier=self.identifier)
self._graph = Graph(self.__store, identifier=self.identifier)
elif self.store_type == "fuseki":
self.__query_endpoint = f'http://{self.config["host"]}:{self.config["port"]}/{self.config["database"]}'
self.__update_endpoint = f'http://{self.config["host"]}:{self.config["port"]}/{self.config["database"]}/update'
self.__store = SPARQLUpdateStore(
queryEndpoint=self.__query_endpoint + '/sparql',
update_endpoint=self.__update_endpoint,
context_aware=True,
postAsEncoded=False,
node_to_sparql=my_bnode_ext)
self.__query_endpoint += '/query'
self.__store.method = 'POST'
else:
raise Exception(f"Unknown store type {self.store_type}!")
def setup(self, create=False, username: str = None, password: str = None):
"""Sets-up a new database connection. Call this to initialize access to the database.
Parameters
----------
create : bool, optional
Whether the tables should be created (idempotent). Only set to True if creating a new database, by default False.
Setting the object property self.create to anything but None will override this value!
username : str, optional
Database access credentials. Only set this if you didn't set it before (e.g. in the config file), by default None
password : str, optional
Database access credentials. Only set this if you didn't set it before (e.g. in the config file), by default None
"""
if self.store_type == "alchemy":
if self.__create is not None:
create = self.__create
self._graph.open(self.config.getDB_URI(
self.__username if username is None else username,
self.__password if password is None else password),
create=create)
elif self.store_type == "fuseki":
print(
f"Query endpoint: {self.__query_endpoint}\nUpdate endpoint: {self.__update_endpoint}\nIndentifier: {self.identifier}"
)
self.__store.open((self.__query_endpoint, self.__update_endpoint))
self._graph = Graph(self.__store, identifier=self.identifier)
for ns, uri in self.config.namespaces.items():
self._graph.bind(ns.lower(), uri)
def closelink(self):
"""Closes the database connection.
"""
try:
self._graph.close()
except Exception as e:
print("Exception in Closing", e)
def destroy(self, confirmation: str = None):
"""Destroys the store for the Ontology
This will erase/destroy the database (triplestore) used to store the data.
Be very careful when calling this function.
Parameters
----------
confirmation : str, optional
[description], by default None
"""
if confirmation == "I know what I am doing":
self._graph.destroy(self.identifier)
else:
raise ValueError(
"Destroying the DB attempted but failed - wrong confirmation string!"
)
def setCredentials(self, username: str = None, password: str = None):
"""Set access credentials for the database server containing the triplestore.
Parameters
----------
username : str, optional
The username, by default None
password : str, optional
The password. Warning, this will be visible in the DB URL! By default None
"""
self.__username = username
self.__password = password
@interact_with_db
def mergeFileIntoDB(self, filepath: str):
"""Merge an existing ontology file into the current database. This could be used to populate
a new ontology from an existing one stored as a file. The ontology is automatically merged
and stored in the triplestore database server after calling this function.
Parameters
----------
filepath : str
Path to the file containing the ontology. See RDFLib documentation,
specifically, the function Graph.parse for supported formats.
"""
tmpGraph = Graph()
tmpGraph.parse(filepath)
self._graph += tmpGraph
@property
def config(self):
return self.__config
@property
def identifier(self):
return self.__identifier
@property
def store_type(self):
return self.__store_type
@interact_with_db
def bind(self, prefix, namespace, override=True):
self._graph.bind(prefix.lower(), namespace, override)
@interact_with_db
def query(self, *args, **kwargs) -> Generator:
return self._graph.query(*args, **kwargs)
@interact_with_db
def update(self, *args, **kwargs) -> Generator:
return self._graph.update(*args, **kwargs)
@interact_with_db
def add(self, triple: tuple):
"""Adds a triple (s, p, o) into the database.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
triple : tuple
(s, p, o) triple
"""
self._graph.add(triple)
@interact_with_db
def addN(self, triples: list):
"""Adds n-triples into the database. This method is faster than adding
individual triples one-by-one using the "add" method. This function
also automatically adds the current graph as the context (unlike
the original method from rdflib).
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
triples : list
list of (s, p, o) triples to be added into the database
"""
# automatically add self.graph as context if not specified directly
quads = [t + (self._graph, ) for t in triples if len(t) == 3]
self._graph.addN(quads)
@interact_with_db
def remove(self, triple: tuple):
"""Remove the specified triple or triples from the database.
Not all fields of the triple needs to be specified. Omitted parts
shall be replaced with "None" value.
In such case, all triples matching the provided variables
are removed.
E.g., (someEntity, None, None) will remove containing "someEntity"
as subject.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
triple : tuple
(s, p, o) triple
"""
self._graph.remove(triple)
@interact_with_db
def triples(self, triple: tuple):
"""Returns a generator for triples matching the provided pattern.
The pattern/template triple can contain concrete values or None
where the item shall be matched to anything.
E.g., (None, RDF.type, None) will return all triples containing
RDF.type as the predicate.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
triple : tuple
(s, p, o) triple
Returns
-------
generator
generator of matching triples
"""
return self._graph.triples(triple)
@interact_with_db
def subjects(self,
predicate: Identifier = None,
object: Identifier = None):
"""Returns a (list of) subject(s) matching the values provided as predicate
and object. Similarly to triples, wildcard items can be replaced with None.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
predicate : Identifier, optional
p, by default None
object : Identifier, optional
o, by default None
Returns
-------
generator
Subjects matching the query
"""
return self._graph.subjects(predicate, object)
@interact_with_db
def subject_objects(self, predicate: Identifier = None):
"""Returns subjects and objects matching the value provided as predicate.
See "subjects" and "triples" methods for more info.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
predicate : Identifier, optional
p, by default None
Returns
-------
generator
The subjects and objects matching where predicate is set to the provided value
"""
return self._graph.subject_objects(predicate)
@interact_with_db
def subject_predicates(self, object: Identifier = None):
"""Returns subjects and predicates matching the value provided as object.
See "subjects" and "triples" methods for more info.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
object : Identifier, optional
o, by default None
Returns
-------
generator
The subjects and predicates matching the query
"""
return self._graph.subject_predicates(object)
@interact_with_db
def objects(self,
subject: Identifier = None,
predicate: Identifier = None):
"""Returns (a list of) object(s) matching the query.
See "subjects" and "triples" methods for more info.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
subject : Identifier, optional
s, by default None
predicate : Identifier, optional
p, by default None
Returns
-------
generator
The objects matching the query
"""
return self._graph.objects(subject, predicate)
@interact_with_db
def predicates(self,
subject: Identifier = None,
object: Identifier = None):
"""Returns (a list of) predicate(s) matching the query.
See "subjects" and "triples" methods for more info.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
subject : Identifier, optional
s, by default None
object : Identifier, optional
o, by default None
Returns
-------
generator
The predicates matching the query.
"""
return self._graph.predicates(subject, object)
@interact_with_db
def predicate_objects(self, subject: Identifier = None):
"""Returns predicates and objects where the subject matches
the value specified in the function parameter.
See "subjects" and "triples" methods for more info.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
subject : Identifier, optional
s, by default None
Returns
-------
generator
The predicates and objects matching the query
"""
return self._graph.predicate_objects(subject)
@interact_with_db
def transitive_subjects(self, predicate: Identifier, object: Identifier):
"""This function transitively generates subjects for the object,
using only the value specified as predicate as the property.
I.e., it "walks backwards" using only the predicate.
E.g., transitive_subjects(parentOf, entity) will generate
all ancestor of the object "entity".
See "subjects" and "triples" methods for more info.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
predicate : Identifier
p
object : Identifier
o
Returns
-------
generator
Generator of subjects matching the query.
"""
return self._graph.transitive_subjects(predicate, object)
@interact_with_db
def transitive_objects(self, subject: Identifier, property: Identifier):
"""This function generates objects for the subject using only the property.
It is the revers of "transitive_subjects". I.e., it "walks forwards"
in the ontology, using only the property/predicate.
E.g., transitive_objects(entity, hasComponent) will generate all objects
that are part of the entity (i.e., all the components of the entity).
See "subjects" and "triples" methods for more info.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
subject : Identifier
s
property : Identifier
p
Returns
-------
generator
Objects matchting the query
"""
return self._graph.transitive_objects(subject, property)
@interact_with_db
def set(self, triple: set):
"""Convenience function for "set" operations in the database.
Values set by this function are first removed and than assigned,
ensuring there is only one record for the specified subject + property.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
triple : set
(s, p, o) triple
"""
self._graph.set(triple)
@interact_with_db
def value(self,
subject: Identifier = None,
predicate: Identifier = RDF.value,
object: Identifier = None,
default=None,
any=True):
"""Complementery function for the "set" method. It expects that there is only one value
matching the subject + predicate combination. Error is risen otherwise!
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
Parameters
----------
subject : Identifier, optional
s, by default None
predicate : Identifier, optional
p, by default RDF.value
object : Identifier, optional
o, by default None
default : any, optional
Default value to be returned if it is not specified in the database, by default None
any : bool, optional
No idea, see rdflib documentation, by default True
Returns
-------
any
The expected value
"""
return self._graph.value(subject, predicate, object, default, any)
@interact_with_db
def compute_qname(self, uri):
return self._graph.compute_qname(uri)
@interact_with_db
def __getitem__(self, item):
"""Convenience function. Allows queries/triples to be specified via the "object[index]"
notation.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
"""
return self._graph.__getitem__(item)
@interact_with_db
def __len__(self):
"""Allows the use of the len(container) function to return the number of entries in the database
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
"""
return len(self._graph)
@interact_with_db
def __contains__(self, item):
"""Allows the use of "item in container" notation to be used to test if database contains entries
matching the query. The item shall be an (s, p, o) triple, obeying the standard contrains.
This function is only a "safe" re-implementation of the original rdflib graph function.
See rdflib.Graph documentation for more information.
"""
return item in self._graph
@property
def graph(self):
# REMOVE: temporal debugging property, shall not be present at release
return self._graph
class Indexer():
"""
Index the KG in either a sparql engine or in memory. This is required for rule learning
"""
def __init__(self,
store='remote',
endpoint=None,
identifier=None,
graph=None,
batch_size=100,
remove_invalid_ids=True):
self.remove_invalid_ids = remove_invalid_ids
self.batch_size = batch_size
self.store = 'SPARQLUpdateStore' if store == 'remote' or store == 'SPARQLUpdateStore' else 'default'
self.endpoint = endpoint
self.identifier = identifier
self.graph = graph
def index_triples(self,
triples_source: TriplesSource,
prefix='',
safe_urls=False,
drop_old=False):
if drop_old:
logger.info("Drop %s " % self.identifier)
self.drop()
if self.store != 'SPARQLUpdateStore' and not self.graph:
self.graph = Graph(store=self.store, identifier=self.identifier)
# print(self.graph.store)
# if self.store == 'SPARQLUpdateStore':
# self.graph.open(self.endpoint)
# self._index(triples_source, prefix, safe_urls)
self._index_np(triples_source) # , prefix, safe_urls)
return self.graph
def _index_np(self, triples_source, prefix='', safe_urls=False):
logger.info("Start indexing " + triples_source.get_name())
data = triples_source.as_numpy_array()
data_size = triples_source.size()
number_splits = math.ceil(data_size / self.batch_size)
logger.info("data size %i" % data_size)
logger.info("chunks %i" % number_splits)
# ch=0
chunks = np.array_split(data, number_splits)
for chunk in tqdm(chunks):
if self.store == 'SPARQLUpdateStore':
self.insert_sparql(chunk)
else:
self.insert_memory(chunk)
logger.info("Done indexing " + triples_source.get_name())
def drop(self):
if self.store == 'SPARQLUpdateStore':
if self.graph_exists():
return self._drop_sparql()
else:
self.graph = Graph(store=self.store, identifier=self.identifier)
return True
return True
def insert_memory(self, triples):
chunk_context = [(URIRef(s), URIRef(p), URIRef(o), self.graph)
for s, p, o in triples]
self.graph.addN(chunk_context)
return True
def insert_sparql(self, triples):
triples_filtered = filter(
lambda a: data_formating.valid_id_triple(a),
triples) if self.remove_invalid_ids else triples
query = 'INSERT DATA into <%s> {%s}' % (self.identifier, '\n'.join(
map(data_formating.sparql_repr, triples_filtered)))
# print(query)
sparql = SPARQLWrapper(self.endpoint)
sparql.setMethod(POST)
sparql.setReturnFormat(JSON)
sparql.setQuery(query)
results = sparql.query().convert()
return results
def graph_exists(self):
if self.store == 'SPARQLUpdateStore':
query = 'ASK WHERE { GRAPH <%s> { ?s ?p ?o } }' % self.identifier
sparql = SPARQLWrapper(self.endpoint)
sparql.setReturnFormat(JSON)
sparql.setQuery(query)
results = sparql.query().convert()
return results['boolean']
else:
return False
def _drop_sparql(self):
query = 'DROP SILENT GRAPH <%s>' % self.identifier
sparql = SPARQLWrapper(self.endpoint)
sparql.setMethod(POST)
sparql.setReturnFormat(JSON)
sparql.setQuery(query)
results = sparql.query().convert()
# print(results)
result = results['results']['bindings'][0]['callret-0']['value']
if 'triples were removed' in result:
return True
elif 'nothing to do' in result:
return False
raise Exception(
'Problem Dropping the graph using: %s Message from sparql : \"%s\"'
% (query, result))
def get_state(self, parameters=None):
"""I override `~rdfrest.cores.ICore.get_state`:meth:
I support parameter "prop" to enrich the Base description with additional information.
I consider an empty dict as equivalent to no dict.
"""
state = super(Base, self).get_state(parameters)
if not parameters:
return state
enriched_state = Graph()
enriched_state += state
whole = ConjunctiveGraph(self.service.store)
initNs = { '': KTBS, 'rdfs': RDFS, 'skos': SKOS }
initBindings = { 'base': self.uri }
for prop in parameters['prop']:
if prop == 'comment':
enriched_state.addN(
(s, RDFS.comment, o, enriched_state)
for s, o in whole.query('''
SELECT ?s ?o {
GRAPH ?base { ?base :contains ?s }
GRAPH ?t { ?s rdfs:comment ?o }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'hasModel':
enriched_state.addN(
(s, KTBS.hasModel, o, enriched_state)
for s, o in whole.query('''
SELECT ?t ?m {
GRAPH ?base { ?base :contains ?t }
GRAPH ?t { ?t :hasModel ?m }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'hasSource':
enriched_state.addN(
(s, KTBS.hasSource, o, enriched_state)
for s, o in whole.query('''
SELECT ?t1 ?t2 {
GRAPH ?base { ?base :contains ?t1 }
GRAPH ?t1 { ?t1 :hasSource ?t2 }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'label':
enriched_state.addN(
(s, p, o, enriched_state)
for s, p, o in whole.query('''
SELECT ?s ?p ?o {
GRAPH ?base { ?base :contains ?s }
GRAPH ?s { ?s ?p ?o }
VALUES ?p { rdfs:label skos:prefLabel }
}
''', initNs=initNs, initBindings=initBindings)
)
elif prop == 'obselCount':
enriched_state.addN(
(s, KTBS.hasObselCount, o, enriched_state)
for s, o in whole.query('''
SELECT ?t (COUNT(?obs) as ?c) {
GRAPH ?base { ?base :contains ?t. ?t a ?tt }
OPTIONAL { ?obs :hasTrace ?t }
VALUES ?tt { :StoredTrace :ComputedTrace }
} GROUP BY ?t
''', initNs=initNs, initBindings=initBindings)
)
else:
pass # ignoring unrecognized properties
# should we signal them instead (diagnosis?)
return enriched_state
def get_state(self, parameters=None):
"""I override `~rdfrest.cores.ICore.get_state`:meth:
I support parameter "prop" to enrich the Base description with additional information.
I consider an empty dict as equivalent to no dict.
"""
state = super(Base, self).get_state(parameters)
if not parameters:
return state
enriched_state = Graph()
enriched_state += state
whole = ConjunctiveGraph(self.service.store)
initNs = {"": KTBS, "rdfs": RDFS, "skos": SKOS}
initBindings = {"base": self.uri}
for prop in parameters["prop"]:
if prop == "comment":
enriched_state.addN(
(s, RDFS.comment, o, enriched_state)
for s, o in whole.query(
"""
SELECT ?s ?o {
GRAPH ?base { ?base :contains ?s }
GRAPH ?t { ?s rdfs:comment ?o }
}
""",
initNs=initNs,
initBindings=initBindings,
)
)
elif prop == "hasModel":
enriched_state.addN(
(s, KTBS.hasModel, o, enriched_state)
for s, o in whole.query(
"""
SELECT ?t ?m {
GRAPH ?base { ?base :contains ?t }
GRAPH ?t { ?t :hasModel ?m }
}
""",
initNs=initNs,
initBindings=initBindings,
)
)
elif prop == "hasSource":
enriched_state.addN(
(s, KTBS.hasSource, o, enriched_state)
for s, o in whole.query(
"""
SELECT ?t1 ?t2 {
GRAPH ?base { ?base :contains ?t1 }
GRAPH ?t1 { ?t1 :hasSource ?t2 }
}
""",
initNs=initNs,
initBindings=initBindings,
)
)
elif prop == "label":
enriched_state.addN(
(s, p, o, enriched_state)
for s, p, o in whole.query(
"""
SELECT ?s ?p ?o {
GRAPH ?base { ?base :contains ?s }
GRAPH ?s { ?s ?p ?o }
VALUES ?p { rdfs:label skos:prefLabel }
}
""",
initNs=initNs,
initBindings=initBindings,
)
)
elif prop == "obselCount":
enriched_state.addN(
(s, KTBS.hasObselCount, o, enriched_state)
for s, o in whole.query(
"""
SELECT ?t (COUNT(?obs) as ?c) {
GRAPH ?base { ?base :contains ?t. ?t a ?tt }
OPTIONAL { ?obs :hasTrace ?t }
VALUES ?tt { :StoredTrace :ComputedTrace }
} GROUP BY ?t
""",
initNs=initNs,
initBindings=initBindings,
)
)
else:
pass # ignoring unrecognized properties
# should we signal them instead (diagnosis?)
return enriched_state
class GraphTestCase(unittest.TestCase):
storetest = True
identifier = URIRef("rdflib_test")
michel = URIRef(u"michel")
tarek = URIRef(u"tarek")
bob = URIRef(u"bob")
likes = URIRef(u"likes")
hates = URIRef(u"hates")
pizza = URIRef(u"pizza")
cheese = URIRef(u"cheese")
namespace_dc = "http://purl.org/dc/elements/1.1/"
namespace_dct = "http://purl.org/dc/terms/"
namespace_saws = "http://purl.org/saws/ontology#"
def setUp(self, uri="sqlite://", storename=None):
store = plugin.get(storename, Store)(identifier=self.identifier)
self.graph = Graph(store, identifier=self.identifier)
self.graph.open(uri, create=True)
def tearDown(self, uri="sqlite://"):
self.graph.destroy(uri)
self.graph.close()
def addStuff(self):
tarek = self.tarek
michel = self.michel
bob = self.bob
likes = self.likes
hates = self.hates
pizza = self.pizza
cheese = self.cheese
self.graph.add((tarek, likes, pizza))
self.graph.add((tarek, likes, cheese))
self.graph.add((michel, likes, pizza))
self.graph.add((michel, likes, cheese))
self.graph.add((bob, likes, cheese))
self.graph.add((bob, hates, pizza))
self.graph.add((bob, hates, michel)) # gasp!
self.graph.commit()
def removeStuff(self):
tarek = self.tarek
michel = self.michel
bob = self.bob
likes = self.likes
hates = self.hates
pizza = self.pizza
cheese = self.cheese
self.graph.remove((tarek, likes, pizza))
self.graph.remove((tarek, likes, cheese))
self.graph.remove((michel, likes, pizza))
self.graph.remove((michel, likes, cheese))
self.graph.remove((bob, likes, cheese))
self.graph.remove((bob, hates, pizza))
self.graph.remove((bob, hates, michel)) # gasp!
def testAdd(self):
self.addStuff()
def testRemove(self):
self.addStuff()
self.removeStuff()
def testTriples(self):
tarek = self.tarek
michel = self.michel
bob = self.bob
likes = self.likes
hates = self.hates
pizza = self.pizza
cheese = self.cheese
asserte = self.assertEqual
triples = self.graph.triples
Any = None
self.addStuff()
# unbound subjects
asserte(len(list(triples((Any, likes, pizza)))), 2)
asserte(len(list(triples((Any, hates, pizza)))), 1)
asserte(len(list(triples((Any, likes, cheese)))), 3)
asserte(len(list(triples((Any, hates, cheese)))), 0)
# unbound objects
asserte(len(list(triples((michel, likes, Any)))), 2)
asserte(len(list(triples((tarek, likes, Any)))), 2)
asserte(len(list(triples((bob, hates, Any)))), 2)
asserte(len(list(triples((bob, likes, Any)))), 1)
# unbound predicates
asserte(len(list(triples((michel, Any, cheese)))), 1)
asserte(len(list(triples((tarek, Any, cheese)))), 1)
asserte(len(list(triples((bob, Any, pizza)))), 1)
asserte(len(list(triples((bob, Any, michel)))), 1)
# unbound subject, objects
asserte(len(list(triples((Any, hates, Any)))), 2)
asserte(len(list(triples((Any, likes, Any)))), 5)
# unbound predicates, objects
asserte(len(list(triples((michel, Any, Any)))), 2)
asserte(len(list(triples((bob, Any, Any)))), 3)
asserte(len(list(triples((tarek, Any, Any)))), 2)
# unbound subjects, predicates
asserte(len(list(triples((Any, Any, pizza)))), 3)
asserte(len(list(triples((Any, Any, cheese)))), 3)
asserte(len(list(triples((Any, Any, michel)))), 1)
# all unbound
asserte(len(list(triples((Any, Any, Any)))), 7)
self.removeStuff()
asserte(len(list(triples((Any, Any, Any)))), 0)
def testConnected(self):
graph = self.graph
self.addStuff()
self.assertEqual(True, graph.connected())
jeroen = URIRef("jeroen")
unconnected = URIRef("unconnected")
graph.add((jeroen, self.likes, unconnected))
self.assertEqual(False, graph.connected())
def testSub(self):
g1 = Graph()
g2 = Graph()
tarek = self.tarek
# michel = self.michel
bob = self.bob
likes = self.likes
# hates = self.hates
pizza = self.pizza
cheese = self.cheese
g1.add((tarek, likes, pizza))
g1.add((bob, likes, cheese))
g2.add((bob, likes, cheese))
g3 = g1 - g2
self.assertEqual(len(g3), 1)
self.assertEqual((tarek, likes, pizza) in g3, True)
self.assertEqual((tarek, likes, cheese) in g3, False)
self.assertEqual((bob, likes, cheese) in g3, False)
g1 -= g2
self.assertEqual(len(g1), 1)
self.assertEqual((tarek, likes, pizza) in g1, True)
self.assertEqual((tarek, likes, cheese) in g1, False)
self.assertEqual((bob, likes, cheese) in g1, False)
def testGraphAdd(self):
g1 = Graph()
g2 = Graph()
tarek = self.tarek
# michel = self.michel
bob = self.bob
likes = self.likes
# hates = self.hates
pizza = self.pizza
cheese = self.cheese
g1.add((tarek, likes, pizza))
g2.add((bob, likes, cheese))
g3 = g1 + g2
self.assertEqual(len(g3), 2)
self.assertEqual((tarek, likes, pizza) in g3, True)
self.assertEqual((tarek, likes, cheese) in g3, False)
self.assertEqual((bob, likes, cheese) in g3, True)
g1 += g2
self.assertEqual(len(g1), 2)
self.assertEqual((tarek, likes, pizza) in g1, True)
self.assertEqual((tarek, likes, cheese) in g1, False)
self.assertEqual((bob, likes, cheese) in g1, True)
def testGraphIntersection(self):
g1 = Graph()
g2 = Graph()
tarek = self.tarek
michel = self.michel
bob = self.bob
likes = self.likes
# hates = self.hates
pizza = self.pizza
cheese = self.cheese
g1.add((tarek, likes, pizza))
g1.add((michel, likes, cheese))
g2.add((bob, likes, cheese))
g2.add((michel, likes, cheese))
g3 = g1 * g2
self.assertEqual(len(g3), 1)
self.assertEqual((tarek, likes, pizza) in g3, False)
self.assertEqual((tarek, likes, cheese) in g3, False)
self.assertEqual((bob, likes, cheese) in g3, False)
self.assertEqual((michel, likes, cheese) in g3, True)
g1 *= g2
self.assertEqual(len(g1), 1)
self.assertEqual((tarek, likes, pizza) in g1, False)
self.assertEqual((tarek, likes, cheese) in g1, False)
self.assertEqual((bob, likes, cheese) in g1, False)
self.assertEqual((michel, likes, cheese) in g1, True)
def testStoreLiterals(self):
bob = self.bob
says = URIRef(u"says")
hello = Literal(u"hello", lang="en")
konichiwa = Literal(u"こんにちは", lang="ja")
something = Literal(u"something")
self.graph.add((bob, says, hello))
self.graph.add((bob, says, konichiwa))
self.graph.add((bob, says, something))
self.graph.commit()
objs = list(self.graph.objects(subject=bob, predicate=says))
for o in objs:
if o.value == u"hello":
self.assertEqual(o.language, "en")
elif o.value == u"こんにちは":
self.assertEqual(o.language, "ja")
elif o.value == u"something":
self.assertEqual(o.language, None)
else:
self.fail()
self.assertEqual(len(list(objs)), 3)
def testStoreLiteralsXml(self):
bob = self.bob
says = URIRef(u"http://www.rdflib.net/terms/says")
objects = [
Literal(u"I'm the one", lang="en"),
Literal(u"こんにちは", lang="ja"),
Literal(u"les garçons à Noël reçoivent des œufs", lang="fr")
]
testdoc = (PY3 and bytes(xmltestdocXml, "UTF-8")) or xmltestdocXml
self.graph.parse(StringInputSource(testdoc), formal="xml")
objs = list(self.graph)
self.assertEqual(len(objs), 3)
for o in objs:
self.assertEqual(o[0], bob)
self.assertEqual(o[1], says)
self.assertIn(o[2], objects)
def testStoreLiteralsXmlQuote(self):
bob = self.bob
says = URIRef(u"http://www.rdflib.net/terms/says")
imtheone = Literal(u"I'm the one", lang="en")
testdoc = (PY3 and bytes(xmltestdocXmlQuote,
"UTF-8")) or xmltestdocXmlQuote
self.graph.parse(StringInputSource(testdoc), formal="xml")
objs = list(self.graph)
self.assertEqual(len(objs), 1)
o = objs[0]
self.assertEqual(o, (bob, says, imtheone))
def testBindNamespace(self):
""" Check that bound namespaced with prefix (including empty ones) are correctly kept """
self.graph.bind("", self.namespace_dc)
self.graph.bind("dct", self.namespace_dct)
self.assertEqual(self.graph.qname(self.namespace_dct + u"title"),
"dct:title",
"Prefixed namespace should be stored and retrieved")
self.assertEqual(
self.graph.qname(self.namespace_dc + u"title"), "title",
"Empty prefixes for namespace should be stored and retrieved")
self.assertEqual(
self.graph.qname(self.namespace_saws + u"title"), "ns1:title",
"Unknown prefixes for namespace should be transformed to nsX")
def testTriplesChoices(self):
likes = self.likes
pizza = self.pizza
cheese = self.cheese
tarek = self.tarek
michel = self.michel
bob = self.bob
self.addStuff()
trips = self.graph.triples_choices((None, likes, [pizza, cheese]))
self.assertEqual(
set(trips),
set([(tarek, likes, pizza), (tarek, likes, pizza),
(tarek, likes, cheese), (michel, likes, pizza),
(michel, likes, cheese), (bob, likes, cheese)]))
def test_type_add(self):
trip = (URIRef('http://example.org#type-add'), RDF.type,
URIRef('http://example.org/cra'))
self.graph.add(trip)
self.graph.add(trip)
# No exception raised
def test_type_addn(self):
quad = (URIRef('http://example.org#type-addn'), RDF.type,
URIRef('http://example.org/cra'), self.graph)
self.graph.addN([quad, quad])
# No exception raised
def test_add(self):
trip = (URIRef('http://example.org#add'),
URIRef('http://example.org/blah'),
URIRef('http://example.org/cra'))
self.graph.add(trip)
self.graph.add(trip)
# No exception raised
def test_addn(self):
quad = (URIRef('http://example.org#addn'),
URIRef('http://example.org/blah'),
URIRef('http://example.org/cra'), self.graph)
self.graph.addN([quad, quad])
# No exception raised
def test_namespace_change_prefix_binding(self):
nm = self.graph.namespace_manager
nm.bind('change_binding',
URIRef('http://example.org/change-binding-1#'),
replace=True)
nm.bind('change_binding',
URIRef('http://example.org/change-binding-2#'),
replace=True)
self.assertIn(
('change_binding', URIRef('http://example.org/change-binding-2#')),
list(nm.namespaces()))
def test_namespace_rebind_prefix(self):
nm = self.graph.namespace_manager
nm.bind('rebind', URIRef('http://example.org/rebind#'))
nm.bind('rebind', URIRef('http://example.org/rebind#'))
def test_add_duplicate_length(self):
'''
Test that adding duplicate triples doesn't increase the length of the graph
'''
trip = (URIRef('http://example.org#add'),
URIRef('http://example.org/blah'),
URIRef('http://example.org/cra'))
self.graph.add(trip)
self.graph.add(trip)
self.assertEqual(len(self.graph), 1)
