def add_resource(self, uri, types):
# type: (basestring, iter) -> TED
ted = self.ted
matching_resources = filter(lambda r: r.node.toPython() == uri,
ted.ecosystem.non_td_root_resources)
if not matching_resources:
agora = self.agora
if not all([t in agora.fountain.types for t in types]):
raise TypeError('Unknown type')
g = Graph()
prefixes = agora.fountain.prefixes
uri_ref = URIRef(uri)
if not types:
dgw = DataGateway(agora.agora, ted, cache=None)
rg, headers = dgw.loader(uri)
type_uris = set(
[extend_uri(t, prefixes) for t in agora.fountain.types])
resource_types = set(rg.objects(uri_ref, RDF.type))
types = tuple(set.intersection(type_uris, resource_types))
for t in types:
g.add((uri_ref, RDF.type, URIRef(extend_uri(t, prefixes))))
ted = self.learn_descriptions(g)
return ted
else:
raise AttributeError(uri)
def add_description(self, id, types):
try:
self._get_request('descriptions/{}'.format(id),
accept='text/turtle')
except IOError:
pass
else:
raise AttributeError(id)
prefixes = self.agora.fountain.prefixes
types = map(lambda t: URIRef(extend_uri(t, prefixes)), types)
td = TD.from_types(id, types)
try:
response = self._post_request(
'descriptions',
td.to_graph(th_nodes={}).serialize(format='turtle'),
content_type='text/turtle',
accept='text/turtle')
g = Graph()
g.parse(StringIO(response), format='turtle')
ted = TED.from_graph(g, loader=self.__loader)
tds = ted.ecosystem.tds
if len(tds):
return list(tds).pop()
except IOError:
pass
raise AttributeError(id)
def add_resource(self, uri, types):
response = self._get_request('resources', accept='text/turtle')
g = Graph()
g.parse(StringIO(response), format='turtle')
if URIRef(uri) in set(g.subjects()):
raise AttributeError(uri)
prefixes = self.agora.fountain.prefixes
types = map(lambda t: URIRef(extend_uri(t, prefixes)), types)
r = Resource(URIRef(uri), types)
try:
response = self._post_request(
'descriptions',
r.to_graph().serialize(format='turtle'),
content_type='text/turtle',
accept='text/turtle')
g = Graph()
g.parse(StringIO(response), format='turtle')
ted = TED.from_graph(g, loader=self.__loader)
all_resources = ted.ecosystem.non_td_root_resources
if all_resources:
return list(all_resources).pop()
except IOError as e:
raise AttributeError(e.message['text'])
raise AttributeError(uri)
def add_enrichment(self, id, type, td_id, replace=False):
# type: (basestring, basestring, basestring, bool) -> Enrichment
ted = self.ted
matching_enrichments = filter(lambda e: str(e.id) == str(id),
ted.ecosystem.enrichments)
if not matching_enrichments:
agora = self.agora
if type not in agora.fountain.types:
raise TypeError('Unknown type')
prefixes = agora.fountain.prefixes
type_uri = extend_uri(type, prefixes)
td = self.get_description(td_id)
try:
e = Enrichment(id, type_uri, td, replace=replace)
g = e.to_graph(td_nodes={})
self.learn_descriptions(g)
except AttributeError:
raise AttributeError(id)
return e
else:
raise AttributeError(id)
def load(self, resource, graph, uri, b64, r_args):
prefixes = self.fountain.prefixes
bnode_map = {}
for s, p, o in resource.graph:
if o in self.nodes:
o = URIRef(self.url_for(tid=self.nodes[o], b64=b64, **r_args))
elif isinstance(o, BNode):
if o not in bnode_map:
bnode_map[o] = BNode()
o = bnode_map[o]
elif isinstance(o, Literal):
if str(o) in r_args:
o = Literal(r_args[str(o)], datatype=o.datatype)
if s == resource.node:
s = uri
if isinstance(s, BNode):
if s not in self.nodes:
if s not in bnode_map:
bnode_map[s] = BNode()
for t in resource.graph.objects(s, RDF.type):
for supt in self.fountain.get_type(t.n3(
self.ns))['super']:
graph.add((bnode_map[s], RDF.type,
extend_uri(supt, prefixes)))
s = bnode_map[s]
graph.add((s, p, o))
else:
graph.add((s, p, o))
for t in resource.types:
if isinstance(t, URIRef):
t_n3 = t.n3(self.ns)
else:
t_n3 = t
type_dict = self.fountain.get_type(t_n3)
self.resource_props.update(type_dict['properties'])
for st in type_dict['super']:
graph.add((uri, RDF.type, extend_uri(st, prefixes)))
def get_resource(ctx, uri, host, port, turtle, raw):
gw = ctx.obj['gw']
ted = gw.ted
dgw = DataGateway(gw.agora, ted, cache=None, port=port, server_name=host)
g, headers = dgw.loader(uri)
uri_ref = URIRef(uri)
prefixes = gw.agora.fountain.prefixes
type_uris = set([extend_uri(t, prefixes) for t in gw.agora.fountain.types])
resource_types = set(g.objects(uri_ref, RDF.type))
known_types = set.intersection(type_uris, resource_types)
ag = Graph()
for prefix, uri in prefixes.items():
ag.bind(prefix, uri)
if raw:
ag.__iadd__(g)
else:
known_types_n3 = [t.n3(ag.namespace_manager) for t in known_types]
known_props = reduce(
lambda x, y: x.union(
set(gw.agora.fountain.get_type(y)['properties'])),
known_types_n3, set())
known_props_uri = set([extend_uri(p, prefixes) for p in known_props])
known_refs = reduce(
lambda x, y: x.union(set(gw.agora.fountain.get_type(y)['refs'])),
known_types_n3, set())
known_refs_uri = set([extend_uri(p, prefixes) for p in known_refs])
for (s, p, o) in g:
if s == uri_ref and ((p == RDF.type and o in known_types)
or p in known_props_uri):
ag.add((s, p, o))
if o == uri_ref and p in known_refs_uri:
ag.add((s, p, o))
show_thing(ag, format='text/turtle' if turtle else 'application/ld+json')
def materialize_th_types(R, ids):
for th_id in ids:
g = R.pull(th_id)
prev_len = len(g)
prefixes = R.agora.fountain.prefixes
for s, p, o in g.triples((None, RDF.type, None)):
try:
type_n3 = g.qname(o)
type_super = R.agora.fountain.get_type(type_n3)['super']
for ts in type_super:
ts_triple = (s, p, URIRef(extend_uri(ts, prefixes)))
if ts_triple not in g:
g.add(ts_triple)
except Exception:
pass
if len(g) > prev_len:
R.push(g)
def add_enrichment(self, id, type, tdid, replace=False):
try:
self._get_request('enrichments/{}'.format(id),
accept='text/turtle')
except IOError:
pass
else:
# Already exists
raise AttributeError(id)
try:
response = self._get_request('descriptions/{}'.format(tdid),
accept='text/turtle')
except IOError:
# TD does not exist
raise AttributeError(tdid)
g = Graph()
g.parse(StringIO(response), format='turtle')
try:
td_uri = list(g.subjects(CORE.identifier, Literal(tdid))).pop()
td = TD.from_graph(g, td_uri, node_map={})
except IndexError:
# Something wrong happens with the TD RDF
raise AttributeError(tdid)
prefixes = self.agora.fountain.prefixes
type = URIRef(extend_uri(type, prefixes))
e = Enrichment(id, type, td, replace=replace)
try:
response = self._post_request(
'descriptions',
e.to_graph().serialize(format='turtle'),
content_type='text/turtle',
accept='text/turtle')
g = Graph()
g.parse(StringIO(response), format='turtle')
ted = TED.from_graph(g, loader=self.__loader)
all_enrichments = ted.ecosystem.enrichments
if all_enrichments:
return list(all_enrichments).pop()
except IOError as e:
raise AttributeError(e.message['text'])
raise AttributeError(id)
def is_described_reachable(fountain, R, td_network, seed, type):
res = R.query("""
PREFIX core: <http://iot.linkeddata.es/def/core#>
SELECT DISTINCT ?g ?id WHERE {
GRAPH ?g {
[] a core:ThingDescription ;
core:identifier ?id ;
core:describes <%s>
}
}
""" % seed,
cache=True,
expire=300,
infer=True)
rd = generate_dict(R.n3, R.ns(fountain=fountain), res)
try:
td_id = rd.keys().pop()
except IndexError:
return True # what if the seed does not correspond to a described (TD-based) thing?
type_uri = extend_uri(type, fountain.prefixes)
res = R.query("""
PREFIX core: <http://iot.linkeddata.es/def/core#>
SELECT DISTINCT ?id WHERE {
[] a core:ThingDescription ; core:describes ?th ; core:identifier ?id
{
GRAPH ?th { ?s a <%s> }
}
}
""" % type_uri,
cache=True,
expire=300,
infer=True)
target_tds = map(lambda x: x['id']['value'], res)
for target in target_tds:
try:
if nx.shortest_path(td_network, td_id, target):
return True
except nx.NetworkXNoPath:
pass
return False
def add_description(self, id, types):
# type: (basestring, iter) -> TED
agora = self.agora
if not all([t in agora.fountain.types for t in types]):
raise TypeError('Unknown type')
prefixes = agora.fountain.prefixes
type_uris = [extend_uri(t, prefixes) for t in types]
td = TD.from_types(types=type_uris, id=id)
g = td.to_graph(th_nodes={})
try:
self.get_description(id)
except AttributeError:
ted = self.learn_descriptions(g)
tds = ted.ecosystem.tds
if len(tds):
return list(tds).pop()
else:
raise AttributeError(id)
def add_mapping(eco_gw,
id,
amid,
predicate,
key,
jsonpath=None,
root=False,
transformed_by=None):
try:
td = eco_gw.get_description(id)
except AttributeError:
raise NotFoundError(id)
transform_td = None
if transformed_by:
try:
transform_td = eco_gw.get_description(transformed_by)
except AttributeError:
raise NotFoundError(transformed_by)
target_am = [
am for am in td.access_mappings
if str(am.id) == amid or am.endpoint.href.toPython() == amid
]
if not target_am:
raise NotFoundError(amid)
target_am = target_am.pop()
m = Mapping(
key=key,
predicate=URIRef(extend_uri(predicate,
eco_gw.agora.fountain.prefixes)),
root=root,
path=jsonpath,
transform=ResourceTransform(transform_td) if transform_td else None)
target_am.mappings.add(m)
g = td.to_graph(th_nodes={})
eco_gw.learn_descriptions(g)
return m
def describe_resource(self, tid, b64=None, **kwargs):
td = self.__rdict[tid]
g = ConjunctiveGraph()
fountain = self.fountain
ns = get_ns(fountain)
prefixes = fountain.prefixes
for prefix, uri in prefixes.items():
g.bind(prefix, uri)
ttl = 100000
try:
if b64 is not None:
b64 = b64.replace('%3D', '=')
resource_args = eval(base64.b64decode(b64))
else:
resource_args = kwargs
r_uri = self.url_for(tid=tid, b64=b64)
if kwargs:
r_uri = '{}?{}'.format(
r_uri,
'&'.join(['{}={}'.format(k, kwargs[k]) for k in kwargs]))
r_uri = URIRef(r_uri)
bnode_map = {}
for s, p, o in td.resource.graph:
if o in self.__ndict:
o = URIRef(
self.url_for(tid=self.__ndict[o],
b64=b64,
**resource_args))
elif isinstance(o, BNode):
if o not in bnode_map:
bnode_map[o] = BNode()
o = bnode_map[o]
elif isinstance(o, Literal):
if str(o) in resource_args:
o = Literal(resource_args[str(o)], datatype=o.datatype)
if s == td.resource.node:
s = r_uri
if isinstance(s, BNode):
if s not in self.__ndict:
if s not in bnode_map:
bnode_map[s] = BNode()
for t in td.resource.graph.objects(s, RDF.type):
for supt in fountain.get_type(t.n3(ns))['super']:
g.add((bnode_map[s], RDF.type,
extend_uri(supt, prefixes)))
s = bnode_map[s]
g.add((s, p, o))
else:
g.add((s, p, o))
resource_props = set([])
for t in td.resource.types:
if isinstance(t, URIRef):
t_n3 = t.n3(ns)
else:
t_n3 = t
type_dict = fountain.get_type(t_n3)
resource_props.update(type_dict['properties'])
for st in type_dict['super']:
g.add((r_uri, RDF.type, extend_uri(st, prefixes)))
if td.rdf_sources:
for e in td.rdf_sources:
uri = URIRef(e.endpoint.href)
g.add((r_uri, OWL.sameAs, uri))
same_as_g = Graph()
same_as_g.load(source=uri)
for s, p, o in same_as_g:
if p.n3(ns) in resource_props:
if s == uri:
s = r_uri
elif not isinstance(s, BNode):
continue
g.add((s, p, o))
if td.base:
invoked_endpoints = {}
endpoints = list(self.compose_endpoints(td))
endpoints_order = {
am.endpoint: am.order
for am in td.access_mappings
}
for e in sorted(endpoints, key=lambda x: endpoints_order[x]):
if str(e.href) not in invoked_endpoints:
invoked_endpoints[str(e.href)] = e.invoke(
graph=g, subject=r_uri, **resource_args)
response = invoked_endpoints[str(e.href)]
if response.status_code == 200:
data = response.json()
e_mappings = td.endpoint_mappings(e)
mapped_data = apply_mappings(data, e_mappings, ns)
ld = self.enrich(r_uri,
mapped_data,
td.resource.types,
fountain,
ns=ns,
vars=td.vars,
**resource_args)
ld_triples(ld, g)
ttl = min(ttl, extract_ttl(response.headers) or ttl)
except Exception as e:
traceback.print_exc()
log.warn(e.message)
return g, {'Cache-Control': 'max-age={}'.format(ttl)}
def enrich(self,
uri,
data,
types,
fountain,
ns=None,
context=None,
vars=None,
t_dicts=None,
p_dicts=None,
**kwargs):
# type: (URIRef, dict, list, AbstractFountain) -> any
if context is None:
context = {}
if t_dicts is None:
t_dicts = {}
if p_dicts is None:
p_dicts = {}
if vars is None:
vars = set([])
if ns is None:
ns = get_ns(fountain)
for t in types:
self.type_sub_tree(t, fountain, ns, t_dicts=t_dicts)
t_matches = {
t: reduce(lambda x, y: x + int(y in d['properties']), data, 0)
for t, d in t_dicts.items()
}
max_match = max(map(lambda x: t_matches[x],
t_matches)) if t_matches else 0
if not max_match and len(types) > 1:
return
types = filter(lambda x: t_matches[x] == max_match, t_dicts.keys())
target = kwargs.get('$target', None)
if target in types:
types = [target]
else:
common_types = filter(
lambda x: not set.intersection(set(t_dicts[x]['super']), types
), types)
if len(common_types) > 1:
return
elif len(common_types) == 1:
types = common_types
j_types = []
data['@id'] = uri
data['@type'] = j_types
prefixes = dict(ns.graph.namespaces())
for t_n3 in types:
props = t_dicts[t_n3]['properties']
short_type = t_n3.split(':')[1]
context[short_type] = {
'@id': str(extend_uri(t_n3, prefixes)),
'@type': '@id'
}
j_types.append(short_type)
for p_n3 in data:
if p_n3 in props:
p = extend_uri(p_n3, prefixes)
if p_n3 not in p_dicts:
p_dicts[p_n3] = fountain.get_property(p_n3)
pdict = p_dicts[p_n3]
if pdict['type'] == 'data':
# return default data type (string) when not defined
range = pdict['range'][0] if pdict[
'range'] else 'xsd:string'
if range == 'rdfs:Resource':
datatype = Literal(data[p_n3]).datatype
else:
datatype = extend_uri(range, prefixes)
jp = {'@type': datatype, '@id': p}
else:
jp = {'@type': '@id', '@id': p}
context[p_n3] = jp
p_n3_data = data[p_n3]
if isinstance(p_n3_data, dict):
sub = self.enrich(BNode(shortuuid.uuid()).n3(ns),
p_n3_data,
pdict['range'],
fountain,
ns,
context,
vars=vars,
t_dicts=t_dicts,
p_dicts=p_dicts,
**kwargs)
data[p_n3] = sub['@graph']
elif hasattr(p_n3_data, '__call__'):
data[p_n3] = p_n3_data(key=p_n3,
context=context,
uri_provider=self.url_for,
vars=vars,
**kwargs)
elif isinstance(p_n3_data, list):
p_items_res = []
data[p_n3] = p_items_res
for p_item in p_n3_data:
if hasattr(p_item, '__call__'):
p_items_res.extend(
p_item(key=p_n3,
context=context,
uri_provider=self.url_for,
vars=vars,
**kwargs))
elif pdict['type'] != 'data':
if isinstance(p_item, basestring):
p_items_res.append(URIRef(
iriToUri(p_item)))
else:
sub = self.enrich(BNode(
shortuuid.uuid()).n3(ns),
p_item,
pdict['range'],
fountain,
ns=ns,
t_dicts=t_dicts,
p_dicts=p_dicts,
context=context,
vars=vars,
**kwargs)
if sub:
p_items_res.append(sub['@graph'])
else:
p_items_res.append(p_item)
elif pdict['type'] == 'object':
try:
data[p_n3] = URIRef(iriToUri(p_n3_data))
except AttributeError:
if 'rdfs:Resource' in pdict['range']:
datatype = Literal(data[p_n3]).datatype
else:
range = pdict['range'][0] if pdict[
'range'] else 'xsd:string'
datatype = extend_uri(range, prefixes)
context[p_n3] = {'@type': datatype, '@id': p}
return {'@context': context, '@graph': data}
def _build_pred_uri(pred_str, graph):
pred_uri = URIRef(extend_uri(pred_str, dict(graph.namespaces())))
return pred_uri
def __uri(self, n3):
if n3 not in self.n3_uri_map:
self.n3_uri_map[n3] = extend_uri(n3, self.prefixes)
return self.n3_uri_map[n3]
def create_gql_schema(gateway):
log.info('Building GraphQL schema from Agora...')
fountain = Wrapper(gateway.agora.fountain)
types = filter(lambda x: fountain.get_type(x)['properties'],
sorted(fountain.types))
all_type_names = {}
for t in types:
t_title = title(t)
if t_title in all_type_names:
t_title = ''.join(map(lambda x: x.title(), t.split(':')))
all_type_names[t] = t_title
all_type_names[t_title] = t
t_params = {}
prefixes = gateway.agora.fountain.prefixes
for t in types:
t_uri = extend_uri(t, prefixes)
t_ted = gateway.discover("""SELECT * WHERE { [] a <%s>}""" % t_uri,
strict=True,
lazy=False)
if t_ted.ecosystem.roots:
params = set()
for root in t_ted.ecosystem.roots:
if isinstance(root, TD):
root_vars = t_ted.ecosystem.root_vars(root)
td_vars = filter(lambda x: x != '$item' and x != '$parent',
root_vars)
params.update(set(td_vars))
t_params[all_type_names[t]] = params
abstract_types = dict(get_abstract_types(fountain))
for abstract in abstract_types.values():
if abstract['type'] == 'Union':
union_type_names = map(lambda x: title(x), sorted(abstract['of']))
union_name = 'Union' + '_'.join(union_type_names)
all_type_names[union_name] = union_type_names
types_str = '\n'.join(
filter(lambda x: x, [
serialize_type(fountain, t, all_type_names, abstract_types)
for t in types
]))
query_str = serialize_queries(t_params)
schema_str = "schema {\n\tquery: Query\n}"
unions_dict = {
k: v
for k, v in all_type_names.items() if k.startswith('Union')
}
unions = [
'union {}= {}'.format(u, '|'.join(union_types))
for u, union_types in unions_dict.items()
]
unions_str = '\n'.join(unions)
res = '\n'.join([types_str, unions_str, query_str, schema_str])
return res
def resolve(self, next, root, info, **args):
if info.context['introspection']:
return next(root, info, **args)
fountain = info.context['fountain']
try:
non_nullable = isinstance(info.return_type, GraphQLNonNull)
return_type = info.return_type.of_type if non_nullable else info.return_type
if info.field_name == '_uri':
return root
if isinstance(return_type, GraphQLList):
if not root:
log.debug(u'Gathering seeds...')
data_graph_kwargs = args.copy()
data_graph_kwargs.update(self.settings)
dg = data_graph(info.context['query'], self.gateway, data_gw_cache=self.data_gw_cache,
follow_cycles=self.follow_cycles,
**data_graph_kwargs)
info.context['load_fn'] = self.loader(dg)
info.context['locks'] = {}
seeds = dg.roots
else:
seeds = []
for parent_ty in match(info.parent_type.name, fountain.types):
try:
alias_prop = list(match(info.field_name, fountain.get_type(parent_ty)['properties'])).pop()
prop_uri = extend_uri(alias_prop, fountain.prefixes)
seeds = objects(self.data_gw_cache, info, root, prop_uri)
break
except IndexError:
pass
if isinstance(info.return_type.of_type, GraphQLInterfaceType) or isinstance(info.return_type.of_type,
GraphQLUnionType):
seeds = filter(lambda x: self.__filter_abstract_seed(x, info), seeds)
if seeds or non_nullable:
return seeds
elif isinstance(return_type, GraphQLScalarType):
if root:
for parent_ty in match(info.parent_type.name, fountain.types):
try:
alias_prop = list(match(info.field_name, fountain.get_type(parent_ty)['properties'])).pop()
prop_uri = URIRef(extend_uri(alias_prop, fountain.prefixes))
try:
value = objects(self.data_gw_cache, info, root, prop_uri).pop()
return value
except IndexError as e:
if non_nullable:
raise Exception(e.message)
except IndexError:
pass
elif isinstance(return_type, GraphQLObjectType):
if root:
for parent_ty in match(info.parent_type.name, fountain.types):
try:
alias_prop = list(match(info.field_name, fountain.get_type(parent_ty)['properties'])).pop()
prop_uri = URIRef(extend_uri(alias_prop, fountain.prefixes))
try:
uri = objects(self.data_gw_cache, info, root, prop_uri).pop()
if uri:
return uri
except IndexError as e:
if non_nullable:
raise Exception(e.message)
except IndexError:
pass
except Exception as e:
traceback.print_exc()
raise e
