def add_resource(self, uri, types): # type: (basestring, iter) -> TED ted = self.ted matching_resources = filter(lambda r: r.node.toPython() == uri, ted.ecosystem.non_td_root_resources) if not matching_resources: agora = self.agora if not all([t in agora.fountain.types for t in types]): raise TypeError('Unknown type') g = Graph() prefixes = agora.fountain.prefixes uri_ref = URIRef(uri) if not types: dgw = DataGateway(agora.agora, ted, cache=None) rg, headers = dgw.loader(uri) type_uris = set( [extend_uri(t, prefixes) for t in agora.fountain.types]) resource_types = set(rg.objects(uri_ref, RDF.type)) types = tuple(set.intersection(type_uris, resource_types)) for t in types: g.add((uri_ref, RDF.type, URIRef(extend_uri(t, prefixes)))) ted = self.learn_descriptions(g) return ted else: raise AttributeError(uri)
def add_description(self, id, types): try: self._get_request('descriptions/{}'.format(id), accept='text/turtle') except IOError: pass else: raise AttributeError(id) prefixes = self.agora.fountain.prefixes types = map(lambda t: URIRef(extend_uri(t, prefixes)), types) td = TD.from_types(id, types) try: response = self._post_request( 'descriptions', td.to_graph(th_nodes={}).serialize(format='turtle'), content_type='text/turtle', accept='text/turtle') g = Graph() g.parse(StringIO(response), format='turtle') ted = TED.from_graph(g, loader=self.__loader) tds = ted.ecosystem.tds if len(tds): return list(tds).pop() except IOError: pass raise AttributeError(id)
def add_resource(self, uri, types): response = self._get_request('resources', accept='text/turtle') g = Graph() g.parse(StringIO(response), format='turtle') if URIRef(uri) in set(g.subjects()): raise AttributeError(uri) prefixes = self.agora.fountain.prefixes types = map(lambda t: URIRef(extend_uri(t, prefixes)), types) r = Resource(URIRef(uri), types) try: response = self._post_request( 'descriptions', r.to_graph().serialize(format='turtle'), content_type='text/turtle', accept='text/turtle') g = Graph() g.parse(StringIO(response), format='turtle') ted = TED.from_graph(g, loader=self.__loader) all_resources = ted.ecosystem.non_td_root_resources if all_resources: return list(all_resources).pop() except IOError as e: raise AttributeError(e.message['text']) raise AttributeError(uri)
def add_enrichment(self, id, type, td_id, replace=False): # type: (basestring, basestring, basestring, bool) -> Enrichment ted = self.ted matching_enrichments = filter(lambda e: str(e.id) == str(id), ted.ecosystem.enrichments) if not matching_enrichments: agora = self.agora if type not in agora.fountain.types: raise TypeError('Unknown type') prefixes = agora.fountain.prefixes type_uri = extend_uri(type, prefixes) td = self.get_description(td_id) try: e = Enrichment(id, type_uri, td, replace=replace) g = e.to_graph(td_nodes={}) self.learn_descriptions(g) except AttributeError: raise AttributeError(id) return e else: raise AttributeError(id)
def load(self, resource, graph, uri, b64, r_args): prefixes = self.fountain.prefixes bnode_map = {} for s, p, o in resource.graph: if o in self.nodes: o = URIRef(self.url_for(tid=self.nodes[o], b64=b64, **r_args)) elif isinstance(o, BNode): if o not in bnode_map: bnode_map[o] = BNode() o = bnode_map[o] elif isinstance(o, Literal): if str(o) in r_args: o = Literal(r_args[str(o)], datatype=o.datatype) if s == resource.node: s = uri if isinstance(s, BNode): if s not in self.nodes: if s not in bnode_map: bnode_map[s] = BNode() for t in resource.graph.objects(s, RDF.type): for supt in self.fountain.get_type(t.n3( self.ns))['super']: graph.add((bnode_map[s], RDF.type, extend_uri(supt, prefixes))) s = bnode_map[s] graph.add((s, p, o)) else: graph.add((s, p, o)) for t in resource.types: if isinstance(t, URIRef): t_n3 = t.n3(self.ns) else: t_n3 = t type_dict = self.fountain.get_type(t_n3) self.resource_props.update(type_dict['properties']) for st in type_dict['super']: graph.add((uri, RDF.type, extend_uri(st, prefixes)))
def get_resource(ctx, uri, host, port, turtle, raw): gw = ctx.obj['gw'] ted = gw.ted dgw = DataGateway(gw.agora, ted, cache=None, port=port, server_name=host) g, headers = dgw.loader(uri) uri_ref = URIRef(uri) prefixes = gw.agora.fountain.prefixes type_uris = set([extend_uri(t, prefixes) for t in gw.agora.fountain.types]) resource_types = set(g.objects(uri_ref, RDF.type)) known_types = set.intersection(type_uris, resource_types) ag = Graph() for prefix, uri in prefixes.items(): ag.bind(prefix, uri) if raw: ag.__iadd__(g) else: known_types_n3 = [t.n3(ag.namespace_manager) for t in known_types] known_props = reduce( lambda x, y: x.union( set(gw.agora.fountain.get_type(y)['properties'])), known_types_n3, set()) known_props_uri = set([extend_uri(p, prefixes) for p in known_props]) known_refs = reduce( lambda x, y: x.union(set(gw.agora.fountain.get_type(y)['refs'])), known_types_n3, set()) known_refs_uri = set([extend_uri(p, prefixes) for p in known_refs]) for (s, p, o) in g: if s == uri_ref and ((p == RDF.type and o in known_types) or p in known_props_uri): ag.add((s, p, o)) if o == uri_ref and p in known_refs_uri: ag.add((s, p, o)) show_thing(ag, format='text/turtle' if turtle else 'application/ld+json')
def materialize_th_types(R, ids): for th_id in ids: g = R.pull(th_id) prev_len = len(g) prefixes = R.agora.fountain.prefixes for s, p, o in g.triples((None, RDF.type, None)): try: type_n3 = g.qname(o) type_super = R.agora.fountain.get_type(type_n3)['super'] for ts in type_super: ts_triple = (s, p, URIRef(extend_uri(ts, prefixes))) if ts_triple not in g: g.add(ts_triple) except Exception: pass if len(g) > prev_len: R.push(g)
def add_enrichment(self, id, type, tdid, replace=False): try: self._get_request('enrichments/{}'.format(id), accept='text/turtle') except IOError: pass else: # Already exists raise AttributeError(id) try: response = self._get_request('descriptions/{}'.format(tdid), accept='text/turtle') except IOError: # TD does not exist raise AttributeError(tdid) g = Graph() g.parse(StringIO(response), format='turtle') try: td_uri = list(g.subjects(CORE.identifier, Literal(tdid))).pop() td = TD.from_graph(g, td_uri, node_map={}) except IndexError: # Something wrong happens with the TD RDF raise AttributeError(tdid) prefixes = self.agora.fountain.prefixes type = URIRef(extend_uri(type, prefixes)) e = Enrichment(id, type, td, replace=replace) try: response = self._post_request( 'descriptions', e.to_graph().serialize(format='turtle'), content_type='text/turtle', accept='text/turtle') g = Graph() g.parse(StringIO(response), format='turtle') ted = TED.from_graph(g, loader=self.__loader) all_enrichments = ted.ecosystem.enrichments if all_enrichments: return list(all_enrichments).pop() except IOError as e: raise AttributeError(e.message['text']) raise AttributeError(id)
def is_described_reachable(fountain, R, td_network, seed, type): res = R.query(""" PREFIX core: <http://iot.linkeddata.es/def/core#> SELECT DISTINCT ?g ?id WHERE { GRAPH ?g { [] a core:ThingDescription ; core:identifier ?id ; core:describes <%s> } } """ % seed, cache=True, expire=300, infer=True) rd = generate_dict(R.n3, R.ns(fountain=fountain), res) try: td_id = rd.keys().pop() except IndexError: return True # what if the seed does not correspond to a described (TD-based) thing? type_uri = extend_uri(type, fountain.prefixes) res = R.query(""" PREFIX core: <http://iot.linkeddata.es/def/core#> SELECT DISTINCT ?id WHERE { [] a core:ThingDescription ; core:describes ?th ; core:identifier ?id { GRAPH ?th { ?s a <%s> } } } """ % type_uri, cache=True, expire=300, infer=True) target_tds = map(lambda x: x['id']['value'], res) for target in target_tds: try: if nx.shortest_path(td_network, td_id, target): return True except nx.NetworkXNoPath: pass return False
def add_description(self, id, types): # type: (basestring, iter) -> TED agora = self.agora if not all([t in agora.fountain.types for t in types]): raise TypeError('Unknown type') prefixes = agora.fountain.prefixes type_uris = [extend_uri(t, prefixes) for t in types] td = TD.from_types(types=type_uris, id=id) g = td.to_graph(th_nodes={}) try: self.get_description(id) except AttributeError: ted = self.learn_descriptions(g) tds = ted.ecosystem.tds if len(tds): return list(tds).pop() else: raise AttributeError(id)
def add_mapping(eco_gw, id, amid, predicate, key, jsonpath=None, root=False, transformed_by=None): try: td = eco_gw.get_description(id) except AttributeError: raise NotFoundError(id) transform_td = None if transformed_by: try: transform_td = eco_gw.get_description(transformed_by) except AttributeError: raise NotFoundError(transformed_by) target_am = [ am for am in td.access_mappings if str(am.id) == amid or am.endpoint.href.toPython() == amid ] if not target_am: raise NotFoundError(amid) target_am = target_am.pop() m = Mapping( key=key, predicate=URIRef(extend_uri(predicate, eco_gw.agora.fountain.prefixes)), root=root, path=jsonpath, transform=ResourceTransform(transform_td) if transform_td else None) target_am.mappings.add(m) g = td.to_graph(th_nodes={}) eco_gw.learn_descriptions(g) return m
def describe_resource(self, tid, b64=None, **kwargs): td = self.__rdict[tid] g = ConjunctiveGraph() fountain = self.fountain ns = get_ns(fountain) prefixes = fountain.prefixes for prefix, uri in prefixes.items(): g.bind(prefix, uri) ttl = 100000 try: if b64 is not None: b64 = b64.replace('%3D', '=') resource_args = eval(base64.b64decode(b64)) else: resource_args = kwargs r_uri = self.url_for(tid=tid, b64=b64) if kwargs: r_uri = '{}?{}'.format( r_uri, '&'.join(['{}={}'.format(k, kwargs[k]) for k in kwargs])) r_uri = URIRef(r_uri) bnode_map = {} for s, p, o in td.resource.graph: if o in self.__ndict: o = URIRef( self.url_for(tid=self.__ndict[o], b64=b64, **resource_args)) elif isinstance(o, BNode): if o not in bnode_map: bnode_map[o] = BNode() o = bnode_map[o] elif isinstance(o, Literal): if str(o) in resource_args: o = Literal(resource_args[str(o)], datatype=o.datatype) if s == td.resource.node: s = r_uri if isinstance(s, BNode): if s not in self.__ndict: if s not in bnode_map: bnode_map[s] = BNode() for t in td.resource.graph.objects(s, RDF.type): for supt in fountain.get_type(t.n3(ns))['super']: g.add((bnode_map[s], RDF.type, extend_uri(supt, prefixes))) s = bnode_map[s] g.add((s, p, o)) else: g.add((s, p, o)) resource_props = set([]) for t in td.resource.types: if isinstance(t, URIRef): t_n3 = t.n3(ns) else: t_n3 = t type_dict = fountain.get_type(t_n3) resource_props.update(type_dict['properties']) for st in type_dict['super']: g.add((r_uri, RDF.type, extend_uri(st, prefixes))) if td.rdf_sources: for e in td.rdf_sources: uri = URIRef(e.endpoint.href) g.add((r_uri, OWL.sameAs, uri)) same_as_g = Graph() same_as_g.load(source=uri) for s, p, o in same_as_g: if p.n3(ns) in resource_props: if s == uri: s = r_uri elif not isinstance(s, BNode): continue g.add((s, p, o)) if td.base: invoked_endpoints = {} endpoints = list(self.compose_endpoints(td)) endpoints_order = { am.endpoint: am.order for am in td.access_mappings } for e in sorted(endpoints, key=lambda x: endpoints_order[x]): if str(e.href) not in invoked_endpoints: invoked_endpoints[str(e.href)] = e.invoke( graph=g, subject=r_uri, **resource_args) response = invoked_endpoints[str(e.href)] if response.status_code == 200: data = response.json() e_mappings = td.endpoint_mappings(e) mapped_data = apply_mappings(data, e_mappings, ns) ld = self.enrich(r_uri, mapped_data, td.resource.types, fountain, ns=ns, vars=td.vars, **resource_args) ld_triples(ld, g) ttl = min(ttl, extract_ttl(response.headers) or ttl) except Exception as e: traceback.print_exc() log.warn(e.message) return g, {'Cache-Control': 'max-age={}'.format(ttl)}
def enrich(self, uri, data, types, fountain, ns=None, context=None, vars=None, t_dicts=None, p_dicts=None, **kwargs): # type: (URIRef, dict, list, AbstractFountain) -> any if context is None: context = {} if t_dicts is None: t_dicts = {} if p_dicts is None: p_dicts = {} if vars is None: vars = set([]) if ns is None: ns = get_ns(fountain) for t in types: self.type_sub_tree(t, fountain, ns, t_dicts=t_dicts) t_matches = { t: reduce(lambda x, y: x + int(y in d['properties']), data, 0) for t, d in t_dicts.items() } max_match = max(map(lambda x: t_matches[x], t_matches)) if t_matches else 0 if not max_match and len(types) > 1: return types = filter(lambda x: t_matches[x] == max_match, t_dicts.keys()) target = kwargs.get('$target', None) if target in types: types = [target] else: common_types = filter( lambda x: not set.intersection(set(t_dicts[x]['super']), types ), types) if len(common_types) > 1: return elif len(common_types) == 1: types = common_types j_types = [] data['@id'] = uri data['@type'] = j_types prefixes = dict(ns.graph.namespaces()) for t_n3 in types: props = t_dicts[t_n3]['properties'] short_type = t_n3.split(':')[1] context[short_type] = { '@id': str(extend_uri(t_n3, prefixes)), '@type': '@id' } j_types.append(short_type) for p_n3 in data: if p_n3 in props: p = extend_uri(p_n3, prefixes) if p_n3 not in p_dicts: p_dicts[p_n3] = fountain.get_property(p_n3) pdict = p_dicts[p_n3] if pdict['type'] == 'data': # return default data type (string) when not defined range = pdict['range'][0] if pdict[ 'range'] else 'xsd:string' if range == 'rdfs:Resource': datatype = Literal(data[p_n3]).datatype else: datatype = extend_uri(range, prefixes) jp = {'@type': datatype, '@id': p} else: jp = {'@type': '@id', '@id': p} context[p_n3] = jp p_n3_data = data[p_n3] if isinstance(p_n3_data, dict): sub = self.enrich(BNode(shortuuid.uuid()).n3(ns), p_n3_data, pdict['range'], fountain, ns, context, vars=vars, t_dicts=t_dicts, p_dicts=p_dicts, **kwargs) data[p_n3] = sub['@graph'] elif hasattr(p_n3_data, '__call__'): data[p_n3] = p_n3_data(key=p_n3, context=context, uri_provider=self.url_for, vars=vars, **kwargs) elif isinstance(p_n3_data, list): p_items_res = [] data[p_n3] = p_items_res for p_item in p_n3_data: if hasattr(p_item, '__call__'): p_items_res.extend( p_item(key=p_n3, context=context, uri_provider=self.url_for, vars=vars, **kwargs)) elif pdict['type'] != 'data': if isinstance(p_item, basestring): p_items_res.append(URIRef( iriToUri(p_item))) else: sub = self.enrich(BNode( shortuuid.uuid()).n3(ns), p_item, pdict['range'], fountain, ns=ns, t_dicts=t_dicts, p_dicts=p_dicts, context=context, vars=vars, **kwargs) if sub: p_items_res.append(sub['@graph']) else: p_items_res.append(p_item) elif pdict['type'] == 'object': try: data[p_n3] = URIRef(iriToUri(p_n3_data)) except AttributeError: if 'rdfs:Resource' in pdict['range']: datatype = Literal(data[p_n3]).datatype else: range = pdict['range'][0] if pdict[ 'range'] else 'xsd:string' datatype = extend_uri(range, prefixes) context[p_n3] = {'@type': datatype, '@id': p} return {'@context': context, '@graph': data}
def _build_pred_uri(pred_str, graph): pred_uri = URIRef(extend_uri(pred_str, dict(graph.namespaces()))) return pred_uri
def __uri(self, n3): if n3 not in self.n3_uri_map: self.n3_uri_map[n3] = extend_uri(n3, self.prefixes) return self.n3_uri_map[n3]
def create_gql_schema(gateway): log.info('Building GraphQL schema from Agora...') fountain = Wrapper(gateway.agora.fountain) types = filter(lambda x: fountain.get_type(x)['properties'], sorted(fountain.types)) all_type_names = {} for t in types: t_title = title(t) if t_title in all_type_names: t_title = ''.join(map(lambda x: x.title(), t.split(':'))) all_type_names[t] = t_title all_type_names[t_title] = t t_params = {} prefixes = gateway.agora.fountain.prefixes for t in types: t_uri = extend_uri(t, prefixes) t_ted = gateway.discover("""SELECT * WHERE { [] a <%s>}""" % t_uri, strict=True, lazy=False) if t_ted.ecosystem.roots: params = set() for root in t_ted.ecosystem.roots: if isinstance(root, TD): root_vars = t_ted.ecosystem.root_vars(root) td_vars = filter(lambda x: x != '$item' and x != '$parent', root_vars) params.update(set(td_vars)) t_params[all_type_names[t]] = params abstract_types = dict(get_abstract_types(fountain)) for abstract in abstract_types.values(): if abstract['type'] == 'Union': union_type_names = map(lambda x: title(x), sorted(abstract['of'])) union_name = 'Union' + '_'.join(union_type_names) all_type_names[union_name] = union_type_names types_str = '\n'.join( filter(lambda x: x, [ serialize_type(fountain, t, all_type_names, abstract_types) for t in types ])) query_str = serialize_queries(t_params) schema_str = "schema {\n\tquery: Query\n}" unions_dict = { k: v for k, v in all_type_names.items() if k.startswith('Union') } unions = [ 'union {}= {}'.format(u, '|'.join(union_types)) for u, union_types in unions_dict.items() ] unions_str = '\n'.join(unions) res = '\n'.join([types_str, unions_str, query_str, schema_str]) return res
def resolve(self, next, root, info, **args): if info.context['introspection']: return next(root, info, **args) fountain = info.context['fountain'] try: non_nullable = isinstance(info.return_type, GraphQLNonNull) return_type = info.return_type.of_type if non_nullable else info.return_type if info.field_name == '_uri': return root if isinstance(return_type, GraphQLList): if not root: log.debug(u'Gathering seeds...') data_graph_kwargs = args.copy() data_graph_kwargs.update(self.settings) dg = data_graph(info.context['query'], self.gateway, data_gw_cache=self.data_gw_cache, follow_cycles=self.follow_cycles, **data_graph_kwargs) info.context['load_fn'] = self.loader(dg) info.context['locks'] = {} seeds = dg.roots else: seeds = [] for parent_ty in match(info.parent_type.name, fountain.types): try: alias_prop = list(match(info.field_name, fountain.get_type(parent_ty)['properties'])).pop() prop_uri = extend_uri(alias_prop, fountain.prefixes) seeds = objects(self.data_gw_cache, info, root, prop_uri) break except IndexError: pass if isinstance(info.return_type.of_type, GraphQLInterfaceType) or isinstance(info.return_type.of_type, GraphQLUnionType): seeds = filter(lambda x: self.__filter_abstract_seed(x, info), seeds) if seeds or non_nullable: return seeds elif isinstance(return_type, GraphQLScalarType): if root: for parent_ty in match(info.parent_type.name, fountain.types): try: alias_prop = list(match(info.field_name, fountain.get_type(parent_ty)['properties'])).pop() prop_uri = URIRef(extend_uri(alias_prop, fountain.prefixes)) try: value = objects(self.data_gw_cache, info, root, prop_uri).pop() return value except IndexError as e: if non_nullable: raise Exception(e.message) except IndexError: pass elif isinstance(return_type, GraphQLObjectType): if root: for parent_ty in match(info.parent_type.name, fountain.types): try: alias_prop = list(match(info.field_name, fountain.get_type(parent_ty)['properties'])).pop() prop_uri = URIRef(extend_uri(alias_prop, fountain.prefixes)) try: uri = objects(self.data_gw_cache, info, root, prop_uri).pop() if uri: return uri except IndexError as e: if non_nullable: raise Exception(e.message) except IndexError: pass except Exception as e: traceback.print_exc() raise e