def get_context_data(self, **kwargs): context = super(NaveDocumentTemplateView, self).get_context_data(**kwargs) absolute_uri = self.request.build_absolute_uri() target_uri = RDFRecord.get_internal_rdf_base_uri(absolute_uri) if "detail/foldout/" in target_uri: slug = self.kwargs.get('slug') record = ElasticSearchRDFRecord(hub_id=slug) graph = record.get_graph_by_id(self.kwargs.get('slug')) if graph is not None: target_uri = record.source_uri else: logger.warn("Unable to find source_uri for slug: {}".format(slug)) else: target_uri = RDFRecord.get_internal_rdf_base_uri(absolute_uri) record = ElasticSearchRDFRecord(hub_id=self.kwargs.get('slug')) graph = record.get_graph_by_source_uri(target_uri) if graph is None: raise UnknownGraph("URI {} is not known in our graph store".format(target_uri)) if "/resource/cache/" in target_uri: target_uri = target_uri.rstrip('/') cache_resource = CacheResource.objects.filter(document_uri=target_uri) if cache_resource.exists(): graph = cache_resource.first().get_graph() elif settings.RDF_USE_LOCAL_GRAPH: mode = self.request.REQUEST.get('mode', 'default') acceptance = True if mode == 'acceptance' else False context['acceptance'] = acceptance elif '/resource/aggregation' in target_uri: target_named_graph = "{}/graph".format(target_uri.rstrip('/')) graph, nr_levels = RDFModel.get_context_graph(store=rdfstore.get_rdfstore(), named_graph=target_named_graph) else: graph, nr_levels = RDFModel.get_context_graph( store=rdfstore.get_rdfstore(), target_uri=target_uri ) # todo: remove: should no longer be necessary with the addition of common.middleware.ForceLangMiddleware language = self.request.GET.get('lang', None) if language: activate(language) bindings = GraphBindings( about_uri=target_uri, graph=graph, excluded_properties=settings.RDF_EXCLUDED_PROPERTIES ) context['resources'] = bindings context['absolute_uri'] = RDFRecord.get_external_rdf_url(target_uri, self.request) for rdf_type in bindings.get_about_resource().get_types(): search_label = rdf_type.search_label.lower() content_template = settings.RDF_CONTENT_FOLDOUTS.get(search_label) if content_template: self.template_name = content_template break context['points'] = RDFModel.get_geo_points(graph) return context
def test_get_geo_points(self): graph = Graph() subject = URIRef('joachim') graph.add((subject, URIRef("http://www.w3.org/2003/01/geo/wgs84_pos#lat"), Literal("51.54"))) graph.add((subject, URIRef("http://www.w3.org/2003/01/geo/wgs84_pos#long"), Literal("5.2"))) points = RDFModel.get_geo_points(graph) assert points is not None assert points == [[51.54, 5.2]] graph = Graph() points = RDFModel.get_geo_points(graph) assert points == []
def test_bindings_get_graph_from_sparql_results(self): from .resources import sparqlwrapper_result as result sparql_json = result.sparql_result graph, nr_level = RDFModel.get_graph_from_sparql_results( sparql_json, named_graph="http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph" ) assert graph is not None assert graph.identifier == URIRef("http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph") assert len(graph) == 118 predicates = set(graph.predicates()) assert len(predicates) != 0 assert len(predicates) == 67 assert URIRef("http://purl.org/dc/elements/1.1/subject") in list(predicates) rdf_types = sorted(set(graph.objects(predicate=RDF.type))) assert rdf_types == sorted({URIRef('http://www.europeana.eu/schemas/edm/ProvidedCHO'), URIRef('http://www.europeana.eu/schemas/edm/WebResource'), URIRef('http://schemas.delving.eu/nave/terms/DelvingResource'), URIRef('http://www.openarchives.org/ore/terms/Aggregation'), URIRef('http://schemas.delving.eu/narthex/terms/Record'), URIRef('http://schemas.delving.eu/nave/terms/BrabantCloudResource'), URIRef('http://schemas.delving.eu/narthex/terms/Dataset'), URIRef('http://www.w3.org/2004/02/skos/core#Concept')}) assert Literal("bomen", lang="nl") in graph.preferredLabel( subject=URIRef('http://data.cultureelerfgoed.nl/semnet/7403e26d-cf33-4372-ad72-a2f9fcf8f63b') )[0] bnodes_materialized = list(graph.objects(predicate=URIRef("http://www.openarchives.org/ore/terms/aggregates"))) assert len(bnodes_materialized) == 1
def test_bindings_determine_context_levels(self): test_list = [ "s", "p", "o", "p2", "o2", "p3", "o3" ] levels = RDFModel.get_context_triples(test_list) assert len(levels) == 3 assert len(RDFModel.get_context_triples(test_list[:3])) == 1 assert len(RDFModel.get_context_triples(test_list[:5])) == 2 assert levels[0] == ("s", "p", "o") assert levels[1] == ("o", "p2", "o2") assert levels[2] == ("o2", "p3", "o3")
def get_context_data(self, **kwargs): context = super(NaveDocumentDetailView, self).get_context_data(**kwargs) target_uri = self.object.document_uri if "/resource/cache/" in target_uri: target_uri = target_uri.rstrip('/') cache_resource = CacheResource.objects.filter(document_uri=target_uri) if cache_resource.exists(): graph = cache_resource.first().get_graph() elif settings.RDF_USE_LOCAL_GRAPH: mode = self.request.REQUEST.get('mode', 'default') acceptance = True if mode == 'acceptance' else False context['acceptance'] = acceptance if isinstance(self.object, EDMRecord): graph = self.object.get_graph(with_mappings=True, include_mapping_target=True, acceptance=acceptance) else: graph = self.object.get_graph(acceptance=acceptance) elif '/resource/aggregation' in target_uri: target_named_graph = "{}/graph".format(target_uri.rstrip('/')) graph, nr_levels = RDFModel.get_context_graph(store=rdfstore.get_rdfstore(), named_graph=target_named_graph) else: graph, nr_levels = RDFModel.get_context_graph( store=rdfstore.get_rdfstore(), target_uri=target_uri ) # todo: remove: should no longer be necessary with the addition of common.middleware.ForceLangMiddleware language = self.request.GET.get('lang', None) if language: activate(language) bindings = GraphBindings( about_uri=target_uri, graph=graph, excluded_properties=settings.RDF_EXCLUDED_PROPERTIES ) context['resources'] = bindings for rdf_type in bindings.get_about_resource().get_types(): search_label = rdf_type.search_label.lower() content_template = settings.RDF_CONTENT_FOLDOUTS.get(search_label) if content_template: self.template_name = content_template break context['points'] = RDFModel.get_geo_points(graph) return context
def graph_to_record(graph, ds, bulk=False, content_hash=None, force_insert=False, acceptance=False): """Update or create an EDM record from a graph.""" exclude_key = URIRef('http://schemas.delving.eu/nave/terms/excludedFromEuropeana') org_id = settings.ORG_ID spec = ds.spec identifier = re.sub("[/]+graph$", "", graph.identifier) local_id = identifier.split('/')[-1] hub_id = "{}_{}_{}".format(org_id, spec, local_id) exclude_from_europeana = RDFModel.get_first_literal(exclude_key, graph) if exclude_from_europeana is None: exclude_from_europeana = False elif not isinstance(exclude_from_europeana, bool): exclude_from_europeana = True if exclude_from_europeana.lower() in ['true'] else False if content_hash and not force_insert: query_filter = {'hub_id': hub_id} if acceptance: query_filter['acceptance_hash'] = content_hash else: query_filter['source_hash'] = content_hash if EDMRecord.objects.filter(**query_filter).exists(): return None group, _ = Group.objects.get_or_create(name='dataset_admin') # get real identifier (support for with or without final slash) update_values = { "hub_id": hub_id, "dataset": ds, "named_graph": graph.identifier, "document_uri": identifier, "source_uri": identifier, "local_id": local_id, "rdf_in_sync": False, "rdf_sync_error": None, "orphaned": False, "exclude_from_europeana": exclude_from_europeana } # add content hash check if acceptance: update_values["acceptance_rdf"] = graph.serialize(format='nt', encoding="UTF-8") update_values["acceptance_updated"] = timezone.now().strftime(fmt) if content_hash: update_values['acceptance_hash'] = content_hash else: update_values["source_rdf"] = graph.serialize(format='nt', encoding="UTF-8") update_values["source_updated"] = timezone.now().strftime(fmt) if content_hash: update_values['source_hash'] = content_hash if not bulk: edm_record, _ = EDMRecord.objects.update_or_create(hub_id=hub_id, defaults=update_values) edm_record.groups.add(*ds.groups.all()) else: update_values['hub_id'] = hub_id edm_record = EDMRecord(**update_values) return edm_record
def test_binding_with_uri(self): uri_object_dict = { 'o': { 'value': 'http://localhost:8000/resource/dataset/ton-smits-huis', 'type': 'uri' } } obj = RDFModel.get_object_from_sparql_result(uri_object_dict['o']) assert obj is not None assert isinstance(obj, URIRef) assert obj == URIRef(uri_object_dict['o']['value'])
def test_context_graph_inlining(self): named_graph = "http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph" inline_id = "http://data.cultureelerfgoed.nl/semnet/7403e26d-cf33-4372-ad72-a2f9fcf8f63b" context_graph, nr_levels = RDFModel.get_context_graph( store=self.store, named_graph=named_graph ) self.assertIsNotNone(context_graph) self.assertIsInstance(context_graph, Graph) assert URIRef(inline_id) in list(context_graph.subjects()) predicates = set(list(context_graph.predicates())) assert URIRef('http://www.openarchives.org/ore/terms/aggregates') in predicates
def test_get_graph_statistics(self): graph = self.graph stats = RDFModel.get_graph_statistics(graph=graph) assert stats is not None assert isinstance(stats, dict) assert 'language' in stats assert stats['language'] == [('nl', 1)] assert sorted(stats['RDF class']) == sorted( [('skos:Concept', 7), ('nave:BrabantCloudResource', 1), ('narthex:Record', 1), ('edm:ProvidedCHO', 1), ('edm:WebResource', 1), ('nave:DelvingResource', 1), ('ore:Aggregation', 1), ('narthex:Dataset', 1)] ) assert 'property' in stats assert len(stats['property']) > 0
def test_bindings_get_graph_from_sparql_results_without_named_graph(self): from .resources import sparqlwrapper_result_graph_name as result sparql_json = result.sparql_result graph, nr_level = RDFModel.get_graph_from_sparql_results( sparql_json ) assert graph is not None assert graph.identifier == URIRef("http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph") assert len(graph) == 118 predicates = set(graph.predicates()) assert len(predicates) != 0 assert len(predicates) == 67
def synchronise_record(graph_uri, ds, store, es_actions, index=settings.SITE_NAME, acceptance=False): # get graph # graph = store.get_graph_store.get(graph_uri, as_graph=True) graph, levels = RDFModel.get_context_graph(named_graph=graph_uri, store=store) # create EDM record edm_record = EDMRecord.graph_to_record(graph, ds) # create ES insert action index_action = edm_record.create_es_action( index=get_index_name(acceptance), record_type="Aggregated", store=store, context=False, exclude_fields=ds.excluded_index_fields.names() ) es_actions.append(index_action) return edm_record, index_action
def test_with_language_literal(self): literal_object_dict = { 'o3': { 'value': 'bomen', 'xml:lang': 'nl', 'type': 'literal' } } obj = RDFModel.get_object_from_sparql_result(literal_object_dict) assert obj is not None assert isinstance(obj, Literal) assert obj.language == "nl" assert obj.datatype is None assert obj == Literal( literal_object_dict['o3']['value'], lang="nl" )
def test_binding_with_typed_literal(self): object_dict = { 'o2': { 'value': 'false', 'type': 'typed-literal', 'datatype': 'http://www.w3.org/2001/XMLSchema#boolean' } } obj = RDFModel.get_object_from_sparql_result(object_dict) assert obj is not None assert isinstance(obj, Literal) assert obj.language is None assert obj.datatype is not None assert obj.datatype == URIRef("http://www.w3.org/2001/XMLSchema#boolean") assert obj == Literal( object_dict['o2']['value'], datatype="http://www.w3.org/2001/XMLSchema#boolean" )
def setUp(self): from .resources import sparqlwrapper_result as result sparql_json = result.sparql_result self.graph, self.nr_levels = RDFModel.get_graph_from_sparql_results(sparql_json)
def retrieve(self, request, pk=None, format=None, *args, **kwargs): def get_mode(default=None): params = request.GET return params.get('schema', default) self._clean_callback(request) query = NaveESQuery( index_name=self.get_index_name, doc_types=self.doc_types, default_facets=self.facets, cluster_geo=False, size=1, converter=self.get_converter() ) try: query = query.build_item_query(query, request.query_params, pk) except ValueError as ve: logger.error("Unable to build request because: {}".format(ve)) # todo display error message when bad/unknown hubId is given return HttpResponseBadRequest() mlt = True if request.query_params.get('mlt', 'false') == "true" else False mlt_count = int(request.query_params.get('mlt.count', 5)) mlt_filter_queries = request.query_params.getlist('mlt.qf', []) mlt_fq_dict = {} for fq in mlt_filter_queries: if ":" in fq: k, v = fq.split(":", maxsplit=1) mlt_fq_dict[k] = v record = ElasticSearchRDFRecord(hub_id=pk) record.get_graph_by_id(hub_id=pk) response = NaveItemResponse( query, self, index=self.get_index_name, mlt=mlt, mlt_count=mlt_count, mlt_filter_query=mlt_fq_dict, rdf_record=record ) renderer_format = request.accepted_renderer.format if renderer_format in list(EXTENSION_TO_MIME_TYPE.keys()) and renderer_format not in ['xml', 'json']: graph = record.get_graph() graph_string = graph.serialize(format=renderer_format).decode('utf-8') mime_type = EXTENSION_TO_MIME_TYPE.get(renderer_format) return Response(data=graph_string, content_type=mime_type) target_uri = record.document_uri if settings.RDF_USE_LOCAL_GRAPH: graph = record.get_graph() else: store = rdfstore.get_rdfstore() graph, _ = RDFModel.get_context_graph(store, named_graph=record.named_graph) if not graph: from django.http import HttpResponseNotFound return HttpResponseNotFound() mode = get_mode(self.default_converter) bindings = GraphBindings(about_uri=target_uri, graph=graph) delving_fields = False if request.GET.get("delving_fields") == 'false' else True converter = None if mode in ['api', 'api-flat']: index_doc = bindings.to_index_doc() if mode == 'api' else bindings.to_flat_index_doc() elif mode in REGISTERED_CONVERTERS.keys(): converter = REGISTERED_CONVERTERS.get(mode) index_doc = converter( bindings=bindings, graph=graph, about_uri=bindings.about_uri() ).convert(add_delving_fields=delving_fields) elif self.default_converter in REGISTERED_CONVERTERS.keys(): converter = REGISTERED_CONVERTERS.get(self.default_converter) index_doc = converter( bindings=bindings, graph=graph, about_uri=bindings.about_uri() ).convert(add_delving_fields=delving_fields) else: logger.warn("unable to convert results to schema {}".format(mode)) index_doc = bindings.to_index_doc() layout_fields = OrderedDict() layout_fields['layout'] = converter().get_layout_fields() if converter else [] if response.get_mlt(): mlt = {"item": [NaveESItemSerializer(item).data for item in response.get_mlt()]} else: mlt = "" result = {'result': { 'layout': layout_fields, 'item': {'fields': index_doc}, "relatedItems": mlt}} return Response(result)