def get_context_data(self, **kwargs):
context = super(NaveDocumentTemplateView, self).get_context_data(**kwargs)
absolute_uri = self.request.build_absolute_uri()
target_uri = RDFRecord.get_internal_rdf_base_uri(absolute_uri)
if "detail/foldout/" in target_uri:
slug = self.kwargs.get('slug')
record = ElasticSearchRDFRecord(hub_id=slug)
graph = record.get_graph_by_id(self.kwargs.get('slug'))
if graph is not None:
target_uri = record.source_uri
else:
logger.warn("Unable to find source_uri for slug: {}".format(slug))
else:
target_uri = RDFRecord.get_internal_rdf_base_uri(absolute_uri)
record = ElasticSearchRDFRecord(hub_id=self.kwargs.get('slug'))
graph = record.get_graph_by_source_uri(target_uri)
if graph is None:
raise UnknownGraph("URI {} is not known in our graph store".format(target_uri))
if "/resource/cache/" in target_uri:
target_uri = target_uri.rstrip('/')
cache_resource = CacheResource.objects.filter(document_uri=target_uri)
if cache_resource.exists():
graph = cache_resource.first().get_graph()
elif settings.RDF_USE_LOCAL_GRAPH:
mode = self.request.REQUEST.get('mode', 'default')
acceptance = True if mode == 'acceptance' else False
context['acceptance'] = acceptance
elif '/resource/aggregation' in target_uri:
target_named_graph = "{}/graph".format(target_uri.rstrip('/'))
graph, nr_levels = RDFModel.get_context_graph(store=rdfstore.get_rdfstore(), named_graph=target_named_graph)
else:
graph, nr_levels = RDFModel.get_context_graph(
store=rdfstore.get_rdfstore(),
target_uri=target_uri
)
# todo: remove: should no longer be necessary with the addition of common.middleware.ForceLangMiddleware
language = self.request.GET.get('lang', None)
if language:
activate(language)
bindings = GraphBindings(
about_uri=target_uri,
graph=graph,
excluded_properties=settings.RDF_EXCLUDED_PROPERTIES
)
context['resources'] = bindings
context['absolute_uri'] = RDFRecord.get_external_rdf_url(target_uri, self.request)
for rdf_type in bindings.get_about_resource().get_types():
search_label = rdf_type.search_label.lower()
content_template = settings.RDF_CONTENT_FOLDOUTS.get(search_label)
if content_template:
self.template_name = content_template
break
context['points'] = RDFModel.get_geo_points(graph)
return context
def test_get_geo_points(self):
graph = Graph()
subject = URIRef('joachim')
graph.add((subject, URIRef("http://www.w3.org/2003/01/geo/wgs84_pos#lat"), Literal("51.54")))
graph.add((subject, URIRef("http://www.w3.org/2003/01/geo/wgs84_pos#long"), Literal("5.2")))
points = RDFModel.get_geo_points(graph)
assert points is not None
assert points == [[51.54, 5.2]]
graph = Graph()
points = RDFModel.get_geo_points(graph)
assert points == []
def test_bindings_get_graph_from_sparql_results(self):
from .resources import sparqlwrapper_result as result
sparql_json = result.sparql_result
graph, nr_level = RDFModel.get_graph_from_sparql_results(
sparql_json,
named_graph="http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph"
)
assert graph is not None
assert graph.identifier == URIRef("http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph")
assert len(graph) == 118
predicates = set(graph.predicates())
assert len(predicates) != 0
assert len(predicates) == 67
assert URIRef("http://purl.org/dc/elements/1.1/subject") in list(predicates)
rdf_types = sorted(set(graph.objects(predicate=RDF.type)))
assert rdf_types == sorted({URIRef('http://www.europeana.eu/schemas/edm/ProvidedCHO'),
URIRef('http://www.europeana.eu/schemas/edm/WebResource'),
URIRef('http://schemas.delving.eu/nave/terms/DelvingResource'),
URIRef('http://www.openarchives.org/ore/terms/Aggregation'),
URIRef('http://schemas.delving.eu/narthex/terms/Record'),
URIRef('http://schemas.delving.eu/nave/terms/BrabantCloudResource'),
URIRef('http://schemas.delving.eu/narthex/terms/Dataset'),
URIRef('http://www.w3.org/2004/02/skos/core#Concept')})
assert Literal("bomen", lang="nl") in graph.preferredLabel(
subject=URIRef('http://data.cultureelerfgoed.nl/semnet/7403e26d-cf33-4372-ad72-a2f9fcf8f63b')
)[0]
bnodes_materialized = list(graph.objects(predicate=URIRef("http://www.openarchives.org/ore/terms/aggregates")))
assert len(bnodes_materialized) == 1
def test_bindings_determine_context_levels(self):
test_list = [
"s",
"p",
"o",
"p2",
"o2",
"p3",
"o3"
]
levels = RDFModel.get_context_triples(test_list)
assert len(levels) == 3
assert len(RDFModel.get_context_triples(test_list[:3])) == 1
assert len(RDFModel.get_context_triples(test_list[:5])) == 2
assert levels[0] == ("s", "p", "o")
assert levels[1] == ("o", "p2", "o2")
assert levels[2] == ("o2", "p3", "o3")
def get_context_data(self, **kwargs):
context = super(NaveDocumentDetailView, self).get_context_data(**kwargs)
target_uri = self.object.document_uri
if "/resource/cache/" in target_uri:
target_uri = target_uri.rstrip('/')
cache_resource = CacheResource.objects.filter(document_uri=target_uri)
if cache_resource.exists():
graph = cache_resource.first().get_graph()
elif settings.RDF_USE_LOCAL_GRAPH:
mode = self.request.REQUEST.get('mode', 'default')
acceptance = True if mode == 'acceptance' else False
context['acceptance'] = acceptance
if isinstance(self.object, EDMRecord):
graph = self.object.get_graph(with_mappings=True, include_mapping_target=True, acceptance=acceptance)
else:
graph = self.object.get_graph(acceptance=acceptance)
elif '/resource/aggregation' in target_uri:
target_named_graph = "{}/graph".format(target_uri.rstrip('/'))
graph, nr_levels = RDFModel.get_context_graph(store=rdfstore.get_rdfstore(), named_graph=target_named_graph)
else:
graph, nr_levels = RDFModel.get_context_graph(
store=rdfstore.get_rdfstore(),
target_uri=target_uri
)
# todo: remove: should no longer be necessary with the addition of common.middleware.ForceLangMiddleware
language = self.request.GET.get('lang', None)
if language:
activate(language)
bindings = GraphBindings(
about_uri=target_uri,
graph=graph,
excluded_properties=settings.RDF_EXCLUDED_PROPERTIES
)
context['resources'] = bindings
for rdf_type in bindings.get_about_resource().get_types():
search_label = rdf_type.search_label.lower()
content_template = settings.RDF_CONTENT_FOLDOUTS.get(search_label)
if content_template:
self.template_name = content_template
break
context['points'] = RDFModel.get_geo_points(graph)
return context
def graph_to_record(graph, ds, bulk=False, content_hash=None, force_insert=False, acceptance=False):
"""Update or create an EDM record from a graph."""
exclude_key = URIRef('http://schemas.delving.eu/nave/terms/excludedFromEuropeana')
org_id = settings.ORG_ID
spec = ds.spec
identifier = re.sub("[/]+graph$", "", graph.identifier)
local_id = identifier.split('/')[-1]
hub_id = "{}_{}_{}".format(org_id, spec, local_id)
exclude_from_europeana = RDFModel.get_first_literal(exclude_key, graph)
if exclude_from_europeana is None:
exclude_from_europeana = False
elif not isinstance(exclude_from_europeana, bool):
exclude_from_europeana = True if exclude_from_europeana.lower() in ['true'] else False
if content_hash and not force_insert:
query_filter = {'hub_id': hub_id}
if acceptance:
query_filter['acceptance_hash'] = content_hash
else:
query_filter['source_hash'] = content_hash
if EDMRecord.objects.filter(**query_filter).exists():
return None
group, _ = Group.objects.get_or_create(name='dataset_admin')
# get real identifier (support for with or without final slash)
update_values = {
"hub_id": hub_id,
"dataset": ds,
"named_graph": graph.identifier,
"document_uri": identifier,
"source_uri": identifier,
"local_id": local_id,
"rdf_in_sync": False,
"rdf_sync_error": None,
"orphaned": False,
"exclude_from_europeana": exclude_from_europeana
}
# add content hash check
if acceptance:
update_values["acceptance_rdf"] = graph.serialize(format='nt', encoding="UTF-8")
update_values["acceptance_updated"] = timezone.now().strftime(fmt)
if content_hash:
update_values['acceptance_hash'] = content_hash
else:
update_values["source_rdf"] = graph.serialize(format='nt', encoding="UTF-8")
update_values["source_updated"] = timezone.now().strftime(fmt)
if content_hash:
update_values['source_hash'] = content_hash
if not bulk:
edm_record, _ = EDMRecord.objects.update_or_create(hub_id=hub_id, defaults=update_values)
edm_record.groups.add(*ds.groups.all())
else:
update_values['hub_id'] = hub_id
edm_record = EDMRecord(**update_values)
return edm_record
def test_binding_with_uri(self):
uri_object_dict = {
'o': {
'value': 'http://localhost:8000/resource/dataset/ton-smits-huis',
'type': 'uri'
}
}
obj = RDFModel.get_object_from_sparql_result(uri_object_dict['o'])
assert obj is not None
assert isinstance(obj, URIRef)
assert obj == URIRef(uri_object_dict['o']['value'])
def test_context_graph_inlining(self):
named_graph = "http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph"
inline_id = "http://data.cultureelerfgoed.nl/semnet/7403e26d-cf33-4372-ad72-a2f9fcf8f63b"
context_graph, nr_levels = RDFModel.get_context_graph(
store=self.store,
named_graph=named_graph
)
self.assertIsNotNone(context_graph)
self.assertIsInstance(context_graph, Graph)
assert URIRef(inline_id) in list(context_graph.subjects())
predicates = set(list(context_graph.predicates()))
assert URIRef('http://www.openarchives.org/ore/terms/aggregates') in predicates
def test_get_graph_statistics(self):
graph = self.graph
stats = RDFModel.get_graph_statistics(graph=graph)
assert stats is not None
assert isinstance(stats, dict)
assert 'language' in stats
assert stats['language'] == [('nl', 1)]
assert sorted(stats['RDF class']) == sorted(
[('skos:Concept', 7), ('nave:BrabantCloudResource', 1), ('narthex:Record', 1), ('edm:ProvidedCHO', 1),
('edm:WebResource', 1), ('nave:DelvingResource', 1), ('ore:Aggregation', 1), ('narthex:Dataset', 1)]
)
assert 'property' in stats
assert len(stats['property']) > 0
def test_bindings_get_graph_from_sparql_results_without_named_graph(self):
from .resources import sparqlwrapper_result_graph_name as result
sparql_json = result.sparql_result
graph, nr_level = RDFModel.get_graph_from_sparql_results(
sparql_json
)
assert graph is not None
assert graph.identifier == URIRef("http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph")
assert len(graph) == 118
predicates = set(graph.predicates())
assert len(predicates) != 0
assert len(predicates) == 67
def synchronise_record(graph_uri, ds, store, es_actions, index=settings.SITE_NAME, acceptance=False):
# get graph
# graph = store.get_graph_store.get(graph_uri, as_graph=True)
graph, levels = RDFModel.get_context_graph(named_graph=graph_uri, store=store)
# create EDM record
edm_record = EDMRecord.graph_to_record(graph, ds)
# create ES insert action
index_action = edm_record.create_es_action(
index=get_index_name(acceptance),
record_type="Aggregated",
store=store,
context=False,
exclude_fields=ds.excluded_index_fields.names()
)
es_actions.append(index_action)
return edm_record, index_action
def test_with_language_literal(self):
literal_object_dict = {
'o3': {
'value': 'bomen',
'xml:lang': 'nl',
'type': 'literal'
}
}
obj = RDFModel.get_object_from_sparql_result(literal_object_dict)
assert obj is not None
assert isinstance(obj, Literal)
assert obj.language == "nl"
assert obj.datatype is None
assert obj == Literal(
literal_object_dict['o3']['value'],
lang="nl"
)
def test_binding_with_typed_literal(self):
object_dict = {
'o2': {
'value': 'false',
'type': 'typed-literal',
'datatype': 'http://www.w3.org/2001/XMLSchema#boolean'
}
}
obj = RDFModel.get_object_from_sparql_result(object_dict)
assert obj is not None
assert isinstance(obj, Literal)
assert obj.language is None
assert obj.datatype is not None
assert obj.datatype == URIRef("http://www.w3.org/2001/XMLSchema#boolean")
assert obj == Literal(
object_dict['o2']['value'],
datatype="http://www.w3.org/2001/XMLSchema#boolean"
)
def setUp(self):
from .resources import sparqlwrapper_result as result
sparql_json = result.sparql_result
self.graph, self.nr_levels = RDFModel.get_graph_from_sparql_results(sparql_json)
def retrieve(self, request, pk=None, format=None, *args, **kwargs):
def get_mode(default=None):
params = request.GET
return params.get('schema', default)
self._clean_callback(request)
query = NaveESQuery(
index_name=self.get_index_name,
doc_types=self.doc_types,
default_facets=self.facets,
cluster_geo=False,
size=1,
converter=self.get_converter()
)
try:
query = query.build_item_query(query, request.query_params, pk)
except ValueError as ve:
logger.error("Unable to build request because: {}".format(ve))
# todo display error message when bad/unknown hubId is given
return HttpResponseBadRequest()
mlt = True if request.query_params.get('mlt', 'false') == "true" else False
mlt_count = int(request.query_params.get('mlt.count', 5))
mlt_filter_queries = request.query_params.getlist('mlt.qf', [])
mlt_fq_dict = {}
for fq in mlt_filter_queries:
if ":" in fq:
k, v = fq.split(":", maxsplit=1)
mlt_fq_dict[k] = v
record = ElasticSearchRDFRecord(hub_id=pk)
record.get_graph_by_id(hub_id=pk)
response = NaveItemResponse(
query,
self,
index=self.get_index_name,
mlt=mlt,
mlt_count=mlt_count,
mlt_filter_query=mlt_fq_dict,
rdf_record=record
)
renderer_format = request.accepted_renderer.format
if renderer_format in list(EXTENSION_TO_MIME_TYPE.keys()) and renderer_format not in ['xml', 'json']:
graph = record.get_graph()
graph_string = graph.serialize(format=renderer_format).decode('utf-8')
mime_type = EXTENSION_TO_MIME_TYPE.get(renderer_format)
return Response(data=graph_string, content_type=mime_type)
target_uri = record.document_uri
if settings.RDF_USE_LOCAL_GRAPH:
graph = record.get_graph()
else:
store = rdfstore.get_rdfstore()
graph, _ = RDFModel.get_context_graph(store, named_graph=record.named_graph)
if not graph:
from django.http import HttpResponseNotFound
return HttpResponseNotFound()
mode = get_mode(self.default_converter)
bindings = GraphBindings(about_uri=target_uri, graph=graph)
delving_fields = False if request.GET.get("delving_fields") == 'false' else True
converter = None
if mode in ['api', 'api-flat']:
index_doc = bindings.to_index_doc() if mode == 'api' else bindings.to_flat_index_doc()
elif mode in REGISTERED_CONVERTERS.keys():
converter = REGISTERED_CONVERTERS.get(mode)
index_doc = converter(
bindings=bindings,
graph=graph,
about_uri=bindings.about_uri()
).convert(add_delving_fields=delving_fields)
elif self.default_converter in REGISTERED_CONVERTERS.keys():
converter = REGISTERED_CONVERTERS.get(self.default_converter)
index_doc = converter(
bindings=bindings,
graph=graph,
about_uri=bindings.about_uri()
).convert(add_delving_fields=delving_fields)
else:
logger.warn("unable to convert results to schema {}".format(mode))
index_doc = bindings.to_index_doc()
layout_fields = OrderedDict()
layout_fields['layout'] = converter().get_layout_fields() if converter else []
if response.get_mlt():
mlt = {"item": [NaveESItemSerializer(item).data for item in response.get_mlt()]}
else:
mlt = ""
result = {'result': {
'layout': layout_fields,
'item': {'fields': index_doc},
"relatedItems": mlt}}
return Response(result)
