def testDocLocal(self):
uri = rdflib.URIRef('http://id.example.org/foo')
self.assertEqual(doc_forward(uri, format='n3'),
'http://data.example.org/doc/foo.n3')
uri = rdflib.URIRef('http://random.example.org/id/foo')
self.assertEqual(doc_forward(uri),
'http://data.example.org/doc:random/foo')
def testDocLocal(self):
uri = rdflib.URIRef('http://id.example.org/foo')
self.assertEqual(doc_forward(uri, format='n3'),
'http://data.example.org/doc/foo.n3')
uri = rdflib.URIRef('http://random.example.org/id/foo')
self.assertEqual(doc_forward(uri),
'http://data.example.org/doc:random/foo')
def url_for_format(self, request, format):
if 'subject_uri' in self.context:
return doc_forward(self.context['subject_uri'],
described=True,
format=format)
else:
return super(DocView, self).url_for_format(request, format)
def get(self, request):
uri = rdflib.URIRef(request.GET.get('uri', ''))
try:
url = urlparse.urlparse(uri)
except Exception:
raise Http404
if not IRI.match(uri):
return HttpResponseTemporaryRedirect(unicode(uri))
elif request.GET.get('source') == 'purl':
return HttpResponseSeeOther(doc_forward(uri, described=True))
elif self.get_types(uri):
return HttpResponsePermanentRedirect(doc_forward(uri, described=True))
elif url.scheme in ('http', 'https') and url.netloc and url.path.startswith('/'):
return HttpResponseTemporaryRedirect(unicode(uri))
else:
raise Http404
def get(self, request):
uri = rdflib.URIRef(request.GET.get('uri', ''))
try:
url = urlparse.urlparse(uri)
except Exception:
raise Http404
try:
token = request.GET['token']
except KeyError:
raise Http404
if token != self.get_uri_token(uri):
raise Http404
if not IRI.match(uri):
return HttpResponseTemporaryRedirect(unicode(uri))
elif request.GET.get('source') == 'purl':
return HttpResponseSeeOther(doc_forward(uri, described=True))
elif self.get_types(uri):
return HttpResponsePermanentRedirect(
doc_forward(uri, described=True))
elif url.scheme in (
'http', 'https') and url.netloc and url.path.startswith('/'):
return HttpResponseTemporaryRedirect(unicode(uri))
else:
raise Http404
def testDocRemote(self):
uri = rdflib.URIRef('http://remote.example.org/foo')
doc_root = 'http://data.example.org/doc/'
desc_root = 'http://data.example.org/desc/'
qs_with_format = '?uri=http%3A%2F%2Fremote.example.org%2Ffoo&format=nt'
qs_without_format = '?uri=http%3A%2F%2Fremote.example.org%2Ffoo'
# With no indication as to whether we know about it
self.assertEqual(doc_forward(uri),
desc_root + qs_without_format)
self.assertEqual(doc_forward(uri, format='nt'),
desc_root + qs_with_format)
# When we provide a graph that doesn't mention it
graph = rdflib.ConjunctiveGraph()
self.assertEqual(doc_forward(uri, graph=graph),
desc_root + qs_without_format)
self.assertEqual(doc_forward(uri, graph=graph, format='nt'),
desc_root + qs_with_format)
# Now our graph knows something about it
graph.add((uri, rdflib.URIRef('http://example.org/predicate'), rdflib.Literal('foo')))
self.assertEqual(doc_forward(uri, graph=graph),
doc_root + qs_without_format)
self.assertEqual(doc_forward(uri, graph=graph, format='nt'),
doc_root + qs_with_format)
# When we definitely know nothing about it, go straight off-site
self.assertEqual(doc_forward(uri, described=False), unicode(uri))
self.assertEqual(doc_forward(uri, described=False, format='nt'), unicode(uri))
# When we definitely know something about it, go straight off-site
self.assertEqual(doc_forward(uri, described=True),
doc_root + qs_without_format)
self.assertEqual(doc_forward(uri, described=True, format='nt'),
doc_root + qs_with_format)
def testDocRemote(self):
uri = rdflib.URIRef('http://remote.example.org/foo')
doc_root = 'http://data.example.org/doc/'
desc_root = 'http://data.example.org/desc/'
qs_with_format = '?uri=http%3A%2F%2Fremote.example.org%2Ffoo&format=nt'
qs_without_format = '?uri=http%3A%2F%2Fremote.example.org%2Ffoo'
# With no indication as to whether we know about it
self.assertRelativeEqual(doc_forward(uri),
desc_root + qs_without_format)
self.assertRelativeEqual(doc_forward(uri, format='nt'),
desc_root + qs_with_format)
# When we provide a graph that doesn't mention it
graph = rdflib.ConjunctiveGraph()
self.assertRelativeEqual(doc_forward(uri, graph=graph),
desc_root + qs_without_format)
self.assertRelativeEqual(doc_forward(uri, graph=graph, format='nt'),
desc_root + qs_with_format)
# Now our graph knows something about it
graph.add((uri, rdflib.URIRef('http://example.org/predicate'), rdflib.Literal('foo')))
self.assertRelativeEqual(doc_forward(uri, graph=graph),
doc_root + qs_without_format)
self.assertRelativeEqual(doc_forward(uri, graph=graph, format='nt'),
doc_root + qs_with_format)
# When we definitely know nothing about it, go straight off-site
self.assertRelativeEqual(doc_forward(uri, described=False), unicode(uri))
self.assertRelativeEqual(doc_forward(uri, described=False, format='nt'), unicode(uri))
# When we definitely know something about it, go straight off-site
self.assertRelativeEqual(doc_forward(uri, described=True),
doc_root + qs_without_format)
self.assertRelativeEqual(doc_forward(uri, described=True, format='nt'),
doc_root + qs_with_format)
def get(self, request):
uri = rdflib.URIRef(request.build_absolute_uri())
if not IRI.match(uri):
raise Http404("Invalid IRI")
if not self.get_types(uri):
raise Http404("URI has no types; not known around here")
description_url = doc_forward(uri, described=True)
for pattern, target, mimetypes in self.id_mapping_redirects:
match = pattern.match(str(uri))
if match and self.override_redirect(request, description_url, mimetypes):
description_url = target % match.groupdict()
break
return HttpResponseSeeOther(description_url)
def get(self, request):
uri = rdflib.URIRef(request.build_absolute_uri())
if not IRI.match(uri):
raise Http404("Invalid IRI")
if not self.get_types(uri):
raise Http404("URI has no types; not known around here")
description_url = doc_forward(uri, described=True)
for pattern, target, mimetypes in self.id_mapping_redirects:
match = pattern.match(str(uri))
if match and self.override_redirect(request, description_url,
mimetypes):
description_url = target % match.groupdict()
break
return HttpResponseSeeOther(description_url)
def doc_url(uri):
return doc_forward(uri)
def url_for_format(self, request, format):
if 'subject_uri' in self.context:
return doc_forward(self.context['subject_uri'], described=True, format=format)
else:
return super(DocView, self).url_for_format(request, format)
def get(self, request):
additional_headers = {}
doc_url = request.build_absolute_uri()
uri, format, is_local = doc_backward(doc_url, set(self._renderers_by_format))
if not uri:
logger.debug("Could not resolve URL to a URI: %r", doc_url)
raise Http404
expected_doc_url = doc_forward(uri, request, format=format, described=True)
types = self.get_types(uri)
if not types:
logger.debug("Resource has no type, so is probably not known in these parts: %r", uri)
raise Http404
if self.check_canonical and expected_doc_url != doc_url:
logger.debug("Request for a non-canonical doc URL (%r) for %r, redirecting to %r", doc_url, uri, expected_doc_url)
return HttpResponsePermanentRedirect(expected_doc_url)
# If no format was given explicitly (i.e. format parameter or
# extension) we inspect the Content-Type header.
if not format:
renderers = self.get_renderers(request)
if renderers:
format = renderers[0].format
expected_doc_url = doc_forward(uri, request, format=format, described=True)
if expected_doc_url != doc_url:
additional_headers['Content-Location'] = expected_doc_url
doc_uri = rdflib.URIRef(doc_forward(uri, request, format=None, described=True))
context = {
'subject_uri': uri,
'doc_uri': doc_uri,
'format': format,
'types': types,
'show_follow_link': not is_local,
'no_index': not is_local,
'additional_headers': additional_headers,
}
subject_uri, doc_uri = context['subject_uri'], context['doc_uri']
types = context['types']
queries, graph = [], rdflib.ConjunctiveGraph()
for prefix, namespace_uri in NS.iteritems():
graph.namespace_manager.bind(prefix, namespace_uri)
graph += ((subject_uri, NS.rdf.type, t) for t in types)
subject = Resource(subject_uri, graph, self.endpoint)
for query in subject.get_queries():
graph += self.endpoint.query(query)
queries.append(query)
licenses, datasets = set(), set()
for graph_name in graph.subjects(NS['ov'].describes):
graph.add((doc_uri, NS['dcterms'].source, graph_name))
licenses.update(graph.objects(graph_name, NS['dcterms'].license))
datasets.update(graph.objects(graph_name, NS['void'].inDataset))
if len(licenses) == 1:
for license_uri in licenses:
graph.add((doc_uri, NS['dcterms'].license, license_uri))
if not graph:
logger.debug("Graph for %r was empty; 404ing", uri)
raise Http404
for doc_rdf_processor in self._doc_rdf_processors:
additional_context = doc_rdf_processor(request=request,
graph=graph,
doc_uri=doc_uri,
subject_uri=subject_uri,
subject=subject,
endpoint=self.endpoint,
renderers=self._renderers)
if additional_context:
context.update(additional_context)
context.update({
'graph': graph,
'subject': subject,
'licenses': [Resource(uri, graph, self.endpoint) for uri in licenses],
'datasets': [Resource(uri, graph, self.endpoint) for uri in datasets],
'queries': queries,
'template_name': subject.template_name,
})
template_name = subject.template_name or self.template_name
for template_override in self.template_overrides:
tn, types = template_override[0], template_override[1:]
print tn, types, subject.get_all('rdf:type')
if set(subject._graph.objects(subject._identifier, NS.rdf.type)) & set(map(expand, types)):
template_name = tn
break
if context['format']:
try:
return self.render_to_format(request, context, template_name, format)
except KeyError:
raise Http404
else:
return self.render(request, context, template_name)
def testDocLocalNegotiate(self):
uri = rdflib.URIRef('http://id.example.org/foo')
self.assertEqual(doc_forward(uri),
'http://data.example.org/doc/foo')
def testUnicodeBackward(self):
for uri, url in self.TESTS:
if isinstance(uri, unicode):
self.assertRelativeEqual(doc_backward(url)[0], uri)
self.assertRelativeEqual(doc_forward(doc_backward(url)[0], described=True), url)
def testUnicodeForward(self):
for uri, url in self.TESTS:
self.assertRelativeEqual(doc_forward(uri, described=True), url)
def doc_url(self):
if '_doc_url' not in self.__dict__:
self._doc_url = doc_forward(self._identifier, graph=self._graph)
return self._doc_url
def upload_dataset_metadata(sender, store, graphs, when, **kwargs):
if store.slug != DEFAULT_STORE_SLUG:
return
if not graphs:
logger.debug("No graphs updated for %r; aborting", store.slug)
return
if not getattr(settings, 'CKAN_API_KEY', None):
logger.debug("No CKAN_API_KEY setting, not doing anything.")
return
client = ckanclient.CkanClient(api_key=settings.CKAN_API_KEY)
endpoint = Endpoint(settings.ENDPOINT_QUERY)
query = _dataset_query % ' \n'.join('(%s)' % rdflib.URIRef(g).n3() for g in graphs)
graph = endpoint.query(query)
datasets = list(graph.subjects(NS.rdf.type, NS.void.Dataset))
if len(datasets) != 1:
logger.debug("Expected one dataset; got %d", len(datasets))
return
dataset = Resource(datasets[0], graph, endpoint)
find = functools.partial(_find, graph, dataset._identifier)
patterns = settings.CKAN_PATTERNS
package_name = find('skos:notation', HUMFREY.theDataHubDatasetName)
if not package_name:
return
#package_name = patterns.get('name', '%s') % slug
package_title = patterns.get('title', '%s') % dataset.label
author = find('dcterms:publisher/foaf:name|rdfs:label|dc:title|skos:prefLabel|dcterms:title')
if author:
author = patterns.get('author', '%s') % author
description = find('rdfs:comment|dcterms:description',
(NS.xtypes['Fragment-Markdown'],
NS.xtypes['Fragment-PlainText'],
None))
maintainer = find('oo:contact/foaf:name|rdfs:label|dc:title|skos:prefLabel|dcterms:title')
if maintainer:
maintainer = patterns.get('maintainer', '%s') % maintainer
maintainer_email = find('oo:contact/foaf:mbox|v:email')
if maintainer_email:
maintainer_email = maintainer_email.replace('mailto:', '')
license = find('dcterms:license|cc:license')
if license:
license = _licenses.get(unicode(license))
sparql_endpoint = find('void:sparqlEndpoint')
if sparql_endpoint:
sparql_endpoint = unicode(sparql_endpoint)
else:
sparql_endpoint = 'http:' + reverse_full('data', 'sparql:endpoint')
tags = find('humfrey:theDataHubDatasetTag', all=True)
groups = find('humfrey:theDataHubDatasetGroup', all=True)
url = doc_forward(dataset.uri)
logger.debug("Fetching existing record for %r", package_name)
try:
package_entity = client.package_entity_get(package_name)
logger.debug("Record successfully retrieved")
except ckanclient.CkanApiNotFoundError:
package_entity = {'name': package_name}
client.package_register_post(package_entity)
logger.debug("No record found; starting from empty")
original = copy.deepcopy(package_entity)
package_entity.update({'name': package_name,
'title': package_title,
'url': url,
'notes': description,
'license_id': license,
'author': author,
'maintainer': maintainer,
'maintainer_email': dataset.oo_contact.get_one_of('foaf:mbox', 'v:email').replace('mailto:', '', 1)})
package_entity['groups'] = list(settings.CKAN_GROUPS
| set(package_entity.get('groups', ()))
| groups)
package_entity['tags'] = list(settings.CKAN_TAGS
| set(package_entity.get('tags', ()))
| tags)
resources = collections.defaultdict(dict, ((r.get('name'), r) for r in package_entity.get('resources', ())))
resources['SPARQL endpoint'].update({'name': 'SPARQL endpoint',
'format': 'api/sparql',
'url': sparql_endpoint})
package_entity['resources'] = resources.values()
logger.debug("Updated CKAN record")
if original != package_entity:
logger.info("Updating %r at thedatahub.org", package_name)
client.package_entity_put(package_entity)
def testDocLocalNegotiateMissing(self):
uri = rdflib.URIRef('http://id.example.org/foo')
self.assertEqual(doc_forward(uri),
'http://data.example.org/doc/foo')
def get(self, request):
additional_headers = {}
# Apache helpfully(!?) unescapes encoded hash characters. If we get one
# we know that the browser sent a '%23' (or else would have stripped it
# as a fragment identifier. We replace it with a '%23' so that our URI
# canonicalisation doesn't get stuck in an endless redirect loop.
doc_url = request.build_absolute_uri().replace('#', '%23')
# Given a URL 'http://example.org/doc/foo.bar' we check whether 'foo',
# has a type (ergo 'bar' is a format), and if not we assume that
# 'foo.bar' is part of the URI
for formats in (None, ()):
uri, format, is_local = doc_backward(doc_url, formats)
if uri and not IRI.match(uri):
raise Http404("Invalid IRI")
if not uri:
logger.debug("Could not resolve URL to a URI: %r", doc_url)
raise Http404("Could not resolve URL to a URI")
types = self.get_types(uri)
if types:
break
doc_url = doc_url.rsplit('.', 1)[0]
else:
logger.debug("Resource has no type, so is probably not known in these parts: %r", uri)
raise Http404("Resource has no type, so is probably not known in these parts")
expected_doc_url = urlparse.urljoin(doc_url, doc_forward(uri, request, format=format, described=True))
if self.check_canonical and expected_doc_url != doc_url:
logger.debug("Request for a non-canonical doc URL (%r) for %r, redirecting to %r", doc_url, uri, expected_doc_url)
return HttpResponsePermanentRedirect(expected_doc_url)
doc_uri = rdflib.URIRef(doc_forward(uri, request, format=None, described=True))
self.context.update({
'subject_uri': uri,
'doc_uri': doc_uri,
'format': format,
'types': types,
'show_follow_link': not is_local,
'no_index': not is_local,
'additional_headers': additional_headers,
})
subject_uri, doc_uri = self.context['subject_uri'], self.context['doc_uri']
types = self.context['types']
queries, graph = [], rdflib.ConjunctiveGraph()
for prefix, namespace_uri in NS.iteritems():
graph.namespace_manager.bind(prefix, namespace_uri)
graph += ((subject_uri, NS.rdf.type, t) for t in types)
subject = Resource(subject_uri, graph, self.endpoint)
for query in subject.get_queries():
graph += self.endpoint.query(query)
queries.append(query)
licenses, datasets = set(), set()
for graph_name in graph.subjects(NS['ov'].describes):
graph.add((doc_uri, NS['dcterms'].source, graph_name))
licenses.update(graph.objects(graph_name, NS['dcterms'].license))
datasets.update(graph.objects(graph_name, NS['void'].inDataset))
if len(licenses) == 1:
for license_uri in licenses:
graph.add((doc_uri, NS['dcterms'].license, license_uri))
if not graph:
logger.debug("Graph for %r was empty; 404ing", uri)
raise Http404("Graph was empty")
self.template_name = subject.template_name or self.template_name
for template_override in self.template_overrides:
tn, types = template_override[0], template_override[1:]
if set(subject._graph.objects(subject._identifier, NS.rdf.type)) & set(map(expand, types)):
self.template_name = tn
break
self.context.update({
'graph': graph,
'subject': subject,
'licenses': [Resource(uri, graph, self.endpoint) for uri in licenses],
'datasets': [Resource(uri, graph, self.endpoint) for uri in datasets],
'queries': map(self.endpoint.normalize_query, queries),
'template_name': self.template_name,
})
self.set_renderers()
for doc_rdf_processor in self._doc_rdf_processors:
additional_context = doc_rdf_processor(self.request, self.context)
if additional_context:
self.context.update(additional_context)
# If no format was given explicitly (i.e. format parameter or
# extension) we inspect the Content-Type header.
if not format:
if request.renderers:
format = request.renderers[0].format
expected_doc_url = doc_forward(uri, request, format=format, described=True)
if expected_doc_url != doc_url:
additional_headers['Content-Location'] = expected_doc_url
# NOTE: This getattrs every atttr on subject, so would force
# memoization on any cached attributes. We call it as late as
# possible to make sure the graph won't change afterwards, making
# those cached results incorrect.
self.conneg += subject
if self.context['format']:
try:
return self.render_to_format(format=format)
except KeyError:
raise Http404
else:
return self.render()
def testUnicodeForward(self):
for uri, url in self.TESTS:
self.assertEqual(doc_forward(uri, described=True), url)
def doc_url(self):
return doc_forward(self._identifier, graph=self._graph)
def get(self, request):
additional_headers = {}
# Apache helpfully(!?) unescapes encoded hash characters. If we get one
# we know that the browser sent a '%23' (or else would have stripped it
# as a fragment identifier. We replace it with a '%23' so that our URI
# canonicalisation doesn't get stuck in an endless redirect loop.
doc_url = request.build_absolute_uri().replace('#', '%23')
# Given a URL 'http://example.org/doc/foo.bar' we check whether 'foo',
# has a type (ergo 'bar' is a format), and if not we assume that
# 'foo.bar' is part of the URI
for formats in (None, ()):
uri, format, is_local = doc_backward(doc_url, formats)
if uri and not IRI.match(uri):
raise Http404("Invalid IRI")
if not uri:
logger.debug("Could not resolve URL to a URI: %r", doc_url)
raise Http404("Could not resolve URL to a URI")
types = self.get_types(uri)
if types:
break
doc_url = doc_url.rsplit('.', 1)[0]
else:
logger.debug(
"Resource has no type, so is probably not known in these parts: %r",
uri)
raise Http404(
"Resource has no type, so is probably not known in these parts"
)
expected_doc_url = urlparse.urljoin(
doc_url, doc_forward(uri, request, format=format, described=True))
if self.check_canonical and expected_doc_url != doc_url:
logger.debug(
"Request for a non-canonical doc URL (%r) for %r, redirecting to %r",
doc_url, uri, expected_doc_url)
return HttpResponsePermanentRedirect(expected_doc_url)
doc_uri = rdflib.URIRef(
doc_forward(uri, request, format=None, described=True))
self.context.update({
'subject_uri': uri,
'doc_uri': doc_uri,
'format': format,
'types': types,
'show_follow_link': not is_local,
'no_index': not is_local,
'additional_headers': additional_headers,
})
subject_uri, doc_uri = self.context['subject_uri'], self.context[
'doc_uri']
types = self.context['types']
queries, graph = [], rdflib.ConjunctiveGraph()
for prefix, namespace_uri in NS.iteritems():
graph.namespace_manager.bind(prefix, namespace_uri)
graph += ((subject_uri, NS.rdf.type, t) for t in types)
subject = Resource(subject_uri, graph, self.endpoint)
for query in subject.get_queries():
graph += self.endpoint.query(query)
queries.append(query)
licenses, datasets = set(), set()
for graph_name in graph.subjects(NS['ov'].describes):
graph.add((doc_uri, NS['dcterms'].source, graph_name))
licenses.update(graph.objects(graph_name, NS['dcterms'].license))
datasets.update(graph.objects(graph_name, NS['void'].inDataset))
if len(licenses) == 1:
for license_uri in licenses:
graph.add((doc_uri, NS['dcterms'].license, license_uri))
if not graph:
logger.debug("Graph for %r was empty; 404ing", uri)
raise Http404("Graph was empty")
self.template_name = subject.template_name or self.template_name
for template_override in self.template_overrides:
tn, types = template_override[0], template_override[1:]
if set(subject._graph.objects(
subject._identifier, NS.rdf.type)) & set(map(
expand, types)):
self.template_name = tn
break
self.context.update({
'graph':
graph,
'subject':
subject,
'licenses':
[Resource(uri, graph, self.endpoint) for uri in licenses],
'datasets':
[Resource(uri, graph, self.endpoint) for uri in datasets],
'queries':
map(self.endpoint.normalize_query, queries),
'template_name':
self.template_name,
})
self.set_renderers()
for doc_rdf_processor in self._doc_rdf_processors:
additional_context = doc_rdf_processor(self.request, self.context)
if additional_context:
self.context.update(additional_context)
# If no format was given explicitly (i.e. format parameter or
# extension) we inspect the Content-Type header.
if not format:
if request.renderers:
format = request.renderers[0].format
expected_doc_url = doc_forward(uri,
request,
format=format,
described=True)
if expected_doc_url != doc_url:
additional_headers['Content-Location'] = expected_doc_url
# NOTE: This getattrs every atttr on subject, so would force
# memoization on any cached attributes. We call it as late as
# possible to make sure the graph won't change afterwards, making
# those cached results incorrect.
self.conneg += subject
if self.context['format']:
try:
return self.render_to_format(format=format)
except KeyError:
raise Http404
else:
return self.render()
def doc_url(self):
if '_doc_url' not in self.__dict__:
self._doc_url = doc_forward(self._identifier, graph=self._graph)
return self._doc_url
def doc_url(uri):
return doc_forward(uri)