def load_locally(url):
options = test.data.get('option', {})
content_type = options.get('contentType')
url_no_frag = strip_fragment(url)
if not content_type and url_no_frag.endswith('.jsonld'):
content_type = 'application/ld+json'
if not content_type and url_no_frag.endswith('.json'):
content_type = 'application/json'
if not content_type and url_no_frag.endswith('.html'):
content_type = 'text/html'
if not content_type:
content_type = 'application/octet-stream'
doc = {
'contentType': content_type,
'contextUrl': None,
'documentUrl': url,
'document': None
}
if options and url == test.base:
if ('redirectTo' in options and options.get('httpStatus') >= 300):
doc['documentUrl'] = (test.manifest.data['baseIri'] +
options['redirectTo'])
elif 'httpLink' in options:
link_header = options.get('httpLink', '')
if isinstance(link_header, list):
link_header = ','.join(link_header)
linked_context = jsonld.parse_link_header(link_header).get(
'http://www.w3.org/ns/json-ld#context')
if linked_context and content_type != 'application/ld+json':
if isinstance(linked_context, list):
raise Exception('multiple context link headers')
doc['contextUrl'] = linked_context['target']
linked_alternate = jsonld.parse_link_header(link_header).get(
'alternate')
# if not JSON-LD, alternate may point there
if (linked_alternate and linked_alternate.get('type')
== 'application/ld+json' and not re.match(
r'^application\/(\w*\+)?json$', content_type)):
doc['contentType'] = 'application/ld+json'
doc['documentUrl'] = jsonld.prepend_base(
url, linked_alternate['target'])
global ROOT_MANIFEST_DIR
if doc['documentUrl'].find(':') == -1:
filename = os.path.join(ROOT_MANIFEST_DIR, doc['documentUrl'])
doc['documentUrl'] = 'file://' + filename
else:
filename = test.dirname + strip_fragment(
strip_base(doc['documentUrl']))
try:
doc['document'] = read_file(filename)
except:
raise Exception('loading document failed')
return doc
def load_locally(url):
doc = {'contextUrl': None, 'documentUrl': url, 'document': None}
options = test.data.get('option')
if options and url == test.base:
if ('redirectTo' in options and options.get('httpStatus') >= 300):
doc['documentUrl'] = (
test.manifest.data['baseIri'] + options['redirectTo'])
elif 'httpLink' in options:
content_type = options.get('contentType')
if not content_type and url.endswith('.jsonld'):
content_type = 'application/ld+json'
link_header = options.get('httpLink', '')
if isinstance(link_header, list):
link_header = ','.join(link_header)
link_header = jsonld.parse_link_header(
link_header).get('http://www.w3.org/ns/json-ld#context')
if link_header and content_type != 'application/ld+json':
if isinstance(link_header, list):
raise Exception('multiple context link headers')
doc['contextUrl'] = link_header['target']
global ROOT_MANIFEST_DIR
if doc['documentUrl'].find(':') == -1:
filename = os.path.join(ROOT_MANIFEST_DIR, doc['documentUrl'])
doc['documentUrl'] = 'file://' + filename
else:
#filename = os.path.join(
# ROOT_MANIFEST_DIR, doc['documentUrl'][len(base):])
filename = ROOT_MANIFEST_DIR + strip_base(doc['documentUrl'])
try:
doc['document'] = read_json(filename)
except:
raise Exception('loading document failed')
return doc
def load_locally(url):
doc = {"contextUrl": None, "documentUrl": url, "document": None}
options = test.data.get("option")
if options and url == test.base:
if "redirectTo" in options and options.get("httpStatus") >= 300:
doc["documentUrl"] = test.manifest.data["baseIri"] + options["redirectTo"]
elif "httpLink" in options:
content_type = options.get("contentType")
if not content_type and url.endswith(".jsonld"):
content_type = "application/ld+json"
link_header = options.get("httpLink", "")
if isinstance(link_header, list):
link_header = ",".join(link_header)
link_header = jsonld.parse_link_header(link_header).get("http://www.w3.org/ns/json-ld#context")
if link_header and content_type != "application/ld+json":
if isinstance(link_header, list):
raise Exception("multiple context link headers")
doc["contextUrl"] = link_header["target"]
global ROOT_MANIFEST_DIR
if doc["documentUrl"].find(":") == -1:
filename = os.path.join(ROOT_MANIFEST_DIR, doc["documentUrl"])
doc["documentUrl"] = "file://" + filename
else:
# filename = os.path.join(
# ROOT_MANIFEST_DIR, doc['documentUrl'][len(base):])
filename = ROOT_MANIFEST_DIR + doc["documentUrl"][len(base) :]
try:
doc["document"] = read_json(filename)
except:
raise Exception("loading document failed")
return doc
def load_locally(url):
doc = {'contextUrl': None, 'documentUrl': url, 'document': None}
options = test.data.get('option')
if options and url == test.base:
if ('redirectTo' in options and options.get('httpStatus') >= 300):
doc['documentUrl'] = (
test.manifest.data['baseIri'] + options['redirectTo'])
elif 'httpLink' in options:
content_type = options.get('contentType')
if not content_type and url.endswith('.jsonld'):
content_type = 'application/ld+json'
link_header = options.get('httpLink', '')
if isinstance(link_header, list):
link_header = ','.join(link_header)
link_header = jsonld.parse_link_header(
link_header).get('http://www.w3.org/ns/json-ld#context')
if link_header and content_type != 'application/ld+json':
if isinstance(link_header, list):
raise Exception('multiple context link headers')
doc['contextUrl'] = link_header['target']
global ROOT_MANIFEST_DIR
if doc['documentUrl'].find(':') == -1:
filename = os.path.join(ROOT_MANIFEST_DIR, doc['documentUrl'])
doc['documentUrl'] = 'file://' + filename
else:
#filename = os.path.join(
# ROOT_MANIFEST_DIR, doc['documentUrl'][len(base):])
filename = ROOT_MANIFEST_DIR + strip_base(doc['documentUrl'])
try:
doc['document'] = read_json(filename)
except:
raise Exception('loading document failed')
return doc
def loader(url):
"""
Retrieves JSON-LD at the given URL.
:param url: the URL to retrieve.
:return: the RemoteDocument.
"""
try:
# validate URL
pieces = urllib_parse.urlparse(url)
if (not all([pieces.scheme, pieces.netloc]) or
pieces.scheme not in ['http', 'https'] or
set(pieces.netloc) > set(
string.ascii_letters + string.digits + '-.:')):
raise JsonLdError(
'URL could not be dereferenced; only "http" and "https" '
'URLs are supported.',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
if secure and pieces.scheme != 'https':
raise JsonLdError(
'URL could not be dereferenced; secure mode enabled and '
'the URL\'s scheme is not "https".',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
headers = {
'Accept': 'application/ld+json, application/json'
}
response = requests.get(url, headers=headers, **kwargs)
doc = {
'contextUrl': None,
'documentUrl': response.url,
'document': response.json()
}
content_type = response.headers.get('content-type')
link_header = response.headers.get('link')
if link_header and content_type != 'application/ld+json':
link_header = parse_link_header(link_header).get(
LINK_HEADER_REL)
# only 1 related link header permitted
if isinstance(link_header, list):
raise JsonLdError(
'URL could not be dereferenced, it has more than one '
'associated HTTP Link Header.',
'jsonld.LoadDocumentError',
{'url': url},
code='multiple context link headers')
if link_header:
doc['contextUrl'] = link_header['target']
return doc
except JsonLdError as e:
raise e
except Exception as cause:
raise JsonLdError(
'Could not retrieve a JSON-LD document from the URL.',
'jsonld.LoadDocumentError', code='loading document failed',
cause=cause)
def loader(url):
"""
Retrieves JSON-LD at the given URL.
:param url: the URL to retrieve.
:return: the RemoteDocument.
"""
try:
# validate URL
pieces = urllib_parse.urlparse(url)
if (not all([pieces.scheme, pieces.netloc])
or pieces.scheme not in ['http', 'https']
or set(pieces.netloc) >
set(string.ascii_letters + string.digits + '-.:')):
raise JsonLdError(
'URL could not be dereferenced; only "http" and "https" '
'URLs are supported.',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
if secure and pieces.scheme != 'https':
raise JsonLdError(
'URL could not be dereferenced; secure mode enabled and '
'the URL\'s scheme is not "https".',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
headers = {'Accept': 'application/ld+json, application/json'}
response = requests.get(url, headers=headers, **kwargs)
doc = {
'contextUrl': None,
'documentUrl': response.url,
'document': response.json()
}
content_type = response.headers.get('content-type')
link_header = response.headers.get('link')
if link_header and content_type != 'application/ld+json':
link_header = parse_link_header(link_header).get(
LINK_HEADER_REL)
# only 1 related link header permitted
if isinstance(link_header, list):
raise JsonLdError(
'URL could not be dereferenced, it has more than one '
'associated HTTP Link Header.',
'jsonld.LoadDocumentError', {'url': url},
code='multiple context link headers')
if link_header:
doc['contextUrl'] = link_header['target']
return doc
except JsonLdError as e:
raise e
except Exception as cause:
raise JsonLdError(
'Could not retrieve a JSON-LD document from the URL.',
'jsonld.LoadDocumentError',
code='loading document failed',
cause=cause)
def loader(url):
"""
Retrieves JSON-LD at the given URL.
:param url: the URL to retrieve.
:return: the RemoteDocument.
"""
try:
# validate URL
pieces = urllib_parse.urlparse(url)
if (not all([pieces.scheme, pieces.netloc])
or pieces.scheme not in ["http", "https"]
or set(pieces.netloc) >
set(string.ascii_letters + string.digits + "-.:")):
raise JsonLdError(
'URL could not be dereferenced; only "http" and "https" '
"URLs are supported.",
"jsonld.InvalidUrl",
{"url": url},
code="loading document failed",
)
if secure and pieces.scheme != "https":
raise JsonLdError(
"URL could not be dereferenced; secure mode enabled and "
'the URL\'s scheme is not "https".',
"jsonld.InvalidUrl",
{"url": url},
code="loading document failed",
)
headers = {"Accept": "application/ld+json, application/json"}
response = session.get(url, headers=headers, **kwargs)
doc = {
"contextUrl": None,
"documentUrl": response.url,
"document": response.json(),
}
content_type = response.headers.get("content-type")
link_header = response.headers.get("link")
if link_header and content_type != "application/ld+json":
link_header = parse_link_header(link_header).get(
LINK_HEADER_REL)
# only 1 related link header permitted
if isinstance(link_header, list):
raise JsonLdError(
"URL could not be dereferenced, it has more than one "
"associated HTTP Link Header.",
"jsonld.LoadDocumentError",
{"url": url},
code="multiple context link headers",
)
if link_header:
doc["contextUrl"] = link_header["target"]
return doc
except JsonLdError as e:
raise e
except Exception as cause:
raise JsonLdError(
"Could not retrieve a JSON-LD document from the URL.",
"jsonld.LoadDocumentError",
code="loading document failed",
cause=cause,
)
def loader(url, options={}):
"""
Retrieves JSON-LD at the given URL.
:param url: the URL to retrieve.
:return: the RemoteDocument.
"""
try:
# validate URL
pieces = urllib_parse.urlparse(url)
if (not all([pieces.scheme, pieces.netloc])
or pieces.scheme not in ['http', 'https']
or set(pieces.netloc) >
set(string.ascii_letters + string.digits + '-.:')):
raise JsonLdError(
'URL could not be dereferenced; only "http" and "https" '
'URLs are supported.',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
if secure and pieces.scheme != 'https':
raise JsonLdError(
'URL could not be dereferenced; secure mode enabled and '
'the URL\'s scheme is not "https".',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
headers = options.get('headers')
if headers is None:
headers = {'Accept': 'application/ld+json, application/json'}
response = requests.get(url, headers=headers, **kwargs)
content_type = response.headers.get('content-type')
if not content_type:
content_type = 'application/octet-stream'
doc = {
'contentType': content_type,
'contextUrl': None,
'documentUrl': response.url,
}
link_header = response.headers.get('link')
if link_header:
linked_context = parse_link_header(link_header).get(
LINK_HEADER_REL)
# only 1 related link header permitted
if linked_context and content_type != 'application/ld+json':
if isinstance(linked_context, list):
raise JsonLdError(
"URL could not be dereferenced, "
"it has more than one "
"associated HTTP Link Header.",
"jsonld.LoadDocumentError", {"url": url},
code="multiple context link headers")
doc["contextUrl"] = linked_context["target"]
linked_alternate = parse_link_header(link_header).get(
'alternate')
# if not JSON-LD, alternate may point there
if (linked_alternate and linked_alternate.get('type')
== 'application/ld+json' and not re.match(
r'^application\/(\w*\+)?json$', content_type)):
doc['contentType'] = 'application/ld+json'
doc['documentUrl'] = prepend_base(
url, linked_alternate['target'])
return loader(doc['documentUrl'], options=options)
doc["document"] = response.json()
return doc
except JsonLdError as e:
raise e
except Exception as cause:
raise JsonLdError(
'Could not retrieve a JSON-LD document from the URL.',
'jsonld.LoadDocumentError',
code='loading document failed',
cause=cause)
async def async_loader(url, headers):
"""
Retrieves JSON-LD at the given URL asynchronously.
:param url: the URL to retrieve.
:return: the RemoteDocument.
"""
try:
# validate URL
pieces = urllib_parse.urlparse(url)
if (not all([pieces.scheme, pieces.netloc])
or pieces.scheme not in ['http', 'https']
or set(pieces.netloc) >
set(string.ascii_letters + string.digits + '-.:')):
raise JsonLdError(
'URL could not be dereferenced; '
'only "http" and "https" URLs are supported.',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
if secure and pieces.scheme != 'https':
raise JsonLdError(
'URL could not be dereferenced; '
'secure mode enabled and '
'the URL\'s scheme is not "https".',
'jsonld.InvalidUrl', {'url': url},
code='loading document failed')
async with aiohttp.ClientSession(loop=loop) as session:
async with session.get(url, headers=headers,
**kwargs) as response:
# Allow any content_type in trying to parse json
# similar to requests library
json_body = await response.json(content_type=None)
content_type = response.headers.get('content-type')
if not content_type:
content_type = 'application/octet-stream'
doc = {
'contentType': content_type,
'contextUrl': None,
'documentUrl': response.url.human_repr(),
'document': json_body
}
link_header = response.headers.get('link')
if link_header:
linked_context = parse_link_header(link_header).get(
LINK_HEADER_REL)
# only 1 related link header permitted
if linked_context and content_type != 'application/ld+json':
if isinstance(linked_context, list):
raise JsonLdError(
'URL could not be dereferenced, '
'it has more than one '
'associated HTTP Link Header.',
'jsonld.LoadDocumentError', {'url': url},
code='multiple context link headers')
doc['contextUrl'] = linked_context['target']
linked_alternate = parse_link_header(link_header).get(
'alternate')
# if not JSON-LD, alternate may point there
if (linked_alternate and linked_alternate.get('type')
== 'application/ld+json' and
not re.match(r'^application\/(\w*\+)?json$',
content_type)):
doc['contentType'] = 'application/ld+json'
doc['documentUrl'] = jsonld.prepend_base(
url, linked_alternate['target'])
return doc
except JsonLdError as e:
raise e
except Exception as cause:
raise JsonLdError(
'Could not retrieve a JSON-LD document from the URL.',
'jsonld.LoadDocumentError',
code='loading document failed',
cause=cause)