def update(self, om_properties, context_js, schema_graph):
"""See :func:`oldman.parsing.schema.context.OMAttributeMdExtractor.update`."""
context = Context(context_js)
for (property_iri, reversed), om_property in om_properties.iteritems():
# Efficient search
term = context.find_term(property_iri)
if term:
self._update_property(om_property, term)
else:
# May not have been found because of its type
terms = [
t for t in context.terms.values() if t.id == property_iri
]
if len(terms) > 0:
for term in terms:
self._update_property(om_property, term)
# Not declared (worst case)
else:
name = schema_graph.qname(property_iri).replace(":", "_")
self._logger.warn(
u"No short name found for property %s. QName %s used instead"
% (property_iri, name))
om_property.add_attribute_metadata(name)
def test_create_context():
ctx = Context()
ctx.add_term('label', 'http://example.org/ns/label')
term = ctx.terms.get('label')
assert term.name == 'label'
assert ctx.find_term('http://example.org/ns/label') is term
def test_create_context():
ctx = Context()
ctx.add_term('label', 'http://example.org/ns/label')
term = ctx.terms.get('label')
assert term.name == 'label'
assert ctx.find_term('http://example.org/ns/label') is term
def test_create_context():
ctx = Context()
ctx.add_term("label", "http://example.org/ns/label")
term = ctx.terms.get("label")
assert term.name == "label"
assert ctx.find_term("http://example.org/ns/label") is term
def test_accessing_keyword_values_by_alias():
ctx = Context({"iri": "@id", "lang": "@language"})
assert ctx.get_id({"iri": "urn:x:1"}) == "urn:x:1"
assert ctx.get_language({"lang": "en"}) == "en"
# test_standard_keywords_still_work():
assert ctx.get_id({"@id": "urn:x:1"}) == "urn:x:1"
# test_representing_keywords_by_alias():
assert ctx.id_key == "iri"
assert ctx.lang_key == "lang"
def test_accessing_keyword_values_by_alias():
ctx = Context({'iri': '@id', 'lang': '@language'})
assert ctx.get_id({'iri': 'urn:x:1'}) == 'urn:x:1'
assert ctx.get_language({'lang': 'en'}) == 'en'
# test_standard_keywords_still_work():
assert ctx.get_id({'@id': 'urn:x:1'}) == 'urn:x:1'
# test_representing_keywords_by_alias():
assert ctx.id_key == 'iri'
assert ctx.lang_key == 'lang'
def test_accessing_keyword_values_by_alias():
ctx = Context({'iri': '@id', 'lang': '@language'})
assert ctx.get_id({'iri': 'urn:x:1'}) == 'urn:x:1'
assert ctx.get_language({'lang': 'en'}) == 'en'
# test_standard_keywords_still_work():
assert ctx.get_id({'@id': 'urn:x:1'}) == 'urn:x:1'
# test_representing_keywords_by_alias():
assert ctx.id_key == 'iri'
assert ctx.lang_key == 'lang'
def test_loading_contexts():
# Given context data:
source1 = "http://example.org/base.jsonld"
source2 = "http://example.org/context.jsonld"
SOURCES[source1] = {'@context': {"@vocab": "http://example.org/vocab/"}}
SOURCES[source2] = {'@context': [source1, {"n": "name"}]}
# Create a context:
ctx = Context(source2)
assert ctx.expand('n') == 'http://example.org/vocab/name'
# Context can be a list:
ctx = Context([source2])
assert ctx.expand('n') == 'http://example.org/vocab/name'
def test_select_term_based_on_value_characteristics():
ctx = Context()
ctx.add_term('updated', 'http://example.org/ns/updated')
ctx.add_term('updatedDate',
'http://example.org/ns/updated',
coercion='http://www.w3.org/2001/XMLSchema#date')
assert ctx.find_term('http://example.org/ns/updated').name == 'updated'
assert ctx.find_term(
'http://example.org/ns/updated',
coercion='http://www.w3.org/2001/XMLSchema#date').name == 'updatedDate'
def test_select_term_based_on_value_characteristics():
ctx = Context()
ctx.add_term('updated', 'http://example.org/ns/updated')
ctx.add_term('updatedDate', 'http://example.org/ns/updated',
coercion='http://www.w3.org/2001/XMLSchema#date')
assert ctx.find_term('http://example.org/ns/updated').name == 'updated'
assert ctx.find_term('http://example.org/ns/updated',
coercion='http://www.w3.org/2001/XMLSchema#date').name == 'updatedDate'
def test_parsing_a_context_expands_prefixes():
ctx = Context({
'@vocab': 'http://example.org/ns/',
'x': 'http://example.org/ns/',
'label': 'x:label',
'x:updated': {
'@type': 'x:date'
}
})
term = ctx.terms.get('label')
assert term.id == 'http://example.org/ns/label'
term = ctx.terms.get('x:updated')
assert term.id == 'http://example.org/ns/updated'
assert term.type == 'http://example.org/ns/date'
# test_expanding_terms():
assert ctx.expand('term') == 'http://example.org/ns/term'
assert ctx.expand('x:term') == 'http://example.org/ns/term'
# test_shrinking_iris():
assert ctx.shrink_iri('http://example.org/ns/term') == 'x:term'
assert ctx.to_symbol('http://example.org/ns/term') == 'term'
def test_parsing_a_context_expands_prefixes():
ctx = Context({
"@vocab": "http://example.org/ns/",
"x": "http://example.org/ns/",
"label": "x:label",
"x:updated": {
"@type": "x:date"
},
})
term = ctx.terms.get("label")
assert term.id == "http://example.org/ns/label"
term = ctx.terms.get("x:updated")
assert term.id == "http://example.org/ns/updated"
assert term.type == "http://example.org/ns/date"
# test_expanding_terms():
assert ctx.expand("term") == "http://example.org/ns/term"
assert ctx.expand("x:term") == "http://example.org/ns/term"
# test_shrinking_iris():
assert ctx.shrink_iri("http://example.org/ns/term") == "x:term"
assert ctx.to_symbol("http://example.org/ns/term") == "term"
def test_parsing_a_context_expands_prefixes():
ctx = Context({
'@vocab': 'http://example.org/ns/',
'x': 'http://example.org/ns/',
'label': 'x:label',
'x:updated': {'@type': 'x:date'}})
term = ctx.terms.get('label')
assert term.id == 'http://example.org/ns/label'
term = ctx.terms.get('x:updated')
assert term.id == 'http://example.org/ns/updated'
assert term.type == 'http://example.org/ns/date'
# test_expanding_terms():
assert ctx.expand('term') == 'http://example.org/ns/term'
assert ctx.expand('x:term') == 'http://example.org/ns/term'
# test_shrinking_iris():
assert ctx.shrink_iri('http://example.org/ns/term') == 'x:term'
assert ctx.to_symbol('http://example.org/ns/term') == 'term'
def test_select_term_based_on_value_characteristics():
ctx = Context()
ctx.add_term("updated", "http://example.org/ns/updated")
ctx.add_term(
"updatedDate",
"http://example.org/ns/updated",
coercion="http://www.w3.org/2001/XMLSchema#date",
)
assert ctx.find_term("http://example.org/ns/updated").name == "updated"
assert (ctx.find_term(
"http://example.org/ns/updated",
coercion="http://www.w3.org/2001/XMLSchema#date",
).name == "updatedDate")
def update(self, om_properties, context_js, schema_graph):
"""See :func:`oldman.parsing.schema.context.OMAttributeMdExtractor.update`."""
context = Context(context_js)
for (property_iri, reversed), om_property in om_properties.iteritems():
# Efficient search
term = context.find_term(property_iri)
if term:
self._update_property(om_property, term)
else:
# May not have been found because of its type
terms = [t for t in context.terms.values()
if t.id == property_iri]
if len(terms) > 0:
for term in terms:
self._update_property(om_property, term)
# Not declared (worst case)
else:
name = schema_graph.qname(property_iri).replace(":", "_")
self._logger.warn(u"No short name found for property %s. QName %s used instead"
% (property_iri, name))
om_property.add_attribute_metadata(name)
def read(self, jsonld, discussion, admin_user_id, base=None):
if isinstance(jsonld, (str, unicode)):
jsonld = json.loads(jsonld)
c = jsonld['@context']
# Avoid loading the main context.
if c == context_url:
c = local_context_loc
elif context_url in c:
c.remove(context_url)
c.append(local_context_loc)
c = Context(c, base=base)
by_id = dict()
site_iri = None
def find_objects(j):
if isinstance(jsonld, (str, unicode)):
return
if isinstance(j, list):
for x in j:
find_objects(x)
if isinstance(j, dict):
jid = j.get('@id', None)
if jid:
by_id[jid] = j
for x in j.values():
find_objects(x)
find_objects(jsonld)
for json in by_id.itervalues():
if json.get('@type', None) == 'Site':
site_iri = json['@id']
break
site_iri = site_iri or base
assert site_iri is not None
handler = ImportRecordHandler(discussion, site_iri)
for json in by_id.itervalues():
cls = self.class_from_type(json['@type'])
if not cls:
print "missing cls for :", json['@type']
continue
if cls:
cls = get_named_class(cls)
cls.create_from_json(
json, admin_user_id, aliases=handler,
parse_def_name='readcif.json', jsonld=by_id)
def test_loading_contexts():
# Given context data:
source1 = "http://example.org/base.jsonld"
source2 = "http://example.org/context.jsonld"
SOURCES[source1] = {'@context': {"@vocab": "http://example.org/vocab/"}}
SOURCES[source2] = {'@context': [source1, {"n": "name"}]}
# Create a context:
ctx = Context(source2)
assert ctx.expand('n') == 'http://example.org/vocab/name'
# Context can be a list:
ctx = Context([source2])
assert ctx.expand('n') == 'http://example.org/vocab/name'
def read_data(self, jsonld, admin_user_id, base=None):
self.load_previous_records()
if isinstance(jsonld, string_types):
jsonld = json.loads(jsonld)
c = jsonld['@context']
self.remote_context = Context(c)
def find_objects(j):
if isinstance(j, list):
for x in j:
for obj in find_objects(x):
yield obj
elif isinstance(j, dict):
jid = j.get('@id', None)
if jid:
yield j
for x in j.values():
for obj in find_objects(x):
yield obj
self.read_data_gen(find_objects(jsonld), admin_user_id)
self.db.flush()
self.add_missing_links()
def test_invalid_remote_context():
ctx_url = "http://example.org/recursive.jsonld"
SOURCES[ctx_url] = {"key": "value"}
ctx = Context(ctx_url)
def test_recursive_context_inclusion_error():
ctx_url = "http://example.org/recursive.jsonld"
SOURCES[ctx_url] = {'@context': ctx_url}
ctx = Context(ctx_url)
def test_prefix_like_vocab():
ctx = Context({"@vocab": "ex:", "term": "ex:term"})
term = ctx.terms.get("term")
assert term.id == "ex:term"
def jsonld_context(ontology_root=DEFAULT_ROOT):
global _jsonld_context
if _jsonld_context is None:
from rdflib_jsonld.context import Context
_jsonld_context = Context(local_context_loc)
return _jsonld_context
def test_resolving_iris():
ctx = Context({'@base': 'http://example.org/path/leaf'})
assert ctx.resolve('/') == 'http://example.org/'
assert ctx.resolve('/trail') == 'http://example.org/trail'
assert ctx.resolve('../') == 'http://example.org/'
assert ctx.resolve('../../') == 'http://example.org/'
def test_getting_keyword_values_from_nodes():
ctx = Context()
assert ctx.get_id({'@id': 'urn:x:1'}) == 'urn:x:1'
assert ctx.get_language({'@language': 'en'}) == 'en'
def test_resolving_iris():
ctx = Context({"@base": "http://example.org/path/leaf"})
assert ctx.resolve("/") == "http://example.org/"
assert ctx.resolve("/trail") == "http://example.org/trail"
assert ctx.resolve("../") == "http://example.org/"
assert ctx.resolve("../../") == "http://example.org/"
def test_creating_a_subcontext():
ctx = Context()
ctx4 = ctx.subcontext({'lang': '@language'})
assert ctx4.get_language({'lang': 'en'}) == 'en'
# index_name = "open-beelden-beeldengeluid"
index_name = args['<index_name>']
out_file = args['<outfile>']
use_ttl = args[
'--ttl'] # Otherwise --nq should be on args and we use NQ format
knownPrefixes = args['PREFIX_FILE'] if args['--prefixes'] else None
host = args['HOST'] if args['--host'] else None
client = Elasticsearch(host)
scroller = Search(using=client, index=index_name).query()
logger.info('Using index: ' + index_name)
global_context = getContext(knownPrefixes, scroller, out_file)
ctx = Context(global_context)
logger.info('Building graph...')
format = 'turtle' if use_ttl else 'nquads'
logger.info('Serializing to file ' + out_file + '...')
logger.info('Using %s format...' % format)
counter = 0
with open(out_file, 'w') as fout:
# Write headers first times
g = ConjunctiveGraph()
g.parse(data='{}', context=global_context, format='json-ld')
serialized_data = g.serialize(format=format)
fout.write(serialized_data)
def test_getting_keyword_values_from_nodes():
ctx = Context()
assert ctx.get_id({"@id": "urn:x:1"}) == "urn:x:1"
assert ctx.get_language({"@language": "en"}) == "en"
def test_creating_a_subcontext():
ctx = Context()
ctx4 = ctx.subcontext({"lang": "@language"})
assert ctx4.get_language({"lang": "en"}) == "en"
def test_resolving_iris():
ctx = Context({'@base': 'http://example.org/path/leaf'})
assert ctx.resolve('/') == 'http://example.org/'
assert ctx.resolve('/trail') == 'http://example.org/trail'
assert ctx.resolve('../') == 'http://example.org/'
assert ctx.resolve('../../') == 'http://example.org/'
def test_use_base_in_local_context():
ctx = Context({'@base': "/local"})
assert ctx.base == '/local'
def test_getting_keyword_values_from_nodes():
ctx = Context()
assert ctx.get_id({'@id': 'urn:x:1'}) == 'urn:x:1'
assert ctx.get_language({'@language': 'en'}) == 'en'
def test_override_base():
ctx = Context(base="http://example.org/app/data/item",
source={'@base': "http://example.org/"})
assert ctx.base == "http://example.org/"
def test_set_null_base():
ctx = Context(base="http://example.org/app/data/item",
source={'@base': None})
assert ctx.base is None
assert ctx.resolve_iri("../other") == "../other"
def test_resolve_relative_base():
ctx = Context(base="http://example.org/app/data/item",
source={'@base': "../"})
assert ctx.base == "http://example.org/app/"
assert ctx.resolve_iri("../other") == "http://example.org/other"
def test_resolve_relative_base():
ctx = Context(base="http://example.org/app/data/item",
source={'@base': "../"})
assert ctx.base == "http://example.org/app/"
assert ctx.resolve_iri("../other") == "http://example.org/other"
def test_set_null_base():
ctx = Context(base="http://example.org/app/data/item",
source={'@base': None})
assert ctx.base is None
assert ctx.resolve_iri("../other") == "../other"
def test_creating_a_subcontext():
ctx = Context()
ctx4 = ctx.subcontext({'lang': '@language'})
assert ctx4.get_language({'lang': 'en'}) == 'en'
def test_ignore_base_remote_context():
ctx_url = "http://example.org/remote-base.jsonld"
SOURCES[ctx_url] = {'@context': {'@base': "/remote"}}
ctx = Context(ctx_url)
assert ctx.base == None
def read(self, jsonld, discussion, admin_user_id, base=None):
from .idea import Idea, IdeaLink
self.local_context = jsonld_context()
self.instance_by_id = {}
self.json_by_id = {}
self.global_url = get_global_base_url() + "/data/"
# preload
self.import_records_by_id = {
r.external_id: r
for r in self.import_records
}
if isinstance(jsonld, string_types):
jsonld = json.loads(jsonld)
c = jsonld['@context']
self.remote_context = Context(c)
# Avoid loading the main context.
# if c == context_url:
# c = local_context_loc
# elif context_url in c:
# c.remove(context_url)
# c.append(local_context_loc)
# c = Context(c, base=base)
# site_iri = None
def find_objects(j):
if isinstance(jsonld, string_types):
return
if isinstance(j, list):
for x in j:
find_objects(x)
if isinstance(j, dict):
jid = j.get('@id', None)
if jid:
self.json_by_id[jid] = j
for x in j.values():
find_objects(x)
find_objects(jsonld)
# for record in self.json_by_id.values():
# if record.get('@type', None) == 'Site':
# site_iri = record['@id']
# break
# site_iri = site_iri or base
# assert site_iri is not None
ctx = discussion.get_instance_context(user_id=admin_user_id)
for key in self.json_by_id:
if key in self.instance_by_id:
continue
record = self.get_record(key)
cls = self.class_from_json(record)
if not cls:
log.error("missing cls for : " + record['@type'])
continue
instance_ctx = cls.create_from_json(record,
ctx,
parse_def_name='cif_reverse',
object_importer=self)
if instance_ctx:
self.db.add(instance_ctx._instance)
self.db.flush()
# add links from discussion root to roots of idea subtrees
base_ids = self.db.query(Idea.id).outerjoin(
IdeaLink, IdeaLink.target_id == Idea.id).filter(
IdeaLink.id == None,
Idea.discussion_id == self.discussion_id).all()
root_id = self.discussion.root_idea.id
base_ids.remove((root_id, ))
for (id, ) in base_ids:
self.db.add(IdeaLink(source_id=root_id, target_id=id))
self.db.flush()
def test_use_base_in_local_context():
ctx = Context({"@base": "/local"})
assert ctx.base == "/local"