def end_schema(self,
base: Optional[str] = None,
output: Optional[str] = None,
**_) -> None:
comments = f'''Auto generated from {self.schema.source_file} by {self.generatorname} version: {self.generatorversion}
Generation date: {self.schema.generation_date}
Schema: {self.schema.name}
id: {self.schema.id}
description: {be(self.schema.description)}
license: {be(self.schema.license)}
'''
context = JsonObj()
context["_comments"] = comments
context_content = {"_comments": None, "type": "@type"}
if base:
if '://' not in base:
self.context_body['@base'] = os.path.relpath(
base, os.path.dirname(self.schema.source_file))
else:
self.context_body['@base'] = base
for prefix in sorted(self.emit_prefixes):
if self.namespaces[prefix] != self.context_body['@vocab']:
context_content[prefix] = self.namespaces[prefix]
for k, v in self.context_body.items():
context_content[k] = v
for k, v in self.slot_class_maps.items():
context_content[k] = v
context['@context'] = context_content
if output:
with open(output, 'w') as outf:
outf.write(as_json(context))
else:
print(as_json(context))
def do_parse(infilename: str,
jsonfilename: Optional[str],
rdffilename: Optional[str],
rdffmt: str,
context: Optional[str] = None) -> bool:
"""
Parse the jsg in infilename and save the results in outfilename
:param infilename: name of the file containing the ShExC
:param jsonfilename: target ShExJ equivalent
:param rdffilename: target ShExR equivalent
:param rdffmt: target RDF format
:param context: @context to use for rdf generation. If None use what is in the file
:return: true if success
"""
inp = InputStream(load_shex_file(infilename))
shexj = parse(inp)
if shexj is not None:
shexj[
'@context'] = context if context else "http://www.w3.org/ns/shex.jsonld"
if jsonfilename:
with open(jsonfilename, 'w') as outfile:
outfile.write(as_json(shexj))
if rdffilename:
g = Graph().parse(data=as_json(shexj, indent=None),
format="json-ld")
g.serialize(open(rdffilename, "wb"), format=rdffmt)
return True
return False
def end_schema(self, context: str = biolink_context) -> None:
# self._visit(self.schema)
json_str = as_json(self.schema)
json_obj = loads(json_str)
if '://' not in context:
context = urljoin('file:', pathname2url(os.path.abspath(context)))
base_prefix = self.default_uri()
json_obj["@context"] = [context, {'@base': base_prefix}] if base_prefix else context
json_obj["@id"] = self.schema.id
print(as_json(json_obj, indent=" "))
def test_as_json(self):
schema = self.fix_schema_metadata(
load_raw_schema(os.path.join(inputdir, 'schema6.yaml')))
outfile = os.path.join(outputdir, 'schema6.json')
if not os.path.exists(outfile):
with open(outfile, 'w') as f:
f.write(as_json(schema))
self.fail(f"Generated {outfile} - run test again")
else:
self.assertEqual(load(outfile), loads(as_json(schema)))
def check_types(s: SchemaDefinition) -> None:
output = os.path.join(outputdir, 'schema4.json')
if not os.path.exists(output):
with open(output, 'w') as f:
f.write(as_json(JsonObj(**{k: as_dict(loads(as_json(v))) for k, v in s.types.items()})))
self.fail(f"File {output} created - rerun test")
with open(output) as f:
expected = as_dict(load(f))
self.assertEqual(expected, {k: as_dict(loads(as_json(v))) for k, v in s.types.items()})
s.types = None
def compare_shexj(
shex: Union[ShExJ.Schema, str],
shexj: Union[ShExJ.Schema, str]) -> Tuple[bool, StringIO]:
d1 = loads(as_json(shex) if isinstance(shex, ShExJ.Schema) else shex)
d2 = loads(
as_json(shexj) if isinstance(shexj, ShExJ.Schema) else shexj)
log = StringIO()
with redirect_stdout(log):
return compare_dicts(d1._as_dict,
d2._as_dict,
d1name="expected",
d2name="actual ",
filtr=json_filtr), log
def compare_json(shex_url: str, shex_json: str, log: Logger) -> bool:
"""
Compare the JSON generated from shex_url to the JSON in the target directory
:param shex_url: URL where we got the ShExC
:param shex_json: ShExJ equivalent of ShExC
:param log: Where comparison errors are recorded
:return: True if they match, false otherwise. If no match, the offending string is printed
"""
json_url = shex_url.rsplit(".", 1)[0] + ".json"
if ':' in json_url:
resp = requests.get(json_url)
if not resp.ok:
return False
json_text = resp.text
else:
try:
with open(json_url) as f:
json_text = f.read()
except FileNotFoundError:
print(f"****> {json_url} not found. Comparison not done ***")
return True
d1 = jao_loads(json_text)
d2 = jao_loads(shex_json)
if not compare_dicts(as_dict(d1),
as_dict(d2),
d1name="expected",
d2name="actual ",
file=log,
filtr=json_filtr):
print(as_json(d2))
return False
return True
def nb_filter(s: str) -> str:
""" Filter for jupyter (ipynb) notebooks """
# It is easier to deal with notebook content in JSON
s_json = loads(ldcontext_metadata_filter(s))
for cell in s_json.cells:
if hasattr(cell, 'execution_count'):
cell.execution_count = 1
if hasattr(cell, 'metadata'):
delattr(cell, 'metadata')
if hasattr(cell, 'outputs'):
del_outputs = []
for output in cell.outputs:
to_del = []
if hasattr(output, 'text'):
for line in output.text:
if 'WARNING: You are using pip' in line or\
'You should consider upgrading via' in line or\
'Requirement already satisfied:' in line:
to_del.append(line)
for del_line in to_del:
output.text.remove(del_line)
if not output.text:
del_outputs.append(output)
if del_outputs:
for del_output in del_outputs:
cell.outputs.remove(del_output)
if hasattr(s_json.metadata, 'language_info'):
if hasattr(s_json.metadata.language_info, 'version'):
s_json.metadata.language_info.version = '3'
return as_json(s_json)
def end_schema(self, context: str = None, **_) -> None:
self._add_type(self.schema)
json_obj = self.schema
base_prefix = self.default_prefix()
# JSON LD adjusts context reference using '@base'. If context is supplied and not a URI, generate an
# absolute URI for it
if context is None:
context = [METAMODEL_CONTEXT_URI]
elif isinstance(
context,
str): # Some of the older code doesn't do multiple contexts
context = [context]
elif isinstance(context, tuple):
context = list(context)
for imp in list(self.loaded.values())[1:]:
context.append(imp[0] + ".context.jsonld")
json_obj["@context"] = [
context[0]
] if len(context) == 1 and not base_prefix else context
if base_prefix:
json_obj["@context"].append({'@base': base_prefix})
# json_obj["@id"] = self.schema.id
print(as_json(json_obj, indent=" "))
def end_schema(self, context: str = None, **_) -> None:
self._add_type(self.schema)
json_obj = self.schema
base_prefix = self.default_prefix()
# JSON LD adjusts context reference using '@base'. If context is supplied and not a URI, generate an
# absolute URI for it
if context is None:
context = [METAMODEL_CONTEXT_URI]
elif isinstance(
context,
str): # Some of the older code doesn't do multiple contexts
context = [context]
elif isinstance(context, tuple):
context = list(context)
for imp in list(self.loaded.values())[1:]:
context.append(imp[0] + ".context.jsonld")
# Absolute file paths have to have a prefix
for ci in range(0, len(context)):
if context[ci].startswith(
'/'): # TODO: how do we deal with absolute DOS paths?
context[ci] = 'file://' + context[ci]
json_obj["@context"] = [
context[0] if len(context) == 1 and not base_prefix else context
]
if base_prefix:
json_obj["@context"].append({'@base': base_prefix})
# json_obj["@id"] = self.schema.id
print(as_json(json_obj, indent=" "))
def validate_shexc_json(json_str: str, input_fname: str) -> bool:
"""
Validate json_str against ShEx Schema
:param json_str: String to validate
:param input_fname: Name of source file for error reporting
:return: True if pass
"""
logger = StringIO()
# Load the JSON image of the good object and make sure it is valid
shex_json: ShExJ.Schema = jsg_loads(json_str, ShExJ)
if not is_valid(shex_json, logger):
print("File: {} - ".format(input_fname))
print(logger.getvalue())
return False
# Convert the JSON image into ShExC
shexc_str = str(ShExC(shex_json))
# Convert the ShExC back into ShExJ
output_shex_obj = ShExC(shexc_str).schema
if output_shex_obj is None:
for number, line in enumerate(shexc_str.split('\n')):
print(f"{number + 1}: {line}")
return False
output_shex_obj["@context"] = "http://www.w3.org/ns/shex.jsonld"
rval = compare_json(json_str, as_json(output_shex_obj), logger)
if not rval:
print(shexc_str)
print(logger.getvalue())
return rval
def end_schema(self, context: str = None, **_) -> None:
self._add_type(self.schema)
json_obj = self.schema
base_prefix = self.default_prefix()
# JSON LD adjusts context reference using '@base'. If context is supplied and not a URI, generate an
# absolute URI for it
if context is None:
context = [METAMODEL_CONTEXT_URI]
elif isinstance(
context,
str): # Some of the older code doesn't do multiple contexts
context = [context]
elif isinstance(context, tuple):
context = list(context)
for imp in list(self.loaded.values())[1:]:
context.append(imp[0] + ".context.jsonld")
# TODO: For testing purposes we use local contexts. JSON-LD requires absolute URI's for contexts (double check
# that this is true), so we end up recording local path names in the tests. All of these must be converted
# to URL's before this can be submitted and used
# Note: This is NOT true -- relative URI's work find. Fix this next time you have to update local models
abs_contexts = [
'file://' + os.path.abspath(os.path.join(self.base_dir, c))
if '://' not in c else c for c in context
]
json_obj["@context"] = abs_contexts[0] if len(
abs_contexts) == 1 and not base_prefix else abs_contexts
if base_prefix:
json_obj["@context"].append({'@base': base_prefix})
# json_obj["@id"] = self.schema.id
print(as_json(json_obj, indent=" "))
def _verify_schema1_content(
self,
schema: SchemaDefinition,
source_file,
addl_checks: Callable[[SchemaDefinition], None] = None) -> None:
expected = loads(f"""{{
"default_prefix": "http://example.org/{source_file}/",
"name": "{source_file}",
"id": "http://example.org/{source_file}",
"title": "Load Raw Schema Test",
"metamodel_version": "0.5.0",
"source_file": "{source_file}.yaml",
"source_file_date": "Mon Dec 31 11:25:38 2018",
"source_file_size": 76,
"generation_date": "2018-12-31 11:50"
}}""")
schema.source_file = os.path.basename(schema.source_file)
if addl_checks:
addl_checks(schema)
self.assertTrue(isinstance(schema.metamodel_version, str))
expected.metamodel_version = schema.metamodel_version
self.assertTrue(isinstance(schema.source_file_date, str))
expected.source_file_date = schema.source_file_date
self.assertTrue(isinstance(schema.source_file_size, int))
expected.source_file_size = schema.source_file_size
self.assertTrue(isinstance(schema.generation_date, str))
expected.generation_date = schema.generation_date
self.assertEqual(expected, loads(as_json(schema)))
def convert_env_triad(attribute_value):
## replace '_' with ':' in curie
curie_val = attribute_value_to_string(attribute_value)
curie_val = curie_val.replace('_', ':')
obj = ControlledTermValue(has_raw_value=curie_val,
term=OntologyClass(id=curie_val))
return json.loads(jsonasobj.as_json(obj))
def end_schema(self,
base: Optional[str] = None,
output: Optional[str] = None,
**_) -> None:
context = JsonObj()
if self.emit_metadata:
comments = f'''Auto generated from {self.schema.source_file} by {self.generatorname} version: {self.generatorversion}'''
if self.schema.generation_date:
comments += f'''
Generation date: {self.schema.generation_date}
Schema: {self.schema.name}
'''
comments += f'''
id: {self.schema.id}
description: {be(self.schema.description)}
license: {be(self.schema.license)}
'''
context["_comments"] = comments
context_content = {}
if base:
if '://' not in base:
self.context_body['@base'] = os.path.relpath(
base, os.path.dirname(self.schema.source_file))
else:
self.context_body['@base'] = base
for prefix in sorted(self.emit_prefixes):
url = str(self.namespaces[prefix])
if ':' == url[-1] or '/' == url[-1] or '?' == url[-1] or '#' == url[
-1] or '[' == url[-1] or ']' == url[-1] or '@' == url[-1]:
context_content[prefix] = self.namespaces[prefix]
else:
prefix_obj = JsonObj()
prefix_obj["@id"] = self.namespaces[prefix]
prefix_obj["@prefix"] = True
context_content[prefix] = prefix_obj
for k, v in self.context_body.items():
context_content[k] = v
for k, v in self.slot_class_maps.items():
context_content[k] = v
context['@context'] = context_content
if output:
with open(output, 'w') as outf:
outf.write(as_json(context))
else:
print(as_json(context))
def check_types(s: SchemaDefinition) -> None:
self.assertEqual({
'integer': {'base': 'int',
'from_schema': 'http://example.org/schema5',
'name': 'integer'},
'string': {'base': 'str',
'from_schema': 'http://example.org/schema4',
'name': 'string'}},
{k: as_dict(loads(as_json(v))) for k, v in s.types.items()})
s.types = None
def check_types(s: SchemaDefinition) -> None:
output = env.expected_path('schema4.json')
if not os.path.exists(output):
with open(output, 'w') as f:
f.write(
as_json(
JsonObj(
**{
k: as_dict(loads(as_json(v)))
for k, v in s.types.items()
})))
with open(output) as f:
expected = as_dict(load(f))
self.assertEqual(
expected,
{k: as_dict(loads(as_json(v)))
for k, v in s.types.items()})
s.types = None
def eval_loader(self,
base_name: str,
is_sourcedir: bool = False,
source: Optional[str] = None) -> None:
fn = os.path.join(sourcedir if is_sourcedir else datadir, base_name +
'.yaml') if not source else source
loader = SchemaLoader(fn)
schema = as_json(self.fix_schema_metadata(loader.resolve()))
self.eval_output(schema, base_name + '.json', loads)
errors = '\n'.join(loader.synopsis.errors())
self.eval_output(errors, base_name + '.errs')
self.assertFalse(update_all_files, "Updating base files -- rerun")
def test_code_lists(self):
# A value set with a single code ("Appendicitis")
vsd1 = ValueSet(
codes=[
ConceptReference("NCIt:C35145", description="Appendicitis")
],
minus=ValueSet(activecodesfrom="http://snomed.info/sct/"))
print(as_json(vsd1))
# A value set with three codes - "Appendicitis",
vsd2 = ValueSet(codes=[
"http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#C35145",
"NCIt:C78178",
ConceptReference(
about=
"http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#C53500",
code="C53500",
href=
"https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&code=C53500&ns=ncit",
description="Non-Neoplastic Intestinal Disorder")
])
print(as_json(vsd2))
def as_rdf(element: YAMLRoot, contexts: CONTEXTS_PARAM_TYPE = None) -> Graph:
"""
Convert element into an RDF graph guided by the context(s) in contexts
:param element: element to represent in RDF
:param contexts: JSON-LD context(s) in the form of a file or URL name, a json string or a json obj
:return: rdflib Graph containing element
"""
jsonld = as_json_object(element, contexts)
graph = Graph()
graph.parse(data=as_json(jsonld), format="json-ld", prefix=True)
return graph
def output_generator(dirname) -> None:
with open(os.path.join(dirname, 'issue_80.json'), 'w') as f:
f.write(as_json(example))
context = os.path.join(dirname, 'issue_80.context.jsonld')
with open(context, 'w') as f:
f.write(
ContextGenerator(
env.input_path('issue_80.yaml')).serialize())
with open(os.path.join(dirname, 'issue_80.ttl'), 'w') as f:
f.write(
as_rdf(
example,
contexts=context).serialize(format="turtle").decode())
def test_importmap(self):
""" Test the importmap parameter """
fn = env.input_path('import_test_1.yaml')
importmap = {
"http://example.org/import_test_2": "import_test_2",
"loc/imp3": "import_test_3",
"base:import_test_4": "http://example.org/import_test_4",
"http://example.org/import_test_4": "import_test_4",
"types": "http://w3id.org/biolink/biolinkml/types"
}
self.env.generate_single_file(
'import_test_1.json',
lambda: as_json(SchemaLoader(fn, importmap=importmap).resolve()),
filtr=json_metadata_filter)
def add(self, name: str, evaluator: ShExEvaluator) -> ShExEvaluator:
"""
Add ShExEvaluator to the cache
:param name: Name of the shex file (sans suffix)
:param evaluator: Evaluator that has the parsed file
"""
if name not in self.evaluator_cache:
self.evaluator_cache[name] = evaluator
shex_path = self._cache_name(name)
if shex_path and not os.path.exists(shex_path):
with open(shex_path, 'w') as f:
# TODO: Add the context in or create a function in the evaluator that emits the full JSON string
f.write(as_json(evaluator._schema))
return evaluator
def end_schema(self,
base: Optional[str] = None,
output: Optional[str] = None,
**_) -> None:
context = JsonObj()
if base:
if '://' not in base:
self.context_body['@base'] = os.path.relpath(
base, os.path.dirname(self.schema.source_file))
else:
self.context_body['@base'] = base
for prefix in sorted(self.emit_prefixes):
context[prefix] = self.namespaces[prefix]
for k, v in self.context_body.items():
context[k] = v
for k, v in self.slot_class_maps.items():
context[k] = v
if output:
with open(output, 'w') as outf:
outf.write(as_json(context))
else:
print(as_json(context))
def validate_shexc(shexc_str: str, input_fname: str) -> bool:
"""
Validate json_str against ShEx Schema
:param shexc_str: String to validate
:param input_fname: Name of source file for error reporting
:return: True if pass
"""
shexj = parse(shexc_str)
if shexj is None:
return False
shexj['@context'] = "http://www.w3.org/ns/shex.jsonld"
shex_obj = jsg_loads(as_json(shexj), ShExJ)
log = StringIO()
rval = True
with redirect_stdout(log):
if not shex_obj._is_valid():
rval = False
elif not compare_json(input_fname, as_json(shex_obj), log):
rval = False
if not rval:
print("File: {} - ".format(input_fname))
print(log.getvalue())
return rval
def dumps(element: YAMLRoot, contexts: CONTEXTS_PARAM_TYPE = None) -> str:
"""
Return element as a JSON or a JSON-LD string
:param element: LinkML object to be emitted
:param contexts: JSON-LD context(s) in the form of:
* file name
* URL
* JSON String
* dict
* JSON Object
* A list containing elements of any type named above
:return: JSON Object representing the element
"""
return as_json(as_json_object(element, contexts), filtr=remove_empty_items, indent=' ')
def end_schema(self) -> None:
comments = f'''Auto generated from {self.schema.source_file} by {self.generatorname} version: {self.generatorversion}
Generation date: {self.schema.generation_date}
Schema: {self.schema.name}
id: {self.schema.id}
description: {be(self.schema.description)}
license: {be(self.schema.license)}
'''
context = JsonObj(comments=comments)
for k, v in self.slot_class_maps.items():
self.prefixmap[k] = v
context['@context'] = self.prefixmap
print(as_json(context))
def convert_file(ifn: str, ofn: str, opts: Namespace) -> bool:
"""
Convert ifn to ofn
:param ifn: Name of file to convert
:param ofn: Target file to convert to
:param opts: Parameters
:return: True if conversion is successful
"""
if ifn not in opts.converted_files:
out_json = to_r4(opts.in_json, opts.fhirserver, opts.addcontext)
with open(ofn, "w") as outf:
outf.write(as_json(out_json))
opts.converted_files.append(ifn)
return True
def end_schema(self, context: str = None) -> None:
self._add_type(self.schema)
json_obj = self.schema
base_prefix = self.default_prefix()
# JSON LD adjusts context reference using '@base'. If context is supplied and not a URI, generate an
# absolute URI for it
if context is None:
context = METAMODEL_CONTEXT_URI
elif '://' not in context:
context = 'file://' + os.path.abspath(os.path.join(os.getcwd(), context))
json_obj["@context"] = [context, {'@base': base_prefix}] if base_prefix else context
# json_obj["@id"] = self.schema.id
print(as_json(json_obj, indent=" "))
def test_element_slots(self):
""" Test all element slots and their inheritence """
schema = SchemaLoader(env.input_path('resolver3.yaml')).resolve()
x = {
k: v
for k, v in as_dict(schema.slots['s1']).items()
if v is not None and v != []
}
outfile = env.expected_path('resolver3.json')
if not os.path.exists(outfile):
with open(outfile, 'w') as f:
f.write(as_json(JsonObj(**x)))
with open(outfile) as f:
expected = as_dict(load(f))
self.assertEqual(expected, x)
