def compare_json(shex_url: str, shex_json: str, log: Logger) -> bool:
"""
Compare the JSON generated from shex_url to the JSON in the target directory
:param shex_url: URL where we got the ShExC
:param shex_json: ShExJ equivalent of ShExC
:param log: Where comparison errors are recorded
:return: True if they match, false otherwise. If no match, the offending string is printed
"""
json_url = shex_url.rsplit(".", 1)[0] + ".json"
if ':' in json_url:
resp = requests.get(json_url)
if not resp.ok:
return False
json_text = resp.text
else:
try:
with open(json_url) as f:
json_text = f.read()
except FileNotFoundError:
print(f"****> {json_url} not found. Comparison not done ***")
return True
d1 = jao_loads(json_text)
d2 = jao_loads(shex_json)
if not compare_dicts(as_dict(d1),
as_dict(d2),
d1name="expected",
d2name="actual ",
file=log,
filtr=json_filtr):
print(as_json(d2))
return False
return True
def dict_processor(container: JsonObj, resource_type: Optional[str] = None, path: List[str] = None,
id_map: Optional[Dict[str, str]] = None, in_container: bool = False) -> None:
"""
Process the elements in container
:param container: JSON dictionary to be processed
:param resource_type: type of resource that container appears in
:param path: Full path from the base resource type to the actual element
:param id_map: Map from local resource to URI if inside a bundle
:param in_container: If True then don't tack they type onto the identifier
"""
# Rule: Whenever we find an embedded resourceType, we assume that we've encountered a brand new resource
# Update the passed resource type (example: container is Observation, we're processing the subject node
# and the inner resourceType is Patient
#
# Note: If there isn't a declared resourceType, it may be able to be extracted from the URL if the URL matches
# the predefined FHIR structure
if hasattr(container, RESOURCETYPE_KEY):
resource_type = container[RESOURCETYPE_KEY]
path = [resource_type]
# If we've got bundle, build an id map to use in the interior
id_map = bundle_urls(container) # Note that this will also assign ids to bundle entries
# Add any contained resources to the contained URL map
add_contained_urls(container, id_map)
# Process each of the elements in the dictionary
# Note: use keys() and re-look up to prevent losing the JsonObj characteristics of the values
for k in [k for k in as_dict(container).keys() if not k.startswith('_')]:
if is_choice_element(k):
map_element(k, container[k], resource_type, [k[VALUE_KEY_LEN:]], container, id_map, in_container)
else:
map_element(k, container[k], resource_type, path, container, id_map, in_container)
# Merge any extensions (keys that start with '_') into the base
# This happens when either:
# A) there is only an extension and no base
# B) there is a base, but it isn't a JSON object
for ext_key in [k for k in as_dict(container).keys() if k.startswith('_')]:
base_key = ext_key[1:]
ext_value = container[ext_key]
del(container[ext_key])
if not hasattr(container, base_key):
container[base_key] = ext_value # No base -- move the extension in
elif not isinstance(container[base_key], JsonObj):
container[base_key] = to_value(container[base_key]) # Base is not a JSON object
container[base_key]['extension'] = ext_value['extension'] \
if isinstance(ext_value, JsonObj) else ext_value
else:
container[base_key]['extension'] = ext_value['extension']
map_element(base_key, ext_value, EXTENSION_RESOURCE_TYPE, [EXTENSION_RESOURCE_TYPE], container, id_map)
def compare_json(j1: str, j2: str, log: TextIO) -> bool:
"""
Compare two JSON representation
:param j1: first string
:param j2: second string
:param log: log file to record differences
:return: Result of comparison
"""
d1 = jao_loads(j1)
d2 = jao_loads(j2)
return compare_dicts(as_dict(d1), as_dict(d2), file=log)
def check_types(s: SchemaDefinition) -> None:
output = os.path.join(outputdir, 'schema4.json')
if not os.path.exists(output):
with open(output, 'w') as f:
f.write(as_json(JsonObj(**{k: as_dict(loads(as_json(v))) for k, v in s.types.items()})))
self.fail(f"File {output} created - rerun test")
with open(output) as f:
expected = as_dict(load(f))
self.assertEqual(expected, {k: as_dict(loads(as_json(v))) for k, v in s.types.items()})
s.types = None
def test_element_slots(self):
""" Test all element slots and their inheritence """
schema = SchemaLoader(env.input_path('resolver3.yaml')).resolve()
x = {
k: v
for k, v in as_dict(schema.slots['s1']).items()
if v is not None and v != []
}
outfile = env.expected_path('resolver3.json')
if not os.path.exists(outfile):
with open(outfile, 'w') as f:
f.write(as_json(JsonObj(**x)))
with open(outfile) as f:
expected = as_dict(load(f))
self.assertEqual(expected, x)
def make_dict(obj):
"""
transforms an nmdc object into a dict
"""
if obj == None:
return # make sure the object has a value
## check if obj can convert to dict
if not hasattr(obj, "_as_dict"):
return obj
# temp_dict = jsonasobj.as_dict(obj) # convert obj dict
temp_dict = {}
obj_dict = {}
## include only valid values in lists and dicts
for key, val in jsonasobj.as_dict(obj).items():
# print('key:', key, '\n', ' val:', val, '\n')
if type({}) == type(val): # check values in dict
temp_dict[key] = {k: v for k, v in val.items() if is_value(v)}
elif type([]) == type(val): # check values in list
temp_dict[key] = [
element for element in val if is_value(element)
]
else:
temp_dict[key] = val
## check for {} or [] that may resulted from prevous loop
for key, val in temp_dict.items():
if is_value(val):
obj_dict[key] = val
return obj_dict
def from_value(v: Any) -> Any:
""" Extract the actual value from v noting that it may have already been transformed using to_value
Note: This assumes that we aren't going to find a "value" in native FHIR JSON. If this turns out not
to be the case, we should probably use a safe token (say, "V") during the processing and then
transform it as a final step.
"""
return v[VALUE_KEY] if isinstance(v, JsonObj) and VALUE_KEY in as_dict(v) else v
def check_types(s: SchemaDefinition) -> None:
output = env.expected_path('schema4.json')
if not os.path.exists(output):
with open(output, 'w') as f:
f.write(
as_json(
JsonObj(
**{
k: as_dict(loads(as_json(v)))
for k, v in s.types.items()
})))
with open(output) as f:
expected = as_dict(load(f))
self.assertEqual(
expected,
{k: as_dict(loads(as_json(v)))
for k, v in s.types.items()})
s.types = None
def check_types(s: SchemaDefinition) -> None:
self.assertEqual({
'integer': {'base': 'int',
'from_schema': 'http://example.org/schema5',
'name': 'integer'},
'string': {'base': 'str',
'from_schema': 'http://example.org/schema4',
'name': 'string'}},
{k: as_dict(loads(as_json(v))) for k, v in s.types.items()})
s.types = None
def get_element(self, name):
element = as_dict(self.bmt.get_element(name))
# This value is not Json serializable, so we're removing it for now.
if 'local_names' in element:
del element['local_names']
if 'slot_usage' in element:
del element['slot_usage']
#Annotations are also not JSON serializable. The current structure is:
# 'annotations': annotations={'biolink:canonical_predicate':
# Annotation(tag='biolink:canonical_predicate', value='True', extensions={}, annotations={})}
# And we want to turn that into "Tag":"Value"
if 'annotations' in element:
for k,v in element['annotations'].items():
element[v['tag']] = v['value']
del element['annotations']
return element
def test_element_slots(self):
""" Test all element slots and their inheritence """
schema = SchemaLoader(os.path.join(datadir,
'resolver3.yaml')).resolve()
x = {
k: v
for k, v in as_dict(schema.slots['s1']).items()
if v is not None and v != []
}
self.assertEqual(
{
'alt_descriptions': {},
'comments': ["I'm a little comment"],
'description':
'this is s1 it is good',
'domain':
'c1',
'examples': [{
'description': 'an example',
'value': 'test: foo'
}, {
'description': None,
'value': 17
}],
'from_schema':
'http://example.org/yaml4',
'in_subset': ['subset1', 'subset 2'],
'local_names': {},
'inlined':
True,
'name':
's1',
'notes':
['Pay attention here', 'Something might be happening'],
'range':
'c2',
'required':
False,
'see_also': ['http://example.org/e1', 'ex:e2'],
'slot_uri':
'http://example.org/yaml4/s1'
}, x)
def type_last(self, obj: JsonObj) -> JsonObj:
""" Move the type identifiers to the end of the object for print purposes """
def _tl_list(v: List) -> List:
return [
self.type_last(e) if isinstance(e, JsonObj) else
_tl_list(e) if isinstance(e, list) else e for e in v
if e is not None
]
rval = JsonObj()
for k in as_dict(obj).keys():
v = obj[k]
if v is not None and k not in ('type', '_context'):
rval[k] = _tl_list(v) if isinstance(
v, list) else self.type_last(v) if isinstance(
v, JsonObj) else v
if 'type' in obj and obj.type:
rval.type = obj.type
return rval
def adjust_urls(fhir_json: Any, outer_url: Optional[str] = "") -> None:
"""
Traverse the structure adding relative URL's to inner structures
:param fhir_json:
:param outer_url: Containing URL
:return:
"""
outer_url = outer_url.rsplit('#')[0] # Remove any fragment ids - they won't next
if isinstance(fhir_json, JsonObj):
if hasattr(fhir_json, '@id'):
container_id = getattr(fhir_json, '@id')
if str(container_id).startswith('#'):
container_id = outer_url + container_id
fhir_json['@id'] = container_id
outer_url = container_id
for k in as_dict(fhir_json).keys():
adjust_urls(fhir_json[k], outer_url)
elif isinstance(fhir_json, list):
for e in fhir_json:
adjust_urls(e, outer_url)
def dict_processor(d: JsonObj, resource_type: str, resource_type_set: Set[str], full_url: Optional[str] = None) \
-> None:
"""
Process the elements in dictionary d:
1) Ignore keys that begin with "@" - this is already JSON information
2) Add index numbers to lists
3) Recursively process values of type object
4) Convert any elements that are not one of "nodeRole", "index", "id" and "div" to objects
5) Merge
:param d: dictionary to be processed
:param resource_type: unedited resource type
:param resource_type_set: set of all resource types in the model
:param full_url: official ID of the resource from the containing fullURL
:return: List of resourceTypes referenced by the resource
"""
def gen_reference(do: JsonObj) -> JsonObj:
"""
Return the object of a fhir:link based on the reference in d
:param do: object containing the reference
:return: link and optional type element
"""
# TODO: find the official regular expression for the type node. For the moment we make the (incorrect)
# assumption that the type is everything that preceeds the first slash
if "://" not in do.reference and not do.reference.startswith('/'):
if hasattr(do, 'type'):
typ = do.type
else:
typ = do.reference.split('/', 1)[0]
link = '../' + do.reference
else:
link = do.reference
typ = getattr(do, 'type', None)
rval = JsonObj(**{"@id": link})
if typ:
rval['@type'] = "fhir:" + typ
return rval
# Normalize all the elements in d.
# We realize the keys as a list up front to prevent messing with our own updates
# full_url is set if we're an embedded resource that has a full_url
inner_type = getattr(d, 'resourceType', None)
if inner_type:
resource_type = inner_type
if full_url:
d['@id'] = full_url
# If this has a resource type, it overrides what was passed in
for k in list(as_dict(d).keys()):
v = d[k]
if k.startswith('@'): # Ignore JSON-LD components
pass
elif isinstance(v, JsonObj): # Inner object -- process recursively
dict_processor(v, resource_type, resource_type_set,
getattr(d, 'fullUrl', None))
elif isinstance(v, list): # Add ordering to the list
d[k] = list_processor(k, v, resource_type_set)
elif k == "id": # Internal ids are relative to the document
if not full_url:
d['@id'] = \
('#' if not inner_type and not v.startswith('#') else (resource_type + '/')) + v
d[k] = to_value(v)
elif k == "reference": # Link to another document
if not hasattr(d, 'link'):
d["fhir:link"] = gen_reference(d)
d[k] = to_value(v)
elif k == "resourceType" and not (v.startswith('fhir:')):
resource_type_set.add(v)
d[k] = 'fhir:' + v
elif k not in ["nodeRole", "index",
"div"]: # Convert most other nodes to value entries
d[k] = to_value(v)
if k == 'coding':
[add_type_arc(n) for n in v]
# Merge any extensions (keys that start with '_') into the base
for k in [k for k in as_dict(d).keys() if k.startswith('_')]:
base_k = k[1:]
if not hasattr(d, base_k) or not isinstance(d[base_k], JsonObj):
d[base_k] = JsonObj()
else:
for kp, vp in as_dict(d[k]).items():
if kp in d[base_k]:
print(
f"Extension element {kp} is already in the base for {k}"
)
else:
d[base_k][kp] = vp
del (d[k])
def from_value(v: Any) -> Any:
return v[VALUE_TAG] if isinstance(
v, JsonObj) and VALUE_TAG in as_dict(v) else v
def dict_processor(dobj: JsonObj, raw_resource_type: str, inside: Optional[bool] = False) -> None:
"""
Process the elements in dictionary d:
1) Ignore keys that begin with "@" - this is already JSON information
2) Duplicate lists creating one ordered, one not
3) Recursively process values of type 'object'
5) Add a mapping to merge '_' elements
:param dobj: dictionary to be processed
:param raw_resource_type: unedited resource type
:param inside: indicator of a recursive call
"""
def gen_reference(rd: JsonObj) -> List[JsonObj]:
"""
Return the object of a fhir:link based on the reference in d
:param rd: object containing the reference
:return: link and optional type element
"""
# TODO: find the official regular expression for the type node. For the moment we make the (incorrect)
# assumption that the type is everything that preceeds the first slash
if "://" not in rd.reference and not rd.reference.startswith('/'):
if hasattr(rd, 'type'):
typ = rd.type
else:
typ = rd.reference.split('/', 1)[0]
link = '../' + rd.reference
else:
link = rd.reference
typ = getattr(rd, 'type', None)
# "subject": [{"@id": "../Patient/foo1",
# "@type": "fhir:Patient" },
# {"reference": "Patient/f001",
# "display": "P. van de Heuvel"}]
ref = JsonObj(**{"@id": link})
if typ:
ref['@type'] = "fhir:" + typ
return [ref, rd]
# Normalize all the elements in d.
# We realize the keys as a list up front to prevent messing with our own updates
for k in list(as_dict(dobj).keys()):
v = dobj[k]
if k.startswith('@'): # Ignore JSON-LD components
pass
elif isinstance(v, JsonObj): # Inner object -- process recursively
dict_processor(v, raw_resource_type, True)
if hasattr(v, 'reference'):
dobj[k] = gen_reference(v)
elif isinstance(v, list): # Add ordering to the list
dobj[k] = list_processor(k, v)
elif k == "id": # Internal ids are relative to the document
dobj['@id'] = ('#' if inside and not v.startswith('#') else (raw_resource_type + '/')) + v
dobj[k] = to_value(v)
elif k == "resourceType" and not(v.startswith('fhir:')):
dobj[k] = 'fhir:' + v
elif k not in ["nodeRole", "index", "div"]: # Convert most other nodes to value entries
dobj[k] = to_value(v)
if k == 'coding':
[add_type_arc(n) for n in v]
underscore_nodes.update([k for k in as_dict(dobj).keys() if k.startswith('_')])
