def test_repr_expands_deep_refs_by_default(self):
json = {
"a": "string",
"b": {
"$ref": "#/a"
},
"c": {
"$ref": "#/b"
},
"d": {
"$ref": "#/c"
},
"e": {
"$ref": "#/d"
},
"f": {
"$ref": "#/e"
}
}
assert (repr(sorted(JsonRef.replace_refs(json).items())) ==
"[('a', 'string'), ('b', 'string'), ('c', 'string'), "
"('d', 'string'), ('e', 'string'), ('f', 'string')]")
# Should not expand when set to False explicitly
result = JsonRef.replace_refs(json, load_on_repr=False)
assert (
repr(sorted(result.items())) ==
"[('a', 'string'), ('b', JsonRef({'$ref': '#/a'})), "
"('c', JsonRef({'$ref': '#/b'})), ('d', JsonRef({'$ref': '#/c'})), "
"('e', JsonRef({'$ref': '#/d'})), ('f', JsonRef({'$ref': '#/e'}))]"
)
def test_inserts_missing_type(response_spec, pet_model_spec):
ref_obj = {'$ref': '#/definitions/Pet'}
pet_proxy = JsonRef(ref_obj)
pet_proxy.__subject__ = pet_model_spec
response_spec['schema'] = pet_proxy
fix_models_with_no_type_callback(response_spec, key='schema')
assert pet_model_spec['type'] == 'object'
def test_inserts_missing_type(response_spec, pet_model_spec):
ref_obj = {'$ref': '#/definitions/Pet'}
pet_proxy = JsonRef(ref_obj)
pet_proxy.__subject__ = pet_model_spec
response_spec['schema'] = pet_proxy
fix_models_with_no_type_callback(response_spec, key='schema')
assert pet_model_spec['type'] == 'object'
def test_abstract_loader_url_fallbacks(get_record, super_get_r_j, g_p_t_f_e):
with_super = {'SUPER': 'SUPER'}
with_actual = {'ACTUAL': 'ACTUAL'}
g_p_t_f_e = 'pt'
super_get_r_j.return_value = with_super
get_record.return_value = with_actual
# Check against prod SERVER_NAME
config = {'SERVER_NAME': 'http://inspirehep.net'}
with patch.dict(current_app.config, config):
expect_actual = JsonRef({'$ref': 'http://inspirehep.net/api/e/1'},
loader=AbstractRecordLoader())
assert expect_actual == with_actual
expect_actual = JsonRef({'$ref': '/api/e/1'},
loader=AbstractRecordLoader())
assert expect_actual == with_actual
expect_super = JsonRef({'$ref': 'http://otherhost.net/api/e/1'},
loader=AbstractRecordLoader())
assert expect_super == with_super
# Check against dev SERVER_NAME
config = {'SERVER_NAME': 'localhost:5000'}
with patch.dict(current_app.config, config):
expect_actual = JsonRef({'$ref': 'http://localhost:5000/api/e/1'},
loader=AbstractRecordLoader())
assert expect_actual == with_actual
expect_actual = JsonRef({'$ref': '/api/e/1'},
loader=AbstractRecordLoader())
assert expect_actual == with_actual
expect_super = JsonRef({'$ref': 'http://inspirehep.net/api/e/1'},
loader=AbstractRecordLoader())
assert expect_super == with_super
# Check against prod https SERVER_NAME
config = {'SERVER_NAME': 'https://inspirehep.net'}
with patch.dict(current_app.config, config):
expect_actual = JsonRef({'$ref': 'https://inspirehep.net/api/e/1'},
loader=AbstractRecordLoader())
assert expect_actual == with_actual
expect_actual = JsonRef({'$ref': '/api/e/1'},
loader=AbstractRecordLoader())
assert expect_actual == with_actual
# https should be backwards compatible with resources indexed with http://.
expect_actual = JsonRef({'$ref': 'http://inspirehep.net/api/e/1'},
loader=AbstractRecordLoader())
assert expect_actual == with_actual
expect_super = JsonRef({'$ref': 'http://otherhost.net/api/e/1'},
loader=AbstractRecordLoader())
assert expect_super == with_super
def get_versioned_release_schema(schema):
"""
Returns the versioned release schema.
"""
# Update schema metadata.
release_with_underscores = release.replace('.', '__')
schema['id'] = f'https://standard.open-contracting.org/schema/{release_with_underscores}/versioned-release-validation-schema.json' # noqa
schema['title'] = 'Schema for a compiled, versioned Open Contracting Release.'
# Release IDs, dates and tags appear alongside values in the versioned release schema.
remove_omit_when_merged(schema)
# Create unversioned copies of all definitions.
unversioned_definitions = {k + 'Unversioned': deepcopy(v) for k, v in schema['definitions'].items()}
update_refs_to_unversioned_definitions(unversioned_definitions)
# Determine which `id` fields occur on objects in arrays.
unversioned_pointers = set()
get_unversioned_pointers(JsonRef.replace_refs(schema), unversioned_pointers)
# Omit `ocid` from versioning.
ocid = schema['properties'].pop('ocid')
add_versioned(schema, unversioned_pointers)
schema['properties']['ocid'] = ocid
# Add the common versioned definitions.
for name, keywords in common_versioned_definitions.items():
versioned = deepcopy(versioned_template)
for keyword, value in keywords.items():
if value:
versioned['items']['properties']['value'][keyword] = value
schema['definitions'][name] = versioned
# Add missing definitions.
while True:
ref = JsonRef.replace_refs(schema)
try:
repr(ref)
break
except JsonRefError as e:
name = e.cause.args[0]
if name.endswith('VersionedId'):
# Add a copy of an definition with a versioned `id` field, using the same logic as before.
definition = deepcopy(schema['definitions'][name[:-11]])
pointer = f'/definitions/{name[:-11]}/properties/id'
pointers = unversioned_pointers - {pointer}
_add_versioned(definition, pointers, pointer, 'id', definition['properties']['id'])
else:
# Add a copy of an definition with no versioned fields.
definition = unversioned_definitions[name]
schema['definitions'][name] = definition
# Remove all metadata and extended keywords.
remove_metadata_and_extended_keywords(schema)
return schema
def test_repr_does_not_loop(self):
json = {"a": ["aoeu", {"$ref": "#/a"}]}
# By default python repr recursion detection should handle it
assert (repr(JsonRef.replace_refs(json)) == "{'a': ['aoeu', [...]]}")
# If we turn of load_on_repr we should get a different representation
assert (repr(JsonRef.replace_refs(
json,
load_on_repr=False)) == "{'a': ['aoeu', JsonRef({'$ref': '#/a'})]}"
)
def load_spec(spec_uri, base_uri=None, password=None):
if base_uri:
spec_uri = urljoin(base_uri, spec_uri)
loader = YamlLoader(password=password)
spec = JsonRef({'$ref': spec_uri}, loader=loader)
exp = TemplateExpander(spec_uri)
spec = exp(spec)
spec = JsonRef.replace_refs(spec, base_uri=spec_uri, loader=loader)
return spec
def test_abstract_loader_return_none(get_record, current_app):
current_app.config = {'SERVER_NAME': 'http://inspirehep.net'}
expect_none = JsonRef({'$ref': 'http://inspirehep.net'},
loader=AbstractRecordLoader())
assert expect_none == None
expect_none = JsonRef({'$ref': 'http://inspirehep.net/'},
loader=AbstractRecordLoader())
assert expect_none == None
expect_none = JsonRef({'$ref': 'http://inspirehep.net/bad'},
loader=AbstractRecordLoader())
assert expect_none == None
assert get_record.call_count == 0
def test_specific_loaders_return_none(get_db_rec, get_es_rec, current_app):
current_app.config = {'SERVER_NAME': 'http://inspirehep.net'}
get_es_rec.side_effect = RecordGetterError('err', None)
get_db_rec.side_effect = RecordGetterError('err', None)
expect_none = JsonRef({'$ref': 'http://inspirehep.net/api/rt/1'},
loader=DatabaseJsonLoader())
assert expect_none == None
expect_none = JsonRef({'$ref': 'http://inspirehep.net/api/rt/2'},
loader=ESJsonLoader())
assert expect_none == None
assert get_db_rec.call_count == 1
assert get_es_rec.call_count == 1
def test_grants_json_resolving(app, db):
"""Test the loadef for the FundRef dataset."""
load_grants_testdata()
grant_ref = {"$ref": "http://inveniosoftware.org/grants/" "10.13039/501100000923::DP0667033"}
json_resolver = JSONResolver(plugins=["invenio_openaire.resolvers.grants"])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
data = JsonRef.replace_refs(grant_ref, loader=loader)
assert data["title"].startswith("Dispersal and colonisation")
# Invalid grant reference
grant_ref = {"$ref": "http://inveniosoftware.org/grants/" "10.13039/invalid"}
data = JsonRef.replace_refs(grant_ref, loader=loader)
pytest.raises(JsonRefError, dict, data)
def test_abstract_loader_return_none(get_record):
config = {'SERVER_NAME': 'http://inspirehep.net'}
with patch.dict(current_app.config, config):
expect_none = JsonRef({'$ref': 'http://inspirehep.net'},
loader=AbstractRecordLoader())
assert expect_none == None # noqa: E711
expect_none = JsonRef({'$ref': 'http://inspirehep.net/'},
loader=AbstractRecordLoader())
assert expect_none == None # noqa: E711
expect_none = JsonRef({'$ref': 'http://inspirehep.net/bad'},
loader=AbstractRecordLoader())
assert expect_none == None # noqa: E711
assert get_record.call_count == 0
def test_funders_json_resolving(app, db):
"""Test the loadef for the FundRef dataset."""
# Test loading the real FundRef dataset.
# 'grant': {'$ref': 'https://zenodo.org/funders/10.19/11/grants/22'}
load_funders_testdata() # Load test data
example_funder = {
'doi': 'http://dx.doi.org/10.13039/003',
'name': 'Some funder',
'acronyms': [
'SF',
],
'parent': {
'$ref': 'http://dx.doi.org/10.13039/002'
},
'country': "US",
}
json_resolver = JSONResolver(plugins=[
'invenio_openaire.resolvers.funders',
'invenio_openaire.resolvers.grants'
])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
print(PID.query.all())
out_json = JsonRef.replace_refs(example_funder, loader=loader)
assert out_json['parent']['name'] == 'Department of Bar'
assert out_json['parent']['parent']['name'] == 'University of Foo'
def generate(spec_file):
spec = read_spec(spec_file)
spec = _make_valid(spec)
application_name = spec["x-Application"]
open_api_file = spec["x-Spec"]
open_api = read_spec(open_api_file)
structure = read_json("generator/structure.json")
structure["name"] = application_name
resolved_structure = JsonRef.replace_refs(structure)
application = next(
filter(lambda x: x["name"] == application_name,
resolved_structure["sub"]))
test = next(
filter(lambda x: x["name"] == "test", resolved_structure["sub"]))
api = next(filter(lambda x: x["name"] == "api", application["sub"]))
generate_api(structure=api,
test_structure=test,
open_api=open_api,
spec=spec)
domain = next(filter(lambda x: x["name"] == "domain", application["sub"]))
generate_domain(structure=domain, test_structure=test, spec=spec)
store = next(filter(lambda x: x["name"] == "store", application["sub"]))
generate_store(structure=store, test_structure=test, spec=spec)
generate_modules(data=resolved_structure, current_dir=".")
def replace_refs(obj, source='db'):
"""Replaces record refs in obj by bypassing HTTP requests.
Any reference URI that comes from the same server and references a resource
will be resolved directly either from the database or from Elasticsearch.
:param obj:
Dict-like object for which '$ref' fields are recursively replaced.
:param source:
List of sources from which to resolve the references. It can be any of:
* 'db' - resolve from Database
* 'es' - resolve from Elasticsearch
* 'http' - force using HTTP
:returns:
The same obj structure with the '$ref' fields replaced with the object
available at the given URI.
"""
loaders = {
'db': db_record_loader,
'es': es_record_loader,
'http': None
}
if source not in loaders:
raise ValueError('source must be one of {}'.format(loaders.keys()))
loader = loaders[source]
return JsonRef.replace_refs(obj, loader=loader, load_on_repr=False)
def test_specific_loaders_return_none(get_db_rec, get_es_rec, g_p_t_f_e):
g_p_t_f_e.return_value = 'pt'
get_es_rec.side_effect = RecordGetterError('err', None)
get_db_rec.side_effect = RecordGetterError('err', None)
config = {'SERVER_NAME': 'http://inspirehep.net'}
with patch.dict(current_app.config, config):
expect_none = JsonRef({'$ref': 'http://inspirehep.net/api/e/1'},
loader=DatabaseJsonLoader())
assert expect_none == None
expect_none = JsonRef({'$ref': 'http://inspirehep.net/api/e/2'},
loader=ESJsonLoader())
assert expect_none == None
assert get_db_rec.call_count == 1
assert get_es_rec.call_count == 1
def handle(self):
with open(self.input_filename) as fp:
schema = json.load(fp)
schema = JsonRef.replace_refs(schema,
base_uri=pathlib.Path(
os.path.realpath(
self.input_filename)).as_uri())
codelists = {}
self._recurse(schema, codelists)
out = []
for codelist, codelist_data in sorted(codelists.items()):
if codelist_data['open'] and codelist_data['closed']:
type = 'Open AND Closed'
elif codelist_data['open']:
type = 'Open'
else:
type = 'Closed'
out.append({'codelist': codelist, 'type': type})
w = csv.DictWriter(sys.stdout, ['codelist', 'type'])
w.writeheader()
w.writerows(out)
def generate_classes(api_key, api_version="1.0", resources="all"):
""" Dynamically create classes to interface with the Civis API.
The Civis API documents behavior using an OpenAPI/Swagger specification.
This function parses the specification and returns a dictionary of
classes to provide a convenient method to make requests to the Civis
API and to view documentation.
https://github.com/OAI/OpenAPI-Specification
Parameters
----------
api_key : str
Your API key obtained from the Civis Platform.
api_version : string, optional
The version of endpoints to call. May instantiate multiple client
objects with different versions. Currently only "1.0" is supported.
resources : string, optional
When set to "base", only the default endpoints will be exposed in the
client object. Set to "all" to include all endpoints available for
a given user, including those that may be in development and subject
to breaking changes at a later date.
"""
assert api_version in API_VERSIONS, (
"APIClient api_version must be one of {}".format(API_VERSIONS))
assert resources in ["base", "all"], ("resources must be one of {}".format(
["base", "all"]))
raw_spec = get_api_spec(api_key, api_version)
spec = JsonRef.replace_refs(raw_spec)
return parse_api_spec(spec, api_version, resources)
def test_jsonref_mode_non_string_is_not_id(self):
base_uri = "http://foo.com/json"
json = {"id": [1], "$ref": "other"}
loader = mock.Mock(return_value="aoeu")
result = JsonRef.replace_refs(json, base_uri=base_uri, loader=loader)
assert result == "aoeu"
loader.assert_called_once_with("http://foo.com/other")
def deref_flattened_spec(self):
deref_spec_dict = JsonRef.replace_refs(self.flattened_spec)
@memoize_by_id
def descend(obj):
# Inline modification of obj
# This method is needed because JsonRef could produce performance penalties in accessing
# the proxied attributes
if isinstance(obj, JsonRef):
# Extract the proxied value
# http://jsonref.readthedocs.io/en/latest/#jsonref.JsonRef.__subject__
return obj.__subject__
if is_dict_like(obj):
for key in list(iterkeys(obj)):
obj[key] = descend(obj[key])
elif is_list_like(obj):
# obj is list like object provided from flattened_spec specs.
# This guarantees that it cannot be a tuple instance and
# inline object modification are allowed
for index in range(len(obj)):
obj[index] = descend(obj[index])
return obj
try:
return descend(deref_spec_dict)
finally:
# Make sure that all memory allocated, for caching, could be released
descend.cache.clear()
def test_nested_refs(self):
json = {"a": {"$ref": "#/b"}, "b": {"$ref": "#/c"}, "c": {"$ref": "#/foo"}}
result = JsonRef.replace_refs(json)
with pytest.raises(JsonRefError) as excinfo:
print(result["a"])
e = excinfo.value
assert e.path == ["c"]
def get(cls, value):
"""Get object type value."""
cls._load_data()
try:
return JsonRef.replace_refs(cls.index_internal_id[value], jsonschema=True, loader=cls._jsonloader)
except KeyError:
return None
def replace_refs(obj, source='db'):
"""Replaces record refs in obj by bypassing HTTP requests.
Any reference URI that comes from the same server and references a resource
will be resolved directly either from the database or from Elasticsearch.
:param obj:
Dict-like object for which '$ref' fields are recursively replaced.
:param source:
List of sources from which to resolve the references. It can be any of:
* 'db' - resolve from Database
* 'es' - resolve from Elasticsearch
* 'http' - force using HTTP
:returns:
The same obj structure with the '$ref' fields replaced with the object
available at the given URI.
"""
loaders = {
'db': db_record_loader,
'es': es_record_loader,
'http': None
}
if source not in loaders:
raise ValueError('source must be one of {}'.format(loaders.keys()))
loader = loaders[source]
return JsonRef.replace_refs(obj, loader=loader, load_on_repr=False)
def test_jsonref_mode_non_string_is_not_id(self):
base_uri = "http://foo.com/json"
json = {"id": [1], "$ref": "other"}
loader = mock.Mock(return_value="aoeu")
result = JsonRef.replace_refs(json, base_uri=base_uri, loader=loader)
assert result == "aoeu"
loader.assert_called_once_with("http://foo.com/other")
def test_recursive_data_structures_remote(self):
json1 = {"a": {"$ref": "/json2"}}
json2 = {"b": {"$ref": "/json1"}}
loader = mock.Mock(return_value=json2)
result = JsonRef.replace_refs(json1, base_uri="/json1", loader=loader)
assert result["a"]["b"].__subject__ is result
assert result["a"].__subject__ is result["a"]["b"]["a"].__subject__
def check_reload(self) -> None:
# Because importing yaml takes significant time, and we only
# use python-yaml for our API docs, importing it lazily here
# is a significant optimization to `manage.py` startup.
#
# There is a bit of a race here...we may have two processes
# accessing this module level object and both trying to
# populate self.data at the same time. Hopefully this will
# only cause some extra processing at startup and not data
# corruption.
import yaml
from jsonref import JsonRef
with open(self.openapi_path) as f:
mtime = os.fstat(f.fileno()).st_mtime
# Using == rather than >= to cover the corner case of users placing an
# earlier version than the current one
if self.mtime == mtime:
return
openapi = yaml.load(f, Loader=yaml.CSafeLoader)
spec = create_spec(openapi)
self._request_validator = RequestValidator(spec)
self._openapi = naively_merge_allOf_dict(JsonRef.replace_refs(openapi))
self.create_endpoints_dict()
self.mtime = mtime
def test_recursive_data_structures_remote(self):
json1 = {"a": {"$ref": "/json2"}}
json2 = {"b": {"$ref": "/json1"}}
loader = mock.Mock(return_value=json2)
result = JsonRef.replace_refs(json1, base_uri="/json1", loader=loader)
assert result["a"]["b"].__subject__ is result
assert result["a"].__subject__ is result["a"]["b"]["a"].__subject__
def post_process(self, metadata):
"""Post process the data.
add extra data such as title statement.
:param metadata: dictionary version of a record
:return: the modified dictionary
"""
metadata = Document.post_process(metadata)
titles = metadata.get('title', [])
text_title = title_format_text_head(titles, with_subtitle=False)
if text_title:
metadata['ui_title_text'] = text_title
responsibility = metadata.get('responsibilityStatement', [])
text_title = title_format_text_head(titles,
responsibility,
with_subtitle=False)
if text_title:
metadata['ui_title_text_responsibility'] = text_title
contributions = metadata.get('contribution', [])
new_contributions = []
for contribution in contributions:
agent = contribution['agent']
agent_type = agent['type']
agent_data = JsonRef.replace_refs(agent,
loader=None).get('metadata')
if agent_data:
agent_data.pop('$schema', None)
agent = agent_data
agent['type'] = agent_type
new_contributions.append({'agent': agent})
if new_contributions:
metadata['contribution'] = create_contributions(new_contributions)
return metadata
def deref_flattened_spec(self):
deref_spec_dict = JsonRef.replace_refs(self.flattened_spec)
@memoize_by_id
def descend(obj):
# Inline modification of obj
# This method is needed because JsonRef could produce performance penalties in accessing
# the proxied attributes
if isinstance(obj, JsonRef):
# Extract the proxied value
# http://jsonref.readthedocs.io/en/latest/#jsonref.JsonRef.__subject__
return obj.__subject__
if is_dict_like(obj):
for key in list(iterkeys(obj)):
obj[key] = descend(obj[key])
elif is_list_like(obj):
# obj is list like object provided from flattened_spec specs.
# This guarantees that it cannot be a tuple instance and
# inline object modification are allowed
for index in range(len(obj)):
obj[index] = descend(obj[index])
return obj
try:
return descend(deref_spec_dict)
finally:
# Make sure that all memory allocated, for caching, could be released
descend.cache.clear()
def patched_validate(data, schema, *args, **kwargs):
if not isinstance(schema, dict):
schema = {'$ref': schema}
schema = JsonRef.replace_refs(schema,
loader=_records_state.loader_cls())
schema = resolve_schema(schema)
return previous_validate(data, schema, *args, **kwargs)
def test_jsonschema_mode_remote(self):
base_uri = "http://foo.com/schema"
json = {
"a": {"$ref": "otherSchema"},
"b": {
"id": "http://bar.com/a/schema",
"c": {"$ref": "otherSchema"},
"d": {"$ref": "/otherSchema"},
"e": {"id": "/b/schema", "$ref": "otherSchema"},
},
}
counter = itertools.count()
loader = mock.Mock(side_effect=lambda uri: next(counter))
result = JsonRef.replace_refs(json, loader=loader, base_uri=base_uri, jsonschema=True)
assert result["a"] == 0
loader.assert_called_once_with("http://foo.com/otherSchema")
loader.reset_mock()
assert result["b"]["c"] == 1
loader.assert_called_once_with("http://bar.com/a/otherSchema")
loader.reset_mock()
assert result["b"]["d"] == 2
loader.assert_called_once_with("http://bar.com/otherSchema")
loader.reset_mock()
assert result["b"]["e"] == 3
loader.assert_called_once_with("http://bar.com/b/otherSchema")
def generate_cli():
spec = retrieve_spec_dict()
# Replace references in the spec so that we don't have to worry about them
# when making the CLI.
spec = JsonRef.replace_refs(spec)
cli = click.Group()
# Iterate through top-level resources (e.g., Scripts, Files, Models).
groups = {}
for path, path_dict in spec['paths'].items():
resource = path.strip('/').split('/')[0]
grp = groups.get(resource)
if grp is None:
grp = click.Group(munge_name(resource),
short_help='Manage {}'.format(resource))
cli.add_command(grp)
groups[resource] = grp
add_path_commands(path, path_dict, grp, resource)
add_extra_commands(cli)
return cli
def get_unassigned_ports(component_json):
with open(component_json) as fin:
block = json5.load(fin)
block = JsonRef.replace_refs(block)
try:
block['component']['model']['ports']
except:
logging.error(
'obj["component"]["model"]["ports"] not accessible in {}'.format(
component_json))
try:
block['component']['busInterfaces']
except:
logging.error(
'obj["component"]["busInterfaces"] not accessible in {}'.format(
component_json))
all_ports = format_ports(block['component']['model']['ports'])
bus_interfaces = block['component']['busInterfaces']
def get_portnames(interface):
atkey = 'abstractionTypes'
pmkey = 'portMaps'
vkey = 'viewRef'
if interface['busType'] == 'bundle':
pns_ = flatten([
_get_bundle_ports(at[pmkey]) for at in interface[atkey]
if at[vkey] == 'RTLview'
])
else:
pns_ = flatten([
at[pmkey].values() for at in interface[atkey]
if at[vkey] == 'RTLview'
])
nv_pns = [p for p in pns_ if type(p) == str]
# flatten vectors
v_pns = flatten([p for p in pns_ if type(p) != str])
return set(chain(v_pns, nv_pns))
portname_sets = [
set(get_portnames(interface)) for interface in bus_interfaces
]
seen = set()
dups = set()
for pns in portname_sets:
dups |= (pns & seen)
seen |= pns
if len(dups) > 0:
logging.error('ports illegally belong to multiple bus interfaces:')
for port in list(sorted(dups))[:10]:
logging.error(' - {}'.format(port))
if len(dups) > 10:
logging.error(' ...')
assn_portnames = set(seen)
unassn_ports = [p for p in all_ports if p[0] not in assn_portnames]
return unassn_ports
def to_schema(x, fields_to_remove):
out = deepcopy(
JsonRef.replace_refs(x.schema(), jsonschema=True)
) # need deepcopy to create a proper dict, jsonref returns a dict-like object that is not json serializable
_remove_complex_ckan_fields(out, fields_to_remove)
_migrate_to_schema3(
out) # migrate required to schema3 format to comply with jsonform
return dict(out)
def test_base_uri_resolution(self):
json = {"$ref": "foo"}
loader = mock.Mock(return_value=17)
result = JsonRef.replace_refs(json,
base_uri="http://bar.com",
loader=loader)
assert result == 17
loader.assert_called_once_with("http://bar.com/foo")
def test_grants_json_resolving(app, db, es, funders):
"""Test the loadef for the FundRef dataset."""
load_grants_testdata()
grant_ref = {'$ref': 'http://inveniosoftware.org/grants/'
'10.13039/501100000923::DP0667033'}
json_resolver = JSONResolver(
plugins=['invenio_openaire.resolvers.grants'])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
data = JsonRef.replace_refs(grant_ref, loader=loader)
assert data['title'].startswith('Dispersal and colonisation')
# Invalid grant reference
grant_ref = {'$ref': 'http://inveniosoftware.org/grants/'
'10.13039/invalid'}
data = JsonRef.replace_refs(grant_ref, loader=loader)
pytest.raises(JsonRefError, dict, data)
def test_grants_json_resolving(app, db, es, funders):
"""Test the loadef for the FundRef dataset."""
load_grants_testdata()
grant_ref = {'$ref': 'http://inveniosoftware.org/grants/'
'10.13039/501100000923::DP0667033'}
json_resolver = JSONResolver(
plugins=['invenio_openaire.resolvers.grants'])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
data = JsonRef.replace_refs(grant_ref, loader=loader)
assert data['title'].startswith('Dispersal and colonisation')
# Invalid grant reference
grant_ref = {'$ref': 'http://inveniosoftware.org/grants/'
'10.13039/invalid'}
data = JsonRef.replace_refs(grant_ref, loader=loader)
pytest.raises(JsonRefError, dict, data)
def load_spec(spec_uri, base_uri=None, password=None):
if base_uri:
spec_uri = urljoin(base_uri, spec_uri)
loader = YamlLoader(password=password)
spec = JsonRef({'$ref': spec_uri}, loader=loader)
exp = TemplateExpander(spec_uri)
spec = exp(spec)
spec = JsonRef.replace_refs(spec, base_uri=spec_uri, loader=loader)
return spec
def get(cls, value):
"""Get object type value."""
cls._load_data()
try:
return JsonRef.replace_refs(cls.index_internal_id[value],
jsonschema=True,
loader=cls._jsonloader)
except KeyError:
return None
def setUpClass(cls):
descriptor = open(cls.SCHEMA_PATH)
schema = yaml.load(descriptor)
cls.swagger = JsonRef.replace_refs(schema)
descriptor.close()
cls.api_base = cls.swagger['basePath']
cls.class_timestamp = round(time.time() * 1000)
def get_schema(self, schema_path):
schema_data = current_jsonschemas.get_schema(schema_path,
with_refs=False,
resolved=False)
schema_data = JsonRef.replace_refs(
schema_data,
base_uri=current_jsonschemas.path_to_url(schema_path),
loader=internal_invenio_loader)
return current_jsonschemas.resolver_cls(schema_data)
def get_swagger():
schema_file_path = os.path.dirname(
os.path.realpath(__file__)) + '/../../mocks/server.yml'
with open(schema_file_path) as f:
swagger_data = yaml.load(f)
swagger = JsonRef.replace_refs(swagger_data)
f.close()
return swagger
def test_basic_error_properties(self):
json = [{"$ref": "#/x"}]
result = JsonRef.replace_refs(json)
with pytest.raises(JsonRefError) as excinfo:
result[0].__subject__
e = excinfo.value
assert e.reference == json[0]
assert e.uri == "#/x"
assert e.base_uri == ""
assert e.path == [0]
assert type(e.cause) == TypeError
def test_repr_expands_deep_refs_by_default(self):
json = {
"a": "string",
"b": {"$ref": "#/a"},
"c": {"$ref": "#/b"},
"d": {"$ref": "#/c"},
"e": {"$ref": "#/d"},
"f": {"$ref": "#/e"},
}
assert (
repr(sorted(JsonRef.replace_refs(json).items())) == "[('a', 'string'), ('b', 'string'), ('c', 'string'), "
"('d', 'string'), ('e', 'string'), ('f', 'string')]"
)
# Should not expand when set to False explicitly
result = JsonRef.replace_refs(json, load_on_repr=False)
assert (
repr(sorted(result.items())) == "[('a', 'string'), ('b', JsonRef({'$ref': '#/a'})), "
"('c', JsonRef({'$ref': '#/b'})), ('d', JsonRef({'$ref': '#/c'})), "
"('e', JsonRef({'$ref': '#/d'})), ('f', JsonRef({'$ref': '#/e'}))]"
)
def test_local_nonexistent_ref(self):
json = {
"data": [1, 2],
"a": {"$ref": "#/x"},
"b": {"$ref": "#/0"},
"c": {"$ref": "#/data/3"},
"d": {"$ref": "#/data/b"},
}
result = JsonRef.replace_refs(json)
for key in "abcd":
with pytest.raises(JsonRefError):
result[key].__subject__
def test_grants_json_resolving(app):
"""Test the loadef for the FundRef dataset."""
with app.app_context():
load_grants_testdata()
grant_ref = {'$ref': 'http://inveniosoftware.org/grants/'
'10.13039/501100000923::DP0667033'}
json_resolver = JSONResolver(
plugins=['invenio_openaire.resolvers.grants'])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
data = JsonRef.replace_refs(grant_ref, loader=loader)
assert data['title'].startswith('Dispersal and colonisation')
def test_jsonschema_mode_local(self):
json = {
"a": {
"id": "http://foo.com/schema",
"b": "aoeu",
# Reference should now be relative to this inner object, rather
# than the whole document
"c": {"$ref": "#/b"},
}
}
result = JsonRef.replace_refs(json, jsonschema=True)
assert result["a"]["c"] == json["a"]["b"]
def test_same_route_different_hosts():
"""Test orignal resolver."""
example = {
'host1': {'$ref': 'http://localhost:4000/test'},
'host2': {'$ref': 'http://inveniosoftware.org/test'},
}
json_resolver = JSONResolver(plugins=['demo.simple', 'demo.hosts'])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
data = JsonRef.replace_refs(example, loader=loader)
assert data['host1']['test'] == 'test'
assert data['host2']['test'] == 'inveniosoftware.org'
def test_license_jsonref_resolver(app, loaded_example_licenses):
"""Test resolver."""
with app.app_context():
example_license = {
'license': {'$ref': 'http://inveniosoftware.org/licenses/mit'}
}
json_resolver = JSONResolver(plugins=[
'invenio_opendefinition.resolvers'
])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
out_json = JsonRef.replace_refs(example_license, loader=loader)
assert out_json['license'] == loaded_example_licenses['MIT']
def jsonschema(collection):
jsonschema_path = os.path.join(os.path.dirname(__file__), 'jsonschemas',
'records', '{0}.json'.format(collection))
json_resolver = JSONResolver(plugins=['cap.modules.records.resolvers.jsonschemas'])
with open(jsonschema_path) as file:
jsonschema_content = file.read()
try:
result = JsonRef.replace_refs(json.loads(jsonschema_content), loader=json_resolver.resolve)
ren = render_template('records/deposit-base-extended.html', schema=json.dumps(result, indent=4))
return jsonify(json.loads(ren))
except:
return jsonify(json.loads(render_template('records/deposit-base-refs.html', collection=collection)))
def test_custom_loader(self):
data = {"$ref": "foo"}
loader = mock.Mock(return_value=42)
result = JsonRef.replace_refs(data, loader=loader)
# Loading should not occur until we do something with result
assert loader.call_count == 0
# Make sure we got the right result
assert result == 42
# Do several more things with result
result + 3
repr(result)
result *= 2
# Make sure we only called the loader once
loader.assert_called_once_with("foo")
def test_key_resolver():
"""Test JSONSchema validation with custom reference resolver."""
example_schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'object',
'properties': {
'authors': {'$ref': 'http://localhost:4000/schema/authors.json#'}
},
'additionalProperties': False,
}
json_resolver = JSONResolver(plugins=['demo.schema'])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
data = JsonRef.replace_refs(example_schema, loader=loader)
assert data['properties']['authors'] == {'type': 'array'}
def get_schema(schema_path):
"""Retrieve a schema."""
try:
schema_dir = state.get_schema_dir(schema_path)
except JSONSchemaNotFound:
abort(404)
if request.args.get('refs',
current_app.config.get('JSONSCHEMAS_REPLACE_REFS'),
type=int):
with open(os.path.join(schema_dir, schema_path), 'r') as file_:
schema = json.load(file_)
return jsonify(JsonRef.replace_refs(
schema, base_uri=request.base_url,
loader=state.loader_cls() if state.loader_cls else None,
))
else:
return send_from_directory(schema_dir, schema_path)
def test_missing_route():
"""Test orignal resolver."""
example_schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'object',
'properties': {
'authors': {'$ref': 'file:///missing.json#'}
},
'additionalProperties': False,
}
json_resolver = JSONResolver(plugins=['demo.schema'])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
data = JsonRef.replace_refs(example_schema, loader=loader)
with pytest.raises(JsonRefError):
data['properties']['authors']['type']
def jsonschema_deposit(collection):
jsonschema_path = os.path.join(os.path.dirname(__file__), 'jsonschemas',
'records', '{0}.json'.format(collection))
json_resolver = JSONResolver(plugins=['cap.modules.records.resolvers.jsonschemas'])
with open(jsonschema_path) as file:
jsonschema_content = file.read()
try:
result = JsonRef.replace_refs(json.loads(jsonschema_content), loader=json_resolver.resolve)
return jsonify(result)
except:
jsonschema_path = os.path.join(os.path.dirname(__file__), 'jsonschemas_gen',
'records', '{0}.json'.format(collection))
with open(jsonschema_path) as file:
jsonschema_content = json.loads(file.read())
return jsonify(jsonschema_content)
def test_funders_json_resolving(doi_host, app, db):
"""Test the loadef for the FundRef dataset."""
# Test loading the real FundRef dataset.
# 'grant': {'$ref': 'https://zenodo.org/funders/10.19/11/grants/22'}
load_funders_testdata() # Load test data
example_funder = {
"doi": "http://{doi_host}/10.13039/003".format(doi_host=doi_host),
"name": "Some funder",
"acronyms": ["SF"],
"parent": {"$ref": "http://{doi_host}/10.13039/002".format(doi_host=doi_host)},
"country": "US",
}
json_resolver = JSONResolver(plugins=["invenio_openaire.resolvers.funders", "invenio_openaire.resolvers.grants"])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
print(PID.query.all())
out_json = JsonRef.replace_refs(example_funder, loader=loader)
assert out_json["parent"]["name"] == "Department of Bar"
assert out_json["parent"]["parent"]["name"] == "University of Foo"
def load_resolved_schema(name):
"""Load a JSON schema with all references resolved.
Args:
name(str): name of the schema to load.
Returns:
dict: the JSON schema with resolved references.
Examples:
>>> resolved_schema = load_resolved_schema('authors')
"""
schema = load_schema(name)
return JsonRef.replace_refs(
schema,
base_uri=url_for('invenio_jsonschemas.get_schema', schema_path='records/{}.json'.format(name)),
loader=SCHEMA_LOADER_CLS()
)
def test_funders_json_resolving(app, db):
"""Test the loadef for the FundRef dataset."""
# Test loading the real FundRef dataset.
# 'grant': {'$ref': 'https://zenodo.org/funders/10.19/11/grants/22'}
load_funders_testdata() # Load test data
example_funder = {
'doi': 'http://dx.doi.org/10.13039/003',
'name': 'Some funder',
'acronyms': ['SF', ],
'parent': {'$ref': 'http://dx.doi.org/10.13039/002'},
'country': "US",
}
json_resolver = JSONResolver(
plugins=['invenio_openaire.resolvers.funders',
'invenio_openaire.resolvers.grants'])
loader_cls = json_loader_factory(json_resolver)
loader = loader_cls()
print(PID.query.all())
out_json = JsonRef.replace_refs(example_funder, loader=loader)
assert out_json['parent']['name'] == 'Department of Bar'
assert out_json['parent']['parent']['name'] == 'University of Foo'
def run(sheets, schema=None, source_maps=False):
input_headings = OrderedDict()
input_sheets = OrderedDict()
for sheet in sheets:
rows = []
for row in sheet['rows']:
rows.append(OrderedDict(zip(sheet['headings'], row)))
input_sheets[sheet['name']] = rows
input_headings[sheet['name']] = sheet['headings']
if schema is not None:
spreadsheet_input = HeadingListInput(
input_sheets,
input_headings,
root_id='', # QUESTION: I don't understand root_id
convert_titles=True, # Without this, the titles aren't understood
)
# Without this, the $ref entries in the schema aren't resolved.
dereferenced_schema = JsonRef.replace_refs(schema)
# raise Exception(dereferenced_schema)
parser = SchemaParser(
root_schema_dict=dereferenced_schema,
root_id='main',
rollup=True
)
parser.parse()
spreadsheet_input.parser = parser
else:
spreadsheet_input = HeadingListInput(
input_sheets,
input_headings,
root_id='',
)
spreadsheet_input.read_sheets()
if source_maps:
result, cell_source_map_data, heading_source_map_data = spreadsheet_input.fancy_unflatten(with_cell_source_map=True, with_heading_source_map=True)
return result, cell_source_map_data, heading_source_map_data
else:
return spreadsheet_input.unflatten(), None, None
def get_schema(self, path, with_refs=False, resolved=False):
"""Retrieve a schema.
:param path: schema's relative path.
:param with_refs: replace $refs in the schema.
:param resolved: resolve schema using the resolver
:py:const:`invenio_jsonschemas.config.JSONSCHEMAS_RESOLVER_CLS`
:raises invenio_jsonschemas.errors.JSONSchemaNotFound: If no schema
was found in the specified path.
:returns: The schema in a dictionary form.
"""
if path not in self.schemas:
raise JSONSchemaNotFound(path)
with open(os.path.join(self.schemas[path], path)) as file_:
schema = json.load(file_)
if with_refs:
schema = JsonRef.replace_refs(
schema,
base_uri=request.base_url,
loader=self.loader_cls() if self.loader_cls else None,
)
if resolved:
schema = self.resolver_cls(schema)
return schema
from jsonref import JsonRef
import requests
import sys
import copy
try:
r = requests.get(sys.argv[1])
release = r.json()
print("Fetched schema from URL")
except:
print("Using local release schema")
with open('release-schema.json', 'r') as f:
release = json.loads(f.read(), object_pairs_hook=collections.OrderedDict)
release = JsonRef.replace_refs(release)
print(jsonref.dumps(release, indent=3))
# Based on https://stackoverflow.com/questions/30734682/extracting-url-and-anchor-text-from-markdown-using-python
INLINE_LINK_RE = re.compile(r'\[([^\]]+)\]\(([^)]+)\)')
def find_md_links(md):
links = dict(INLINE_LINK_RE.findall(md))
return links
def remove_links(text, links):
for key, link in links.items():
