def test_repr_expands_deep_refs_by_default(self): json = { "a": "string", "b": { "$ref": "#/a" }, "c": { "$ref": "#/b" }, "d": { "$ref": "#/c" }, "e": { "$ref": "#/d" }, "f": { "$ref": "#/e" } } assert (repr(sorted(JsonRef.replace_refs(json).items())) == "[('a', 'string'), ('b', 'string'), ('c', 'string'), " "('d', 'string'), ('e', 'string'), ('f', 'string')]") # Should not expand when set to False explicitly result = JsonRef.replace_refs(json, load_on_repr=False) assert ( repr(sorted(result.items())) == "[('a', 'string'), ('b', JsonRef({'$ref': '#/a'})), " "('c', JsonRef({'$ref': '#/b'})), ('d', JsonRef({'$ref': '#/c'})), " "('e', JsonRef({'$ref': '#/d'})), ('f', JsonRef({'$ref': '#/e'}))]" )
def get_versioned_release_schema(schema): """ Returns the versioned release schema. """ # Update schema metadata. release_with_underscores = release.replace('.', '__') schema['id'] = f'https://standard.open-contracting.org/schema/{release_with_underscores}/versioned-release-validation-schema.json' # noqa schema['title'] = 'Schema for a compiled, versioned Open Contracting Release.' # Release IDs, dates and tags appear alongside values in the versioned release schema. remove_omit_when_merged(schema) # Create unversioned copies of all definitions. unversioned_definitions = {k + 'Unversioned': deepcopy(v) for k, v in schema['definitions'].items()} update_refs_to_unversioned_definitions(unversioned_definitions) # Determine which `id` fields occur on objects in arrays. unversioned_pointers = set() get_unversioned_pointers(JsonRef.replace_refs(schema), unversioned_pointers) # Omit `ocid` from versioning. ocid = schema['properties'].pop('ocid') add_versioned(schema, unversioned_pointers) schema['properties']['ocid'] = ocid # Add the common versioned definitions. for name, keywords in common_versioned_definitions.items(): versioned = deepcopy(versioned_template) for keyword, value in keywords.items(): if value: versioned['items']['properties']['value'][keyword] = value schema['definitions'][name] = versioned # Add missing definitions. while True: ref = JsonRef.replace_refs(schema) try: repr(ref) break except JsonRefError as e: name = e.cause.args[0] if name.endswith('VersionedId'): # Add a copy of an definition with a versioned `id` field, using the same logic as before. definition = deepcopy(schema['definitions'][name[:-11]]) pointer = f'/definitions/{name[:-11]}/properties/id' pointers = unversioned_pointers - {pointer} _add_versioned(definition, pointers, pointer, 'id', definition['properties']['id']) else: # Add a copy of an definition with no versioned fields. definition = unversioned_definitions[name] schema['definitions'][name] = definition # Remove all metadata and extended keywords. remove_metadata_and_extended_keywords(schema) return schema
def test_repr_does_not_loop(self): json = {"a": ["aoeu", {"$ref": "#/a"}]} # By default python repr recursion detection should handle it assert (repr(JsonRef.replace_refs(json)) == "{'a': ['aoeu', [...]]}") # If we turn of load_on_repr we should get a different representation assert (repr(JsonRef.replace_refs( json, load_on_repr=False)) == "{'a': ['aoeu', JsonRef({'$ref': '#/a'})]}" )
def test_grants_json_resolving(app, db): """Test the loadef for the FundRef dataset.""" load_grants_testdata() grant_ref = {"$ref": "http://inveniosoftware.org/grants/" "10.13039/501100000923::DP0667033"} json_resolver = JSONResolver(plugins=["invenio_openaire.resolvers.grants"]) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() data = JsonRef.replace_refs(grant_ref, loader=loader) assert data["title"].startswith("Dispersal and colonisation") # Invalid grant reference grant_ref = {"$ref": "http://inveniosoftware.org/grants/" "10.13039/invalid"} data = JsonRef.replace_refs(grant_ref, loader=loader) pytest.raises(JsonRefError, dict, data)
def replace_refs(obj, source='db'): """Replaces record refs in obj by bypassing HTTP requests. Any reference URI that comes from the same server and references a resource will be resolved directly either from the database or from Elasticsearch. :param obj: Dict-like object for which '$ref' fields are recursively replaced. :param source: List of sources from which to resolve the references. It can be any of: * 'db' - resolve from Database * 'es' - resolve from Elasticsearch * 'http' - force using HTTP :returns: The same obj structure with the '$ref' fields replaced with the object available at the given URI. """ loaders = { 'db': db_record_loader, 'es': es_record_loader, 'http': None } if source not in loaders: raise ValueError('source must be one of {}'.format(loaders.keys())) loader = loaders[source] return JsonRef.replace_refs(obj, loader=loader, load_on_repr=False)
def deref_flattened_spec(self): deref_spec_dict = JsonRef.replace_refs(self.flattened_spec) @memoize_by_id def descend(obj): # Inline modification of obj # This method is needed because JsonRef could produce performance penalties in accessing # the proxied attributes if isinstance(obj, JsonRef): # Extract the proxied value # http://jsonref.readthedocs.io/en/latest/#jsonref.JsonRef.__subject__ return obj.__subject__ if is_dict_like(obj): for key in list(iterkeys(obj)): obj[key] = descend(obj[key]) elif is_list_like(obj): # obj is list like object provided from flattened_spec specs. # This guarantees that it cannot be a tuple instance and # inline object modification are allowed for index in range(len(obj)): obj[index] = descend(obj[index]) return obj try: return descend(deref_spec_dict) finally: # Make sure that all memory allocated, for caching, could be released descend.cache.clear()
def test_recursive_data_structures_remote(self): json1 = {"a": {"$ref": "/json2"}} json2 = {"b": {"$ref": "/json1"}} loader = mock.Mock(return_value=json2) result = JsonRef.replace_refs(json1, base_uri="/json1", loader=loader) assert result["a"]["b"].__subject__ is result assert result["a"].__subject__ is result["a"]["b"]["a"].__subject__
def generate_classes(api_key, api_version="1.0", resources="all"): """ Dynamically create classes to interface with the Civis API. The Civis API documents behavior using an OpenAPI/Swagger specification. This function parses the specification and returns a dictionary of classes to provide a convenient method to make requests to the Civis API and to view documentation. https://github.com/OAI/OpenAPI-Specification Parameters ---------- api_key : str Your API key obtained from the Civis Platform. api_version : string, optional The version of endpoints to call. May instantiate multiple client objects with different versions. Currently only "1.0" is supported. resources : string, optional When set to "base", only the default endpoints will be exposed in the client object. Set to "all" to include all endpoints available for a given user, including those that may be in development and subject to breaking changes at a later date. """ assert api_version in API_VERSIONS, ( "APIClient api_version must be one of {}".format(API_VERSIONS)) assert resources in ["base", "all"], ("resources must be one of {}".format( ["base", "all"])) raw_spec = get_api_spec(api_key, api_version) spec = JsonRef.replace_refs(raw_spec) return parse_api_spec(spec, api_version, resources)
def test_funders_json_resolving(app, db): """Test the loadef for the FundRef dataset.""" # Test loading the real FundRef dataset. # 'grant': {'$ref': 'https://zenodo.org/funders/10.19/11/grants/22'} load_funders_testdata() # Load test data example_funder = { 'doi': 'http://dx.doi.org/10.13039/003', 'name': 'Some funder', 'acronyms': [ 'SF', ], 'parent': { '$ref': 'http://dx.doi.org/10.13039/002' }, 'country': "US", } json_resolver = JSONResolver(plugins=[ 'invenio_openaire.resolvers.funders', 'invenio_openaire.resolvers.grants' ]) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() print(PID.query.all()) out_json = JsonRef.replace_refs(example_funder, loader=loader) assert out_json['parent']['name'] == 'Department of Bar' assert out_json['parent']['parent']['name'] == 'University of Foo'
def generate(spec_file): spec = read_spec(spec_file) spec = _make_valid(spec) application_name = spec["x-Application"] open_api_file = spec["x-Spec"] open_api = read_spec(open_api_file) structure = read_json("generator/structure.json") structure["name"] = application_name resolved_structure = JsonRef.replace_refs(structure) application = next( filter(lambda x: x["name"] == application_name, resolved_structure["sub"])) test = next( filter(lambda x: x["name"] == "test", resolved_structure["sub"])) api = next(filter(lambda x: x["name"] == "api", application["sub"])) generate_api(structure=api, test_structure=test, open_api=open_api, spec=spec) domain = next(filter(lambda x: x["name"] == "domain", application["sub"])) generate_domain(structure=domain, test_structure=test, spec=spec) store = next(filter(lambda x: x["name"] == "store", application["sub"])) generate_store(structure=store, test_structure=test, spec=spec) generate_modules(data=resolved_structure, current_dir=".")
def generate_cli(): spec = retrieve_spec_dict() # Replace references in the spec so that we don't have to worry about them # when making the CLI. spec = JsonRef.replace_refs(spec) cli = click.Group() # Iterate through top-level resources (e.g., Scripts, Files, Models). groups = {} for path, path_dict in spec['paths'].items(): resource = path.strip('/').split('/')[0] grp = groups.get(resource) if grp is None: grp = click.Group(munge_name(resource), short_help='Manage {}'.format(resource)) cli.add_command(grp) groups[resource] = grp add_path_commands(path, path_dict, grp, resource) add_extra_commands(cli) return cli
def get(cls, value): """Get object type value.""" cls._load_data() try: return JsonRef.replace_refs(cls.index_internal_id[value], jsonschema=True, loader=cls._jsonloader) except KeyError: return None
def test_jsonschema_mode_remote(self): base_uri = "http://foo.com/schema" json = { "a": {"$ref": "otherSchema"}, "b": { "id": "http://bar.com/a/schema", "c": {"$ref": "otherSchema"}, "d": {"$ref": "/otherSchema"}, "e": {"id": "/b/schema", "$ref": "otherSchema"}, }, } counter = itertools.count() loader = mock.Mock(side_effect=lambda uri: next(counter)) result = JsonRef.replace_refs(json, loader=loader, base_uri=base_uri, jsonschema=True) assert result["a"] == 0 loader.assert_called_once_with("http://foo.com/otherSchema") loader.reset_mock() assert result["b"]["c"] == 1 loader.assert_called_once_with("http://bar.com/a/otherSchema") loader.reset_mock() assert result["b"]["d"] == 2 loader.assert_called_once_with("http://bar.com/otherSchema") loader.reset_mock() assert result["b"]["e"] == 3 loader.assert_called_once_with("http://bar.com/b/otherSchema")
def patched_validate(data, schema, *args, **kwargs): if not isinstance(schema, dict): schema = {'$ref': schema} schema = JsonRef.replace_refs(schema, loader=_records_state.loader_cls()) schema = resolve_schema(schema) return previous_validate(data, schema, *args, **kwargs)
def test_nested_refs(self): json = {"a": {"$ref": "#/b"}, "b": {"$ref": "#/c"}, "c": {"$ref": "#/foo"}} result = JsonRef.replace_refs(json) with pytest.raises(JsonRefError) as excinfo: print(result["a"]) e = excinfo.value assert e.path == ["c"]
def test_jsonref_mode_non_string_is_not_id(self): base_uri = "http://foo.com/json" json = {"id": [1], "$ref": "other"} loader = mock.Mock(return_value="aoeu") result = JsonRef.replace_refs(json, base_uri=base_uri, loader=loader) assert result == "aoeu" loader.assert_called_once_with("http://foo.com/other")
def check_reload(self) -> None: # Because importing yaml takes significant time, and we only # use python-yaml for our API docs, importing it lazily here # is a significant optimization to `manage.py` startup. # # There is a bit of a race here...we may have two processes # accessing this module level object and both trying to # populate self.data at the same time. Hopefully this will # only cause some extra processing at startup and not data # corruption. import yaml from jsonref import JsonRef with open(self.openapi_path) as f: mtime = os.fstat(f.fileno()).st_mtime # Using == rather than >= to cover the corner case of users placing an # earlier version than the current one if self.mtime == mtime: return openapi = yaml.load(f, Loader=yaml.CSafeLoader) spec = create_spec(openapi) self._request_validator = RequestValidator(spec) self._openapi = naively_merge_allOf_dict(JsonRef.replace_refs(openapi)) self.create_endpoints_dict() self.mtime = mtime
def handle(self): with open(self.input_filename) as fp: schema = json.load(fp) schema = JsonRef.replace_refs(schema, base_uri=pathlib.Path( os.path.realpath( self.input_filename)).as_uri()) codelists = {} self._recurse(schema, codelists) out = [] for codelist, codelist_data in sorted(codelists.items()): if codelist_data['open'] and codelist_data['closed']: type = 'Open AND Closed' elif codelist_data['open']: type = 'Open' else: type = 'Closed' out.append({'codelist': codelist, 'type': type}) w = csv.DictWriter(sys.stdout, ['codelist', 'type']) w.writeheader() w.writerows(out)
def post_process(self, metadata): """Post process the data. add extra data such as title statement. :param metadata: dictionary version of a record :return: the modified dictionary """ metadata = Document.post_process(metadata) titles = metadata.get('title', []) text_title = title_format_text_head(titles, with_subtitle=False) if text_title: metadata['ui_title_text'] = text_title responsibility = metadata.get('responsibilityStatement', []) text_title = title_format_text_head(titles, responsibility, with_subtitle=False) if text_title: metadata['ui_title_text_responsibility'] = text_title contributions = metadata.get('contribution', []) new_contributions = [] for contribution in contributions: agent = contribution['agent'] agent_type = agent['type'] agent_data = JsonRef.replace_refs(agent, loader=None).get('metadata') if agent_data: agent_data.pop('$schema', None) agent = agent_data agent['type'] = agent_type new_contributions.append({'agent': agent}) if new_contributions: metadata['contribution'] = create_contributions(new_contributions) return metadata
def get_unassigned_ports(component_json): with open(component_json) as fin: block = json5.load(fin) block = JsonRef.replace_refs(block) try: block['component']['model']['ports'] except: logging.error( 'obj["component"]["model"]["ports"] not accessible in {}'.format( component_json)) try: block['component']['busInterfaces'] except: logging.error( 'obj["component"]["busInterfaces"] not accessible in {}'.format( component_json)) all_ports = format_ports(block['component']['model']['ports']) bus_interfaces = block['component']['busInterfaces'] def get_portnames(interface): atkey = 'abstractionTypes' pmkey = 'portMaps' vkey = 'viewRef' if interface['busType'] == 'bundle': pns_ = flatten([ _get_bundle_ports(at[pmkey]) for at in interface[atkey] if at[vkey] == 'RTLview' ]) else: pns_ = flatten([ at[pmkey].values() for at in interface[atkey] if at[vkey] == 'RTLview' ]) nv_pns = [p for p in pns_ if type(p) == str] # flatten vectors v_pns = flatten([p for p in pns_ if type(p) != str]) return set(chain(v_pns, nv_pns)) portname_sets = [ set(get_portnames(interface)) for interface in bus_interfaces ] seen = set() dups = set() for pns in portname_sets: dups |= (pns & seen) seen |= pns if len(dups) > 0: logging.error('ports illegally belong to multiple bus interfaces:') for port in list(sorted(dups))[:10]: logging.error(' - {}'.format(port)) if len(dups) > 10: logging.error(' ...') assn_portnames = set(seen) unassn_ports = [p for p in all_ports if p[0] not in assn_portnames] return unassn_ports
def test_grants_json_resolving(app, db, es, funders): """Test the loadef for the FundRef dataset.""" load_grants_testdata() grant_ref = {'$ref': 'http://inveniosoftware.org/grants/' '10.13039/501100000923::DP0667033'} json_resolver = JSONResolver( plugins=['invenio_openaire.resolvers.grants']) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() data = JsonRef.replace_refs(grant_ref, loader=loader) assert data['title'].startswith('Dispersal and colonisation') # Invalid grant reference grant_ref = {'$ref': 'http://inveniosoftware.org/grants/' '10.13039/invalid'} data = JsonRef.replace_refs(grant_ref, loader=loader) pytest.raises(JsonRefError, dict, data)
def to_schema(x, fields_to_remove): out = deepcopy( JsonRef.replace_refs(x.schema(), jsonschema=True) ) # need deepcopy to create a proper dict, jsonref returns a dict-like object that is not json serializable _remove_complex_ckan_fields(out, fields_to_remove) _migrate_to_schema3( out) # migrate required to schema3 format to comply with jsonform return dict(out)
def test_base_uri_resolution(self): json = {"$ref": "foo"} loader = mock.Mock(return_value=17) result = JsonRef.replace_refs(json, base_uri="http://bar.com", loader=loader) assert result == 17 loader.assert_called_once_with("http://bar.com/foo")
def load_spec(spec_uri, base_uri=None, password=None): if base_uri: spec_uri = urljoin(base_uri, spec_uri) loader = YamlLoader(password=password) spec = JsonRef({'$ref': spec_uri}, loader=loader) exp = TemplateExpander(spec_uri) spec = exp(spec) spec = JsonRef.replace_refs(spec, base_uri=spec_uri, loader=loader) return spec
def setUpClass(cls): descriptor = open(cls.SCHEMA_PATH) schema = yaml.load(descriptor) cls.swagger = JsonRef.replace_refs(schema) descriptor.close() cls.api_base = cls.swagger['basePath'] cls.class_timestamp = round(time.time() * 1000)
def get_schema(self, schema_path): schema_data = current_jsonschemas.get_schema(schema_path, with_refs=False, resolved=False) schema_data = JsonRef.replace_refs( schema_data, base_uri=current_jsonschemas.path_to_url(schema_path), loader=internal_invenio_loader) return current_jsonschemas.resolver_cls(schema_data)
def get_test(): test_file_name = os.path.dirname(__file__) test_schema_file_path = os.path.join(test_file_name, 'schema.json') master_schema_file_path = os.path.join(test_file_name, 'master_schema.json') with open(test_schema_file_path) as test, open(master_schema_file_path) as master: test_schema = json.load(test) test_schema = JsonRef.replace_refs(test_schema) master_schema = json.load(master) validate(test_schema, master_schema) return test_schema
def get_swagger(): schema_file_path = os.path.dirname( os.path.realpath(__file__)) + '/../../mocks/server.yml' with open(schema_file_path) as f: swagger_data = yaml.load(f) swagger = JsonRef.replace_refs(swagger_data) f.close() return swagger
def test_basic_error_properties(self): json = [{"$ref": "#/x"}] result = JsonRef.replace_refs(json) with pytest.raises(JsonRefError) as excinfo: result[0].__subject__ e = excinfo.value assert e.reference == json[0] assert e.uri == "#/x" assert e.base_uri == "" assert e.path == [0] assert type(e.cause) == TypeError
def test_repr_expands_deep_refs_by_default(self): json = { "a": "string", "b": {"$ref": "#/a"}, "c": {"$ref": "#/b"}, "d": {"$ref": "#/c"}, "e": {"$ref": "#/d"}, "f": {"$ref": "#/e"}, } assert ( repr(sorted(JsonRef.replace_refs(json).items())) == "[('a', 'string'), ('b', 'string'), ('c', 'string'), " "('d', 'string'), ('e', 'string'), ('f', 'string')]" ) # Should not expand when set to False explicitly result = JsonRef.replace_refs(json, load_on_repr=False) assert ( repr(sorted(result.items())) == "[('a', 'string'), ('b', JsonRef({'$ref': '#/a'})), " "('c', JsonRef({'$ref': '#/b'})), ('d', JsonRef({'$ref': '#/c'})), " "('e', JsonRef({'$ref': '#/d'})), ('f', JsonRef({'$ref': '#/e'}))]" )
def parse_data(cls, file, image_path): results = [] with open(file, 'r') as f: data = JsonRef.replace_refs(json.load(f)) data = data.get('understandingResults', None) if data is None or not isinstance(data, list): raise ValueError('Wrong schema.') for unit in data: results.append(cls.GTSchema.from_json(unit, image_path)) return results
def test_duplicate_names_generated_from_api_spec(): resolved_civis_api_spec = JsonRef.replace_refs(API_SPEC) paths = resolved_civis_api_spec['paths'] classes = defaultdict(list) for path, ops in paths.items(): class_name, methods = _resources.parse_path(path, ops, "1.0", "all") method_names = [x[0] for x in methods] classes[class_name].extend(method_names) for cls, names in classes.items(): err_msg = "Duplicate methods in {}: {}".format(cls, sorted(names)) assert len(set(names)) == len(names), err_msg
def test_grants_json_resolving(app): """Test the loadef for the FundRef dataset.""" with app.app_context(): load_grants_testdata() grant_ref = {'$ref': 'http://inveniosoftware.org/grants/' '10.13039/501100000923::DP0667033'} json_resolver = JSONResolver( plugins=['invenio_openaire.resolvers.grants']) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() data = JsonRef.replace_refs(grant_ref, loader=loader) assert data['title'].startswith('Dispersal and colonisation')
def generate_api(structure, test_structure, open_api, spec): resolved_open_api = JsonRef.replace_refs(open_api) api_data = get_open_api_data(resolved_open_api) apis = make_api(spec["x-apis"], api_data, application=spec["x-Application"]) structure["sub"].extend(apis) models = make_models("models.py", open_api) structure["sub"].append(models)
def validate_ref(path, data): ref = JsonRef.replace_refs(data) try: # `repr` causes the references to be loaded, if possible. repr(ref) except JsonRefError as e: warnings.warn('ERROR: {} has {} at {}'.format(path, e.message, '/'.join(e.path))) return 1 return 0
def test_local_nonexistent_ref(self): json = { "data": [1, 2], "a": {"$ref": "#/x"}, "b": {"$ref": "#/0"}, "c": {"$ref": "#/data/3"}, "d": {"$ref": "#/data/b"}, } result = JsonRef.replace_refs(json) for key in "abcd": with pytest.raises(JsonRefError): result[key].__subject__
def test_jsonschema_mode_local(self): json = { "a": { "id": "http://foo.com/schema", "b": "aoeu", # Reference should now be relative to this inner object, rather # than the whole document "c": {"$ref": "#/b"}, } } result = JsonRef.replace_refs(json, jsonschema=True) assert result["a"]["c"] == json["a"]["b"]
def test_same_route_different_hosts(): """Test orignal resolver.""" example = { 'host1': {'$ref': 'http://localhost:4000/test'}, 'host2': {'$ref': 'http://inveniosoftware.org/test'}, } json_resolver = JSONResolver(plugins=['demo.simple', 'demo.hosts']) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() data = JsonRef.replace_refs(example, loader=loader) assert data['host1']['test'] == 'test' assert data['host2']['test'] == 'inveniosoftware.org'
def jsonschema(collection): jsonschema_path = os.path.join(os.path.dirname(__file__), 'jsonschemas', 'records', '{0}.json'.format(collection)) json_resolver = JSONResolver(plugins=['cap.modules.records.resolvers.jsonschemas']) with open(jsonschema_path) as file: jsonschema_content = file.read() try: result = JsonRef.replace_refs(json.loads(jsonschema_content), loader=json_resolver.resolve) ren = render_template('records/deposit-base-extended.html', schema=json.dumps(result, indent=4)) return jsonify(json.loads(ren)) except: return jsonify(json.loads(render_template('records/deposit-base-refs.html', collection=collection)))
def test_custom_loader(self): data = {"$ref": "foo"} loader = mock.Mock(return_value=42) result = JsonRef.replace_refs(data, loader=loader) # Loading should not occur until we do something with result assert loader.call_count == 0 # Make sure we got the right result assert result == 42 # Do several more things with result result + 3 repr(result) result *= 2 # Make sure we only called the loader once loader.assert_called_once_with("foo")
def test_license_jsonref_resolver(app, loaded_example_licenses): """Test resolver.""" with app.app_context(): example_license = { 'license': {'$ref': 'http://inveniosoftware.org/licenses/mit'} } json_resolver = JSONResolver(plugins=[ 'invenio_opendefinition.resolvers' ]) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() out_json = JsonRef.replace_refs(example_license, loader=loader) assert out_json['license'] == loaded_example_licenses['MIT']
def test_key_resolver(): """Test JSONSchema validation with custom reference resolver.""" example_schema = { '$schema': 'http://json-schema.org/draft-04/schema#', 'type': 'object', 'properties': { 'authors': {'$ref': 'http://localhost:4000/schema/authors.json#'} }, 'additionalProperties': False, } json_resolver = JSONResolver(plugins=['demo.schema']) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() data = JsonRef.replace_refs(example_schema, loader=loader) assert data['properties']['authors'] == {'type': 'array'}
def test_missing_route(): """Test orignal resolver.""" example_schema = { '$schema': 'http://json-schema.org/draft-04/schema#', 'type': 'object', 'properties': { 'authors': {'$ref': 'file:///missing.json#'} }, 'additionalProperties': False, } json_resolver = JSONResolver(plugins=['demo.schema']) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() data = JsonRef.replace_refs(example_schema, loader=loader) with pytest.raises(JsonRefError): data['properties']['authors']['type']
def get_schema(schema_path): """Retrieve a schema.""" try: schema_dir = state.get_schema_dir(schema_path) except JSONSchemaNotFound: abort(404) if request.args.get('refs', current_app.config.get('JSONSCHEMAS_REPLACE_REFS'), type=int): with open(os.path.join(schema_dir, schema_path), 'r') as file_: schema = json.load(file_) return jsonify(JsonRef.replace_refs( schema, base_uri=request.base_url, loader=state.loader_cls() if state.loader_cls else None, )) else: return send_from_directory(schema_dir, schema_path)
def jsonschema_deposit(collection): jsonschema_path = os.path.join(os.path.dirname(__file__), 'jsonschemas', 'records', '{0}.json'.format(collection)) json_resolver = JSONResolver(plugins=['cap.modules.records.resolvers.jsonschemas']) with open(jsonschema_path) as file: jsonschema_content = file.read() try: result = JsonRef.replace_refs(json.loads(jsonschema_content), loader=json_resolver.resolve) return jsonify(result) except: jsonschema_path = os.path.join(os.path.dirname(__file__), 'jsonschemas_gen', 'records', '{0}.json'.format(collection)) with open(jsonschema_path) as file: jsonschema_content = json.loads(file.read()) return jsonify(jsonschema_content)
def test_funders_json_resolving(doi_host, app, db): """Test the loadef for the FundRef dataset.""" # Test loading the real FundRef dataset. # 'grant': {'$ref': 'https://zenodo.org/funders/10.19/11/grants/22'} load_funders_testdata() # Load test data example_funder = { "doi": "http://{doi_host}/10.13039/003".format(doi_host=doi_host), "name": "Some funder", "acronyms": ["SF"], "parent": {"$ref": "http://{doi_host}/10.13039/002".format(doi_host=doi_host)}, "country": "US", } json_resolver = JSONResolver(plugins=["invenio_openaire.resolvers.funders", "invenio_openaire.resolvers.grants"]) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() print(PID.query.all()) out_json = JsonRef.replace_refs(example_funder, loader=loader) assert out_json["parent"]["name"] == "Department of Bar" assert out_json["parent"]["parent"]["name"] == "University of Foo"
def load_resolved_schema(name): """Load a JSON schema with all references resolved. Args: name(str): name of the schema to load. Returns: dict: the JSON schema with resolved references. Examples: >>> resolved_schema = load_resolved_schema('authors') """ schema = load_schema(name) return JsonRef.replace_refs( schema, base_uri=url_for('invenio_jsonschemas.get_schema', schema_path='records/{}.json'.format(name)), loader=SCHEMA_LOADER_CLS() )
def test_funders_json_resolving(app, db): """Test the loadef for the FundRef dataset.""" # Test loading the real FundRef dataset. # 'grant': {'$ref': 'https://zenodo.org/funders/10.19/11/grants/22'} load_funders_testdata() # Load test data example_funder = { 'doi': 'http://dx.doi.org/10.13039/003', 'name': 'Some funder', 'acronyms': ['SF', ], 'parent': {'$ref': 'http://dx.doi.org/10.13039/002'}, 'country': "US", } json_resolver = JSONResolver( plugins=['invenio_openaire.resolvers.funders', 'invenio_openaire.resolvers.grants']) loader_cls = json_loader_factory(json_resolver) loader = loader_cls() print(PID.query.all()) out_json = JsonRef.replace_refs(example_funder, loader=loader) assert out_json['parent']['name'] == 'Department of Bar' assert out_json['parent']['parent']['name'] == 'University of Foo'
def run(sheets, schema=None, source_maps=False): input_headings = OrderedDict() input_sheets = OrderedDict() for sheet in sheets: rows = [] for row in sheet['rows']: rows.append(OrderedDict(zip(sheet['headings'], row))) input_sheets[sheet['name']] = rows input_headings[sheet['name']] = sheet['headings'] if schema is not None: spreadsheet_input = HeadingListInput( input_sheets, input_headings, root_id='', # QUESTION: I don't understand root_id convert_titles=True, # Without this, the titles aren't understood ) # Without this, the $ref entries in the schema aren't resolved. dereferenced_schema = JsonRef.replace_refs(schema) # raise Exception(dereferenced_schema) parser = SchemaParser( root_schema_dict=dereferenced_schema, root_id='main', rollup=True ) parser.parse() spreadsheet_input.parser = parser else: spreadsheet_input = HeadingListInput( input_sheets, input_headings, root_id='', ) spreadsheet_input.read_sheets() if source_maps: result, cell_source_map_data, heading_source_map_data = spreadsheet_input.fancy_unflatten(with_cell_source_map=True, with_heading_source_map=True) return result, cell_source_map_data, heading_source_map_data else: return spreadsheet_input.unflatten(), None, None
def get_schema(self, path, with_refs=False, resolved=False): """Retrieve a schema. :param path: schema's relative path. :param with_refs: replace $refs in the schema. :param resolved: resolve schema using the resolver :py:const:`invenio_jsonschemas.config.JSONSCHEMAS_RESOLVER_CLS` :raises invenio_jsonschemas.errors.JSONSchemaNotFound: If no schema was found in the specified path. :returns: The schema in a dictionary form. """ if path not in self.schemas: raise JSONSchemaNotFound(path) with open(os.path.join(self.schemas[path], path)) as file_: schema = json.load(file_) if with_refs: schema = JsonRef.replace_refs( schema, base_uri=request.base_url, loader=self.loader_cls() if self.loader_cls else None, ) if resolved: schema = self.resolver_cls(schema) return schema
from jsonref import JsonRef import requests import sys import copy try: r = requests.get(sys.argv[1]) release = r.json() print("Fetched schema from URL") except: print("Using local release schema") with open('release-schema.json', 'r') as f: release = json.loads(f.read(), object_pairs_hook=collections.OrderedDict) release = JsonRef.replace_refs(release) print(jsonref.dumps(release, indent=3)) # Based on https://stackoverflow.com/questions/30734682/extracting-url-and-anchor-text-from-markdown-using-python INLINE_LINK_RE = re.compile(r'\[([^\]]+)\]\(([^)]+)\)') def find_md_links(md): links = dict(INLINE_LINK_RE.findall(md)) return links def remove_links(text, links): for key, link in links.items():
def test_recursive_data_structures_local(self): json = {"a": "foobar", "b": {"$ref": "#"}} result = JsonRef.replace_refs(json) assert result["b"].__subject__ is result