def resolve_schema_references(schema,
loaded_schemas,
schema_url=None,
refs=None):
""" Resolves and replaces json-schema $refs with the appropriate dict.
Recursively walks the given schema dict, converting every instance
of $ref in a 'properties' structure with a resolved dict.
This modifies the input schema and also returns it.
:param schema: the schema dict
:param loaded_schemas: a recursive dictionary that stores the path of
already loaded schemas to prevent circularity issues
:param refs: a dict of <string, dict> which forms a store of referenced schemata
:param schema_url: the URL of the schema
:return: schema
"""
schema = OrderedDict(schema)
refs = refs or {}
if schema_url:
return _resolve_schema_references(
schema, RefResolver(schema_url, schema, store=refs),
loaded_schemas, '#')
else:
return _resolve_schema_references(schema,
RefResolver("", schema, store=refs),
loaded_schemas, '#')
def resolver(self):
# type: () -> RefResolver
return RefResolver(
base_uri=self.origin_url or '',
referrer=self.spec_dict,
handlers=self.get_ref_handlers(),
)
def make_validation(self):
""" Method to run the mapping for the given number of items
:return: a dictionary containing the list of errors for all processed items
"""
valid = {}
invalid = {}
user_accessible_ids = self.get_user_content_id()
# print(user_accessible_ids)
if isinstance(user_accessible_ids, Exception):
return Exception("Error with client ID " + self.clientID)
else:
schema = json.loads(requests.get(self.schema_url).text)
resolver = RefResolver(self.schema_url, schema, {})
validator = Draft4Validator(schema, resolver=resolver)
content = self.get_all_experiments(self.item_number, user_accessible_ids)
for raw_experiment in content:
experiment = self.preprocess_content(content[raw_experiment])
try:
validation = validator.validate(experiment)
if validation is None:
valid[raw_experiment] = experiment
else:
invalid[raw_experiment] = validation
except Exception as e:
invalid[raw_experiment] = "Unexpected error: " + str(e)
return valid, invalid
def __init__(self,
spec_dict,
origin_url=None,
http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client))
def validate_instance(schemapath, schemafile, instancepath, instancefile,
error_printing, store):
""" Validate a JSON instance against a JSON schema.
:param schemapath: the path to the schema directory
:param schemafile: the name of the schema file
:param instancepath: the path of the instance direvotyr
:param instancefile: the name of the instance path
:param error_printing: the error log
:param store: a store required by RefResolver
:return: errors
"""
instancefile_fullpath = os.path.join(instancepath, instancefile)
instance_file = open(instancefile_fullpath)
instance = json.load(instance_file)
instance_file.close()
schemafile_fullpath = os.path.join(schemapath, schemafile)
schema_file = open(schemafile_fullpath)
schema = json.load(schema_file)
schema_file.close()
resolver = RefResolver('file://' + schemapath + '/' + schemafile, schema,
store)
return validate_instance_against_schema(instance, resolver, schema)
def test_if_you_give_it_junk_you_get_a_resolution_error(self):
ref = "foo://bar"
foo_handler = mock.Mock(side_effect=ValueError("Oh no! What's this?"))
resolver = RefResolver("", {}, handlers={"foo": foo_handler})
with self.assertRaises(RefResolutionError) as err:
with resolver.resolving(ref):
pass
self.assertEqual(str(err.exception), "Oh no! What's this?")
def test_custom_uri_scheme_handlers(self):
schema = {"foo": "bar"}
ref = "foo://bar"
foo_handler = mock.Mock(return_value=schema)
resolver = RefResolver("", {}, handlers={"foo": foo_handler})
with resolver.resolving(ref) as resolved:
self.assertEqual(resolved, schema)
foo_handler.assert_called_once_with(ref)
def get_validator(schema_name: str):
store = _get_schema_store()
schema = store[_schema_id_base + schema_name]
return validator_for(schema)(
schema,
resolver=RefResolver("", schema, store),
format_checker=draft7_format_checker,
)
def test_ref():
ref_dict = {'$ref': '#/definitions/Foo'}
definitions = {
'definitions': {
'Foo': 'bar'
}
}
assert deref(ref_dict, RefResolver('', definitions)) == 'bar'
def test_cache_remote_off(self):
ref = "foo://bar"
foo_handler = mock.Mock()
resolver = RefResolver(
"", {}, cache_remote=False, handlers={"foo" : foo_handler},
)
with resolver.resolving(ref):
pass
self.assertEqual(foo_handler.call_count, 1)
def test_it_delegates_to_a_ref_resolver(self):
resolver = RefResolver("", {})
schema = {"$ref": mock.Mock()}
with mock.patch.object(resolver, "resolve") as resolve:
resolve.return_value = "url", {"type": "integer"}
with self.assertRaises(ValidationError):
self.validator_class(schema, resolver=resolver).validate(None)
resolve.assert_called_once_with(schema["$ref"])
def validate_json(spec_dict, schema_path, spec_url='', http_handlers=None):
"""Validate a json document against a json schema.
:param spec_dict: json document in the form of a list or dict.
:param schema_path: package relative path of the json schema file.
:param spec_url: base uri to use when creating a
RefResolver for the passed in spec_dict.
:param http_handlers: used to download any remote $refs in spec_dict with
a custom http client. Defaults to None in which case the default
http client built into jsonschema's RefResolver is used. This
is a mapping from uri scheme to a callable that takes a
uri.
:return: RefResolver for spec_dict with cached remote $refs used during
validation.
:rtype: :class:`jsonschema.RefResolver`
"""
schema_path = resource_filename('swagger_spec_validator', schema_path)
schema = read_file(schema_path)
schema_resolver = RefResolver(
base_uri=get_uri_from_file_path(schema_path),
referrer=schema,
handlers=default_handlers,
)
spec_resolver = RefResolver(
base_uri=spec_url,
referrer=spec_dict,
handlers=http_handlers or default_handlers,
)
ref_validators.validate(
instance=spec_dict,
schema=schema,
resolver=schema_resolver,
instance_cls=ref_validators.create_dereffing_validator(spec_resolver),
cls=Draft4Validator,
)
# Since remote $refs were downloaded, pass the resolver back to the caller
# so that its cached $refs can be re-used.
return spec_resolver
def get_schema_from_uri(cls, schema_uri):
schema = cls._schema_cache.get(schema_uri)
if schema is None:
raise Exception('Could not read schema from {}'.format(schema_uri))
resolver = RefResolver(base_uri=schema_uri,
referrer=schema,
store=SchemaValidator._schema_cache)
return (schema, resolver)
def test_cache_remote_on(self):
ref = "foo://bar"
foo_handler = mock.Mock()
resolver = RefResolver(
"", {}, cache_remote=True, handlers={"foo" : foo_handler},
)
with resolver.resolving(ref):
pass
with resolver.resolving(ref):
pass
foo_handler.assert_called_once_with(ref)
def create_spec(
spec_dict, spec_url='', handlers=default_handlers,
validate_spec=True,
):
if validate_spec:
openapi_v3_spec_validator.validate(spec_dict, spec_url=spec_url)
spec_resolver = RefResolver(
spec_url, spec_dict, handlers=handlers)
spec_factory = SpecFactory(spec_resolver)
return spec_factory.create(spec_dict, spec_url=spec_url)
def test_resolve_schema_ref(self):
expected_output = self.load_expected_input()[0]['schemas']
network_schemas = {}
schema_content = json.loads(
requests.get(self.network_1_schema_url).text)
network_schemas[get_name(schema_content['id'])] = schema_content
resolver = RefResolver(self.network_1_schema_url,
schema_content,
store={})
data = prepare_fulldiff_input.resolve_schema_ref(
schema_content, resolver, network_schemas)
self.assertTrue(DeepDiff(data, expected_output) == {})
def create_spec(
spec_dict,
spec_url="",
handlers=default_handlers,
validate_spec=True,
):
if validate_spec:
openapi_v3_spec_validator.validate(spec_dict, spec_url=spec_url)
spec_resolver = RefResolver(spec_url, spec_dict, handlers=handlers)
dereferencer = Dereferencer(spec_resolver)
return SpecPath.from_spec(spec_dict, dereferencer)
def resolve_network_file(schema_file):
network_schemas = {}
try:
with open(schema_file.replace("file:/", '')) as f:
schema_content = json.load(f)
network_schemas[get_name(schema_content['id'])] = schema_content
resolver = RefResolver(schema_file, schema_content, store={})
return resolve_schema_ref(schema_content, resolver,
network_schemas)
except Exception as e:
raise Exception("There is a problem with your url or schema: ",
schema_file, ", ", e)
def _validate_parameter_examples(schema, examples, base_uri):
base_uri = _get_base_uri(schema, base_uri)
spec_resolver = RefResolver(base_uri, schema, handlers=handlers)
for example in itervalues(examples):
ref_validators.validate(
instance=example,
schema=schema,
resolver=spec_resolver,
instance_cls=ref_validators.create_dereffing_validator(
spec_resolver),
cls=Draft4Validator)
def from_dict(
cls,
data,
*args,
url="",
ref_resolver_handlers=default_handlers,
separator=SPEC_SEPARATOR,
):
ref_resolver = RefResolver(url, data, handlers=ref_resolver_handlers)
dereferencer = Dereferencer(ref_resolver)
accessor = SpecAccessor(data, dereferencer)
return cls(accessor, *args, separator=separator)
def test__resolve_schema_references(self):
schema_url = 'https://w3id.org/dats/schema/person_schema.json#'
processed_schemas = {compile_schema.get_name(schema_url): '#'}
schema = compile_schema.resolve_reference(schema_url)
resolver = RefResolver(schema_url, schema, store={})
data = compile_schema._resolve_schema_references(
schema, resolver, processed_schemas, '#')
output_value = json.loads(json.dumps(data))
expected_output = json.loads(json.dumps(self.expected_output))
self.assertTrue(DeepDiff(output_value, expected_output) == {})
def get_schema(self, obj_type):
schema_uri = SchemaValidator.schemas.get(obj_type)
if schema_uri is None:
raise Exception('No schema for type {}'.format(obj_type))
schema = self.schema_cache.get(obj_type)
if schema is None:
schema = json.loads(STAC_IO.read_text(schema_uri))
self.schema_cache[obj_type] = schema
resolver = RefResolver(base_uri=schema_uri, referrer=schema)
return (schema, resolver)
def validate_json(spec_dict,
schema_url=SCHEMA_URL,
spec_url='',
spec_url_base_path=None,
schema_url_base_path=None):
spec_url, schema_url = _normalize_urls(spec_url, schema_url,
spec_url_base_path,
schema_url_base_path)
schema = read_url(schema_url)
schema_resolver = RefResolver(base_uri=schema_url,
referrer=schema,
handlers=handlers)
spec_resolver = RefResolver(base_uri=spec_url,
referrer=spec_dict,
handlers=handlers)
ref_validators.validate(
instance=spec_dict,
schema=schema,
resolver=schema_resolver,
instance_cls=ref_validators.create_dereffing_validator(spec_resolver),
cls=Draft4Validator)
return spec_resolver
def resolve_network_url(schema_url):
""" Function that triggers the resolved_schema_ref function
:param schema_url: a schema URL
:return: a fully resolved network
"""
network_schemas = {}
try:
schema_content = json.loads(requests.get(schema_url).text)
network_schemas[get_name(schema_content['id'])] = schema_content
resolver = RefResolver(schema_url, schema_content, store={})
return resolve_schema_ref(schema_content, resolver, network_schemas)
except Exception as e:
raise Exception("There is a problem with your url or schema",
schema_url, "exception ", e)
def test_it_delegates_to_a_ref_resolver(self):
resolver = RefResolver("", {})
schema = {"$ref": mock.Mock()}
@contextmanager
def resolving():
yield {"type": "integer"}
with mock.patch.object(resolver,
"resolving") as resolve: # @UndefinedVariable
resolve.return_value = resolving()
with self.assertRaises(ValidationError):
self.validator_class(schema, resolver=resolver).validate(None)
resolve.assert_called_once_with(schema["$ref"])
def validate_instance(self, user_input):
""" Validates an instance against a schema
:param user_input: a dictionary containing the schema_url and instance_url attributes
:type user_input: dict
:return: a validation str or a list of errors
"""
try:
schema_url = user_input['schema_url']
instance_url = user_input['instance_url']
schema = requests.get(schema_url)
instance = requests.get(instance_url)
if schema.status_code != 200:
raise falcon.HTTPError(falcon.HTTP_400, "verifiy your URL ",
schema_url)
elif instance.status_code != 200:
raise falcon.HTTPError(falcon.HTTP_400, "verifiy your URL ",
instance_url)
else:
try:
resolver = RefResolver(schema_url, schema, {})
drafter = Draft4Validator(json.loads(schema.text),
resolver=resolver)
errors_array = sorted(drafter.iter_errors(
json.loads(instance.text)),
key=lambda e: e.path)
errors = {}
for i in range(len(errors_array)):
errors[i] = errors_array[i].message
if len(errors) > 0:
return json.dumps(errors, indent=4)
else:
return json.dumps("Your json is valid")
except Exception:
raise falcon.HTTPError(
falcon.HTTP_400, "Malformed JSON, "
"please verify your schema and your instance")
except requests.RequestException as e:
raise falcon.HTTPError(
falcon.HTTP_400,
"Problem loading your schema or your instance: ", str(e))
def __init__(self,
spec_dict,
origin_url=None,
http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# Cached copy of spec_dict with x-scope metadata removed.
# See @property client_spec_dict().
self._client_spec_dict = None
# Cached copy of dereferenced spec_dict with x-scope metadata removed.
# See @property dereferenced_spec().
self._flattened_spec = None
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client),
)
# generated by @property to avoid multiple swagger validations
self._security_definitions = None
def load_core_schema(version):
'''
Given a query schema version, loads the schema from the schema_base_path directory.
:param version: the version to load
:return: the loaded schema (as a dict) and a jsonschmea validator object for the schema
'''
schema_file = os.path.join(schema_base_path, version,
u'{}.json'.format(version))
with io.open(schema_file, u'r', encoding=u'utf-8') as f:
schema = json.load(f)
validator_cls = validator_for(schema)
validator_cls.check_schema(schema)
# create a resolver which can resolve refs relative to the schema
resolver = RefResolver(base_uri='file://{}'.format(schema_file),
referrer=schema)
validator = validator_cls(schema, resolver=resolver)
return schema, validator
def __init__(self,
spec_dict,
origin_url=None,
http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client),
)
self._validate_config()
if self.config['internally_dereference_refs']:
# If internally_dereference_refs is enabled we do NOT need to resolve references anymore
# it's useless to evaluate is_ref every time
self.deref = lambda ref_dict: ref_dict
else:
self.deref = self._force_deref
def __init__(
self,
spec_dict,
origin_url=None,
http_client=None,
config=None,
):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=self.get_ref_handlers(),
)
# spec dict used to build resources, in case internally_dereference_refs config is enabled
# it will be overridden by the dereferenced specs (by build method). More context in PR#263
self._internal_spec_dict = spec_dict
