def test_cache_remote_off(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver("", {}, cache_remote=False, handlers={"foo": foo_handler}) with resolver.resolving(ref): pass with resolver.resolving(ref): pass self.assertEqual(foo_handler.call_count, 2)
def test_cache_remote_on(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver("", {}, cache_remote=True, handlers={"foo": foo_handler}) with resolver.resolving(ref): pass with resolver.resolving(ref): pass foo_handler.assert_called_once_with(ref)
def test_cache_remote_on(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver( "", {}, cache_remote=True, handlers={"foo" : foo_handler}, ) with resolver.resolving(ref): pass with resolver.resolving(ref): pass foo_handler.assert_called_once_with(ref)
def test_cache_remote_off(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver( "", {}, cache_remote=False, handlers={"foo" : foo_handler}, ) with resolver.resolving(ref): pass with resolver.resolving(ref): pass self.assertEqual(foo_handler.call_count, 2)
def test_if_you_give_it_junk_you_get_a_resolution_error(self): ref = "foo://bar" foo_handler = mock.Mock(side_effect=ValueError("Oh no! What's this?")) resolver = RefResolver("", {}, handlers={"foo": foo_handler}) with self.assertRaises(RefResolutionError) as err: with resolver.resolving(ref): pass self.assertEqual(str(err.exception), "Oh no! What's this?")
def test_custom_uri_scheme_handlers(self): schema = {"foo": "bar"} ref = "foo://bar" foo_handler = mock.Mock(return_value=schema) resolver = RefResolver("", {}, handlers={"foo": foo_handler}) with resolver.resolving(ref) as resolved: self.assertEqual(resolved, schema) foo_handler.assert_called_once_with(ref)
def test_if_you_give_it_junk_you_get_a_resolution_error(self): ref = "foo://bar" foo_handler = mock.Mock(side_effect=ValueError("Oh no! What's this?")) resolver = RefResolver("", {}, handlers={"foo" : foo_handler}) with self.assertRaises(RefResolutionError) as err: with resolver.resolving(ref): pass self.assertEqual(str(err.exception), "Oh no! What's this?")
def test_custom_uri_scheme_handlers(self): schema = {"foo": "bar"} ref = "foo://bar" foo_handler = mock.Mock(return_value=schema) resolver = RefResolver("", {}, handlers={"foo": foo_handler}) with resolver.resolving(ref) as resolved: self.assertEqual(resolved, schema) foo_handler.assert_called_once_with(ref)
class WalkInstance(Walk): def __init__(self, merger, base, head): Walk.__init__(self, merger) self.base_resolver = RefResolver("", base.val) self.head_resolver = RefResolver("", head.val) def add_meta(self, head, meta): if meta is None: rv = dict() else: rv = dict(meta) rv['value'] = head return rv def default_strategy(self, schema, base, head, meta, **kwargs): if self.is_type(head, "object"): return "objectMerge" else: return "overwrite" def work(self, strategy, schema, base, head, meta, **kwargs): assert isinstance(schema, JSONValue) assert isinstance(base, JSONValue) assert isinstance(head, JSONValue) log.debug("work : %sbase %s, head %s" % (self._indent(), base.ref, head.ref)) if not base.is_undef(): with self.base_resolver.resolving(base.ref) as resolved: assert base.val == resolved if not head.is_undef(): with self.head_resolver.resolving(head.ref) as resolved: assert head.val == resolved rv = strategy.merge(self, base, head, schema, meta, **kwargs) assert isinstance(rv, JSONValue) return rv
class WalkInstance(Walk): def __init__(self, merger, base, head): Walk.__init__(self, merger) self.base_resolver = RefResolver("", base.val) self.head_resolver = RefResolver("", head.val) def add_meta(self, head, meta): if meta is None: rv = dict() else: rv = dict(meta) rv['value'] = head return rv def default_strategy(self, schema, base, head, meta, **kwargs): if self.is_type(head, "object"): return "objectMerge" else: return "overwrite" def work(self, strategy, schema, base, head, meta, **kwargs): assert isinstance(schema, JSONValue) assert isinstance(base, JSONValue) assert isinstance(head, JSONValue) log.debug("work : %sbase %s, head %s" % (self._indent(), base.ref, head.ref)) if not base.is_undef(): with self.base_resolver.resolving(base.ref) as resolved: assert base.val == resolved if not head.is_undef(): with self.head_resolver.resolving(head.ref) as resolved: assert head.val == resolved rv = strategy.merge(self, base, head, schema, meta, **kwargs) assert isinstance(rv, JSONValue) return rv
class TestRefResolver(unittest.TestCase): base_uri = "" stored_uri = "foo://stored" stored_schema = {"stored": "schema"} def setUp(self): self.referrer = {} self.store = {self.stored_uri: self.stored_schema} self.resolver = RefResolver(self.base_uri, self.referrer, self.store) def test_it_does_not_retrieve_schema_urls_from_the_network(self): ref = Draft3Validator.META_SCHEMA["id"] with mock.patch.object(self.resolver, "resolve_remote") as remote: with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, Draft3Validator.META_SCHEMA) self.assertFalse(remote.called) def test_it_resolves_local_refs(self): ref = "#/properties/foo" self.referrer["properties"] = {"foo": object()} with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, self.referrer["properties"]["foo"]) def test_it_resolves_local_refs_with_id(self): schema = {"id": "foo://bar/schema#", "a": {"foo": "bar"}} resolver = RefResolver.from_schema(schema) with resolver.resolving("#/a") as resolved: self.assertEqual(resolved, schema["a"]) with resolver.resolving("foo://bar/schema#/a") as resolved: self.assertEqual(resolved, schema["a"]) def test_it_retrieves_stored_refs(self): with self.resolver.resolving(self.stored_uri) as resolved: self.assertIs(resolved, self.stored_schema) self.resolver.store["cached_ref"] = {"foo": 12} with self.resolver.resolving("cached_ref#/foo") as resolved: self.assertEqual(resolved, 12) def test_it_retrieves_unstored_refs_via_requests(self): ref = "http://bar#baz" schema = {"baz": 12} with mock.patch("jsonschema.validators.requests") as requests: requests.get.return_value.json.return_value = schema with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, 12) requests.get.assert_called_once_with("http://bar") def test_it_retrieves_unstored_refs_via_urlopen(self): ref = "http://bar#baz" schema = {"baz": 12} with mock.patch("jsonschema.validators.requests", None): with mock.patch("jsonschema.validators.urlopen") as urlopen: urlopen.return_value.read.return_value = ( json.dumps(schema).encode("utf8")) with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, 12) urlopen.assert_called_once_with("http://bar") def test_it_can_construct_a_base_uri_from_a_schema(self): schema = {"id": "foo"} resolver = RefResolver.from_schema(schema) self.assertEqual(resolver.base_uri, "foo") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("foo") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("foo#") as resolved: self.assertEqual(resolved, schema) def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): schema = {} resolver = RefResolver.from_schema(schema) self.assertEqual(resolver.base_uri, "") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: self.assertEqual(resolved, schema) def test_custom_uri_scheme_handlers(self): schema = {"foo": "bar"} ref = "foo://bar" foo_handler = mock.Mock(return_value=schema) resolver = RefResolver("", {}, handlers={"foo": foo_handler}) with resolver.resolving(ref) as resolved: self.assertEqual(resolved, schema) foo_handler.assert_called_once_with(ref) def test_cache_remote_on(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver( "", {}, cache_remote=True, handlers={"foo": foo_handler}, ) with resolver.resolving(ref): pass with resolver.resolving(ref): pass foo_handler.assert_called_once_with(ref) def test_cache_remote_off(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver( "", {}, cache_remote=False, handlers={"foo": foo_handler}, ) with resolver.resolving(ref): pass with resolver.resolving(ref): pass self.assertEqual(foo_handler.call_count, 2) def test_if_you_give_it_junk_you_get_a_resolution_error(self): ref = "foo://bar" foo_handler = mock.Mock(side_effect=ValueError("Oh no! What's this?")) resolver = RefResolver("", {}, handlers={"foo": foo_handler}) with self.assertRaises(RefResolutionError) as err: with resolver.resolving(ref): pass self.assertEqual(str(err.exception), "Oh no! What's this?")
class TestRefResolver(unittest.TestCase): base_uri = "" stored_uri = "foo://stored" stored_schema = {"stored" : "schema"} def setUp(self): self.referrer = {} self.store = {self.stored_uri : self.stored_schema} self.resolver = RefResolver(self.base_uri, self.referrer, self.store) def test_it_does_not_retrieve_schema_urls_from_the_network(self): ref = Draft3Validator.META_SCHEMA["id"] with mock.patch.object(self.resolver, "resolve_remote") as remote: with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, Draft3Validator.META_SCHEMA) self.assertFalse(remote.called) def test_it_resolves_local_refs(self): ref = "#/properties/foo" self.referrer["properties"] = {"foo" : object()} with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, self.referrer["properties"]["foo"]) def test_it_resolves_local_refs_with_id(self): schema = {"id": "foo://bar/schema#", "a": {"foo": "bar"}} resolver = RefResolver.from_schema(schema) with resolver.resolving("#/a") as resolved: self.assertEqual(resolved, schema["a"]) with resolver.resolving("foo://bar/schema#/a") as resolved: self.assertEqual(resolved, schema["a"]) def test_it_retrieves_stored_refs(self): with self.resolver.resolving(self.stored_uri) as resolved: self.assertIs(resolved, self.stored_schema) self.resolver.store["cached_ref"] = {"foo" : 12} with self.resolver.resolving("cached_ref#/foo") as resolved: self.assertEqual(resolved, 12) def test_it_retrieves_unstored_refs_via_requests(self): ref = "http://bar#baz" schema = {"baz" : 12} with mock.patch("jsonschema.validators.requests") as requests: requests.get.return_value.json.return_value = schema with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, 12) requests.get.assert_called_once_with("http://bar") def test_it_retrieves_unstored_refs_via_urlopen(self): ref = "http://bar#baz" schema = {"baz" : 12} with mock.patch("jsonschema.validators.requests", None): with mock.patch("jsonschema.validators.urlopen") as urlopen: urlopen.return_value.read.return_value = ( json.dumps(schema).encode("utf8")) with self.resolver.resolving(ref) as resolved: self.assertEqual(resolved, 12) urlopen.assert_called_once_with("http://bar") def test_it_can_construct_a_base_uri_from_a_schema(self): schema = {"id" : "foo"} resolver = RefResolver.from_schema(schema) self.assertEqual(resolver.base_uri, "foo") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("foo") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("foo#") as resolved: self.assertEqual(resolved, schema) def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): schema = {} resolver = RefResolver.from_schema(schema) self.assertEqual(resolver.base_uri, "") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: self.assertEqual(resolved, schema) def test_custom_uri_scheme_handlers(self): schema = {"foo": "bar"} ref = "foo://bar" foo_handler = mock.Mock(return_value=schema) resolver = RefResolver("", {}, handlers={"foo": foo_handler}) with resolver.resolving(ref) as resolved: self.assertEqual(resolved, schema) foo_handler.assert_called_once_with(ref) def test_cache_remote_on(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver( "", {}, cache_remote=True, handlers={"foo" : foo_handler}, ) with resolver.resolving(ref): pass with resolver.resolving(ref): pass foo_handler.assert_called_once_with(ref) def test_cache_remote_off(self): ref = "foo://bar" foo_handler = mock.Mock() resolver = RefResolver( "", {}, cache_remote=False, handlers={"foo" : foo_handler}, ) with resolver.resolving(ref): pass with resolver.resolving(ref): pass self.assertEqual(foo_handler.call_count, 2) def test_if_you_give_it_junk_you_get_a_resolution_error(self): ref = "foo://bar" foo_handler = mock.Mock(side_effect=ValueError("Oh no! What's this?")) resolver = RefResolver("", {}, handlers={"foo" : foo_handler}) with self.assertRaises(RefResolutionError) as err: with resolver.resolving(ref): pass self.assertEqual(str(err.exception), "Oh no! What's this?")
class SchemaTools(): """ The schema tools class does all the handling, validation and transformation of schemas in the optimal framework. """ # The json_schema_objects keeps the parsed and instantiated schema objects. json_schema_objects = None # json_schema_folder is the folder of where the json schemas are kept json_schema_folders = None # The resolve is responsible for resolving URI prefixes referencing other schemas, resolver = None # The URI-handlers are a dict of callbacks, used when resolving URI:s. uri_handlers = None # An instance of the mongodb JSON validator. mongodb_validator = None # Default handler, will only look at the cache def cache_handler(self, _uri): print ("fetching " + _uri) if _uri in self.json_schema_objects: return self.json_schema_objects[_uri] else: return None # noinspection PyDefaultArgument def __init__(self, _json_schema_folders=[], _uri_handlers=None): """ Initiate the SchemaTools class :param _json_schema_folders: A list of folders where schema files are stored :param : _uri_handlers: A dict of uri_handlers, resolves a URI prefix to a actual schema. """ if not _json_schema_folders: _json_schema_folders = [] if not _uri_handlers: self.uri_handlers = {} else: self.uri_handlers = _uri_handlers # All methods that have no handlers should use the cache handler. for _curr_key, _curr_value in _uri_handlers.items(): if _curr_value is None: _uri_handlers[_curr_key] = self.cache_handler self.resolver = RefResolver(base_uri="", handlers=self.uri_handlers, referrer=None, cache_remote=True) self.mongodb_validator = MongodbValidator(resolver= self.resolver) self.json_schema_objects = {} # Load application specific schemas for _curr_folder in _json_schema_folders: _loaded_uris = self.load_schemas_from_directory(os.path.abspath(_curr_folder)) # Resolve all the schemas for _curr_uri in _loaded_uris: self.json_schema_objects[_curr_uri] = self.resolveSchema(self.json_schema_objects[_curr_uri]) write_to_log("Schemas loaded and resolved: " + str.join(", ", ["\"" +_curr_schema["title"] + "\"" for _curr_schema in self.json_schema_objects.values()]) , _category=EC_NOTIFICATION, _severity=SEV_DEBUG) @staticmethod def check_schema_fields(_curr_schema_obj, _curr_file): """ Check so all mandatory fields are in the schema :param _curr_schema_obj: Schema to check :param _curr_file: File name use in error message """ def raise_field_error(_collection): raise Exception("Schematools.check_schema_fields: The \"" + _collection + "\"" + " field is not in the schema-\"" + _curr_file + "\"") if "version" not in _curr_schema_obj: raise_field_error("version") def load_schema_from_file(self, _file_name): """ Loads a specified schema from a file, checks it and stores it in the schema cache. :param _file_name: The name of the schema file """ try: _curr_file = open(_file_name, "r") except Exception as e: raise Exception("load_schema_from_file: Error loading \"" + _file_name + "\": " + str(e)) try: _json_schema_obj = json.load(_curr_file) except Exception as e: raise Exception("load_schema_from_file: Error parsing \"" + _file_name + "\"" + str(e)) _curr_file.close() try: self.check_schema_fields(_json_schema_obj, _file_name) except SchemaError as scherr: raise Exception("load_schema_from_file: SchemaError in " + _file_name + " at path:" + str( scherr.path) + "\nMessage:\n" + str(scherr.message)) except Exception as e: raise Exception("load_schema_from_file: schema validation in " + _file_name + ", error :" + str(e)) return _json_schema_obj def load_schemas_from_directory(self, _schema_folder, _destination = None): """ Load and validate all schemas in a folder structure, add to json_schema_objects :param _schema_folder: Where to look """ _loaded_uris = [] if _destination == None: _destination = self.json_schema_objects def _recurse(_folder): for _root, _dirs, _files in os.walk(_folder): for _file in _files: if _file[-5:].lower() == ".json": _ref = "ref://" + ".".join(os.path.relpath(_root, _schema_folder).split(os.path.sep) + [_file[0:-5]]) _destination[_ref] = self.load_schema_from_file (os.path.join(_root, _file)) if _ref not in _loaded_uris: _loaded_uris.append(_ref) for _dir in _dirs: _recurse(os.path.join(_folder, _dir)) _recurse(_schema_folder) return _loaded_uris def apply(self, _data, _schema_ref=None): """ Validate the JSON in _data against a JSON schema. :param _data: The JSON data to validate :param _schema_ref: If set, validate against the specified schema, and not the one in the data. :return: the schema object that was validated against. """ if _schema_ref is not None: _json_schema_obj = self.json_schema_objects[_schema_ref] else: if "schemaRef" in _data: try: _json_schema_obj = self.json_schema_objects[_data["schemaRef"]] except KeyError as e: raise Exception("SchemaTools.apply, invalid schemaRef: " + _data["schemaRef"]) else: raise Exception("SchemaTools.apply, data must have a schemaRef attribute") self.mongodb_validator.apply(_data, _json_schema_obj) return _data, _json_schema_obj def validate(self, _data, _schema_ref=None): """ Validate the JSON in _data against a JSON schema. :param _data: The JSON data to validate :param _schema_ref: If set, validate against the specified schema, and not the one in the data. :return: the schema object that was validated against. """ if _schema_ref is not None: _json_schema_obj = self.json_schema_objects[_schema_ref] else: _json_schema_obj = self.json_schema_objects[_data["schemaRef"]] self.mongodb_validator.validate(_data, _json_schema_obj) return _data, _json_schema_obj def non_base_type(self, _type): if _type not in ["array", "string", "integer", "object"]: return [_type] else: return [] def resolveSchema(self, _schema): """ Recursively resolve all I{$ref} JSON references in a JSON Schema. :param _schema: A L{dict} with a JSON Schema. :return: The resolved JSON Schema, a L{dict}. """ _result = deepcopy(_schema) def local_resolve(_obj, _ref_history): """ Recurse the JSON-tree and see where there are unresolved remote references. :param _obj: The node to resolve :param _ref_history: The previously resolved remote references, for cyclical check """ if isinstance(_obj, list): # Loop any list for item in _obj: local_resolve(item, _ref_history) return if isinstance(_obj, dict): if "$ref" in _obj: _curr_ref = _obj["$ref"] # Do not resolve local references if _curr_ref[0] == "#": return # Check for cyclical references if _curr_ref in _ref_history: raise Exception("Error, cyclical remote reference: " + str(_curr_ref) + ": Formers " + str(_ref_history)) with self.resolver.resolving(_curr_ref) as resolved: # Resolve the resolved schema local_resolve(resolved, _ref_history + [_curr_ref]) # Remove the reference del _obj["$ref"] # Add the resolved fragment to the schema _obj.update(resolved) else: # Loop all properties for _key, _value in _obj.items(): if isinstance(_value, dict) and "type" in _value: local_resolve(_value, _ref_history) else: local_resolve(_value, _ref_history) try: local_resolve(_result, []) except Exception as e: raise Exception("schemaTools.resolveSchema: Error resolving schema:" + str(e) + "Schema " + json.dumps(_schema, indent=4)) # Make top allOf into properties if "allOf" in _result: _new_properties = {} for _curr_properties in _result["allOf"]: _new_properties.update(_curr_properties["properties"]) _result["properties"] = _new_properties del _result["allOf"] _result["$schema"] = "http://json-schema.org/draft-04/schema#" try: self.mongodb_validator.check_schema(_result) except Exception as e: raise Exception("schemaTools.resolveSchema: error validating resolved schema:" + str(e) + "Schema " + json.dumps(_result, indent=4)) return _result