def test_it_can_register_checkers(self):
checker = FormatChecker()
checker.checks("boom")(boom)
self.assertEqual(
checker.checkers,
dict(FormatChecker.checkers, boom=(boom, ()))
)
def test_it_can_register_checkers(self):
checker = FormatChecker()
checker.checks("new")(self.fn)
self.assertEqual(
checker.checkers,
dict(FormatChecker.checkers, new=(self.fn, ()))
)
def test_validate_with_format(app, db):
"""Test that validation can accept custom format rules."""
with app.app_context():
checker = FormatChecker()
checker.checks("foo")(lambda el: el.startswith("foo"))
data = {"bar": "foo", "$schema": {"properties": {"bar": {"format": "foo"}}}}
# test record creation with valid data
assert data == Record.create(data)
record = Record.create(data, format_checker=checker)
# test direct call to validate with valid data
assert record.validate(format_checker=checker) is None
# test commit with valid data
record.commit(format_checker=checker)
record["bar"] = "bar"
# test direct call to validate with invalid data
with pytest.raises(ValidationError) as excinfo:
record.validate(format_checker=checker)
assert "'bar' is not a 'foo'" in str(excinfo.value)
# test commit with invalid data
with pytest.raises(ValidationError) as excinfo:
record.commit(format_checker=checker)
assert "'bar' is not a 'foo'" in str(excinfo.value)
data["bar"] = "bar"
# test record creation with invalid data
with pytest.raises(ValidationError) as excinfo:
record = Record.create(data, format_checker=checker)
assert "'bar' is not a 'foo'" in str(excinfo.value)
def test_it_can_register_cls_checkers(self):
original = dict(FormatChecker.checkers)
self.addCleanup(FormatChecker.checkers.pop, "boom")
FormatChecker.cls_checks("boom")(boom)
self.assertEqual(
FormatChecker.checkers,
dict(original, boom=(boom, ())),
)
def test_format_checkers_come_with_defaults(self):
# This is bad :/ but relied upon.
# The docs for quite awhile recommended people do things like
# validate(..., format_checker=FormatChecker())
# We should change that, but we can't without deprecation...
checker = FormatChecker()
with self.assertRaises(FormatError):
checker.check(instance="not-an-ipv4", format="ipv4")
def get_schema_validation_errors(json_data, schema_url, current_app):
schema = requests.get(schema_url).json()
validation_errors = collections.defaultdict(list)
format_checker = FormatChecker()
if current_app == 'cove-360':
format_checker.checkers['date-time'] = (datetime_or_date, ValueError)
for n, e in enumerate(validator(schema, format_checker=format_checker).iter_errors(json_data)):
validation_errors[e.message].append("/".join(str(item) for item in e.path))
return dict(validation_errors)
def is_valid_json(data, schema):
checker = FormatChecker();
# add the "interval" format
checker.checks("interval")(parse_iso8601_interval)
validator = Draft4Validator(schema, format_checker=checker)
errors = []
for error in validator.iter_errors(data):
errors.append(error.message)
return errors
def test_format_error_causes_become_validation_error_causes(self):
checker = FormatChecker()
checker.checks("boom", raises=ValueError)(boom)
validator = Draft4Validator({"format": "boom"}, format_checker=checker)
with self.assertRaises(ValidationError) as cm:
validator.validate("BOOM")
self.assertIs(cm.exception.cause, BOOM)
self.assertIs(cm.exception.__cause__, BOOM)
def test_format_error_causes_become_validation_error_causes(self):
checker = FormatChecker()
checker.checks("foo", raises=ValueError)(self.fn)
cause = self.fn.side_effect = ValueError()
validator = Draft4Validator({"format": "foo"}, format_checker=checker)
with self.assertRaises(ValidationError) as cm:
validator.validate("bar")
self.assertIs(cm.exception.__cause__, cause)
def test_invalid_format_default_message(self):
checker = FormatChecker(formats=())
check_fn = mock.Mock(return_value=False)
checker.checks("thing")(check_fn)
schema = {"format" : "thing"}
message = self.message_for("bla", schema, format_checker=checker)
self.assertIn(repr("bla"), message)
self.assertIn(repr("thing"), message)
self.assertIn("is not a", message)
def __init__(self, spec_dict, origin_url=None, http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# Cached copy of spec_dict with x-scope metadata removed.
# See @property client_spec_dict().
self._client_spec_dict = None
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client))
def __init__(self, spec_dict, origin_url=None, http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = None
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
def test_validate_with_format(app, db):
"""Test that validation can accept custom format rules."""
with app.app_context():
checker = FormatChecker()
checker.checks('foo')(lambda el: el.startswith('foo'))
record = Record.create({
'bar': 'foo',
'$schema': {
'properties': {
'bar': {'format': 'foo'}
}
}
})
assert record.validate(format_checker=checker) is None
record['bar'] = 'bar'
with pytest.raises(ValidationError) as excinfo:
record.validate(format_checker=checker)
assert "'bar' is not a 'foo'" in str(excinfo.value)
def test_it_catches_registered_errors(self):
checker = FormatChecker()
checker.checks("boom", raises=type(BOOM))(boom)
with self.assertRaises(FormatError) as cm:
checker.check(instance=12, format="boom")
self.assertIs(cm.exception.cause, BOOM)
self.assertIs(cm.exception.__cause__, BOOM)
# Unregistered errors should not be caught
with self.assertRaises(type(BANG)):
checker.check(instance="bang", format="boom")
def test_it_catches_registered_errors(self):
checker = FormatChecker()
checker.checks("foo", raises=ValueError)(self.fn)
# Registered errors should be caught and turned into FormatErrors
cause = ValueError()
self.fn.side_effect = cause
with self.assertRaises(FormatError) as cm:
checker.check("bar", "foo")
# Original exception should be attached to cause attribute
self.assertIs(cm.exception.cause, cause)
# Unregistered errors should not be caught
self.fn.side_effect = AttributeError
with self.assertRaises(AttributeError):
checker.check("bar", "foo")
def test_it_catches_registered_errors(self):
checker = FormatChecker()
cause = self.fn.side_effect = ValueError()
checker.checks("foo", raises=ValueError)(self.fn)
with self.assertRaises(FormatError) as cm:
checker.check("bar", "foo")
self.assertIs(cm.exception.cause, cause)
self.assertIs(cm.exception.__cause__, cause)
# Unregistered errors should not be caught
self.fn.side_effect = AttributeError
with self.assertRaises(AttributeError):
checker.check("bar", "foo")
def __init__(self, spec_dict, origin_url=None, http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client),
)
self._validate_config()
if self.config['internally_dereference_refs']:
# If internally_dereference_refs is enabled we do NOT need to resolve references anymore
# it's useless to evaluate is_ref every time
self.deref = lambda ref_dict: ref_dict
else:
self.deref = self._force_deref
def test_it_can_register_cls_checkers(self):
with mock.patch.dict(FormatChecker.checkers, clear=True):
FormatChecker.cls_checks("new")(self.fn)
self.assertEqual(FormatChecker.checkers, {"new": (self.fn, ())})
# Copyright (C) 2020 @HirMtsd. All Rights Reserved.
# coding: utf-8
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND.
# 2020/08/08
import json
from jsonschema import validate, ValidationError, FormatChecker
with open('Prefecture_schema.json', encoding="utf-8") as file_schema:
json_schema = json.load(file_schema)
with open('Prefecture_list.json', encoding="utf-8") as file_json:
json_data = json.load(file_json)
try:
validate(json_data, json_schema, format_checker=FormatChecker())
except ValidationError as e:
print(e.message)
print('CHECK END')
class Spec(object):
"""Represents a Swagger Specification for a service.
:param spec_dict: Swagger API specification in json-like dict form
:param origin_url: URL from which the spec was retrieved.
:param http_client: Used to retrive the spec via http/https.
:type http_client: :class:`bravado.http_client.HTTPClient`
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
def __init__(self, spec_dict, origin_url=None, http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client))
@classmethod
def from_dict(cls, spec_dict, origin_url=None, http_client=None,
config=None):
"""Build a :class:`Spec` from Swagger API Specificiation
:param spec_dict: swagger spec in json-like dict form.
:param origin_url: the url used to retrieve the spec, if any
:type origin_url: str
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
spec = cls(spec_dict, origin_url, http_client, config)
spec.build()
return spec
def build(self):
if self.config['validate_swagger_spec']:
self.resolver = validator20.validate_spec(
self.spec_dict, spec_url=self.origin_url or '',
http_handlers=build_http_handlers(self.http_client))
post_process_spec(
self,
on_container_callbacks=[
functools.partial(
tag_models, visited_models={}, swagger_spec=self),
functools.partial(
collect_models, models=self.definitions,
swagger_spec=self)
])
for format in self.config['formats']:
self.register_format(format)
self.api_url = build_api_serving_url(self.spec_dict, self.origin_url)
self.resources = build_resources(self)
def deref(self, ref_dict):
"""Dereference ref_dict (if it is indeed a ref) and return what the
ref points to.
:param ref_dict: {'$ref': '#/blah/blah'}
:return: dereferenced value of ref_dict
:rtype: scalar, list, dict
"""
if ref_dict is None or not is_ref(ref_dict):
return ref_dict
# Restore attached resolution scope before resolving since the
# resolver doesn't have a traversal history (accumulated scope_stack)
# when asked to resolve.
with in_scope(self.resolver, ref_dict):
log.debug('Resolving {0} with scope {1}: {2}'.format(
ref_dict['$ref'],
len(self.resolver._scopes_stack),
self.resolver._scopes_stack))
_, target = self.resolver.resolve(ref_dict['$ref'])
return target
def get_op_for_request(self, http_method, path_pattern):
"""Return the Swagger operation for the passed in request http method
and path pattern. Makes it really easy for server-side implementations
to map incoming requests to the Swagger spec.
:param http_method: http method of the request
:param path_pattern: request path pattern. e.g. /foo/{bar}/baz/{id}
:returns: the matching operation or None if a match couldn't be found
:rtype: :class:`bravado_core.operation.Operation`
"""
if self._request_to_op_map is None:
# lazy initialization
self._request_to_op_map = {}
base_path = self.spec_dict.get('basePath', '').rstrip('/')
for resource in self.resources.values():
for op in resource.operations.values():
full_path = base_path + op.path_name
key = (op.http_method, full_path)
self._request_to_op_map[key] = op
key = (http_method.lower(), path_pattern)
return self._request_to_op_map.get(key)
def register_format(self, user_defined_format):
"""Registers a user-defined format to be used with this spec.
:type user_defined_format:
:class:`bravado_core.formatter.SwaggerFormat`
"""
name = user_defined_format.format
self.user_defined_formats[name] = user_defined_format
validate = return_true_wrapper(user_defined_format.validate)
self.format_checker.checks(
name, raises=(SwaggerValidationError,))(validate)
def get_format(self, name):
"""
:param name: Name of the format to retrieve
:rtype: :class:`bravado_core.formatters.SwaggerFormat`
"""
if name in formatter.DEFAULT_FORMATS:
return formatter.DEFAULT_FORMATS[name]
format = self.user_defined_formats.get(name)
if format is None:
warnings.warn('{0} format is not registered with bravado-core!'
.format(name), Warning)
return format
def test_it_returns_true_for_formats_it_does_not_know_about(self):
validator = self.validator_class(
{"format": "carrot"}, format_checker=FormatChecker(),
)
validator.validate("bugs")
class Spec(object):
"""Represents a Swagger Specification for a service.
:param spec_dict: Swagger API specification in json-like dict form
:param origin_url: URL from which the spec was retrieved.
:param http_client: Used to retrive the spec via http/https.
:type http_client: :class:`bravado.http_client.HTTPClient`
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
def __init__(self, spec_dict, origin_url=None, http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = None
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
@classmethod
def from_dict(cls, spec_dict, origin_url=None, http_client=None,
config=None):
"""
Build a :class:`Spec` from Swagger API Specificiation
:param spec_dict: swagger spec in json-like dict form.
:param origin_url: the url used to retrieve the spec, if any
:type origin_url: str
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
tag_models(spec_dict)
fix_malformed_model_refs(spec_dict)
spec_dict = jsonref.JsonRef.replace_refs(spec_dict,
base_uri=origin_url or '')
replace_jsonref_proxies(spec_dict)
spec = cls(spec_dict, origin_url, http_client, config)
spec.build()
return spec
def build(self):
if self.config['validate_swagger_spec']:
validator20.validate_spec(self.spec_dict)
self.api_url = build_api_serving_url(self.spec_dict, self.origin_url)
self.definitions = build_models(self.spec_dict.get('definitions', {}))
self.resources = build_resources(self)
def get_op_for_request(self, http_method, path_pattern):
"""
Return the Swagger operation for the passed in request http method
and path pattern. Makes it really easy for server-side implementations
to map incoming requests to the Swagger spec.
:param http_method: http method of the request
:param path_pattern: request path pattern. e.g. /foo/{bar}/baz/{id}
:returns: the matching operation or None if a match couldn't be found
:rtype: :class:`bravado_core.operation.Operation`
"""
if self._request_to_op_map is None:
# lazy initialization
self._request_to_op_map = {}
base_path = self.spec_dict.get('basePath', '').rstrip('/')
for resource in self.resources.values():
for op in resource.operations.values():
full_path = base_path + op.path_name
key = (op.http_method, full_path)
self._request_to_op_map[key] = op
key = (http_method.lower(), path_pattern)
return self._request_to_op_map.get(key)
def register_format(self, user_defined_format):
"""Registers a user-defined format to be used with this spec.
:type user_defined_format:
:class:`bravado_core.formatter.SwaggerFormat`
"""
name = user_defined_format.format
self.user_defined_formats[name] = user_defined_format
validate = return_true_wrapper(user_defined_format.validate)
self.format_checker.checks(
name, raises=(SwaggerValidationError,))(validate)
def get_format(self, name):
"""
:param name: Name of the format to retrieve
:rtype: :class:`bravado_core.formatters.SwaggerFormat`
"""
if name in formatter.DEFAULT_FORMATS:
return formatter.DEFAULT_FORMATS[name]
format = self.user_defined_formats.get(name)
if format is None:
warnings.warn('{0} format is not registered with bravado-core!'
.format(name), Warning)
return format
def test_it_can_register_cls_checkers(self):
with mock.patch.dict(FormatChecker.checkers, clear=True):
FormatChecker.cls_checks("new")(self.fn)
self.assertEqual(FormatChecker.checkers, {"new": (self.fn, ())})
def test_it_can_validate_no_formats(self):
checker = FormatChecker(formats=())
self.assertFalse(checker.checkers)
class Spec(object):
"""Represents a Swagger Specification for a service.
:param spec_dict: Swagger API specification in json-like dict form
:param origin_url: URL from which the spec was retrieved.
:param http_client: Used to retrive the spec via http/https.
:type http_client: :class:`bravado.http_client.HTTPClient`
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
def __init__(self,
spec_dict,
origin_url=None,
http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# Cached copy of spec_dict with x-scope metadata removed.
# See @property client_spec_dict().
self._client_spec_dict = None
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client))
# generated by @property to avoid multiple swagger validations
self._security_definitions = None
@property
def client_spec_dict(self):
"""Return a copy of spec_dict with x-scope metadata removed so that it
is suitable for consumption by Swagger clients.
You may be asking, "Why is there a difference between the Swagger spec
a client sees and the one used internally?". Well, as part of the
ingestion process, x-scope metadata is added to spec_dict so that
$refs can be de-reffed successfully during requset/response validation
and marshalling. This medatadata is specific to the context of the
server and contains files and paths that are not relevant to the
client. This is required so the client does not re-use (and in turn,
re-creates) the invalid x-scope metadata created by the server.
For example, a section of spec_dict that contains a ref would change
as folows.
Before:
'MON': {
'$ref': '#/definitions/DayHours',
'x-scope': [
'file:///happyhour/api_docs/swagger.json',
'file:///happyhour/api_docs/swagger.json#/definitions/WeekHours'
]
}
After:
'MON': {
'$ref': '#/definitions/DayHours'
}
"""
if self._client_spec_dict is None:
self._client_spec_dict = strip_xscope(self.spec_dict)
return self._client_spec_dict
@classmethod
def from_dict(cls,
spec_dict,
origin_url=None,
http_client=None,
config=None):
"""Build a :class:`Spec` from Swagger API Specificiation
:param spec_dict: swagger spec in json-like dict form.
:param origin_url: the url used to retrieve the spec, if any
:type origin_url: str
:param: http_client: http client used to download remote $refs
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
spec = cls(spec_dict, origin_url, http_client, config)
spec.build()
return spec
def build(self):
if self.config['validate_swagger_spec']:
self.resolver = validator20.validate_spec(
self.spec_dict,
spec_url=self.origin_url or '',
http_handlers=build_http_handlers(self.http_client))
post_process_spec(self,
on_container_callbacks=[
functools.partial(tag_models,
visited_models={},
swagger_spec=self),
functools.partial(collect_models,
models=self.definitions,
swagger_spec=self)
])
for format in self.config['formats']:
self.register_format(format)
self.api_url = build_api_serving_url(self.spec_dict, self.origin_url)
self.resources = build_resources(self)
def deref(self, ref_dict):
"""Dereference ref_dict (if it is indeed a ref) and return what the
ref points to.
:param ref_dict: {'$ref': '#/blah/blah'}
:return: dereferenced value of ref_dict
:rtype: scalar, list, dict
"""
if ref_dict is None or not is_ref(ref_dict):
return ref_dict
# Restore attached resolution scope before resolving since the
# resolver doesn't have a traversal history (accumulated scope_stack)
# when asked to resolve.
with in_scope(self.resolver, ref_dict):
log.debug('Resolving {0} with scope {1}: {2}'.format(
ref_dict['$ref'], len(self.resolver._scopes_stack),
self.resolver._scopes_stack))
_, target = self.resolver.resolve(ref_dict['$ref'])
return target
def get_op_for_request(self, http_method, path_pattern):
"""Return the Swagger operation for the passed in request http method
and path pattern. Makes it really easy for server-side implementations
to map incoming requests to the Swagger spec.
:param http_method: http method of the request
:param path_pattern: request path pattern. e.g. /foo/{bar}/baz/{id}
:returns: the matching operation or None if a match couldn't be found
:rtype: :class:`bravado_core.operation.Operation`
"""
if self._request_to_op_map is None:
# lazy initialization
self._request_to_op_map = {}
base_path = self.spec_dict.get('basePath', '').rstrip('/')
for resource in self.resources.values():
for op in resource.operations.values():
full_path = base_path + op.path_name
key = (op.http_method, full_path)
self._request_to_op_map[key] = op
key = (http_method.lower(), path_pattern)
return self._request_to_op_map.get(key)
def register_format(self, user_defined_format):
"""Registers a user-defined format to be used with this spec.
:type user_defined_format:
:class:`bravado_core.formatter.SwaggerFormat`
"""
name = user_defined_format.format
self.user_defined_formats[name] = user_defined_format
validate = return_true_wrapper(user_defined_format.validate)
self.format_checker.checks(name,
raises=(SwaggerValidationError, ))(validate)
def get_format(self, name):
"""
:param name: Name of the format to retrieve
:rtype: :class:`bravado_core.formatters.SwaggerFormat`
"""
if name in formatter.DEFAULT_FORMATS:
return formatter.DEFAULT_FORMATS[name]
format = self.user_defined_formats.get(name)
if format is None:
warnings.warn(
'{0} format is not registered with bravado-core!'.format(name),
Warning)
return format
@property
def security_definitions(self):
if self._security_definitions is None:
self._security_definitions = {}
for security_name, security_def in iteritems(
self.spec_dict.get('securityDefinitions', {})):
self._security_definitions[security_name] = SecurityDefinition(
self, security_def)
return self._security_definitions
for trace in trace_seq:
validate_timestamp(trace['timestamp'])
# FIXME: Be more specific with errors.
except Exception as e:
print(e)
abort(400)
try:
for trace in trace_seq:
# FIXME: If not unique timestamp, it's a user error 400.
write_trace_in_db(session_id, trace)
# FIXME: Or 200, as the resource is not shown to user?
return make_response(jsonify({"status": "OK"}), 201)
except Exception as e:
print(e)
abort(500)
# Adds date-time checking.
# FIXME: Is this really how it's supposed to be done?
FormatChecker.cls_checks("date-time", ())(validate_rfc3339)
json_schema = open_json_schema()
if __name__ == "__main__":
app.run()
def checker(ontology):
modelname_checker = FormatChecker()
# Pass the ontology object as a closure
modelname_checker.checks("modelname", raises=ValidationError)(is_modelname(ontology))
# Returns the checker after configuring it
return modelname_checker
def post(self):
"""
Create new object version from the file in the given path.
Verify first, if the file validates against the maDMP schema.
Then store its metadata.
:returns: Created Record View.
"""
global json_data
path = os.path.dirname(os.path.abspath(__file__))
filename = 'maDMP-schema.json'
try:
with open(os.path.join(path, filename)) as json_file:
schema_data = json.load(json_file)
except IOError as io_exc:
response = jsonify({'message': 'Something went wrong', 'status': 500})
response.status_code = 500
print(io_exc)
return response
try:
if 'file' not in request.files and not request.json:
raise BadRequest('No file or json data in request')
if 'file' in request.files and request.json:
raise BadRequest('Only file or data must be in request')
if 'file' in request.files:
file = request.files['file'] # FileStorage Object
if file.filename == '':
raise BadRequest('No file selected')
if len(file.read()) is 0:
raise BadRequest('File is empty')
json_data = json.load(file)
elif request.json:
json_data = request.json
if json_data is None:
raise BadRequest('JSON data is empty')
validate(instance=json_data, schema=schema_data, format_checker=FormatChecker())
except BadRequest as bad_req_exc:
response = jsonify({'message': bad_req_exc.description, 'status': 400})
response.status_code = 400
return response
except ValidationError as validation_exc:
response = jsonify({
'message': 'JSON does not validate against the schema',
'details': validation_exc.message,
'status': 400
})
response.status_code = 400
return response
except JSONDecodeError as json_exc:
response = jsonify({'message': 'JSON syntax error: ' + json_exc.msg, 'status': 400})
response.status_code = 400
print(json_exc)
return response
except IOError:
response = jsonify({'message': 'Error processing file', 'status': 500})
response.status_code = 500
return response
except Exception as exc:
response = jsonify({'message': 'Something went wrong', 'status': 500})
response.status_code = 500
print(exc.__str__())
return response
else:
try:
data = UploadMaDMP.extract_data(json_data)
if not data:
raise BadRequest
except BadRequest:
response = jsonify({
'message': 'Could not extract any data. Check again your json structure',
'status': 400
})
response.status_code = 400
return response
except Exception as exc:
response = jsonify({'message': 'Something went wrong', 'status': 500})
response.status_code = 500
print('Extact data method: ' + exc.__str__())
return response
calls = 1
responses = {'responses': []}
for rec in data:
try:
_ = UploadMaDMP.create_record(**rec)
except Exception as exc:
response = {'id': calls, 'message': 'Something went wrong', 'status': 500}
responses['responses'].append(response)
print("Error inserting record: " + exc.__str__())
else:
response = {'id': calls, 'message': 'Metadata created successfully', 'status': 201}
responses['responses'].append(response)
calls += 1
resp = jsonify(responses)
resp.status_code = 201 if not any(item['status'] == 500 for item in responses['responses']) else 500
return resp
return errorsMap
@staticmethod
def getSchema(inputtype):
errorLog = list()
validSchema = None
if inputtype == 'NAMED_ENTITY':
try:
request_schema = requests.get(
'http://europepmc.org/docs/ne_annotation_schema.json')
ne_schemaObj = request_schema.json()
validSchema = Draft4Validator(ne_schemaObj,
format_checker=FormatChecker())
except Exception, nescherr:
errorLog.append(repr(nescherr))
elif inputtype == 'SENTENCE':
try:
request_schema = requests.get(
'http://europepmc.org/docs/sentence_annotation_schema.json'
)
sent_schemaObj = request_schema.json()
validSchema = Draft4Validator(sent_schemaObj,
format_checker=FormatChecker())
except Exception, sentscherr:
errorLog.append(repr(sentscherr))
return (validSchema, errorLog)
from dalet import is_domain, is_url
from jsonschema import Draft4Validator, FormatChecker, RefResolver
from aleph.core import get_config
resolver = RefResolver('core.json#', {})
SCHEMA_DIR = os.path.join(os.path.dirname(__file__), 'validation')
for (root, dirs, files) in os.walk(SCHEMA_DIR):
for schema_file in files:
with open(os.path.join(root, schema_file), 'r') as fh:
schema = json.load(fh)
resolver.store[schema['id']] = schema
format_checker = FormatChecker()
format_checker.checks('country-code')(is_country_code)
format_checker.checks('partial-date')(is_partial_date)
format_checker.checks('language-code')(is_language_code)
format_checker.checks('url')(is_url)
format_checker.checks('domain')(is_domain)
@format_checker.checks('collection-category')
def is_collection_category(cat):
categories = get_config('COLLECTION_CATEGORIES', {})
return cat in categories.keys()
def validate(data, schema):
_, schema = resolver.resolve(schema)
def test_it_raises_a_key_error_for_unknown_formats(self):
with self.assertRaises(KeyError):
FormatChecker(formats=["o noes"])
class Spec(object):
"""Represents a Swagger Specification for a service.
:param spec_dict: Swagger API specification in json-like dict form
:param origin_url: URL from which the spec was retrieved.
:param http_client: Used to retrieve the spec via http/https.
:type http_client: :class:`bravado.http_client.HTTPClient`
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
def __init__(
self,
spec_dict,
origin_url=None,
http_client=None,
config=None,
):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=self.get_ref_handlers(),
)
# spec dict used to build resources, in case internally_dereference_refs config is enabled
# it will be overridden by the dereferenced specs (by build method). More context in PR#263
self._internal_spec_dict = spec_dict
def is_equal(self, other):
# Not implemented as __eq__ otherwise we would need to implement __hash__ to preserve
# hashability of the class and it would not necessarily be performance effective
if id(self) == id(other):
return True
if not isinstance(other, self.__class__):
return False
# If self and other are of the same type but not pointing to the same memory location then we're going to inspect
# all the attributes.
for attr_name in set(
chain(
iterkeys(self.__dict__),
iterkeys(other.__dict__),
), ):
# Few attributes have recursive references to Spec or do not define an equality method we're going to ignore them
if attr_name in {
'definitions', # Recursively point back to self (Spec instance). It is built via Spec.build so we're ignoring it
'format_checker', # jsonschema.FormatChecker does not define an equality method
'resolver', # jsonschema.validators.RefResolver does not define an equality method
'resources', # Recursively point back to self (Spec instance). It is built via Spec.build so we're ignoring it
'security_definitions', # Recursively point back to self (Spec instance). It is a cached property so ignore it
}:
continue
try:
if getattr(self, attr_name) != getattr(other, attr_name):
return False
except AttributeError:
return False
return True
def __deepcopy__(self, memo=None):
if memo is None:
memo = {}
copied_self = self.__class__(spec_dict=None)
memo[id(self)] = copied_self
# Copy the attributes that are built via Spec.build
for attr_name, attr_value in iteritems(self.__dict__):
setattr(copied_self, attr_name, deepcopy(attr_value, memo=memo))
return copied_self
@cached_property
def client_spec_dict(self):
"""Return a copy of spec_dict with x-scope metadata removed so that it
is suitable for consumption by Swagger clients.
You may be asking, "Why is there a difference between the Swagger spec
a client sees and the one used internally?". Well, as part of the
ingestion process, x-scope metadata is added to spec_dict so that
$refs can be de-reffed successfully during request/response validation
and marshalling. This metadata is specific to the context of the
server and contains files and paths that are not relevant to the
client. This is required so the client does not re-use (and in turn,
re-creates) the invalid x-scope metadata created by the server.
For example, a section of spec_dict that contains a ref would change
as follows.
Before:
'MON': {
'$ref': '#/definitions/DayHours',
'x-scope': [
'file:///happyhour/api_docs/swagger.json',
'file:///happyhour/api_docs/swagger.json#/definitions/WeekHours'
]
}
After:
'MON': {
'$ref': '#/definitions/DayHours'
}
"""
return strip_xscope(self.spec_dict)
@classmethod
def from_dict(cls,
spec_dict,
origin_url=None,
http_client=None,
config=None):
"""Build a :class:`Spec` from Swagger API Specification
:param spec_dict: swagger spec in json-like dict form.
:param origin_url: the url used to retrieve the spec, if any
:type origin_url: str
:param http_client: http client used to download remote $refs
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
spec = cls(spec_dict, origin_url, http_client, config)
spec.build()
return spec
def _validate_spec(self):
if self.config['validate_swagger_spec']:
self.resolver = validator20.validate_spec(
spec_dict=self.spec_dict,
spec_url=self.origin_url or '',
http_handlers=self.get_ref_handlers(),
)
def build(self):
self._validate_spec()
model_discovery(self)
if self.config['internally_dereference_refs']:
# Avoid to evaluate is_ref every time, no references are possible at this time
self.deref = lambda ref_dict: ref_dict
self._internal_spec_dict = self.deref_flattened_spec
for user_defined_format in self.config['formats']:
self.register_format(user_defined_format)
self.resources = build_resources(self)
self.api_url = build_api_serving_url(
spec_dict=self.spec_dict,
origin_url=self.origin_url,
use_spec_url_for_base_path=self.
config['use_spec_url_for_base_path'],
)
def get_ref_handlers(self):
"""Get mapping from URI schemes to handlers that takes a URI.
The handlers (callables) are used by the RefResolver to retrieve
remote specification $refs.
:returns: dict like {'http': callable, 'https': callable}
:rtype: dict
"""
return build_http_handlers(self.http_client)
def _force_deref(self, ref_dict):
# type: (T) -> T
"""Dereference ref_dict (if it is indeed a ref) and return what the
ref points to.
:param ref_dict: {'$ref': '#/blah/blah'}
:return: dereferenced value of ref_dict
:rtype: scalar, list, dict
"""
if ref_dict is None or not is_ref(ref_dict):
return ref_dict
# Restore attached resolution scope before resolving since the
# resolver doesn't have a traversal history (accumulated scope_stack)
# when asked to resolve.
with in_scope(self.resolver, ref_dict):
reference_value = ref_dict['$ref'] # type: ignore
_, target = self.resolver.resolve(reference_value)
return target
# NOTE: deref gets overridden, if internally_dereference_refs is enabled, after calling build
deref = _force_deref
def get_op_for_request(self, http_method, path_pattern):
"""Return the Swagger operation for the passed in request http method
and path pattern. Makes it really easy for server-side implementations
to map incoming requests to the Swagger spec.
:param http_method: http method of the request
:param path_pattern: request path pattern. e.g. /foo/{bar}/baz/{id}
:returns: the matching operation or None if a match couldn't be found
:rtype: :class:`bravado_core.operation.Operation`
"""
if self._request_to_op_map is None:
# lazy initialization
self._request_to_op_map = {}
base_path = self.spec_dict.get('basePath', '').rstrip('/')
for resource in self.resources.values():
for op in resource.operations.values():
full_path = base_path + op.path_name
key = (op.http_method, full_path)
self._request_to_op_map[key] = op
key = (http_method.lower(), path_pattern)
return self._request_to_op_map.get(key)
def register_format(self, user_defined_format):
"""Registers a user-defined format to be used with this spec.
:type user_defined_format:
:class:`bravado_core.formatter.SwaggerFormat`
"""
name = user_defined_format.format
self.user_defined_formats[name] = user_defined_format
validate = return_true_wrapper(user_defined_format.validate)
self.format_checker.checks(name,
raises=(SwaggerValidationError, ))(validate)
def get_format(self, name):
# type: (typing.Text) -> SwaggerFormat
"""
:param name: Name of the format to retrieve
:rtype: :class:`bravado_core.formatter.SwaggerFormat`
"""
user_defined_format = self.user_defined_formats.get(name)
if user_defined_format is None:
user_defined_format = formatter.DEFAULT_FORMATS.get(name)
if name == 'byte' and self.config['use_base64_for_byte_format']:
user_defined_format = formatter.BASE64_BYTE_FORMAT
if user_defined_format is None:
warnings.warn(
message='{0} format is not registered with bravado-core!'.
format(name),
category=Warning,
)
return user_defined_format
@cached_property
def security_definitions(self):
security_defs = {}
for security_name, security_def in iteritems(
self.spec_dict.get('securityDefinitions', {})):
security_defs[security_name] = SecurityDefinition(
self, security_def)
return security_defs
@cached_property
def flattened_spec(self):
"""
Representation of the current swagger specs that could be written to a single file.
:rtype: dict
"""
if not self.config['validate_swagger_spec']:
log.warning(
'Flattening unvalidated specs could produce invalid specs. '
'Use it at your risk or enable `validate_swagger_specs`', )
return strip_xscope(spec_dict=flattened_spec(swagger_spec=self), )
@cached_property
def deref_flattened_spec(self):
deref_spec_dict = JsonRef.replace_refs(self.flattened_spec)
@memoize_by_id
def descend(obj):
# Inline modification of obj
# This method is needed because JsonRef could produce performance penalties in accessing
# the proxied attributes
if isinstance(obj, JsonRef):
# Extract the proxied value
# http://jsonref.readthedocs.io/en/latest/#jsonref.JsonRef.__subject__
return obj.__subject__
if is_dict_like(obj):
for key in list(iterkeys(obj)):
obj[key] = descend(obj=obj[key])
elif is_list_like(obj):
# obj is list like object provided from flattened_spec specs.
# This guarantees that it cannot be a tuple instance and
# inline object modification are allowed
for index in range(len(obj)):
obj[index] = descend(obj=obj[index])
return obj
try:
return descend(obj=deref_spec_dict)
finally:
# Make sure that all memory allocated, for caching, could be released
descend.cache.clear()
def test_it_can_register_checkers(self):
checker = FormatChecker()
checker.checks("new")(self.fn)
self.assertEqual(checker.checkers,
dict(FormatChecker.checkers, new=(self.fn, ())))
def test_pass_a_junos(self):
"""Name only."""
config = {"name": "et-0/0/0"}
validate(config, self.interface_schema, format_checker=FormatChecker())
def validate_json(instance, schema):
"""Validate a dictionary using the provided json schema."""
Validator(schema, format_checker=FormatChecker()).validate(instance)
def create_validator(self, name: str, schema: Dict[str, Any]) -> Draft4Validator:
"""Create a validator from a schema
:param name: The name of this validator
:param schema: A dict which is a valid Openapi 3.0 schema
:return: the validator object
You can use references to models that are in the :attr:`base_model` as that will be used to resolve
any references such as `#/components/requestBodies/sample`.
.. todo:: Ensure that models with the same name in different categories don't conflict with each other
.. seealso:: The :meth:`expect` decorator
"""
validator = Draft4Validator(schema, resolver=self._ref_resolver, format_checker=FormatChecker())
self._validators[name] = validator
return validator
def validate_json_schema(schema_def, instance):
DefaultValidatingDraft4Validator(
schema_def, format_checker=FormatChecker()).validate(instance)
def validate_interface_config(config):
"""Interface schema validation."""
with open("./configmodel/schemas/interface.schema.json") as f:
interface_schema = json.load(f)
validate(config, interface_schema, format_checker=FormatChecker())
class JSONSchemaField(JSONBField):
"""
A JSONB field that self-validates against a defined JSON schema
(http://json-schema.org). This base class is intended to be overwritten by
defining `self.schema`.
"""
format_checker = FormatChecker()
# If an empty {} is provided, we still want to perform this schema
# validation
empty_values = (None, '')
def get_default(self):
return copy.deepcopy(super(JSONBField, self).get_default())
def schema(self, model_instance):
raise NotImplementedError()
def validate(self, value, model_instance):
super(JSONSchemaField, self).validate(value, model_instance)
errors = []
for error in Draft4Validator(
self.schema(model_instance),
format_checker=self.format_checker).iter_errors(value):
# strip Python unicode markers from jsonschema validation errors
error.message = re.sub(r'\bu(\'|")', r'\1', error.message)
if error.validator == 'pattern' and 'error' in error.schema:
error.message = six.text_type(
error.schema['error']).format(instance=error.instance)
elif error.validator == 'type':
expected_type = error.validator_value
if expected_type == 'object':
expected_type = 'dict'
if error.path:
error.message = _(
'{type} provided in relative path {path}, expected {expected_type}'
).format(path=list(error.path),
type=type(error.instance).__name__,
expected_type=expected_type)
else:
error.message = _(
'{type} provided, expected {expected_type}').format(
path=list(error.path),
type=type(error.instance).__name__,
expected_type=expected_type)
elif error.validator == 'additionalProperties' and hasattr(
error, 'path'):
error.message = _(
'Schema validation error in relative path {path} ({error})'
).format(path=list(error.path), error=error.message)
errors.append(error)
if errors:
raise django_exceptions.ValidationError(
[e.message for e in errors],
code='invalid',
params={'value': value},
)
def get_db_prep_value(self, value, connection, prepared=False):
if connection.vendor == 'sqlite':
# sqlite (which we use for tests) does not support jsonb;
return json.dumps(value)
return super(JSONSchemaField,
self).get_db_prep_value(value, connection, prepared)
def from_db_value(self, value, expression, connection, context):
# Work around a bug in django-jsonfield
# https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
if isinstance(value, six.string_types):
return json.loads(value)
return value
def validate_bgppeer_config(config):
"""BGP peer schema validation."""
with open("./configmodel/schemas/bgppeer.schema.json") as f:
bgppeer_schema = json.load(f)
validate(config, bgppeer_schema, format_checker=FormatChecker())
def test_pass_a(self):
"""Name only."""
config = {"name": "Ethernet1"}
validate(config, self.interface_schema, format_checker=FormatChecker())
def uri_blank(value):
return value == '' or FormatChecker().conforms(value, 'uri')
def test(self, format=format):
v = Draft4Validator({"format": format}, format_checker=FormatChecker())
v.validate(123)
def test_it_can_register_checkers(self):
checker = FormatChecker()
checker.checks("boom")(boom)
self.assertEqual(checker.checkers,
dict(FormatChecker.checkers, boom=(boom, ())))
def persist_lines(config, lines, table_cache=None) -> None:
state = None
flushed_state = None
schemas = {}
key_properties = {}
validators = {}
records_to_load = {}
row_count = {}
stream_to_sync = {}
total_row_count = {}
batch_size_rows = config.get('batch_size_rows', DEFAULT_BATCH_SIZE_ROWS)
# Loop over lines from stdin
for line in lines:
try:
o = json.loads(line)
except json.decoder.JSONDecodeError:
LOGGER.error("Unable to parse:\n{}".format(line))
raise
if 'type' not in o:
raise Exception(
"Line is missing required key 'type': {}".format(line))
t = o['type']
if t == 'RECORD':
if 'stream' not in o:
raise Exception(
"Line is missing required key 'stream': {}".format(line))
if o['stream'] not in schemas:
raise Exception(
"A record for stream {} was encountered before a corresponding schema"
.format(o['stream']))
# Get schema for this record's stream
stream = o['stream']
adjust_timestamps_in_record(o['record'], schemas[stream])
# Validate record
if config.get('validate_records'):
try:
validators[stream].validate(float_to_decimal(o['record']))
except Exception as ex:
if type(ex).__name__ == "InvalidOperation":
raise InvalidValidationOperationException(
f"Data validation failed and cannot load to destination. RECORD: {o['record']}\n"
"multipleOf validations that allows long precisions are not supported (i.e. with 15 digits"
"or more) Try removing 'multipleOf' methods from JSON schema."
)
raise RecordValidationException(
f"Record does not pass schema validation. RECORD: {o['record']}"
)
primary_key_string = stream_to_sync[
stream].record_primary_key_string(o['record'])
if not primary_key_string:
primary_key_string = 'RID-{}'.format(total_row_count[stream])
if stream not in records_to_load:
records_to_load[stream] = {}
# increment row count only when a new PK is encountered in the current batch
if primary_key_string not in records_to_load[stream]:
row_count[stream] += 1
total_row_count[stream] += 1
# append record
if config.get('add_metadata_columns') or config.get('hard_delete'):
records_to_load[stream][
primary_key_string] = add_metadata_values_to_record(
o, stream_to_sync[stream])
else:
records_to_load[stream][primary_key_string] = o['record']
if row_count[stream] >= batch_size_rows:
# flush all streams, delete records if needed, reset counts and then emit current state
if config.get('flush_all_streams'):
filter_streams = None
else:
filter_streams = [stream]
# Flush and return a new state dict with new positions only for the flushed streams
flushed_state = flush_streams(records_to_load,
row_count,
stream_to_sync,
config,
state,
flushed_state,
filter_streams=filter_streams)
# emit last encountered state
emit_state(copy.deepcopy(flushed_state))
elif t == 'SCHEMA':
if 'stream' not in o:
raise Exception(
"Line is missing required key 'stream': {}".format(line))
stream = o['stream']
new_schema = float_to_decimal(o['schema'])
# Update and flush only if the the schema is new or different than
# the previously used version of the schema
if stream not in schemas or schemas[stream] != new_schema:
schemas[stream] = new_schema
validators[stream] = Draft7Validator(
schemas[stream], format_checker=FormatChecker())
# flush records from previous stream SCHEMA
# if same stream has been encountered again, it means the schema might have been altered
# so previous records need to be flushed
if row_count.get(stream, 0) > 0:
flushed_state = flush_streams(records_to_load, row_count,
stream_to_sync, config,
state, flushed_state)
# emit latest encountered state
emit_state(flushed_state)
# key_properties key must be available in the SCHEMA message.
if 'key_properties' not in o:
raise Exception("key_properties field is required")
# Log based and Incremental replications on tables with no Primary Key
# cause duplicates when merging UPDATE events.
# Stop loading data by default if no Primary Key.
#
# If you want to load tables with no Primary Key:
# 1) Set ` 'primary_key_required': false ` in the target-snowflake config.json
# or
# 2) Use fastsync [postgres-to-snowflake, mysql-to-snowflake, etc.]
if config.get('primary_key_required', True) and len(
o['key_properties']) == 0:
LOGGER.critical(
"Primary key is set to mandatory but not defined in the [{}] stream"
.format(stream))
raise Exception("key_properties field is required")
key_properties[stream] = o['key_properties']
if config.get('add_metadata_columns') or config.get(
'hard_delete'):
stream_to_sync[stream] = DbSync(
config, add_metadata_columns_to_schema(o), table_cache)
else:
stream_to_sync[stream] = DbSync(config, o, table_cache)
stream_to_sync[stream].create_schema_if_not_exists()
stream_to_sync[stream].sync_table()
row_count[stream] = 0
total_row_count[stream] = 0
elif t == 'ACTIVATE_VERSION':
LOGGER.debug('ACTIVATE_VERSION message')
elif t == 'STATE':
LOGGER.debug('Setting state to {}'.format(o['value']))
state = o['value']
# Initially set flushed state
if not flushed_state:
flushed_state = copy.deepcopy(state)
else:
raise Exception("Unknown message type {} in message {}".format(
o['type'], o))
# if some bucket has records that need to be flushed but haven't reached batch size
# then flush all buckets.
if sum(row_count.values()) > 0:
# flush all streams one last time, delete records if needed, reset counts and then emit current state
flushed_state = flush_streams(records_to_load, row_count,
stream_to_sync, config, state,
flushed_state)
# emit latest state
emit_state(copy.deepcopy(flushed_state))
def validate_configuration(configuration):
"""
Validate a provided configuration.
:param dict configuration: A desired configuration.
:raises: jsonschema.ValidationError if the configuration is invalid.
"""
schema = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"required": ["scenarios", "operations", "metrics"],
"properties": {
"scenarios": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": ["name", "type"],
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string"
},
},
"additionalProperties": "true",
},
},
"operations": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": ["name", "type"],
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string"
},
},
"additionalProperties": "true",
},
},
"metrics": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": ["name", "type"],
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string"
},
},
"additionalProperties": "true",
},
}
}
}
v = Draft4Validator(schema, format_checker=FormatChecker())
v.validate(configuration)
class Spec(object):
"""Represents a Swagger Specification for a service.
:param spec_dict: Swagger API specification in json-like dict form
:param origin_url: URL from which the spec was retrieved.
:param http_client: Used to retrive the spec via http/https.
:type http_client: :class:`bravado.http_client.HTTPClient`
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
def __init__(self,
spec_dict,
origin_url=None,
http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client),
)
self._validate_config()
if self.config['internally_dereference_refs']:
# If internally_dereference_refs is enabled we do NOT need to resolve references anymore
# it's useless to evaluate is_ref every time
self.deref = lambda ref_dict: ref_dict
else:
self.deref = self._force_deref
def _validate_config(self):
"""
Validates the correctness of the configurations injected and makes sure that:
- no extra config keys are available on the config dictionary
- dependent configs are checked
:return: True if the initial configs are valid, False otherwise
:rtype: bool
"""
are_config_changed = False
extraneous_keys = set(iterkeys(self.config)) - set(
iterkeys(CONFIG_DEFAULTS))
if extraneous_keys:
are_config_changed = True
for key in extraneous_keys:
warnings.warn(
message='config {} is not a recognized config key'.format(
key),
category=Warning,
)
if self.config['internally_dereference_refs'] and not self.config[
'validate_swagger_spec']:
are_config_changed = True
self.config['internally_dereference_refs'] = False
warnings.warn(
message=
'internally_dereference_refs config disabled because validate_swagger_spec has to be enabled',
category=Warning,
)
return not are_config_changed
@cached_property
def client_spec_dict(self):
"""Return a copy of spec_dict with x-scope metadata removed so that it
is suitable for consumption by Swagger clients.
You may be asking, "Why is there a difference between the Swagger spec
a client sees and the one used internally?". Well, as part of the
ingestion process, x-scope metadata is added to spec_dict so that
$refs can be de-reffed successfully during requset/response validation
and marshalling. This medatadata is specific to the context of the
server and contains files and paths that are not relevant to the
client. This is required so the client does not re-use (and in turn,
re-creates) the invalid x-scope metadata created by the server.
For example, a section of spec_dict that contains a ref would change
as folows.
Before:
'MON': {
'$ref': '#/definitions/DayHours',
'x-scope': [
'file:///happyhour/api_docs/swagger.json',
'file:///happyhour/api_docs/swagger.json#/definitions/WeekHours'
]
}
After:
'MON': {
'$ref': '#/definitions/DayHours'
}
"""
return strip_xscope(self.spec_dict)
@classmethod
def from_dict(cls,
spec_dict,
origin_url=None,
http_client=None,
config=None):
"""Build a :class:`Spec` from Swagger API Specificiation
:param spec_dict: swagger spec in json-like dict form.
:param origin_url: the url used to retrieve the spec, if any
:type origin_url: str
:param: http_client: http client used to download remote $refs
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
spec = cls(spec_dict, origin_url, http_client, config)
spec.build()
return spec
def _validate_spec(self):
if self.config['validate_swagger_spec']:
self.resolver = validator20.validate_spec(
spec_dict=self.spec_dict,
spec_url=self.origin_url or '',
http_handlers=build_http_handlers(self.http_client),
)
def build(self):
self._validate_spec()
post_process_spec(
self,
on_container_callbacks=[
functools.partial(
tag_models,
visited_models={},
swagger_spec=self,
),
functools.partial(
collect_models,
models=self.definitions,
swagger_spec=self,
),
],
)
for format in self.config['formats']:
self.register_format(format)
self.api_url = build_api_serving_url(self.spec_dict, self.origin_url)
self.resources = build_resources(self)
@cached_property
def _internal_spec_dict(self):
if self.config['internally_dereference_refs']:
return self.deref_flattened_spec
else:
return self.spec_dict
def _force_deref(self, ref_dict):
"""Dereference ref_dict (if it is indeed a ref) and return what the
ref points to.
:param ref_dict: {'$ref': '#/blah/blah'}
:return: dereferenced value of ref_dict
:rtype: scalar, list, dict
"""
if ref_dict is None or not is_ref(ref_dict):
return ref_dict
# Restore attached resolution scope before resolving since the
# resolver doesn't have a traversal history (accumulated scope_stack)
# when asked to resolve.
with in_scope(self.resolver, ref_dict):
_, target = self.resolver.resolve(ref_dict['$ref'])
return target
def deref(self, ref_dict):
# This method is actually set in __init__
pass
def get_op_for_request(self, http_method, path_pattern):
"""Return the Swagger operation for the passed in request http method
and path pattern. Makes it really easy for server-side implementations
to map incoming requests to the Swagger spec.
:param http_method: http method of the request
:param path_pattern: request path pattern. e.g. /foo/{bar}/baz/{id}
:returns: the matching operation or None if a match couldn't be found
:rtype: :class:`bravado_core.operation.Operation`
"""
if self._request_to_op_map is None:
# lazy initialization
self._request_to_op_map = {}
base_path = self.spec_dict.get('basePath', '').rstrip('/')
for resource in self.resources.values():
for op in resource.operations.values():
full_path = base_path + op.path_name
key = (op.http_method, full_path)
self._request_to_op_map[key] = op
key = (http_method.lower(), path_pattern)
return self._request_to_op_map.get(key)
def register_format(self, user_defined_format):
"""Registers a user-defined format to be used with this spec.
:type user_defined_format:
:class:`bravado_core.formatter.SwaggerFormat`
"""
name = user_defined_format.format
self.user_defined_formats[name] = user_defined_format
validate = return_true_wrapper(user_defined_format.validate)
self.format_checker.checks(name,
raises=(SwaggerValidationError, ))(validate)
def get_format(self, name):
"""
:param name: Name of the format to retrieve
:rtype: :class:`bravado_core.formatters.SwaggerFormat`
"""
if name in formatter.DEFAULT_FORMATS:
return formatter.DEFAULT_FORMATS[name]
format = self.user_defined_formats.get(name)
if format is None:
warnings.warn(
message='{0} format is not registered with bravado-core!'.
format(name),
category=Warning,
)
return format
@cached_property
def security_definitions(self):
security_defs = {}
for security_name, security_def in iteritems(
self.spec_dict.get('securityDefinitions', {})):
security_defs[security_name] = SecurityDefinition(
self, security_def)
return security_defs
@cached_property
def flattened_spec(self):
"""
Representation of the current swagger specs that could be written to a single file.
NOTE: The representation strips out all the definitions that are not referenced
:return:
"""
if not self.config['validate_swagger_spec']:
raise RuntimeError(
'Swagger Specs have to be validated before flattening.')
# If resources are defined it means that Spec has been built and so swagger specs have been validated
if self.resources is None:
self._validate_spec()
return strip_xscope(spec_dict=flattened_spec(
spec_dict=self.spec_dict,
spec_resolver=self.resolver,
spec_url=self.origin_url,
http_handlers=build_http_handlers(self.http_client),
spec_definitions=self.definitions,
), )
@cached_property
def deref_flattened_spec(self):
deref_spec_dict = JsonRef.replace_refs(self.flattened_spec)
@memoize_by_id
def descend(obj):
# Inline modification of obj
# This method is needed because JsonRef could produce performance penalties in accessing
# the proxied attributes
if isinstance(obj, JsonRef):
# Extract the proxied value
# http://jsonref.readthedocs.io/en/latest/#jsonref.JsonRef.__subject__
return obj.__subject__
if is_dict_like(obj):
for key in list(iterkeys(obj)):
obj[key] = descend(obj[key])
elif is_list_like(obj):
# obj is list like object provided from flattened_spec specs.
# This guarantees that it cannot be a tuple instance and
# inline object modification are allowed
for index in range(len(obj)):
obj[index] = descend(obj[index])
return obj
try:
return descend(deref_spec_dict)
finally:
# Make sure that all memory allocated, for caching, could be released
descend.cache.clear()
def __testValidateOpts(self,
databaseNameD,
inputPathList=None,
schemaLevel="full",
mergeContentTypeD=None):
#
eCount = 0
for databaseName in databaseNameD:
mergeContentTypes = mergeContentTypeD[
databaseName] if databaseName in mergeContentTypeD else None
_ = self.__schP.makeSchemaDef(databaseName,
dataTyping="ANY",
saveSchema=True)
pthList = inputPathList if inputPathList else self.__rpP.getLocatorObjList(
databaseName, mergeContentTypes=mergeContentTypes)
for collectionName in databaseNameD[databaseName]:
cD = self.__schP.makeSchema(databaseName,
collectionName,
encodingType="JSON",
level=schemaLevel,
saveSchema=True,
extraOpts=None)
#
dL, cnL = self.__testPrepDocumentsFromContainers(
pthList,
databaseName,
collectionName,
styleType="rowwise_by_name_with_cardinality",
mergeContentTypes=mergeContentTypes)
# Raises exceptions for schema compliance.
try:
Draft4Validator.check_schema(cD)
except Exception as e:
logger.error("%s %s schema validation fails with %s",
databaseName, collectionName, str(e))
#
valInfo = Draft4Validator(cD, format_checker=FormatChecker())
logger.info("Validating %d documents from %s %s", len(dL),
databaseName, collectionName)
for ii, dD in enumerate(dL):
logger.debug("Schema %s collection %s document %d",
databaseName, collectionName, ii)
try:
cCount = 0
for error in sorted(valInfo.iter_errors(dD), key=str):
logger.info(
"schema %s collection %s (%s) path %s error: %s",
databaseName, collectionName, cnL[ii],
error.path, error.message)
logger.debug("Failing document %d : %r", ii,
list(dD.items()))
eCount += 1
cCount += 1
if cCount > 0:
logger.info(
"schema %s collection %s container %s error count %d",
databaseName, collectionName, cnL[ii], cCount)
except Exception as e:
logger.exception("Validation processing error %s",
str(e))
return eCount
def parse(filename):
try:
schema = json.loads(open("pl.schema").read())
schema = Draft4Validator(schema, format_checker=FormatChecker())
except ValueError as e:
post_error("pl.schema - " + str(e))
return
try:
pl = json.loads(open(filename).read())
except ValueError as e:
post_error(filename + " - " + str(e))
return
for error in schema.iter_errors(pl):
post_error(error.message)
foldernames = []
displaynames = []
repositories = []
os.mkdir("./" + bitness_from_input)
for plugin in pl["npp-plugins"]:
print(plugin["display-name"])
try:
response = requests.get(plugin["repository"])
except requests.exceptions.RequestException as e:
post_error(str(e))
continue
if response.status_code != 200:
post_error(
f'{plugin["display-name"]}: failed to download plugin. Returned code {response.status_code}'
)
continue
# Hash it and make sure its what is expected
hash = sha256(response.content).hexdigest()
if plugin["id"].lower() != hash.lower():
post_error(
f'{plugin["display-name"]}: Invalid hash. Got {hash.lower()} but expected {plugin["id"]}'
)
continue
# Make sure its a valid zip file
try:
zip = zipfile.ZipFile(io.BytesIO(response.content))
except zipfile.BadZipFile as e:
post_error(f'{plugin["display-name"]}: Invalid zip file')
continue
# The expected DLL name
dll_name = f'{plugin["folder-name"]}.dll'.lower()
# Notepad++ is not case sensitive, but extracting files from the zip is,
# so find the exactfile name to use
for file in zip.namelist():
if dll_name == file.lower():
dll_name = file
break
else:
post_error(
f'{plugin["display-name"]}: Zip file does not contain {plugin["folder-name"]}.dll'
)
continue
with zip.open(dll_name) as dll_file, open(
"./" + bitness_from_input + "/" + dll_name, 'wb') as f:
f.write(dll_file.read())
version = plugin["version"]
# Fill in any of the missing numbers as zeros
version = version + (3 - version.count('.')) * ".0"
try:
dll_version = get_version_number("./" + bitness_from_input + "/" +
dll_name)
except win32api.error:
post_error(
f'{plugin["display-name"]}: Does not contain any version information'
)
continue
if dll_version != version:
post_error(
f'{plugin["display-name"]}: Unexpected DLL version. DLL is {dll_version} but expected {version}'
)
continue
#check uniquess of json folder-name, display-name and repository
found = False
for name in displaynames:
if plugin["display-name"] == name:
post_error(
f'{plugin["display-name"]}: non unique display-name entry')
found = True
if found == False:
displaynames.append(plugin["display-name"])
found = False
for folder in foldernames:
if plugin["folder-name"] == folder:
post_error(
f'{plugin["folder-name"]}: non unique folder-name entry')
found = True
if found == False:
foldernames.append(plugin["folder-name"])
found = False
for repo in repositories:
if plugin["repository"] == repo:
post_error(
f'{plugin["repository"]}: non unique repository entry')
found = True
if found == False:
repositories.append(plugin["repository"])
class JSONSchemaField(models.JSONField):
"""
A JSONB field that self-validates against a defined JSON schema
(http://json-schema.org). This base class is intended to be overwritten by
defining `self.schema`.
"""
format_checker = FormatChecker()
# If an empty {} is provided, we still want to perform this schema
# validation
empty_values = (None, '')
def __init__(self, encoder=None, decoder=None, **options):
if encoder is None:
encoder = DjangoJSONEncoder
super().__init__(encoder=encoder, decoder=decoder, **options)
def get_default(self):
return copy.deepcopy(super(models.JSONField, self).get_default())
def schema(self, model_instance):
raise NotImplementedError()
def validate(self, value, model_instance):
super(JSONSchemaField, self).validate(value, model_instance)
errors = []
for error in Draft4Validator(
self.schema(model_instance),
format_checker=self.format_checker).iter_errors(value):
if error.validator == 'pattern' and 'error' in error.schema:
error.message = error.schema['error'].format(
instance=error.instance)
elif error.validator == 'type':
expected_type = error.validator_value
if expected_type == 'object':
expected_type = 'dict'
if error.path:
error.message = _(
'{type} provided in relative path {path}, expected {expected_type}'
).format(path=list(error.path),
type=type(error.instance).__name__,
expected_type=expected_type)
else:
error.message = _(
'{type} provided, expected {expected_type}').format(
path=list(error.path),
type=type(error.instance).__name__,
expected_type=expected_type)
elif error.validator == 'additionalProperties' and hasattr(
error, 'path'):
error.message = _(
'Schema validation error in relative path {path} ({error})'
).format(path=list(error.path), error=error.message)
errors.append(error)
if errors:
raise django_exceptions.ValidationError(
[e.message for e in errors],
code='invalid',
params={'value': value},
)
'services-digital-outcomes-and-specialists-user-research-participants',
'services-inoket-1-iaas',
'services-inoket-1-saas',
'services-inoket-1-paas',
'services-inoket-1-scs',
'services-inoket-1-supply_teachers',
'services-inoket-2-supply_teachers',
'services-update',
'users',
'users-auth',
'suppliers',
'new-supplier',
'contact-information',
'orders-create',
]
FORMAT_CHECKER = FormatChecker()
def load_schemas(schemas_path, schema_names):
loaded_schemas = {}
for schema_name in schema_names:
schema_path = os.path.join(schemas_path, '{}.json'.format(schema_name))
with open(schema_path) as f:
schema = json.load(f)
validator = validator_for(schema)
validator.check_schema(schema)
loaded_schemas[schema_name] = schema
return loaded_schemas
class Spec(object):
"""Represents a Swagger Specification for a service.
:param spec_dict: Swagger API specification in json-like dict form
:param origin_url: URL from which the spec was retrieved.
:param http_client: Used to retrive the spec via http/https.
:type http_client: :class:`bravado.http_client.HTTPClient`
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
def __init__(self,
spec_dict,
origin_url=None,
http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client))
@classmethod
def from_dict(cls,
spec_dict,
origin_url=None,
http_client=None,
config=None):
"""Build a :class:`Spec` from Swagger API Specificiation
:param spec_dict: swagger spec in json-like dict form.
:param origin_url: the url used to retrieve the spec, if any
:type origin_url: str
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
spec = cls(spec_dict, origin_url, http_client, config)
spec.build()
return spec
def build(self):
if self.config['validate_swagger_spec']:
self.resolver = validator20.validate_spec(
self.spec_dict,
spec_url=self.origin_url or '',
http_handlers=build_http_handlers(self.http_client))
post_process_spec(self,
on_container_callbacks=[
functools.partial(tag_models,
visited_models={},
swagger_spec=self),
functools.partial(collect_models,
models=self.definitions,
swagger_spec=self)
])
for format in self.config['formats']:
self.register_format(format)
self.api_url = build_api_serving_url(self.spec_dict, self.origin_url)
self.resources = build_resources(self)
def deref(self, ref_dict):
"""Dereference ref_dict (if it is indeed a ref) and return what the
ref points to.
:param ref_dict: {'$ref': '#/blah/blah'}
:return: dereferenced value of ref_dict
:rtype: scalar, list, dict
"""
if ref_dict is None or not is_ref(ref_dict):
return ref_dict
# Restore attached resolution scope before resolving since the
# resolver doesn't have a traversal history (accumulated scope_stack)
# when asked to resolve.
with in_scope(self.resolver, ref_dict):
log.debug('Resolving {0} with scope {1}: {2}'.format(
ref_dict['$ref'], len(self.resolver._scopes_stack),
self.resolver._scopes_stack))
_, target = self.resolver.resolve(ref_dict['$ref'])
return target
def get_op_for_request(self, http_method, path_pattern):
"""Return the Swagger operation for the passed in request http method
and path pattern. Makes it really easy for server-side implementations
to map incoming requests to the Swagger spec.
:param http_method: http method of the request
:param path_pattern: request path pattern. e.g. /foo/{bar}/baz/{id}
:returns: the matching operation or None if a match couldn't be found
:rtype: :class:`bravado_core.operation.Operation`
"""
if self._request_to_op_map is None:
# lazy initialization
self._request_to_op_map = {}
base_path = self.spec_dict.get('basePath', '').rstrip('/')
for resource in self.resources.values():
for op in resource.operations.values():
full_path = base_path + op.path_name
key = (op.http_method, full_path)
self._request_to_op_map[key] = op
key = (http_method.lower(), path_pattern)
return self._request_to_op_map.get(key)
def register_format(self, user_defined_format):
"""Registers a user-defined format to be used with this spec.
:type user_defined_format:
:class:`bravado_core.formatter.SwaggerFormat`
"""
name = user_defined_format.format
self.user_defined_formats[name] = user_defined_format
validate = return_true_wrapper(user_defined_format.validate)
self.format_checker.checks(name,
raises=(SwaggerValidationError, ))(validate)
def get_format(self, name):
"""
:param name: Name of the format to retrieve
:rtype: :class:`bravado_core.formatters.SwaggerFormat`
"""
if name in formatter.DEFAULT_FORMATS:
return formatter.DEFAULT_FORMATS[name]
format = self.user_defined_formats.get(name)
if format is None:
warnings.warn(
'{0} format is not registered with bravado-core!'.format(name),
Warning)
return format
def formatchecker_factory(**checkerdict):
"""Converts a dictionary of strings:checkers into a formatchecker object"""
fc = FormatChecker()
for format_name, checker in checkerdict.items():
fc.checks(format_name)(checker)
return fc
class Spec(object):
"""Represents a Swagger Specification for a service.
:param spec_dict: Swagger API specification in json-like dict form
:param origin_url: URL from which the spec was retrieved.
:param http_client: Used to retrive the spec via http/https.
:type http_client: :class:`bravado.http_client.HTTPClient`
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
def __init__(self, spec_dict, origin_url=None, http_client=None,
config=None):
self.spec_dict = spec_dict
self.origin_url = origin_url
self.http_client = http_client
self.api_url = None
self.config = dict(CONFIG_DEFAULTS, **(config or {}))
# (key, value) = (simple format def name, Model type)
# (key, value) = (#/ format def ref, Model type)
self.definitions = {}
# (key, value) = (simple resource name, Resource)
# (key, value) = (#/ format resource ref, Resource)
self.resources = None
# (key, value) = (simple ref name, param_spec in dict form)
# (key, value) = (#/ format ref name, param_spec in dict form)
self.params = None
# Built on-demand - see get_op_for_request(..)
self._request_to_op_map = None
# (key, value) = (format name, SwaggerFormat)
self.user_defined_formats = {}
self.format_checker = FormatChecker()
self.resolver = RefResolver(
base_uri=origin_url or '',
referrer=self.spec_dict,
handlers=build_http_handlers(http_client),
)
self._validate_config()
if self.config['internally_dereference_refs']:
# If internally_dereference_refs is enabled we do NOT need to resolve references anymore
# it's useless to evaluate is_ref every time
self.deref = lambda ref_dict: ref_dict
else:
self.deref = self._force_deref
def _validate_config(self):
"""
Validates the correctness of the configurations injected and makes sure that:
- no extra config keys are available on the config dictionary
- dependent configs are checked
:return: True if the initial configs are valid, False otherwise
:rtype: bool
"""
are_config_changed = False
extraneous_keys = set(iterkeys(self.config)) - set(iterkeys(CONFIG_DEFAULTS))
if extraneous_keys:
are_config_changed = True
for key in extraneous_keys:
warnings.warn(
message='config {} is not a recognized config key'.format(key),
category=Warning,
)
if self.config['internally_dereference_refs'] and not self.config['validate_swagger_spec']:
are_config_changed = True
self.config['internally_dereference_refs'] = False
warnings.warn(
message='internally_dereference_refs config disabled because validate_swagger_spec has to be enabled',
category=Warning,
)
return not are_config_changed
@cached_property
def client_spec_dict(self):
"""Return a copy of spec_dict with x-scope metadata removed so that it
is suitable for consumption by Swagger clients.
You may be asking, "Why is there a difference between the Swagger spec
a client sees and the one used internally?". Well, as part of the
ingestion process, x-scope metadata is added to spec_dict so that
$refs can be de-reffed successfully during requset/response validation
and marshalling. This medatadata is specific to the context of the
server and contains files and paths that are not relevant to the
client. This is required so the client does not re-use (and in turn,
re-creates) the invalid x-scope metadata created by the server.
For example, a section of spec_dict that contains a ref would change
as folows.
Before:
'MON': {
'$ref': '#/definitions/DayHours',
'x-scope': [
'file:///happyhour/api_docs/swagger.json',
'file:///happyhour/api_docs/swagger.json#/definitions/WeekHours'
]
}
After:
'MON': {
'$ref': '#/definitions/DayHours'
}
"""
return strip_xscope(self.spec_dict)
@classmethod
def from_dict(cls, spec_dict, origin_url=None, http_client=None, config=None):
"""Build a :class:`Spec` from Swagger API Specificiation
:param spec_dict: swagger spec in json-like dict form.
:param origin_url: the url used to retrieve the spec, if any
:type origin_url: str
:param: http_client: http client used to download remote $refs
:param config: Configuration dict. See CONFIG_DEFAULTS.
"""
spec = cls(spec_dict, origin_url, http_client, config)
spec.build()
return spec
def _validate_spec(self):
if self.config['validate_swagger_spec']:
self.resolver = validator20.validate_spec(
spec_dict=self.spec_dict,
spec_url=self.origin_url or '',
http_handlers=build_http_handlers(self.http_client),
)
def build(self):
self._validate_spec()
post_process_spec(
self,
on_container_callbacks=[
functools.partial(
tag_models,
visited_models={},
swagger_spec=self,
),
functools.partial(
collect_models,
models=self.definitions,
swagger_spec=self,
),
],
)
for format in self.config['formats']:
self.register_format(format)
self.api_url = build_api_serving_url(self.spec_dict, self.origin_url)
self.resources = build_resources(self)
@cached_property
def _internal_spec_dict(self):
if self.config['internally_dereference_refs']:
return self.deref_flattened_spec
else:
return self.spec_dict
def _force_deref(self, ref_dict):
"""Dereference ref_dict (if it is indeed a ref) and return what the
ref points to.
:param ref_dict: {'$ref': '#/blah/blah'}
:return: dereferenced value of ref_dict
:rtype: scalar, list, dict
"""
if ref_dict is None or not is_ref(ref_dict):
return ref_dict
# Restore attached resolution scope before resolving since the
# resolver doesn't have a traversal history (accumulated scope_stack)
# when asked to resolve.
with in_scope(self.resolver, ref_dict):
_, target = self.resolver.resolve(ref_dict['$ref'])
return target
def deref(self, ref_dict):
# This method is actually set in __init__
pass
def get_op_for_request(self, http_method, path_pattern):
"""Return the Swagger operation for the passed in request http method
and path pattern. Makes it really easy for server-side implementations
to map incoming requests to the Swagger spec.
:param http_method: http method of the request
:param path_pattern: request path pattern. e.g. /foo/{bar}/baz/{id}
:returns: the matching operation or None if a match couldn't be found
:rtype: :class:`bravado_core.operation.Operation`
"""
if self._request_to_op_map is None:
# lazy initialization
self._request_to_op_map = {}
base_path = self.spec_dict.get('basePath', '').rstrip('/')
for resource in self.resources.values():
for op in resource.operations.values():
full_path = base_path + op.path_name
key = (op.http_method, full_path)
self._request_to_op_map[key] = op
key = (http_method.lower(), path_pattern)
return self._request_to_op_map.get(key)
def register_format(self, user_defined_format):
"""Registers a user-defined format to be used with this spec.
:type user_defined_format:
:class:`bravado_core.formatter.SwaggerFormat`
"""
name = user_defined_format.format
self.user_defined_formats[name] = user_defined_format
validate = return_true_wrapper(user_defined_format.validate)
self.format_checker.checks(
name, raises=(SwaggerValidationError,))(validate)
def get_format(self, name):
"""
:param name: Name of the format to retrieve
:rtype: :class:`bravado_core.formatters.SwaggerFormat`
"""
if name in formatter.DEFAULT_FORMATS:
return formatter.DEFAULT_FORMATS[name]
format = self.user_defined_formats.get(name)
if format is None:
warnings.warn(
message='{0} format is not registered with bravado-core!'.format(name),
category=Warning,
)
return format
@cached_property
def security_definitions(self):
security_defs = {}
for security_name, security_def in iteritems(self.spec_dict.get('securityDefinitions', {})):
security_defs[security_name] = SecurityDefinition(self, security_def)
return security_defs
@cached_property
def flattened_spec(self):
"""
Representation of the current swagger specs that could be written to a single file.
NOTE: The representation strips out all the definitions that are not referenced
:return:
"""
if not self.config['validate_swagger_spec']:
raise RuntimeError('Swagger Specs have to be validated before flattening.')
# If resources are defined it means that Spec has been built and so swagger specs have been validated
if self.resources is None:
self._validate_spec()
return strip_xscope(
spec_dict=flattened_spec(
spec_dict=self.spec_dict,
spec_resolver=self.resolver,
spec_url=self.origin_url,
http_handlers=build_http_handlers(self.http_client),
spec_definitions=self.definitions,
),
)
@cached_property
def deref_flattened_spec(self):
deref_spec_dict = JsonRef.replace_refs(self.flattened_spec)
@memoize_by_id
def descend(obj):
# Inline modification of obj
# This method is needed because JsonRef could produce performance penalties in accessing
# the proxied attributes
if isinstance(obj, JsonRef):
# Extract the proxied value
# http://jsonref.readthedocs.io/en/latest/#jsonref.JsonRef.__subject__
return obj.__subject__
if is_dict_like(obj):
for key in list(iterkeys(obj)):
obj[key] = descend(obj[key])
elif is_list_like(obj):
# obj is list like object provided from flattened_spec specs.
# This guarantees that it cannot be a tuple instance and
# inline object modification are allowed
for index in range(len(obj)):
obj[index] = descend(obj[index])
return obj
try:
return descend(deref_spec_dict)
finally:
# Make sure that all memory allocated, for caching, could be released
descend.cache.clear()
