def process_record(self, dataset):
dataset_id = dataset['id']
record = self.redis_client.get(dataset_id)
portal = dataset['extras'].get('metadata_original_portal', 'null')
if record is None:
record = {'id': dataset_id, 'metadata_original_portal': portal}
else:
record = eval(record)
record['schema'] = []
broken_rules = []
errors = Draft3Validator(self.schema).iter_errors(dataset)
if not Draft3Validator(self.schema).is_valid(dataset):
errors = Draft3Validator(self.schema).iter_errors(dataset)
for error in errors:
path = [e for e in error.path if isinstance(e, basestring)]
path = str('.'.join(map((lambda e: str(e)), path)))
field_path_message = [path, error.message]
broken_rules.append(field_path_message)
record['schema'] = broken_rules
self.redis_client.set(dataset_id, record)
return not broken_rules
def validate_datasets(self, dataset, data):
normalize_action_dataset(dataset)
identifier = dataset['id']
portal = dataset['extras'].get('metadata_original_portal', 'null')
data['broken_rules'][portal][identifier] = []
broken_rules = data['broken_rules'][portal][identifier]
data['datasets_per_portal'][portal].add(identifier)
errors = Draft3Validator(self.schema).iter_errors(dataset)
if Draft3Validator(self.schema).is_valid(dataset):
data['valid_datasets'] += 1
else:
data['invalid_datasets'] += 1
errors = Draft3Validator(self.schema).iter_errors(dataset)
for error in errors:
path = [e for e in error.path if isinstance(e, basestring)]
path = str(' -> '.join(map((lambda e: str(e)), path)))
data['field_paths'][path] += 1
field_path_message = [path, error.message]
broken_rules.append(field_path_message)
def validate_incoming_request(route_mapper, request, schema_map, resolver):
"""Validates an incoming request against our schemas.
:param request: the request object to validate
:type request: Pyramid request object passed into a view
:param schema_map: our mapping from request data to schemas (see
load_schema)
:type schema_map: dict
:param resolver: the request object to validate
:type resolver: Pyramid request object passed into a view
:returns: None
"""
if schema_map.request_query_schema:
# You'll notice we use Draft3 some places and Draft4 in others.
# Unfortunately this is just Swagger's inconsistency showing. It
# may be nice in the future to do the necessary munging to make
# everything Draft4 compatible, although the Swagger UI will
# probably never truly support Draft4.
request_query_params = dict(
(k, cast_request_param(schema_map.request_query_schema, k, v))
for k, v in request.GET.items())
Draft3Validator(
schema_map.request_query_schema,
resolver=resolver,
types=EXTENDED_TYPES,
).validate(request_query_params)
if schema_map.request_path_schema:
# We don't have access to this yet but let's go ahead and build the
# matchdict so we can validate it.
info = route_mapper(request)
matchdict = dict(
(k, cast_request_param(schema_map.request_path_schema, k, v))
for k, v in info.get('match', {}).items())
Draft3Validator(
schema_map.request_path_schema,
resolver=resolver,
types=EXTENDED_TYPES,
).validate(matchdict)
# Body validation
if schema_map.request_body_schema:
body = getattr(request, 'json_body', {})
Draft4Validator(
schema_map.request_body_schema,
resolver=resolver,
types=EXTENDED_TYPES,
).validate(body)
def test_multiple_nesting(self):
instance = [1, {"foo": 2, "bar": {"baz": [1]}}, "quux"]
schema = {
"type": "string",
"items": {
"type": ["string", "object"],
"properties": {
"foo": {
"enum": [1, 3]
},
"bar": {
"type": "array",
"properties": {
"bar": {
"required": True
},
"baz": {
"minItems": 2
},
}
}
}
}
}
validator = Draft3Validator(schema)
errors = validator.iter_errors(instance)
e1, e2, e3, e4, e5, e6 = sorted_errors(errors)
self.assertEqual(e1.path, deque([]))
self.assertEqual(e2.path, deque([0]))
self.assertEqual(e3.path, deque([1, "bar"]))
self.assertEqual(e4.path, deque([1, "bar", "bar"]))
self.assertEqual(e5.path, deque([1, "bar", "baz"]))
self.assertEqual(e6.path, deque([1, "foo"]))
self.assertEqual(e1.schema_path, deque(["type"]))
self.assertEqual(e2.schema_path, deque(["items", "type"]))
self.assertEqual(
list(e3.schema_path),
["items", "properties", "bar", "type"],
)
self.assertEqual(
list(e4.schema_path),
["items", "properties", "bar", "properties", "bar", "required"],
)
self.assertEqual(
list(e5.schema_path),
["items", "properties", "bar", "properties", "baz", "minItems"])
self.assertEqual(
list(e6.schema_path),
["items", "properties", "foo", "enum"],
)
self.assertEqual(e1.validator, "type")
self.assertEqual(e2.validator, "type")
self.assertEqual(e3.validator, "type")
self.assertEqual(e4.validator, "required")
self.assertEqual(e5.validator, "minItems")
self.assertEqual(e6.validator, "enum")
def test_single_nesting(self):
instance = {"foo": 2, "bar": [1], "baz": 15, "quux": "spam"}
schema = {
"properties": {
"foo": {
"type": "string"
},
"bar": {
"minItems": 2
},
"baz": {
"maximum": 10,
"enum": [2, 4, 6, 8]
},
}
}
validator = Draft3Validator(schema)
errors = validator.iter_errors(instance)
e1, e2, e3, e4 = sorted_errors(errors)
self.assertEqual(list(e1.path), ["bar"])
self.assertEqual(list(e2.path), ["baz"])
self.assertEqual(list(e3.path), ["baz"])
self.assertEqual(list(e4.path), ["foo"])
self.assertEqual(e1.validator, "minItems")
self.assertEqual(e2.validator, "enum")
self.assertEqual(e3.validator, "maximum")
self.assertEqual(e4.validator, "type")
def test_additionalProperties(self):
instance = {"bar": "bar", "foo": 2}
schema = {"additionalProperties": {"type": "integer", "minimum": 5}}
validator = Draft3Validator(schema)
errors = validator.iter_errors(instance)
e1, e2 = sorted_errors(errors)
self.assertEqual(e1.path, deque(["bar"]))
self.assertEqual(e2.path, deque(["foo"]))
self.assertEqual(e1.validator, "type")
self.assertEqual(e2.validator, "minimum")
def test_type(self):
instance = {"foo": 1}
schema = {
"type": [
{"type": "integer"},
{
"type": "object",
"properties": {
"foo": {"enum": [2]}
}
}
]
}
validator = Draft3Validator(schema)
errors = list(validator.iter_errors(instance))
self.assertEqual(len(errors), 1)
e = errors[0]
self.assertEqual(e.validator, "type")
self.assertEqual(list(e.schema_path), ["type"])
self.assertEqual(e.validator_value, schema["type"])
self.assertEqual(e.instance, instance)
self.assertEqual(e.schema, schema)
self.assertEqual(list(e.path), [])
self.assertEqual(len(e.context), 2)
e1, e2 = sorted_errors(e.context)
self.assertEqual(e1.validator, "type")
self.assertEqual(list(e1.schema_path), [0, "type"])
self.assertEqual(e1.validator_value, schema["type"][0]["type"])
self.assertEqual(e1.instance, instance)
self.assertEqual(e1.schema, schema["type"][0])
self.assertEqual(list(e1.path), [])
self.assertEqual(len(e1.context), 0)
self.assertEqual(e2.validator, "enum")
self.assertEqual(
list(e2.schema_path),
[1, "properties", "foo", "enum"]
)
self.assertEqual(
e2.validator_value,
schema["type"][1]["properties"]["foo"]["enum"]
)
self.assertEqual(e2.instance, instance["foo"])
self.assertEqual(e2.schema, schema["type"][1]["properties"]["foo"])
self.assertEqual(list(e2.path), ["foo"])
self.assertEqual(len(e2.context), 0)
def test_additionalItems_with_items(self):
instance = ["foo", "bar", 1]
schema = {
"items": [{}],
"additionalItems" : {"type": "integer", "minimum": 5}
}
validator = Draft3Validator(schema)
errors = validator.iter_errors(instance)
e1, e2 = sorted_errors(errors)
self.assertEqual(e1.path, deque([1]))
self.assertEqual(e2.path, deque([2]))
self.assertEqual(e1.validator, "type")
self.assertEqual(e2.validator, "minimum")
def test_patternProperties(self):
instance = {"bar": 1, "foo": 2}
schema = {
"patternProperties" : {
"bar": {"type": "string"},
"foo": {"minimum": 5}
}
}
validator = Draft3Validator(schema)
errors = validator.iter_errors(instance)
e1, e2 = sorted_errors(errors)
self.assertEqual(e1.path, deque(["bar"]))
self.assertEqual(e2.path, deque(["foo"]))
self.assertEqual(e1.validator, "type")
self.assertEqual(e2.validator, "minimum")
def _validate_schema(self, schema, machine_readable):
'''
validates given machine_readable data against given schema
with jsonschema draft03 validator.
:param schema: json schema to check against
:type schema: dict
:param machine_readable: xarf machine readable part
:type machine_readable: dict
:raises: :py:class:`ValidationError`: if validation fails
'''
errors = []
validator = Draft3Validator(schema)
result = validator.iter_errors(machine_readable)
for error in result:
msg = '%s %s' % (error.path[0], error.message)
errors.append(msg)
if len(errors):
raise ValidationError(', '.join(errors))
def setUp(self):
self.validator = Draft3Validator({})
def test_type(self):
instance = {"foo": 1}
schema = {
"type": [{
"type": "integer"
}, {
"type": "object",
"properties": {
"foo": {
"enum": [2]
}
}
}]
}
validator = Draft3Validator(schema)
errors = list(validator.iter_errors(instance))
self.assertEqual(len(errors), 1)
e = errors[0]
self.assertEqual(e.validator, "type")
self.assertEqual(e.validator_value, schema["type"])
self.assertEqual(e.instance, instance)
self.assertEqual(e.schema, schema)
self.assertIsNone(e.parent)
self.assertEqual(e.path, deque([]))
self.assertEqual(e.relative_path, deque([]))
self.assertEqual(e.absolute_path, deque([]))
self.assertEqual(e.schema_path, deque(["type"]))
self.assertEqual(e.relative_schema_path, deque(["type"]))
self.assertEqual(e.absolute_schema_path, deque(["type"]))
self.assertEqual(len(e.context), 2)
e1, e2 = sorted_errors(e.context)
self.assertEqual(e1.validator, "type")
self.assertEqual(e1.validator_value, schema["type"][0]["type"])
self.assertEqual(e1.instance, instance)
self.assertEqual(e1.schema, schema["type"][0])
self.assertIs(e1.parent, e)
self.assertEqual(e1.path, deque([]))
self.assertEqual(e1.relative_path, deque([]))
self.assertEqual(e1.absolute_path, deque([]))
self.assertEqual(e1.schema_path, deque([0, "type"]))
self.assertEqual(e1.relative_schema_path, deque([0, "type"]))
self.assertEqual(e1.absolute_schema_path, deque(["type", 0, "type"]))
self.assertFalse(e1.context)
self.assertEqual(e2.validator, "enum")
self.assertEqual(e2.validator_value, [2])
self.assertEqual(e2.instance, 1)
self.assertEqual(e2.schema, {u"enum": [2]})
self.assertIs(e2.parent, e)
self.assertEqual(e2.path, deque(["foo"]))
self.assertEqual(e2.relative_path, deque(["foo"]))
self.assertEqual(e2.absolute_path, deque(["foo"]))
self.assertEqual(
e2.schema_path,
deque([1, "properties", "foo", "enum"]),
)
self.assertEqual(
e2.relative_schema_path,
deque([1, "properties", "foo", "enum"]),
)
self.assertEqual(
e2.absolute_schema_path,
deque(["type", 1, "properties", "foo", "enum"]),
)
self.assertFalse(e2.context)
def _validate(schema, data):
reqv = Draft3Validator(schema)
errors = []
for e in reqv.iter_errors(data):
errors.append(dict(name=e.path[0], reason=e.validator))
return errors
