def _pprint_validation_error(error):
"""
A version of jsonschema's ValidationError __str__ method that doesn't
include the schema fragment that failed. This makes the error messages
much more succinct.
It also shows any subschemas of anyOf/allOf that failed, if any (what
jsonschema calls "context").
"""
essential_for_verbose = (
error.validator,
error.validator_value,
error.instance,
error.schema,
)
if any(m is _unset for m in essential_for_verbose):
return error.message
pinstance = pprint.pformat(error.instance, width=72)
parts = [
'On {}{}:'.format(
error._word_for_instance_in_error_message,
_utils.format_as_index(error.relative_path),
),
_utils.indent(pinstance), '', error.message
]
if error.context:
parts.extend(_utils.indent(x.message) for x in error.context)
return '\n'.join(parts)
def __unicode__(self):
essential_for_verbose = (
self.validator, self.validator_value, self.instance, self.schema,
)
if any(m is _unset for m in essential_for_verbose):
return self.message
pschema = pprint.pformat(self.schema, width=72)
pinstance = pprint.pformat(self.instance, width=72)
return self.message + textwrap.dedent("""
Failed validating %r in %s%s:
%s
On %s%s:
%s
""".rstrip()
) % (
self.validator,
self._word_for_schema_in_error_message,
_utils.format_as_index(list(self.relative_schema_path)[:-1]),
_utils.indent(pschema),
self._word_for_instance_in_error_message,
_utils.format_as_index(self.relative_path),
_utils.indent(pinstance),
)
def __unicode__(self):
essential_for_verbose = (
self.validator,
self.validator_value,
self.instance,
self.schema,
)
if any(m is _unset for m in essential_for_verbose):
return self.message
pschema = pprint.pformat(self.schema, width=72)
pinstance = pprint.pformat(self.instance, width=72)
return self.message + textwrap.dedent("""
Failed validating %r in %s %s:
%s
On %s %s:
%s
""".rstrip()) % (
self.validator,
self._word_for_schema_in_error_message,
_utils.format_as_index(list(self.relative_schema_path)[:-1]),
_utils.indent(pschema),
self._word_for_instance_in_error_message,
_utils.format_as_index(self.relative_path),
_utils.indent(pinstance),
)
def __unicode__(self):
if _unset in (
self.validator, self.validator_value, self.instance, self.schema,
):
return self.message
path = _utils.format_as_index(self.path)
schema_path = _utils.format_as_index(list(self.schema_path)[:-1])
pschema = pprint.pformat(self.schema, width=72)
pinstance = pprint.pformat(self.instance, width=72)
return self.message + textwrap.dedent("""
Failed validating %r in schema%s:
%s
On instance%s:
%s
""".rstrip()
) % (
self.validator,
schema_path,
_utils.indent(pschema),
path,
_utils.indent(pinstance),
)
def validate_schema(self, data_file, file_base):
return_value = True
schema_file = self.get_schema_path(file_base)
try:
schema_contents = CPConfigFile.parse(schema_file)
data_contents = CPConfigFile.parse(data_file)
# We've already checked the input files for errors, but not the
# schema files
schema_errors = CPConfigFile.errors(schema_file)
schema_warnings = CPConfigFile.warnings(schema_file)
if schema_warnings:
self._warnings += schema_warnings
if schema_errors:
self._errors += schema_errors
return False
except Exception as e:
msg = (
'Syntax errors detected, data:"%s" schema "%s" errors "%s" ' %
(data_file, schema_file, e))
self.add_error(msg)
return_value = False
try:
schema_check = validators.validator_for(schema_contents)
schema_check.check_schema(schema_contents)
except SchemaError as e:
self.add_error('Schema "%s" is invalid: %s' % (schema_contents, e))
return False
try:
validate = Draft4Validator(schema_contents)
error_list = [err for err in validate.iter_errors(data_contents)]
if len(error_list) > 0:
error_string = '\n\nInput\n%s\n\nCould not be validated - list of errors:\n' % (
_utils.indent(
yaml.dump(data_contents,
default_flow_style=False,
indent=4)))
for e in error_list:
error_string += "%s\n%s\n%s\n" % (
_utils.indent("Index of error: %s" %
_utils.format_as_index(deque(e.path))),
_utils.indent(" Erroneous value: %s" %
pprint.pformat(e.instance, width=72)),
_utils.indent(
" Expected type: %s" % e.validator_value))
self.add_error(error_string)
return_value = False
except Exception as e:
self.add_error('File "%s" or "%s" could not be loaded: %s' %
(schema_file, data_file, e))
return_value = False
return return_value
def __unicode__(self):
pschema = pprint.pformat(self.schema, width=72)
pinstance = pprint.pformat(self.instance, width=72)
return textwrap.dedent("""
Unknown type %r for validator with schema:
%s
While checking instance:
%s
""".rstrip()
) % (self.type, _utils.indent(pschema), _utils.indent(pinstance))
def __unicode__(self):
pschema = pprint.pformat(self.schema, width=72)
pinstance = pprint.pformat(self.instance, width=72)
return textwrap.dedent("""
Unknown type %r for validator with schema:
%s
While checking instance:
%s
""".rstrip()) % (self.type, _utils.indent(pschema),
_utils.indent(pinstance))
def validate_schema(self, data_file, file_base):
return_value = True
schema_file = self.get_schema_path(file_base)
try:
schema_contents = CPConfigFile.parse(schema_file)
data_contents = CPConfigFile.parse(data_file)
# We've already checked the input files for errors, but not the
# schema files
schema_errors = CPConfigFile.errors(schema_file)
schema_warnings = CPConfigFile.warnings(schema_file)
if schema_warnings:
self._warnings += schema_warnings
if schema_errors:
self._errors += schema_errors
return False
except Exception as e:
msg = ('Syntax errors detected, data:"%s" schema "%s" errors "%s" '
% (data_file, schema_file, e))
self.add_error(msg)
return_value = False
try:
schema_check = validators.validator_for(schema_contents)
schema_check.check_schema(schema_contents)
except SchemaError as e:
self.add_error('Schema "%s" is invalid: %s' % (
schema_contents, e))
return False
try:
validate = Draft4Validator(schema_contents)
error_list = [err for err in validate.iter_errors(data_contents)]
if len(error_list) > 0:
error_string = '\n\nInput\n%s\n\nCould not be validated - list of errors:\n' % (
_utils.indent(yaml.dump(data_contents, default_flow_style=False, indent=4)))
for e in error_list:
error_string += "%s\n%s\n%s\n" % (
_utils.indent("Index of error: %s" %
_utils.format_as_index(deque(e.path))),
_utils.indent(" Erroneous value: %s" %
pprint.pformat(e.instance, width=72)),
_utils.indent(" Expected type: %s" %
e.validator_value))
self.add_error(error_string)
return_value = False
except Exception as e:
self.add_error('File "%s" or "%s" could not be loaded: %s' % (
schema_file, data_file, e))
return_value = False
return return_value
def __unicode__(self):
pschema = self.schema
pinstance = self.instance
return textwrap.dedent("""
Unknown type %r for validator with schema:
%s
While checking instance:
%s
""".rstrip()
) % (self.type, _utils.indent(pschema), _utils.indent(pinstance))
def format_error(filepath, header, content):
"""
Format a jsonshema validation error.
"""
if isinstance(filepath, Path):
filepath = filepath.resolve()
else:
filepath = "<string>"
if header:
return "{}: {}\n{}".format(filepath, header, _utils.indent(content))
else:
return "{}:\n{}".format(filepath, _utils.indent(content))
def pprint_validation_error(error):
"""
A version of jsonschema's ValidationError __str__ method that doesn't
include the schema fragment that failed. This makes the error messages
much more succinct.
It also shows any subschemas of anyOf/allOf that failed, if any (what
jsonschema calls "context").
"""
essential_for_verbose = (
error.validator,
error.validator_value,
error.instance,
error.schema,
)
if any(m is _unset for m in essential_for_verbose):
return textwrap.fill(error.message)
instance = error.instance
for path in list(error.relative_path)[::-1]:
if isinstance(path, str):
instance = {path: instance}
else:
instance = [instance]
yaml_instance = ordered_yaml_dump(instance,
width=72,
default_flow_style=False)
parts = [
"```",
yaml_instance.rstrip(), "```", "",
textwrap.fill(error.message)
]
if error.context:
parts.extend(
textwrap.fill(
x.message, initial_indent=" ", subsequent_indent=" ")
for x in error.context)
description = error.schema.get("description")
if description:
parts.extend(
["", "Documentation for this node:",
_utils.indent(description)])
return "\n".join(parts)
def validator_form(validator, form, instance, schema, _from_items=False):
"""
- Silk form validators
- Every "form" property must be either absent or present
- If present, the form must have exactly that value
- Or, the "form" property must be a list, and have one of the values in the list
Or, if numeric, it must have one of the values between any two adjacent ascending list items
- The above applies a bit differently for shape, as it is already a list:
- Or, the "form" property must be a list of lists. The property must have the same length.
For each item in the lists-of-lists, the property must have one of the values, or
be between any two adjacent ascending list items.
- Validation on "strides" works through exact match of the strides value.
Note that "strides" is only present if "contiguous" is absent (and vice versa)
"""
def _allowed_value(schema_value, instance_value):
if isinstance(schema_value, (list, tuple)):
if instance_value in schema_value:
return True
for n in range(len(schema_value)-1):
left, right = schema_value[n:n+2]
if instance_value > left and instance_value < right:
return True
return False
else:
return schema_value == instance_value
if isinstance(instance, FormWrapper):
instance_storage, instance_form = instance._storage, instance._form
else:
#TODO:BAD
instance_storage, instance_form = get_form(instance)
form_str = indent(pprint.pformat(instance_form, width=72))
if instance_form is not None and "storage" in instance_form:
storage_form = instance_form["storage"]
for error in _validator_storage(storage_form, instance_storage, form_str):
yield error
if instance_storage is None:
instance_storage = storage_form.get("form")
if instance_storage is None:
return
if _from_items:
form_str += "\n(on items)"
binary_form_props = ("unsigned", "shape", "bytesize", "strides", "ndim")
for key, value in sorted(form.items(),key=lambda item:item[0]):
if key in binary_form_props and not instance_storage.endswith("binary"):
continue
missing_key = None
if key == "ndim":
if "shape" not in instance_form:
missing_key = "'shape' (needed by ndim)"
else:
if key not in instance_form:
missing_key = "'" + key + "'"
if missing_key:
msg = textwrap.dedent("""
No form property '%s'
On form:
%s
""".rstrip()
) % (missing_key, form_str)
yield ValidationError(msg)
if key != "ndim":
instance_value = instance_form[key]
ok = True
if key == "ndim":
instance_value = len(instance_form["shape"])
if instance_value != value:
ok = False
elif key == "strides":
if value != instance_value:
ok = False
elif key == "shape":
assert len(value) == len(instance_value) #TODO: check before for inconsistent shape/ndim requirement
for schema_dim, instance_dim in zip(value, instance_value):
if schema_dim == -1:
continue
if not _allowed_value(schema_dim, instance_dim):
ok = False
elif isinstance(instance_value, _types["number"]):
if isinstance(value, _types["number"]):
if not _allowed_value(value, instance_value):
ok = False
else:
if value != instance_value:
ok = False
if not ok:
msg = textwrap.dedent("""
Form property '%s' has value %r, not %r
On form:
%s
""".rstrip()
) % (key, instance_value, value, form_str)
yield ValidationError(msg)
if not _from_items and is_numpy_structure_schema(schema):
assert instance_storage is not None, schema
if "items" not in instance_form:
msg = textwrap.dedent("""
No form property 'items'
On form:
%s
""".rstrip()
) % (form_str,)
yield ValidationError(msg)
return
form_wrapper = FormWrapper(None, instance_form["items"], instance_storage)
items_form = schema.get("items", {}).get("form" , {})
for error in validator_form(
validator,
items_form, form_wrapper,
schema, _from_items=True
):
yield error
def validator_form(validator, form, instance, schema, _from_items=False):
"""
- Silk form validators
- Every "form" property must be either absent or present
- If present, the form must have exactly that value
- Or, the "form" property must be a list, and have one of the values in the list
Or, if numeric, it must have one of the values between any two adjacent ascending list items
- The above applies a bit differently for shape, as it is already a list:
- Or, the "form" property must be a list of lists. The property must have the same length.
For each item in the lists-of-lists, the property must have one of the values, or
be between any two adjacent ascending list items.
- Validation on "strides" works through exact match of the strides value.
Note that "strides" is only present if "contiguous" is absent (and vice versa)
"""
def _allowed_value(schema_value, instance_value):
if isinstance(schema_value, (list, tuple)):
if instance_value in schema_value:
return True
for n in range(len(schema_value)-1):
left, right = schema_value[n:n+2]
if instance_value > left and instance_value < right:
return True
return False
else:
return schema_value == instance_value
if isinstance(instance, FormWrapper):
instance_storage, instance_form = instance._storage, instance._form
else:
#TODO:BAD
instance_storage, instance_form = get_form(instance)
form_str = indent(pprint.pformat(instance_form, width=72))
if instance_form is not None and "storage" in instance_form:
storage_form = instance_form["storage"]
for error in _validator_storage(storage_form, instance_storage, form_str):
yield error
if instance_storage is None:
instance_storage = storage_form.get("form")
if instance_storage is None:
return
if _from_items:
form_str += "\n(on items)"
binary_form_props = ("unsigned", "shape", "bytesize", "strides", "ndim")
for key, value in sorted(form.items(),key=lambda item:item[0]):
if key in binary_form_props and not instance_storage.endswith("binary"):
continue
missing_key = None
if key == "ndim":
if "shape" not in instance_form:
missing_key = "'shape' (needed by ndim)"
else:
if key not in instance_form:
missing_key = "'" + key + "'"
if missing_key:
msg = textwrap.dedent("""
No form property '%s'
On form:
%s
""".rstrip()
) % (missing_key, form_str)
yield ValidationError(msg)
if key != "ndim":
instance_value = instance_form[key]
ok = True
if key == "ndim":
instance_value = len(instance_form["shape"])
if instance_value != value:
ok = False
elif key == "strides":
if value != instance_value:
ok = False
elif key == "shape":
assert len(value) == len(instance_value) #TODO: check before for inconsistent shape/ndim requirement
for schema_dim, instance_dim in zip(value, instance_value):
if schema_dim == -1:
continue
if not _allowed_value(schema_dim, instance_dim):
ok = False
elif isinstance(instance_value, _types["number"]):
if isinstance(value, _types["number"]):
if not _allowed_value(value, instance_value):
ok = False
else:
if value != instance_value:
ok = False
if not ok:
msg = textwrap.dedent("""
Form property '%s' has value %r, not %r
On form:
%s
""".rstrip()
) % (key, instance_value, value, form_str)
yield ValidationError(msg)
if not _from_items and is_numpy_structure_schema(schema):
assert instance_storage is not None, schema
if "items" not in instance_form:
msg = textwrap.dedent("""
No form property 'items'
On form:
%s
""".rstrip()
) % (form_str,)
yield ValidationError(msg)
return
form_wrapper = FormWrapper(None, instance_form["items"], instance_storage)
items_form = schema.get("items", {}).get("form" , {})
for error in validator_form(
validator,
items_form, form_wrapper,
schema, _from_items=True
):
yield error
def test_parser_schema_violation():
"""1507792"""
all_metrics = parser.parse_objects(ROOT / "data" / "schema-violation.yaml")
errors = list(all_metrics)
found_errors = set(
str(error).split("\n", 1)[1].strip().replace(" ", "")
for error in errors)
expected_errors = [
"""
```
gleantest:
test_event:
type: event
```
Missing required properties: bugs, data_reviews, description, expires,
notification_emails
Documentation for this node:
Describes a single metric.
See https://mozilla.github.io/glean_parser/metrics-yaml.html
""",
"""
```
gleantest.lifetime:
test_event_inv_lt:
lifetime: user2
```
'user2' is not one of ['ping', 'user', 'application']
Documentation for this node:
Definesthelifetimeofthe metric. It must be one of the following values:
- `ping` (default): The metric is reset each time it is sent in a ping.
- `user`: The metric contains a property that is part of the user's
profile and is never reset.
- `application`: The metric contains a property that is related to the
application, and is reset only at application restarts.
""",
"""
```
gleantest.foo:
a: b
```
'b' is not of type 'object'
Documentation for this node:
Describes a single metric.
See https://mozilla.github.io/glean_parser/metrics-yaml.html
""",
"""
```
gleantest.with.way.too.long.category.name
...
```
'gleantest.with.way.too.long.category.name' is not valid under any of
the given schemas
'gleantest.with.way.too.long.category.name' is too long
'gleantest.with.way.too.long.category.name' is not one of
['$schema']
""",
"""
```
gleantest.short.category:very_long_metric_name_this_is_too_long_s_well
```
'very_long_metric_name_this_is_too_long_s_well' is not valid under any
of the given schemas
'very_long_metric_name_this_is_too_long_s_well' is too long
""",
]
expected_errors = set(
_utils.indent(textwrap.dedent(x)).strip().replace(" ", "")
for x in expected_errors)
assert sorted(list(found_errors)) == sorted(list(expected_errors))
def test_parser_schema_violation():
"""1507792"""
all_metrics = parser.parse_objects(ROOT / "data" / "schema-violation.yaml")
errors = list(all_metrics)
found_errors = set(
re.sub(r"\s", "",
str(error).split("\n", 1)[1].strip()) for error in errors)
expected_errors = [
"""
```
gleantest:
test_event:
type: event
```
Missing required properties: bugs, data_reviews, description, expires,
notification_emails
Documentation for this node:
Describes a single metric.
See https://mozilla.github.io/glean_parser/metrics-yaml.html
""",
"""
```
gleantest.lifetime:
test_counter_inv_lt:
lifetime: user2
```
'user2' is not one of ['ping', 'user', 'application']
Documentation for this node:
Definesthelifetimeofthe metric. It must be one of the following values:
- `ping` (default): The metric is reset each time it is sent in a ping.
- `user`: The metric contains a property that is part of the user's
profile and is never reset.
- `application`: The metric contains a property that is related to the
application, and is reset only at application restarts.
""",
"""
```
gleantest.foo:
a: b
```
'b' is not of type 'object'
Documentation for this node:
Describes a single metric.
See https://mozilla.github.io/glean_parser/metrics-yaml.html
""",
"""
```
gleantest.with.way.too.long.category.name
...
```
'gleantest.with.way.too.long.category.name' is not valid under any of
the given schemas
'gleantest.with.way.too.long.category.name' is too long
'gleantest.with.way.too.long.category.name' is not one of
['$schema']
""",
"""
```
gleantest.short.category:very_long_metric_name_this_is_too_long_s_well
```
'very_long_metric_name_this_is_too_long_s_well' is not valid under any
of the given schemas
'very_long_metric_name_this_is_too_long_s_well' is too long
""",
]
# The validator reports a different error based on the python version, so
# we need to provide two copies for this to work.
if sys.version_info < (3, 7):
expected_errors.append("""
```
gleantest.event:
event_too_many_extras:
extra_keys:
key_1:
description: Sample extra key
key_2:
description: Sample extra key
key_3:
description: Sample extra key
key_4:
description: Sample extra key
key_5:
description: Sample extra key
key_6:
description: Sample extra key
key_7:
description: Sample extra key
key_8:
description: Sample extra key
key_9:
description: Sample extra key
key_10:
description: Sample extra key
key_11:
description: Sample extra key
```
OrderedDict([('key_1', OrderedDict([('description', 'Sample extra
key')])), ('key_2', OrderedDict([('description', 'Sample extra
key')])), ('key_3', OrderedDict([('description', 'Sample extra
key')])), ('key_4', OrderedDict([('description', 'Sample extra
key')])), ('key_5', OrderedDict([('description', 'Sample extra
key')])), ('key_6', OrderedDict([('description', 'Sample extra
key')])), ('key_7', OrderedDict([('description', 'Sample extra
key')])), ('key_8', OrderedDict([('description', 'Sample extra
key')])), ('key_9', OrderedDict([('description', 'Sample extra
key')])), ('key_10', OrderedDict([('description', 'Sample extra
key')])), ('key_11', OrderedDict([('description', 'Sample extra
key')]))]) has too many properties
Documentation for this node:
The acceptable keys on the "extra" object sent with events. This is an
object mapping the key to an object containing metadata about the key.
A maximum of 10 extra keys is allowed.
This metadata object has the following keys:
- `description`: **Required.** A description of the key.
Valid when `type`_ is `event`.
""")
else:
expected_errors.append("""
```
gleantest.event:
event_too_many_extras:
extra_keys:
key_1:
description: Sample extra key
key_10:
description: Sample extra key
key_11:
description: Sample extra key
key_2:
description: Sample extra key
key_3:
description: Sample extra key
key_4:
description: Sample extra key
key_5:
description: Sample extra key
key_6:
description: Sample extra key
key_7:
description: Sample extra key
key_8:
description: Sample extra key
key_9:
description: Sample extra key
```
{'key_1': {'description': 'Sample extra key'}, 'key_2':
{'description': 'Sample extra key'}, 'key_3': {'description': 'Sample
extra key'}, 'key_4': {'description': 'Sample extra key'}, 'key_5':
{'description': 'Sample extra key'}, 'key_6': {'description': 'Sample
extra key'}, 'key_7': {'description': 'Sample extra key'}, 'key_8':
{'description': 'Sample extra key'}, 'key_9': {'description': 'Sample
extra key'}, 'key_10': {'description': 'Sample extra key'}, 'key_11':
{'description': 'Sample extra key'}} has too many properties
Documentation for this node:
The acceptable keys on the "extra" object sent with events. This is an
object mapping the key to an object containing metadata about the key.
A maximum of 10 extra keys is allowed.
This metadata object has the following keys:
- `description`: **Required.** A description of the key.
Valid when `type`_ is `event`.
""")
expected_errors = set(
re.sub(r"\s", "",
_utils.indent(textwrap.dedent(x)).strip())
for x in expected_errors)
assert sorted(list(found_errors)) == sorted(list(expected_errors))
