def matches_schema(self, schema):
'''
Check if this JsonDict matches the provided schema
Arguments:
schema: Python dict schema or `JsonSchema` object.
Returns:
A 2-element tuple (True/False, List of error strings in matching)
'''
if isinstance(schema, JsonSchema):
schema = schema.as_dict()
from jsonschema import Draft7Validator
validator = Draft7Validator(schema)
errors = []
for error in sorted(validator.iter_errors(self.store), key=str):
errors.append("Key >>{}<<: {}".format(
".".join([str(i) for i in error.path]), error.message))
if errors:
return False, errors
else:
return True, None
def validate(cls, to_validate, model, field=None):
cls.FILE_NAME = model.lower() + '_schema.json'
try:
schema = cls.get_schema_for_field(field, model)
validator = Draft7Validator(schema)
found_invalid_field = False
errors = []
for error in sorted(validator.iter_errors(to_validate), key=str):
found_invalid_field = True
errors.append(error.message)
if found_invalid_field:
raise ValidationError(f"Instance {to_validate} is not proper. Errors: {errors}")
except ValidationError as ve:
raise ve
except SchemaError as se:
raise se
except BaseException as e:
# one of many errors specified in helper methods
raise e
def put(self, person_id):
person = Person.find_by_id(person_id)
print(person)
if not person:
response = {}
return make_response(response=response, code=404)
# print(person_id)
person_json = request.json
error = best_match(Draft7Validator(schema).iter_errors(person_json))
if error:
response = {'error': error.message}
return make_response(response=response, code=400)
# print(person_id)
if person_json.get('birthdate'):
try:
birthdate = datetime.datetime.strptime(
person_json.get('birthdate'), "%Y-%m-%d")
person_json['birthdate'] = birthdate
except:
response = {'error': "INVALID_BIRTHDATE"}
return make_response(response=response, code=400)
# print(person_id)
if person_json.get("children"):
children = Person.query.filter(
Person.person_id.in_(person_json.get("children"))).all()
person_json['children'] = children
if person_json.get("parents"):
parents = Person.query.filter(
Person.person_id.in_(person_json.get("parents"))).all()
person_json['parents'] = parents
print(person_id)
person.patch(person_json)
print(person_id)
person.save_to_db()
print(person_id)
response = {"result": "ok", "data": [person_obj(person)]}
print(response)
return make_response(response=response, code=200)
def test_valid_data_all_fields_specified(set_test_variables):
"""
:test_id: ASL-SCHEMA-TEST-001
:req_id: D2N-FUN-REQ-003
:description: This test verifies that a valid structure following exactly the definition set in the json schema gets
validated correctly.
:inputs: A dictionary object following exactly the structure defined in the asl schema, with all fields mandatory and
optional specified.
:criteria: This test is considered passed if the validation method returns True.
"""
asl_schema = set_test_variables["asl_schema"]
asl_valid = set_test_variables["asl_valid_full"]
with open(asl_schema, "r", encoding="utf-8") as fp:
schema = json.load(fp)
with open(asl_valid, "r", encoding="utf-8") as fp:
data = json.load(fp)
validator = Draft7Validator(schema=schema)
validator.check_schema(schema)
valid = validator.is_valid(instance=data)
assert valid is True
def test_valid_data_conditional_type_array(set_test_variables):
"""
:test_id: ASL-SCHEMA-TEST-003
:req_id: D2N-FUN-REQ-003
:description: This test verifies that a valid structure following the definition set in the json schema gets
validated correctly.
:inputs: A dictionary object following largely the structure defined in the asl schema. The twist here is the
LabelingDuration field is an array instead of a single int, as this is allowed in the schema this should not affect
the validation.
:criteria: This test is considered passed if the validation method returns True.
"""
asl_schema = set_test_variables["asl_schema"]
asl_valid = set_test_variables["asl_valid_labeling_duration_array"]
with open(asl_schema, "r", encoding="utf-8") as fp:
schema = json.load(fp)
with open(asl_valid, "r", encoding="utf-8") as fp:
data = json.load(fp)
validator = Draft7Validator(schema=schema)
valid = validator.is_valid(instance=data)
assert valid is True
def is_valid_xsoar_config_file(self):
"""Runs the schema validation on the configuration data, with the schema data.
Returns:
bool. Whether the configuration file's schema is valid or not.
"""
validator = Draft7Validator(schema=self.schema_json)
errors = validator.iter_errors(self.configuration_json)
errors_table, errors_found = self.create_schema_validation_results_table(
errors)
if errors_found:
error_message, error_code = Errors.xsoar_config_file_malformed(
self.configuration_file_path,
self.schema_path,
errors_table,
)
if self.handle_error(error_message,
error_code,
file_path=self.configuration_file_path):
self._is_valid = False
return self._is_valid
def checks(validator: jsonschema.Draft7Validator, checks: Dict, instance: Any,
schema: Dict) -> Iterator[jsonschema.ValidationError]:
"""
checks is a simple extension that returns a specific error if a subschema fails to match.
The keys of the "checks" dictionary are the user-facing messages, and the values are the
subschemas that must match.
Example:
"checks": {
"you must specify an entrypoint that references the trial class":{
... (schema which allows Native API or requires that entrypoint is set) ...
},
"you requested a bayesian search but hyperband is way better": {
... (schema which checks if you try searcher.name=baysian) ...
}
}
"""
for msg, subschema in schema["checks"].items():
errors = list(validator.descend(instance, schema=subschema))
if errors:
yield jsonschema.ValidationError(msg)
def importDialog(self):
def updateAfterImport(new_injection):
self.setupUi(deserialize_injection(new_injection))
dirpath = Path(f'{os.path.dirname(os.path.realpath(__file__))}', '../../json_schemas/inj.json')
schema_path = dirpath.absolute().as_uri()
with dirpath.open('r') as jsonfile:
schema = json.load(jsonfile)
resolver = RefResolver(
schema_path,
schema,
)
validator = Draft7Validator(schema, resolver=resolver, format_checker=None)
dial = SRSettingUpdate(mw)
dial.setupUi(
json.dumps(serialize_injection(self.exportData()), sort_keys=True, indent=4),
validator,
updateAfterImport,
)
dial.exec_()
def check2(data_fn, schema_url):
print(f'Checking {data_fn} against {schema_url}')
with open(data_fn) as data_file, urllib.request.urlopen(schema_url) as schema_file:
# with open('main.json') as schema_file, open('json/address1.json') as json_file:
schema = json.load(schema_file)
data = json.load(data_file)
# print(json.dumps(schema))
try:
validator = Draft7Validator(schema)
# print(*validator.iter_errors(data))
print('\n'.join(str(x) for x in sorted(validator.iter_errors(data), key=str)))
except ValidationError as e:
print(f'message: {e.message}')
print(f'validator: {e.validator}')
print(f'validator_value: {e.validator_value}')
print(f'relative_schema_path: {e.relative_schema_path}')
print(f'absolute_schema_path: {e.absolute_schema_path}')
print(f'relative_path: {e.relative_path}')
print(f'absolute_path: {e.absolute_path}')
print(f'instance: {e.instance}')
print(f'context: {e.context}')
print(f'cause: {e.cause}')
print(f'parent: {e.parent}')
def validate_schema(schema, json):
schema = get_json(schema)
json = get_json(json)
v = Draft7Validator(schema, format_checker=draft7_format_checker)
errors = sorted(v.iter_errors(json), key=lambda e: e.path)
print(json['id'], ": Number of validation errors = ", len(errors))
data = []
for error in errors:
err = {}
if len(list(error.path)):
err['attribute'] = list(error.path)[0]
print(err['attribute'], error.message, sep=": ")
err['message'] = ": ".join([err['attribute'], error.message])
for suberror in sorted(error.context, key=lambda e: e.schema_path):
print(" ", list(suberror.schema_path)[1], ": ", suberror.message)
err['suberrors'] = " " + list(suberror.schema_path)[1] + ": " + suberror.message
else:
print(error.message)
err['attribute'] = re.findall(r"(.*?)'", error.message)[1]
err['message'] = error.message
data.append(err)
return data
def __init__(self, source):
if source.startswith('{'):
self.data = json.loads(source)
else:
filename = os.path.expandvars(util.ospath(source))
din = open(filename).read()
self.data = json.loads(jsmin(din))
try:
from jsonschema import Draft7Validator
# Validate configuration against schema
schemaPath = os.path.dirname(__file__) + '/schema.json'
schema = json.load(open(schemaPath))
v = Draft7Validator(schema)
errors = sorted(v.iter_errors(self.data), key=lambda e: e.path)
if errors != []:
for e in errors:
sys.stderr.write("%s @ %s\n" % (e.message, e.path))
sys.exit(1)
except ImportError as err:
sys.stderr.write(
"\n** WARNING! %s: Cannot validate config '%s', please run `make python-requirements` **\n\n"
% (str(err), filename))
def run(self, tmp=None, task_vars=None):
del tmp # tmp no longer has any effect
self._supports_check_mode = True
result = super(ActionModule, self).run(task_vars=task_vars)
result["changed"] = False
result["validated"] = False
schema = self._task.args.get("schema")
instance = self._task.args.get("instance")
fatal = self._task.args.get("fatal", False)
# input validation
if schema is None:
result["failed"] = True
result["msg"] = "missing required argument: schema"
return result
if instance is None:
result["failed"] = True
result["msg"] = "missing required argument: instance"
return result
try:
Draft7Validator(schema).validate(instance)
result["validated"] = True
return result
except ValidationError as e:
result["msg"] = '{0}: {1}'.format(e.path, e.message)
if fatal:
result["failed"] = True
return result
except Exception as e:
result["msg"] = e
if fatal:
result["failed"] = True
return result
def index_schema_check(self, index_content):
def get_schema_json_obj(path):
return self.get_json_obj_from_file(
os.path.join("tools/index_schema_check", path))
index_all_schema = get_schema_json_obj("index_all_schema.json")
rtt_source_schema = get_schema_json_obj("rtt_source_schema.json")
rtt_source_releases_schema = get_schema_json_obj(
"rtt_source_releases_schema.json")
csp_schema = get_schema_json_obj("csp_schema.json")
csp_dvendor_schema = get_schema_json_obj("csp_dvendor_schema.json")
csp_dvendor_package_schema = get_schema_json_obj(
"csp_dvendor_package_schema.json")
csp_dvendor_package_releases_schema = get_schema_json_obj(
"csp_dvendor_package_releases_schema.json")
schema_store = {
index_all_schema['$id']:
index_all_schema,
rtt_source_releases_schema['$id']:
rtt_source_releases_schema,
rtt_source_schema['$id']:
rtt_source_schema,
csp_schema['$id']:
csp_schema,
csp_dvendor_schema['$id']:
csp_dvendor_schema,
csp_dvendor_package_schema['$id']:
csp_dvendor_package_schema,
csp_dvendor_package_releases_schema['$id']:
csp_dvendor_package_releases_schema
}
resolver = RefResolver.from_schema(rtt_source_releases_schema,
store=schema_store)
validator = Draft7Validator(index_all_schema, resolver=resolver)
validator.validate(index_content)
def validate_schema(schema: Dict, recipe: Dict):
errors = []
try:
with jsonsempai.imports():
from ocli.aikp.cluster import recipe_schema
deep_update(schema, recipe_schema.properties.envelope)
v = Draft7Validator(schema, format_checker=draft7_format_checker)
_errors = sorted(v.iter_errors(recipe), key=lambda e: e.path)
errors.extend([
f'Recipe schema "{".".join(e.absolute_path)}" invalid : {e.message}'
if e.absolute_path else f"Recipe schema {e.message}"
for e in _errors
])
try:
_t = numpy.load(os.path.join(recipe['PREDICTOR_DIR'], 'tnorm.npy'))
_mlc = max(recipe['learn_channels'])
if _t.shape[0] < _mlc:
errors.append(
f"Predictor is invalid: learning chanel {_mlc} could not be used with tnorm.npy shape [0..{_t.shape[0] - 1}]"
)
with open(os.path.join(recipe['PREDICTOR_DIR'], 'gm.pkl'),
'rb') as _f:
_t = pickle.load(_f)
if recipe['num_clusters'] != _t.n_components:
errors.append(
f"Predictor is invalid: gm.pkl has {_t.n_components} clusters, \
recipe has {recipe['num_clusters']} clusters ")
except Exception as e:
errors.append(
f"Predictor is invalid: Could not validate tnorm.npy: {e}")
except Exception as e:
# log.exception(e)
errors.append(f'Could not perform validation: {e}')
return errors
def __validate_schemas(self):
"""
Validates the given plugin schemas are valid.
Raises:
UserError: If the schemas are not valid.
"""
plugin_meta_schema = {}
try:
with open(self.__plugin_meta_schema, "r") as f:
try:
plugin_meta_schema = json.load(f)
except ValueError as err:
raise exceptions.UserError(
"Failed to load schemas because '{}' is not a "
"valid json file. Error: {}".format(
self.__plugin_meta_schema, err))
except (IOError, OSError) as err:
raise exceptions.UserError(
"Unable to read plugin schema file '{}'"
"\nError code: {}. Error message: {}".format(
self.__plugin_meta_schema, err.errno,
os.strerror(err.errno)))
# Validate the plugin schema against the meta schema
v = Draft7Validator(plugin_meta_schema)
#
# This will do lazy validation so that we can consolidate all the
# validation errors and report everything wrong with the schema.
#
validation_errors = sorted(v.iter_errors(self.__plugin_schemas),
key=lambda e: e.path)
if validation_errors:
raise exceptions.SchemaValidationError(self.__schema_file,
validation_errors)
def test_invalid_data_multiple_errors(set_test_variables):
"""
:test_id: ASL-SCHEMA-TEST-010
:req_id: D2N-FUN-REQ-003
:description: This test verifies that an invalid structure does not get validated against our schema.
:inputs: A dictionary object following largely the structure defined in the asl schema. The object gathers all the
mistakes tested previously.
:criteria: This test is considered passed if the validation method returns False. and the number of errors is
correctly estimated (6)
"""
asl_schema = set_test_variables["asl_schema"]
asl_valid = set_test_variables["asl_valid_full"]
with open(asl_schema, "r", encoding="utf-8") as fp:
schema = json.load(fp)
with open(asl_valid, "r", encoding="utf-8") as fp:
data = json.load(fp)
# Mess up data so that the validation fails
# Let's take all examples from the tests above
data["LocationOfLabelingPlane"] = [[12, 11], [10, 12], [7, 6, 5]]
data["LabelingOrientation"] = [[12, 11], [10, 12], [7, 6], [2, 2]]
data["ASLContext"] = "And still the type is right"
data["FlipAngles"] = [0, "yeah messing around", 14, 60]
data["BackgroundSuppressionPulseTime"] = 16
data.pop("EchoTime", 0)
validator = Draft7Validator(schema=schema)
valid = validator.is_valid(instance=data)
assert valid is False
error_count = 0
for error in validator.iter_errors(instance=data):
error_count += 1
assert error_count == 6
def eventually(
validator: jsonschema.Draft7Validator,
eventually: Any,
instance: List,
schema: Dict,
) -> Iterator[jsonschema.ValidationError]:
"""
eventually allows for two-step validation, by only enforcing the specified subschemas
during the completeness validation phase. This is a requirement specific to Determined.
One use case is when it is necessary to enforce a `oneOf` on two fields that are
`eventuallyRequired`. If the `oneOf` is evaluated during the sanity validation phase, it will
always fail, if for example, the user is using cluster default values, but if validation
for this subschema is held off until completeness validation, it will validate correctly.
Example: eventually require one of connection string and account url to be specified:
"eventually": {
"checks": {
"Exactly one of connection_string or account_url must be set": {
"oneOf": [
{
"eventuallyRequired": [
"connection_string"
]
},
{
"eventuallyRequired": [
"account_url"
]
}
]
}
}
}
"""
yield from validator.descend(instance, schema=eventually, schema_path="eventually")
def oauth_provider_config(o):
"""
:param o: either:
- A JSON object containing the full OAuth provider configuration
- A JSON or YAML file containing a dictionary that will be merged
"""
try:
cfg = json.loads(o)
except ValueError:
cfgfile = o
with open(cfgfile) as f:
if cfgfile.endswith('.yml') or cfgfile.endswith('.yaml'):
cfg = yaml.load(f, Loader=yaml.FullLoader)
else:
cfg = json.load(f, Loader=yaml.FullLoader)
schemastr = get_data('omero_oauth', 'schema/provider-schema.yaml')
schema = yaml.load(schemastr, Loader=yaml.FullLoader)
v = Draft7Validator(schema)
if not v.is_valid(cfg):
errs = '\n\n** '.join(
['Invalid provider configuration'] +
['\n\n'.join(str(e) for e in v.iter_errors(cfg))])
raise ValueError(errs)
return cfg
def mosdex_open_and_test(self, do_print=False):
problem_file = self.problem_file
schema_file = self.schema_file
with open(problem_file) as f:
problem_json = json.load(f)
with open(schema_file) as f:
schema_json = json.load(f)
validator = Draft7Validator(schema_json)
valid = validator.is_valid(problem_json)
if do_print:
if not valid:
print("NO: Mosdex problem {} is not a valid instance of Mosdex schema {}".
format(problem_file, schema_file))
pp = pprint.PrettyPrinter(indent=4)
for error in sorted(validator.iter_errors(problem_json), key=str):
print()
pp.pprint(error.message)
else:
print("YES: Mosdex problem {} is a valid instance of Mosdex schema {}".
format(problem_file, schema_file))
print()
self.json = problem_json
self.is_valid = valid
def create_ref_validator(self, name: str,
category: str) -> Draft7Validator:
"""Use the :attr:`base_model` to resolve the component category and name and create a
:class:`~jsonschema.Draft7Validator`
:param name: The name of the model
:param category: The category under 'components' to look in
:return: The validator object
The resulting validator can be passed into decorators like :meth:`param` or :meth:`response`
and will be used to create the schema in the openapi json output or passed into :meth:`expect`
to use it for actually validating a request against the schema. It will be output as a '$ref'
object in the resulting openapi json output.
.. seealso:: The :meth:`expect` decorator
.. todo:: Ensure that models with the same name in different categories don't conflict
"""
validator = Draft7Validator(
{'$ref': f'#/components/{category}/{name}'},
resolver=self._ref_resolver,
format_checker=FormatChecker())
self._validators[name] = validator
return validator
def check_if_valid(self):
"""Check if the schema definition is valid against JsonSchema draft7.
Returns:
List[ValidationResult]: A list of validation result objects.
"""
validator = Draft7Validator(v7schema)
results = []
has_error = False
for err in validator.iter_errors(self.data):
has_error = True
results.append(
ValidationResult(
schema_id=self.id,
result=RESULT_FAIL,
message=err.message,
absolute_path=list(err.absolute_path),
instance_type="SCHEMA",
instance_name=self.id,
instance_location="",
))
if not has_error:
results.append(
ValidationResult(
schema_id=self.id,
result=RESULT_PASS,
instance_type="SCHEMA",
instance_name=self.id,
instance_location="",
))
return results
def validate_object(obj, schema_name): # type: (dict, str) -> List[str]
schema = os.path.join(SCHEMA_DIR, "{}.json".format(schema_name))
if not os.path.exists(schema):
raise ValueError("Schema {} does not exist.".format(schema_name))
with open(schema, encoding="utf-8") as f:
schema = json.loads(f.read())
validator = Draft7Validator(schema)
validation_errors = sorted(validator.iter_errors(obj),
key=lambda e: e.path)
errors = []
for error in validation_errors:
message = error.message
if error.path:
message = "[{}] {}".format(
".".join(str(x) for x in error.absolute_path), message)
errors.append(message)
return errors
def disallowProperties(
validator: jsonschema.Draft7Validator, disallowed: Dict, instance: Any, schema: Dict
) -> Iterator[jsonschema.ValidationError]:
"""
disallowProperties is for restricting which properties are allowed in an object with
per-property, such as when we allow a k8s pod spec with some fields disallowed.
Example: The "pod_spec" property of the environment config:
"pod_spec": {
"type": "object",
"disallowProperties": {
"name": "pod Name is not a configurable option",
"name_space": "pod NameSpace is not a configurable option"
}
}
"""
if not validator.is_type(instance, "object"):
return
for prop in instance:
if prop in disallowed:
msg = disallowed[prop]
yield jsonschema.ValidationError(msg)
def optionalRef(
validator: jsonschema.Draft7Validator, optionalRef: Dict, instance: Any, schema: Dict
) -> Iterator[jsonschema.ValidationError]:
"""
optionalRef behaves like $ref, except that it also allows the value to be null.
This is logically equivalent to an anyOf with a {"type": "null"} element, but it has better
error messages.
Example: The "internal" property of the experiment config may be a literal null:
"internal": {
"type": [
"object",
"null"
],
"optionalRef": "http://determined.ai/schemas/expconf/v0/internal.json",
"default": null
}
"""
if instance is None:
return
yield from validator.descend(instance, schema={"$ref": optionalRef}, schema_path="optionalRef")
def validate(schema, jsonData):
try:
validator = Draft7Validator(schema)
results = validator.iter_errors(jsonData)
except SchemaError as err:
print('Problem with the supplied schema:\n')
print(err)
errors = sorted(results, key=lambda e: e.path)
if errors:
print('Validation Failed')
errorCount = 1
for err in errors:
print('Error {} of {}.\n Message: {}'.format(
errorCount, len(errors), err.message))
if 'title' in err.schema:
print(' Test message: {}'.format(err.schema['title']))
if 'description' in err.schema:
print(' Test description: {}'.format(
err.schema['description']))
print('\n Path for error: {}'.format(
printPath(err.path, err.message)))
context = err.instance
if isinstance(context, dict):
for key in context:
if isinstance(context[key], list):
context[key] = '[ ... ]'
elif isinstance(context[key], dict):
context[key] = '{ ... }'
print(json.dumps(err.instance, indent=4))
errorCount += 1
return False
else:
return True
def post(self):
person_json = request.json
error = best_match(Draft7Validator(schema).iter_errors(person_json))
if error:
response = {'error': error.message}
return make_response(response=response, code=400)
try:
birthdate = datetime.datetime.strptime(
person_json.get('birthdate'), "%Y-%m-%d")
except:
response = {'error': "INVALID_BIRTHDATE"}
return make_response(response=response, code=400)
if person_json.get("children"):
children = Person.query.filter(
Person.person_id.in_(person_json.get("children"))).all()
person_json['children'] = children
if person_json.get("parents"):
parents = Person.query.filter(
Person.person_id.in_(person_json.get("parents"))).all()
person_json['parents'] = parents
person = Person(person_id=str(uuid.uuid4()),
firstname=person_json.get('firstname'),
lastname=person_json.get('lastname'),
birthdate=birthdate,
email=person_json.get('email'),
street_address=person_json.get('street_address'),
city=person_json.get('city'),
state=person_json.get('state'),
zip=person_json.get('zip'),
phone=person_json.get('phone'),
children=person_json.get('children', []),
parents=person_json.get('parents', []))
person.save_to_db()
response = {"result": "ok", "data": [person_obj(person)]}
return make_response(response=response, code=201)
class JsonInlineLoader(Loader):
_validator = Draft7Validator(
loadSchema(os.path.join(curDir, "json_inline.yaml")))
@property
def data(self):
return self._data
def __init__(self, jsonString):
inlineSchema = json.loads(jsonString)
if not JsonInlineLoader._validator.is_valid(inlineSchema):
error = "Invalid inline schema: jsonString[:128]={}".format(
jsonString[:128])
for err in JsonInlineLoader._validator.iter_errors(inlineSchema):
print("Error: {}".format(err.message))
print("Context: {}".format(err.absolute_schema_path))
raise SchemaException(error)
self._data = inlineSchema["data"]
super().__init__(inlineSchema["filePath"])
def validate(self, data):
"""
Validate that the Serializer contains valid data.
Set the DebRepository based on the RepositoryVersion if only the latter is provided.
Set the RepositoryVersion based on the DebRepository if only the latter is provided.
Convert the human-friendly names of the content types into what Pulp needs to query on.
"""
super().validate(data)
if hasattr(self, "initial_data"):
validate_unknown_fields(self.initial_data, self.fields)
if "config" in data:
validator = Draft7Validator(COPY_CONFIG_SCHEMA)
err = []
for error in sorted(validator.iter_errors(data["config"]),
key=str):
err.append(error.message)
if err:
raise serializers.ValidationError(
_("Provided copy criteria is invalid:'{}'".format(err)))
return data
def check_schema(data):
try:
DraftValidator.check_schema(data)
except SchemaError as e:
message = e.path.pop() + e.message
raise ValidationError(message)
def validate(json_to_validate, schema):
validator = Draft7Validator(schema, format_checker=format_checker)
errors = list(validator.iter_errors(json_to_validate))
if errors.__len__() > 0:
raise ValidationError(build_error_message(errors))
return json_to_validate