def validate_node(self, json_node):
if not jsonschema.Draft3Validator(self.schema).is_valid(json_node):
exception_data = ['Validation error in: {}'.format(json_node)]
for error in sorted(jsonschema.Draft3Validator(
self.schema).iter_errors(json_node),
key=str):
exception_data.append(error.message)
raise ValueError(exception_data)
def on_post(self, req, resp):
resp.content_type = 'application/json'
response = ReviewResponse()
try:
body = json.loads(req.stream.read())
# Try to get apiVersion and uid even if request doesn't comply to
# schema
response.api_version = body.get('apiVersion')
review_request = body.get('request', {})
response.uid = review_request.get('uid')
# Validate admission request against schema
jsonschema.Draft3Validator(self.schema_dict).validate(body)
except Exception as e:
response.set_error(
400, f'Exception parsing the body of request: {e}.')
else:
if review_request.get('kind', {}).get(
'kind') == 'OpenStackDeployment':
features = review_request.get('object', {}).get(
'spec', {}).get('features', {})
for service in features.get('services', []):
if service in _VALIDATORS:
# Validate all the enabled services, if there is a
# corresponding validator
_VALIDATORS[service].validate(
review_request, response)
if not response.is_allowed:
break
resp.body = json.dumps(response.to_json())
def validate_jsonschema(schema, response):
try:
jsonschema.Draft3Validator(json.loads(schema)).validate(
json.loads(response.text))
except jsonschema.ValidationError as e:
error = e.message + " " + str(e.absolute_schema_path)
pytest.fail(error)
def payload_validator(self, schema, data):
try:
if type(data) is not dict:
data = json.loads(data)
if type(schema) is not dict:
schema = json.loads(schema)
v = jsonschema.Draft3Validator(schema)
validation_error = []
for error in sorted(v.iter_errors(data), key=str):
property = str(error.path).replace("deque", "", 1)
if "required" not in error.message:
message = str("value {} for property {}".format(
error.message, property))
print(message)
validation_error.append(message)
else:
print(error.message)
validation_error.append(error.message)
# print(validation_error)
except jsonschema.ValidationError as e:
print(e.message)
return validation_error
def validate_json_schema(data, schema):
"""
validate json schema using jsonschema Draft3Validator
"""
logger._info("Validating JSON schema")
validate_ok = True
if not isinstance(data, dict):
data = json.loads(data)
if not isinstance(schema, dict):
schema = json.loads(schema)
try:
jsonschema.Draft3Validator.check_schema(schema)
v = jsonschema.Draft3Validator(schema)
for error in sorted(v.iter_errors(data), key=str):
logger._warn("Invalid JSON data: For key '%s', %s" %
(error.path.pop(), error.message))
validate_ok = False
except jsonschema.ValidationError as e:
logger._warn("Invalid data: %s" % e.message)
validate_ok = False
except jsonschema.SchemaError as e:
logger._warn("Invalid schema: %s" % e.message)
validate_ok = False
return validate_ok
def _validate_jsonschema(self):
error_messages = None
draft = self._get_sub_plugin_options("draft")
error_messages = []
for criteria in self._criteria:
if draft == "draft3":
validator = jsonschema.Draft3Validator(
criteria, format_checker=jsonschema.draft3_format_checker)
elif draft == "draft4":
validator = jsonschema.Draft4Validator(
criteria, format_checker=jsonschema.draft4_format_checker)
elif draft == "draft6":
validator = jsonschema.Draft6Validator(
criteria, format_checker=jsonschema.draft6_format_checker)
else:
validator = jsonschema.Draft7Validator(
criteria, format_checker=jsonschema.draft7_format_checker)
validation_errors = sorted(validator.iter_errors(self._data),
key=lambda e: e.path)
if validation_errors:
if "errors" not in self._result:
self._result["errors"] = []
for validation_error in validation_errors:
if isinstance(validation_error,
jsonschema.ValidationError):
error = {
"message":
validation_error.message,
"data_path":
to_path(validation_error.absolute_path),
"json_path":
json_path(validation_error.absolute_path),
"schema_path":
to_path(validation_error.relative_schema_path),
"relative_schema":
validation_error.schema,
"expected":
validation_error.validator_value,
"validator":
validation_error.validator,
"found":
validation_error.instance,
}
self._result["errors"].append(error)
error_message = "At '{schema_path}' {message}. ".format(
schema_path=error["schema_path"],
message=error["message"],
)
error_messages.append(error_message)
if error_messages:
if "msg" not in self._result:
self._result["msg"] = "\n".join(error_messages)
else:
self._result["msg"] += "\n".join(error_messages)
def testCase700(self):
"""Validate by MODE_SCHEMA_DRAFT3 API: jsonschema.MODE_SCHEMA_DRAFT3Validator
"""
global sval
global jval
# validate data
jsonschema.Draft3Validator(jval, sval)
def validJson(rowData):
jsonRec=json.loads(rowData)
jsonSc=json.loads('''{ "title": "usecase", "type": "object", "properties": { "TID" : { "type": [ "string" ] },"CUR" : { "type": [ "string" ] , "maxLength" : 3 },"DAT1" : { "type": [ "number","null"] , "maximum" : 99999999999 },"DAT2" : { "type": [ "number" ,"null"] , "maximum" : 112 } }, "required": [ "TID", "CUR", "DAT1", "DAT2" ] }''')
val = jsonschema.Draft3Validator(jsonSc).is_valid(jsonRec)
print "is records valid ::"+str( val)
va=jsonschema.validate(jsonRec, jsonSc)
print "records validated "+str(va)
return json.dumps(jsonRec)
def schema_check(self, request_jsons, schema):
try:
schema_check = jsonschema.Draft3Validator(schema).validate(
request_jsons)
except jsonschema.ValidationError as e:
schema_check = str(e)
return schema_check
def main():
"""Валидируем все файлы и собираем результаты"""
result = list()
for jsonfile in jsonfiles:
one = dict()
with open(jsonfile, 'r') as f:
data = json.load(f)
try:
schema = choice_schema(data['data'].keys())
v = jsonschema.Draft3Validator(schema)
errors = v.iter_errors(data['data'])
for error in errors:
message = convert_error(error)
one = {
'File': jsonfile,
'Result': 'error',
'Field': error.path[0],
'Message': message
}
result.append(one)
if not one:
one = {
'File': jsonfile,
'Result': 'ok',
'Field': 'ok',
'Message': "ok"
}
result.append(one)
except (KeyError, TypeError) as e:
one = {
'File': jsonfile,
'Result': 'no data',
'Field': 'all',
'Message': 'В файле нет данных'
}
result.append(one)
except AttributeError as e:
one = {
'File': jsonfile,
'Result': 'no data',
'Field': 'data',
'Message': 'В файле недостаточно данных'
}
result.append(one)
"""Преобразуем результаты валидации в датафрейм"""
df = pd.DataFrame.from_records(result)
print(df)
"""Раскрашиваем датафрейм и конвертим в html"""
html = df.style.apply(
highlight, axis=1
).set_caption('<h1>Валидация json</h1>').set_table_attributes(
'border="0" class="table table-striped table-bordered table-condensed"'
).render()
with open('test.html', 'w', encoding="utf-8") as f:
f.write(html)
def message_is_valid(message, schema):
"""
EXPERIMENTAL
Returns true if the message is valid under the given schema
:param message:
:param schema:
:return:
"""
return jsonschema.Draft3Validator(schema).is_valid(message)
def valid(msg, schema):
try:
log.debug('Validating message %s with schema %s', c.jsonify(msg),
c.jsonify(schema))
jsonschema.Draft3Validator(schema).validate(msg)
except jsonschema.ValidationError as e:
log.warning("Message is not valid %s", e.message)
return False
return True
def validate_config(config, schema):
print("Validating configuration")
v = jsonschema.Draft3Validator(schema)
if not v.is_valid(config):
print("Error(s) in configuration:")
for error in sorted(v.iter_errors(config), key=str):
print(error)
else:
print("config ok")
def set_repo_path(path):
global repo_path
global schema_validator
repo_path = path
# Create schema validator
with open(repo_path + os.sep + 'test_data.schema.json') as f:
schema = json.load(f)
schema_validator = jsonschema.Draft3Validator(schema)
def test_analyze(self, mock_requests):
domain = "http://www.mock{0}.com".format(uuid.uuid4())
sitemap = domain + "/sitemap.xml"
report = json.loads(cmd.analyze(domain, sitemap))
jsonschema.validate(report, self.report_schema)
self.assertTrue(jsonschema.Draft3Validator(
self.report_schema).is_valid(report))
def validate_config(config):
schema = json.load(file(os.path.join(sys.path[0], "config_schema.json")))
log.info("Validating configuration")
v = jsonschema.Draft3Validator(schema)
if not v.is_valid(config):
log.error("Error(s) in configuration:")
for error in sorted(v.iter_errors(config), key=str):
log.error(error)
return False
log.info("Config ok")
return True
def validate(capsule):
"""Validates an encapsulated event.
:raises :exc:`jsonschema.ValidationError`: If event is invalid.
"""
try:
scid = capsule['schema'], capsule['revision']
except KeyError as ex:
# If `schema` or `revision` keys are missing, a KeyError
# exception will be raised. We re-raise it as a
# :exc:`ValidationError` to provide a simpler API for callers.
raise jsonschema.ValidationError('Missing key: %s' % ex)
schema = get_schema(scid, encapsulate=True)
jsonschema.Draft3Validator(schema).validate(capsule)
def test_is_keycloak_bridge_online(self, test_settings,
default_route_json_schema):
"""
Test if keycloak bridge service is running
:param test_settings: pytest config loaded from config file
:return:
"""
#Challange value
component_name = test_settings['keycloak_bridge']['component_name']
#Settings
keycloak_bridge_hostname = test_settings['keycloak_bridge']['hostname']
keycloak_bridge_scheme = test_settings['keycloak_bridge'][
'http_scheme']
keycloak_bridge_ip = test_settings['keycloak_bridge']['ip']
keycloak_bridge_port = test_settings['keycloak_bridge']['port']
#Test
s = Session()
headers = {
'Accept': "application/json'",
'Host': '{host}'.format(host=keycloak_bridge_hostname)
}
req = Request(method='GET',
url="{scheme}://{ip}:{port}/".format(
scheme=keycloak_bridge_scheme,
ip=keycloak_bridge_ip,
port=keycloak_bridge_port),
headers=headers)
prepared_request = req.prepare()
logger.debug(
json.dumps(prepared_request_to_json(req),
sort_keys=True,
indent=4,
separators=(',', ': ')))
response = s.send(prepared_request, verify=False)
response_json = response.json()
assert jsonschema.Draft3Validator(default_route_json_schema).is_valid(
response_json)
assert re.search(component_name, response_json['name'])
def test_are_all_services_running(self, test_settings,
services_json_schema):
"""
Test if all the services are up by doing a health check
:param test_settings: pytest config loaded from config file
:return:
"""
#Challange value
fail_value = "KO"
#Settings
keycloak_bridge_hostname = test_settings['keycloak_bridge']['hostname']
keycloak_bridge_scheme = test_settings['keycloak_bridge'][
'http_scheme']
keycloak_bridge_ip = test_settings['keycloak_bridge']['ip']
keycloak_bridge_port = test_settings['keycloak_bridge']['port']
#Test
s = Session()
headers = {
'Accept': "application/json'",
'Host': '{host}'.format(host=keycloak_bridge_hostname)
}
req = Request(method='GET',
url="{scheme}://{ip}:{port}/health".format(
scheme=keycloak_bridge_scheme,
ip=keycloak_bridge_ip,
port=keycloak_bridge_port),
headers=headers)
prepared_request = req.prepare()
logger.debug(
json.dumps(prepared_request_to_json(req),
sort_keys=True,
indent=4,
separators=(',', ': ')))
response = s.send(prepared_request, verify=False)
response_json = response.json()
assert jsonschema.Draft3Validator(services_json_schema).is_valid(
response_json)
assert re.search(fail_value, response.text) is None
def is_valid_flavor_configuration(flavor, schema):
if schema is not None:
errors_list = list(
jsonschema.Draft3Validator(schema).iter_errors(flavor))
if len(errors_list) > 0:
details = dict(errors=[{
'message':
'-'.join([
"[%s]" % "][".join(repr(p) for p in error.path),
str(getattr(error, "message", error))
])
} for error in errors_list])
raise exceptions.ValidationFailed(json.dumps(details))
return
def schemaValidate(inputDict,schema):
inputJson=json.loads(json.dumps(inputDict))
if schema["$schema"] == "http://json-schema.org/draft-04/schema#":
v = jsonschema.Draft4Validator(schema)
elif schema["$schema"] == "http://json-schema.org/draft-03/schema#":
v = jsonschema.Draft3Validator(schema)
else:
msg="Schema needs to be a recoginized meta-schema [Draft 3, Draft 4]"
error(LOG,msg,error_codes.EX_IOERR)
msg=None
errs = sorted(v.iter_errors(inputDict), key=lambda e: e.path)
if errs:
msg = ""
for err in errs:
msg += err.message
return(msg)
def json_matches_schema_inner(request, schema=None):
errors_list = []
try:
data = json.loads(request.body.decode('utf-8'))
except ValueError:
raise exceptions.ValidationFailed('Invalid JSON string')
if schema is not None:
errors_list = list(
jsonschema.Draft3Validator(schema).iter_errors(data))
if len(errors_list) > 0:
details = dict(errors=[{
'message': str(getattr(error, "message", error))
} for error in errors_list])
raise exceptions.ValidationFailed(json.dumps(details))
else:
return
def wrapped(*args, **kwargs):
validation_failed = False
v_error = None
errors_list = []
if request.method in ('POST', 'PUT', 'PATCH') and \
schema is not None:
try:
data = json.loads(request.body.decode('utf-8'))
errors_list = list(
jsonschema.Draft3Validator(schema).iter_errors(data))
except ValueError:
validation_failed = True
v_error = ["Invalid JSON body in request"]
if len(errors_list) > 0:
validation_failed = True
v_error = errors_list
if not validation_failed:
return f(*args, **kwargs)
else:
return handler(v_error)
def load_machine_config(configfile):
with open(scriptpath + os.sep + ".." + os.sep + ".." + os.sep \
+ "machine_configs" + os.sep + "machine_config.schema.json") as f:
json_schema = json.load(f)
json_validator = jsonschema.Draft3Validator(json_schema)
with open(configfile) as f:
machine_config = json.load(f)
errors = sorted(json_validator.iter_errors(machine_config),
key=lambda e: e.path)
if len(errors) == 0:
return machine_config
print "'" + configfile + "' is not valid!"
print
for error in errors:
print(error)
exit(1)
def on_post(self, req, resp):
resp.content_type = "application/json"
response = ReviewResponse()
try:
body = json.loads(req.stream.read(req.content_length or 0))
# Try to get apiVersion and uid even if request doesn't comply to
# schema
response.api_version = body.get("apiVersion")
review_request = body.get("request", {})
response.uid = review_request.get("uid")
# Validate admission request against schema
jsonschema.Draft3Validator(self.schema_dict).validate(body)
except Exception as e:
response.set_error(
400, f"Exception parsing the body of request: {e}."
)
else:
if (
review_request.get("kind", {}).get("kind")
== "OpenStackDeployment"
):
validators = ["openstack", "nodes"]
spec = review_request.get("object", {}).get("spec", {})
# Validate all the enabled services, if there is a
# corresponding validator.
# Not validating services that are about to be deleted
validators.extend(
layers.services(spec, LOG)[0] & _VALIDATORS.keys()
)
for service in validators:
try:
_VALIDATORS[service].validate(review_request)
except exception.OsDplValidationFailed as e:
response.set_error(e.code, e.message)
break
resp.body = json.dumps(response.to_json())
def with_schema_falcon(request, schema=None):
"""Use to decorate a falcon style controller route
:param request: A falcon request
:param schema: a Json schema to validate against
"""
validation_failed = False
v_error = None
if schema is not None:
errors_list = []
try:
data = json.loads(request.body)
errors_list = list(
jsonschema.Draft3Validator(schema).iter_errors(data))
except ValueError:
validation_failed = True
v_error = ["Invalid JSON body in request"]
if len(errors_list) > 0:
validation_failed = True
v_error = errors_list
if validation_failed:
raise exceptions.ValidationFailed(repr(v_error))
def is_valid_service_configuration(service, schema):
if schema is not None:
errors_list = list(
jsonschema.Draft3Validator(schema).iter_errors(service))
if len(errors_list) > 0:
details = dict(errors=[{
'message':
'-'.join([
"[%s]" % "][".join(repr(p) for p in error.path),
str(getattr(error, "message", error))
])
} for error in errors_list])
raise exceptions.ValidationFailed(json.dumps(details))
# Schema structure is valid. Check the functional rules.
# 1. origins and origin rules must be unique
if 'origins' in service:
origin_rules = []
origins = []
for origin in service['origins']:
origin_ssl = 'https' if origin.get('ssl') else 'http'
origin_value = u"{0}://{1}".format(origin_ssl,
origin.get('origin'))
if origin_value in origins:
raise exceptions.ValidationFailed(
'The origin {0} already exists for another '
'origin on this service'.format(origin_value))
else:
origins.append(origin_value)
if 'rules' in origin:
for rule in origin['rules']:
request_url = rule['request_url']
if not request_url.startswith('/'):
request_url = ('/' + request_url)
if request_url in origin_rules:
raise exceptions.ValidationFailed(
'The path {0} already exists for another '
'origin on this service'.format(request_url))
else:
origin_rules.append(request_url)
# 2. caching rules must be unique
if 'caching' in service:
caching_rules = []
for caching in service['caching']:
if 'rules' in caching:
for rule in caching['rules']:
request_url = rule['request_url']
if request_url in caching_rules:
raise exceptions.ValidationFailed(
'The path {0} already exists for another '
'caching rule on this service'.format(request_url))
else:
caching_rules.append(request_url)
# 3. domains must be unique
if 'domains' in service:
domains = []
for domain in service['domains']:
domain_value = u"{0}://{1}".format(domain.get('protocol', 'http'),
domain.get('domain'))
if domain_value in domains:
raise exceptions.ValidationFailed(
'The domain {0} already exists on another service'.format(
domain_value))
else:
domains.append(domain_value)
# We allow multiple restrictions rules on the same path
# 5. domains must be valid
if 'domains' in service:
for domain in service['domains']:
if not is_valid_domain(domain):
raise exceptions.ValidationFailed(
u'Domain {0} is not a valid domain'.format(
domain.get('domain')))
# 6. origins and domains cannot be the same
if 'origins' in service and 'domains' in service:
origins = set()
for origin in service['origins']:
origin_name = origin.get('origin').lower().strip()
origins.add(origin_name)
domains = set()
for domain in service['domains']:
domain_name = domain.get('domain').lower().strip()
domains.add(domain_name)
if origins.intersection(domains):
raise exceptions.ValidationFailed(
u'Domains and origins cannot be same: {0}'.format(origin))
# 7. origins must be valid
if 'origins' in service:
for origin in service['origins']:
if not is_valid_origin(origin):
raise exceptions.ValidationFailed(
u'Origin {0} is not valid'.format(origin.get('origin')))
# 8. domains must not be root domains
if 'domains' in service:
for domain in service['domains']:
protocol = domain.get('protocol', 'http')
certificate = domain.get('certificate')
# for a shared SSL domains, domain name is a single segment
# so, root domain validation does not apply to it
if protocol == "https" and certificate == "shared":
continue
if is_root_domain(domain):
raise exceptions.ValidationFailed(
u'{0} is a root domain. Most DNS providers do not allow '
'setting a CNAME on a root domain. Please add a subdomain '
'(e.g. www.{0})'.format(domain.get('domain')))
# 9. Hostheadervalue must be valid
if 'origins' in service:
for origin in service['origins']:
if 'hostheadervalue' in origin:
hostheadervalue = origin.get('hostheadervalue')
if hostheadervalue is not None:
if not is_valid_domain_name(hostheadervalue):
raise exceptions.ValidationFailed(
u'The host header {0} is not valid'.format(
hostheadervalue))
# 10. Need to validate restriction correctness here
# Cannot allow one restriction rule entity to have both
# "blacklist" and "whitelist" restriction type
whitelist_restriction_entities = {}
blacklist_restriction_entities = {}
if 'restrictions' in service:
for restriction in service['restrictions']:
if restriction.get('access', 'blacklist') == 'blacklist':
for rule in restriction['rules']:
entity = None
request_url = '/*'
for key in rule:
if key == 'name':
pass
elif key == 'request_url':
request_url = rule['request_url']
else:
entity = key
# validate country code is valid
# if key == 'geography':
# rule[key] is valid
if request_url not in blacklist_restriction_entities:
blacklist_restriction_entities[request_url] = []
blacklist_restriction_entities[request_url].append(entity)
elif restriction.get('access', 'whitelist') == 'whitelist':
for rule in restriction['rules']:
entity = None
request_url = '/*'
for key in rule:
if key == 'name':
pass
elif key == 'request_url':
request_url = rule['request_url']
else:
entity = key
# validate country code is valid
# if key == 'geography':
# rule[key] is valid
if request_url in blacklist_restriction_entities and \
entity in blacklist_restriction_entities[
request_url]:
raise exceptions.ValidationFailed(
'Cannot blacklist and whitelist [%s] on %s'
' at the same time' % key, request_url)
if request_url not in whitelist_restriction_entities:
whitelist_restriction_entities[request_url] = []
whitelist_restriction_entities[request_url].append(entity)
for request_url in whitelist_restriction_entities:
if request_url in blacklist_restriction_entities:
intersect_entities = set(
blacklist_restriction_entities[request_url]
).intersection(whitelist_restriction_entities[request_url])
if len(intersect_entities) > 0:
raise exceptions.ValidationFailed(
'Cannot blacklist and whitelist %s on %s'
' at the same time' %
(str(list(intersect_entities)), request_url))
# referrer domains must be valid
for rule in restriction['rules']:
if rule.get("referrer"):
referrer = rule.get("referrer")
if not is_valid_domain_name(referrer):
raise exceptions.ValidationFailed(
u'Referrer {0} is not a valid domain'.format(
referrer))
return
def validateJsonResponseVsSchema(self, jsonResponce, schema={}):
return jsonschema.Draft3Validator(schema).is_valid(jsonResponce)
#!bin/python
import json
import yaml
import jsonschema
print("Loading config..")
config = yaml.load(file("config.yml"))
print("Loading json schema..")
schema = json.load(file("pyfibot/config_schema.json"))
print("Validating configuration")
v = jsonschema.Draft3Validator(schema)
if not v.is_valid(config):
print("Error(s) in configuration:")
for error in sorted(v.iter_errors(config), key=str):
print(error)
else:
print("config ok")
with open(test_file) as f:
test_data = json.load(f)
errors = sorted(v.iter_errors(test_data), key=lambda e: e.path)
if len(errors) == 0:
return
# print "'" + test_file + "' is valid."
# print
# return
print "'" + test_file + "' is not valid!"
print
for error in errors:
print(error)
exit(1)
if __name__ == "__main__":
with open('test_data.schema.json') as f:
test_schema = json.load(f)
test_validator = jsonschema.Draft3Validator(test_schema)
for path, subdirs, files in os.walk('test_data'):
for testfile in files:
if testfile.endswith('.json'):
validate(path + os.sep + testfile, test_validator)