def validate(self):
try:
Draft3Validator.check_schema(self.schema)
except SchemaError as err:
return 'schema is invalid: {}'.format(err)
return None
def __init__(self):
super(RabbitMQingressProcessorTests, self).__init__(self.MODULE_NAME,
self.PRIORITY)
# Read in the actuator schema for validating messages
fileName = os.path.join(RESOURCE_PATH + '/actuators',
self.JSON_ACTUATOR_SCHEMA)
with open(fileName, 'r') as f:
_schema = f.read()
# Remove tabs and newlines
self._actuator_schema = json.loads(' '.join(_schema.split()))
# Validate the schema
Draft3Validator.check_schema(self._actuator_schema)
# Read in the sensor schema for validating messages
fileName = os.path.join(RESOURCE_PATH + '/sensors',
self.JSON_SENSOR_SCHEMA)
with open(fileName, 'r') as f:
_schema = f.read()
# Remove tabs and newlines
self._sensor_schema = json.loads(' '.join(_schema.split()))
# Validate the schema
Draft3Validator.check_schema(self._sensor_schema)
def __init__(self, train_set):
if type(train_set) is not str:
raise TypeError(
'Train set has to be path to .json file, '
'not {}'.format(train_set)
)
if not train_set.endswith('.json') or not isfile(train_set):
raise ValueError(
'Train set has to be path to .json file, '
'not {}'.format(train_set)
)
self.schema = {
'type': 'array',
'items': {
'type': 'string'
}
}
schema = {
'type': 'object',
'required': ['data'],
'properties': {
'data': {'type': 'array'}
}
}
with open(train_set, 'r', encoding='utf-8') as train_file:
train_data = load(train_file)
if not Draft3Validator(schema).is_valid(train_data):
exception_data = ['Validation error in: {}'.format(train_data)]
for error in sorted(
Draft3Validator(self.schema).iter_errors(train_data),
key=str):
exception_data.append(error.message)
raise ValueError(exception_data)
self.train_set = train_set
def __init__(self):
"""Reads in the json schema for all actuator response messages"""
super(BaseActuatorMsg, self).__init__()
# Read in the schema for validating messages
fileName = os.path.join(RESOURCE_PATH + '/actuators',
self.JSON_ACTUATOR_SCHEMA)
with open(fileName, 'r') as f:
self._schema = json.load(f)
# Validate the schema
Draft3Validator.check_schema(self._schema)
def _load_schema(self, schema_file):
"""Loads a schema from a file and validates
@param string schema_file location of schema on the file system
@return string Trimmed and validated schema
"""
with open(schema_file, 'r') as f:
schema = json.load(f)
# Validate the schema to conform to Draft 3 specification
Draft3Validator.check_schema(schema)
return schema
def __init__(self):
"""Reads in the json schema for all sensor response messages"""
super(BaseSensorMsg, self).__init__()
# Read in the sensor schema for validating messages
fileName = os.path.join(RESOURCE_PATH + '/sensors',
self.JSON_SENSOR_SCHEMA)
with open(fileName, 'r') as f:
_schema = f.read()
# Remove tabs and newlines
self._schema = json.loads(' '.join(_schema.split()))
# Validate the schema
Draft3Validator.check_schema(self._schema)
def test_rudimentary_ref_support(self):
schema = {
"type": "object",
"properties": {
"nn": {
"$ref": "http://www.example.com/schemas#/neatoNumber"
},
"ss": {
"$ref": "http://www.example.com/schemas#/superString"
}
}
}
schema_store = {
"http://www.example.com/schemas": {
"neatoNumber": {
"type": "number"
},
"superString": {
"type": "string"
}
}
}
validator = Draft3Validator(schema, schema_store=schema_store)
validator.validate({"nn": 1})
validator.validate({"ss": "hello"})
with self.assertRaises(ValidationError):
validator.validate({"nn": "hello"})
with self.assertRaises(ValidationError):
validator.validate({"ss": 1})
def _load_schema(self, schema_file):
"""Loads a schema from a file and validates
@param string schema_file location of schema on the file system
@return string Trimmed and validated schema
"""
with open(schema_file, 'r') as f:
schema = f.read()
# Remove tabs and newlines
schema_trimmed = json.loads(' '.join(schema.split()))
# Validate the actuator schema
Draft3Validator.check_schema(schema_trimmed)
return schema_trimmed
def verify(df, table_name):
""" Takes a Pandas DataFrame and a Microsoft SQL Server table name to compare the number of rows in each. This function only works if the to_sql function replaces the table (if it appends the number of rows will obviously be off)
Parameters:
csv_df (Pandas Dataframe): dataframe of the original data from CSV
table_name (string): name of table that you want to compare against
Returns:
nothing
"""
print("Verifying schema...")
df = df.where(pd.notnull(df), None)
v = Draft3Validator(schemas[table_name])
errors = set()
for row in df.to_dict(orient='records'):
for error in sorted(v.iter_errors(row), key=str):
errors.add(str(error))
if errors:
print('Validation errors when running schema check on {}'.format(
table_name))
with open("/tmp/{}_validation_errors.txt".format(table_name),
'w+') as fp:
for error in errors:
fp.write("{}\n\n\n".format(error))
return False
return True
def validate(self, schema=None):
"""
Validate that we have a valid object.
On error, this will raise a `ScrapeValueError`
This also expects that the schemas assume that omitting required
in the schema asserts the field is optional, not required. This is
due to upstream schemas being in JSON Schema v3, and not validictory's
modified syntax.
^ TODO: FIXME
"""
if schema is None:
schema = self._schema
validator = Draft3Validator(
schema,
types={'datetime': (datetime.date, datetime.datetime)},
format_checker=FormatChecker())
errors = [
str(error) for error in validator.iter_errors(self.as_dict())
]
if errors:
raise ScrapeValueError('validation of {} {} failed: {}'.format(
self.__class__.__name__, self._id,
'\n\t' + '\n\t'.join(errors)))
def validate_data(path_schema, path_event, path_log):
schemas = []
with os.scandir(path_schema) as entries:
for entry in entries:
with open(os.path.join(path_schema, entry.name)) as file:
schema = json.load(file)
schemas.append(dict(schema_file=entry.name, schema=schema))
log = []
for schema in schemas:
with os.scandir(path_event) as entries:
for entry in entries:
with open(os.path.join(path_event, entry.name)) as f:
data = json.load(f)
try:
Draft3Validator(schema['schema']).validate(data)
except jsonschema.exceptions.ValidationError as err:
log.append(dict(file=entry.name, schema_file=schema['schema_file'], err=err.message))
# Вывод для README
print('Файл данных | Файл схемы | Ошибки')
print('----------- | ---------- | ------')
for line in log:
print(f"{line['file']} | {line['schema_file']} | {line['err']}")
# Вывод в файл
with open(path_log, 'w+') as file:
for line in log:
file.write(str(line) + '\n')
def updateALL_product(uuid):
for header in request.headers:
print(header)
data = request.get_json()
data = json.dumps(data)
data = json.loads(data)
data2 = conn.get(uuid)
data2 = json.loads(data2)
print(data2)
print()
print(data)
print()
try:
jwt.decode(data2["token"],
'secret',
leeway=10,
algorithms=['HS256'],
verify=True)
except jwt.ExpiredSignatureError:
return 'Signature expired. Please log in again.'
except jwt.InvalidTokenError:
return 'Invalid token. Please log in again.'
# except Exception as e:
# print("Token Expired")
# return jsonify({'Token Expired'})
encoded = 'Bearer ' + data2["token"]
if (request.headers['Authorization'] != encoded):
return "Authorization Error"
# print(data)
# print(type(data))
# Loading Schema
with open(r'''schema.txt''') as json_schema:
schema = json.load(json_schema)
#validating data against schema
myJSONValidation = Draft3Validator(schema).is_valid(data)
if (myJSONValidation == True):
uniqueId = data2['uuid']
uniqueId = str(uniqueId)
data['uuid'] = uniqueId
only_token = data2["token"]
only_token = str(only_token)
data['token'] = only_token
data3 = json.dumps(data)
if request.method == 'PUT':
old_etag = request.headers.get('If-None-Match', '')
# Generate hash
# data_n = json.dumps(data2)
new_etag = md5(data3.encode('utf-8')).hexdigest()
if new_etag == old_etag:
# Resource has not changed
return '', 304
else:
conn.set(uuid, data3)
return jsonify({'product': data}), 200, {'ETag': new_etag}
return 'test'
def validate_input(input, schema):
try:
validation_errors = Draft3Validator(schema).validate(input)
return True
except Exception as e:
print str(e)
return False
def validate(nbjson):
"""Checks whether the given notebook JSON conforms to the current
notebook format schema, and returns the list of errors.
"""
# load the schema file
with open(schema_path, 'r') as fh:
schema_json = json.load(fh)
# resolve internal references
schema = resolve_ref(schema_json)
schema = jsonpointer.resolve_pointer(schema, '/notebook')
# count how many errors there are
v = Validator(schema)
errors = list(v.iter_errors(nbjson))
return errors
def post(self):
validator = Draft3Validator(json.loads(self.request.body))
any_error = False
for x in validator.iter_errors({}):
any_error = True
break
self.write({"status": "WORKING", "errors": any_error})
def validate(nbjson):
"""Checks whether the given notebook JSON conforms to the current
notebook format schema, and returns the list of errors.
"""
# load the schema file
with open(schema_path, 'r') as fh:
schema_json = json.load(fh)
# resolve internal references
schema = resolve_ref(schema_json)
schema = jsonpointer.resolve_pointer(schema, '/notebook')
# count how many errors there are
v = Validator(schema)
errors = list(v.iter_errors(nbjson))
return errors
def test_it_delegates_to_a_ref_resolver(self):
resolver = mock.Mock()
resolver.resolve.return_value = {"type": "integer"}
schema = {"$ref": mock.Mock()}
with self.assertRaises(ValidationError):
Draft3Validator(schema, resolver=resolver).validate(None)
resolver.resolve.assert_called_once_with(schema, schema["$ref"])
def is_rule_valid(rule):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
return Draft3Validator(rules_schema).is_valid(rule)
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return False
finally:
context.end_span()
def validate_config(config):
filename = get_config_path(config)
try:
to_validate = json.load(open(filename))
except ValueError as e:
print('{}: {}'.format(filename, e))
return
validator = Draft3Validator(schema)
for error in sorted(validator.iter_errors(to_validate), key=str):
print('{}: {}'.format(filename, error))
def verifySchema(schema, instance):
try:
validator = Draft3Validator(schema)
validator.validate(instance)
except ValidationError as e:
print 'Data did not comply with jsonschema. Schema: "' + str(schema) + '"' + \
' Response: "' + str(instance) + '"'
raise e
except (SchemaError, UnknownType, TypeError) as e:
print 'Error in the jsonschema. Schema: "' + str(schema) + '"'
raise e
def hello_world():
text = input("Json File Name:") # form1.json form2.json
inp = open(text, "r")
data = json.loads(inp.read())
instance = data
v = Draft3Validator(schema)
errors = sorted(v.iter_errors(instance), key=lambda e: e.path)
for error in errors:
print(error.message)
createjavascript(data)
intiliazedbfile(data)
createhtmlfile(data)
return print()
def main():
usage = 'Usage: %prog [ --all --cont ]'
parser = optparse.OptionParser(usage=usage)
parser.add_option('-f', '--filepath', action='store', default=None,
help='Path to files, e.g. paraguay/sample')
parser.add_option('-v', '--version', action='store', default='1.1.0',
help='Version, e.g. 1.1.0')
parser.add_option('-V', '--verbose', action='store_true', default=False,
help='Print verbose output')
parser.add_option(
'-t', '--type', action='store', default='release',
help='File type: release-package, record-package or release')
(options, args) = parser.parse_args()
if not options.filepath:
parser.error('You must supply a filepath, using the -f argument')
schema = get_schema('%s-schema.json' % options.type, options.version)
if options.type == 'record-package' and options.version == '1.1.0':
# Fix v1.1 schema error - wrong item is required.
schema['required'].remove('releases')
schema['required'].append('records')
count = 0
files = glob.glob('%s/*.json' % options.filepath)
for filename in files:
count += 1
if not count % 1000:
print('Validating file %s of %s' % (count, len(files)))
if not filename.endswith('.json'):
print('Skipping non-JSON file %s' % filename)
continue
with open(filename, 'r') as file:
if options.verbose:
print('\nValidating %s' % filename)
try:
data = json.load(file)
except Exception as e:
print('Problem loading', filename)
print(e)
continue
v = Draft3Validator(schema)
errors = sorted(v.iter_errors(data), key=str)
data['validationErrors'] = ''
for error in errors:
location = '/'.join(error.absolute_schema_path)
message = "%s: %s\n" % (location, error.message)
data['validationErrors'] += message
if options.verbose:
print(message)
with open(filename, 'w') as writefile:
writefile.write(json.dumps(data, indent=2))
def validator(self, data, schema):
messages = []
is_valid = True
validator = Draft3Validator(schema)
for i in data:
try:
validator.validate({str(i): data[i]})
except Exception as e:
print(e.message)
messages.append('ERROR: ' + e.message)
if messages:
is_valid = False
return (is_valid, messages)
return is_valid
def validate_json(event, schema):
"""Function to validate json format
Arguments:
event {str} -- Whole message content
schema {dict} -- The json schema to use for validation
"""
logger.debug("Validating message {}".format(event))
try:
Draft3Validator(json.loads(schema)).validate(event)
except ValidationError:
logger.exception("Error in JSON data")
raise
def importConfigJson(fullPath, pathToSchema=None):
"""
Reads JSON formatted master configuration file
:param fullPath: full path of the JSON file to load
:type fullPath: str
:raises: jsonschema.exceptions.ValidationError
:returns: dictionary of master configuration information
"""
configDict = {}
file = open(fullPath, 'r')
configDict = json.loads(file.read())
if (pathToSchema == None):
validator = Draft3Validator(appNetworkConfigSchema.schema)
else:
schema = separatePathAndModule(pathToSchema)
validator = Draft3Validator(schema)
# this will raise jsonschema.exceptions.ValidationError exception
validator.validate(configDict)
file.close()
return configDict
def custom_endpoint_body_rules(self):
if self.__input == "":
return True
try:
data = json.loads(self.__input)
except Exception as e:
return False
draft3 = True
draft4 = True
try:
Draft3Validator.check_schema(data)
except Exception as e:
draft3 = False
try:
Draft4Validator.check_schema(data)
except Exception as e:
draft4 = False
return draft3 or draft4
def validate_params(params, instance, action):
root = cherrypy.request.app.root
if hasattr(root, 'api_schema'):
api_schema = root.api_schema
else:
return
operation = model_fn(instance, action)
validator = Draft3Validator(api_schema, format_checker=FormatChecker())
request = {operation: params}
try:
validator.validate(request)
except ValidationError, e:
raise InvalidParameter(e.schema['error'], {'value': str(e.instance)})
def validate(self, json, schema):
"""Validates catalog entry via JSON Schema. The expected_author param
will prevent entry overrides from others than the original author.
.. seealso:: http://json-schema.org/
.. seealso:: http://tools.ietf.org/html/draft-zyp-json-schema-03
.. seealso:: https://github.com/json-schema/json-schema
"""
errors = []
validator = Draft3Validator(schema)
for e in validator.iter_errors(json):
e.path.reverse()
errors.append(e.path)
return errors
async def post(self):
"""Создание объявления. Для валидации передавемых данных используем jsonschema"""
data = await self.request.json()
v = Draft3Validator(schema, format_checker=FormatChecker())
if v.is_valid(data):
async with self.request.app['db'].acquire() as conn:
cursor = await conn.execute(db.posts.insert().values(**data))
post_id = await cursor.fetchone()
return web.HTTPCreated(body=json.dumps({'id': post_id[0]}),
content_type='application/json')
else:
response_data = {
'errors':
dict((err.path.pop(), err.message)
for err in v.iter_errors(data))
}
return web.HTTPBadRequest(body=json.dumps(response_data),
content_type='application/json')
def validate_json(json_data, f):
"""REF: https://json-schema.org/ """
# Describe what kind of json you expect.
execute_api_schema = get_schema(json_data['SchemaName'])
errCount = 0
v = Draft3Validator(execute_api_schema)
errors = v.iter_errors(json_data)
error_text = ''
for error in sorted(errors, key=str):
#print("Line {} --- ".format(get_error_line(error, json_data)) + error.message)
#error_text += "Line {}: ".format(get_error_line(error, json_data)) + error.message + "\n"
error_text += ''.join([str(elem) + " " for elem in error.path]) + ": "
error_text += error.message + "\n"
errCount += 1
if errCount > 0:
message = "Given JSON data is InValid"
return False, message, error_text
else:
message = "Given JSON data is Valid"
return True, message, error_text
def validate_received_responses(schema_loc, response_loc, array=False):
# assuming 10to1 will do tests to validate api output against swagger defined json schema,
# TODO it turns out they don't, so I'll do it myself
for response in glob(response_loc):
test_name = splitext(basename(response))[0]
with open(response, "r", "utf-8") as f:
v = Draft3Validator(definitions[schema_loc])
full_doc = load(f)
if array:
doc = full_doc[0] if len(full_doc) > 0 else {}
else:
doc = full_doc
validation_errors = sorted(v.iter_errors(doc), key=str)
filtered_errors = []
for error in validation_errors:
if "None is not of type" not in error.message:
filtered_errors.append(str(error))
if len(filtered_errors) > 0:
with open("validation_errors/" + test_name + ".txt", "w",
"utf-8") as f:
f.write(
"\n\n--------------------------------------\n\n".join(
[error for error in filtered_errors]))
def __init__(self, name, mode="pass", schema=''):
Actor.__init__(self, name)
self.name=name
self.mode=mode
self.schema=schema
if mode == "decode":
self.convert = self.__loads
elif mode == "encode":
self.convert = self.__dumps
elif mode == "pass":
self.convert = self.__pass
else:
raise Exception ("mode should be either 'encode' or 'decode'.")
if schema != "":
self.logging.debug("Validation schema defined. Doing validation.")
schema_data = self.__loadValidationSchema(schema)
self.validate = self.__validate
self.validator=Validator(schema_data)
else:
self.logging.debug("No validation schema defined. No validation.")
self.validate = self.__noValidate
def test_schema_valid(self):
"""
The schema itself is a valid Draft 3 schema
"""
Draft3Validator.check_schema(group_schemas.policy)
msg = " check effect %s ... " % filename
try:
effect = json.loads(f.read())
script = path.basename(effect['script'])
if not path.exists(jsonFiles+'/'+script):
raise ValueError('script file: '+script+' not found.')
schema = path.splitext(script)[0]+'.schema.json'
if not path.exists(jsonFiles+'/schema/'+schema):
raise ValueError('schema file: '+schema+' not found.')
schema = jsonFiles+'/schema/'+schema
# validate against schema
with open(schema) as s:
effectSchema = json.loads(s.read())
Draft3Validator.check_schema(effectSchema)
validator = Draft3Validator(effectSchema)
baseValidator.validate(effect)
validator.validate(effect['args'])
#print(msg + "ok")
except Exception as e:
print(msg + 'error ('+str(e)+')')
errors += 1
retval = 1
print(" checked effect files: %s success: %s errors: %s" % (total,(total-errors),errors))
sys.exit(retval)
# How do I validate a JSON Schema schema, in Python? from jsonschema import Draft3Validator my_schema = json.loads(my_text_file) #or however else you end up with a dict of the schema Draft3Validator.check_schema(schema)
# Loading example files
client_example_file = open(WORKSPACE_DIR + "schema/communication/examples/client-message-example.json")
server_example_file = open(WORKSPACE_DIR + "schema/communication/examples/server-message-example.json")
client_configuration_example_file = open(WORKSPACE_DIR + "schema/configuration/examples/client-configuration-example.json")
client_logging_example_file = open(WORKSPACE_DIR + "schema/communication/examples/client-logging-example.json")
# Loading into JSON
client_schema = json.load(client_schema_file)
server_schema = json.load(server_schema_file)
client_configuration_schema = json.load(client_configuration_schema_file)
client_logging_schema = json.load(client_logging_schema_file)
client_example = json.load(client_example_file)
server_example = json.load(server_example_file)
client_configuration_example = json.load(client_configuration_example_file)
client_logging_example = json.load(client_logging_example_file)
# Running verification
logging.info("Testing schemes for compliance against the JSON Schema format")
Draft3Validator.check_schema(client_schema)
Draft3Validator.check_schema(server_schema)
Draft3Validator.check_schema(client_configuration_schema)
Draft3Validator.check_schema(client_logging_schema)
logging.info("Testing example files for compliance against respective schemes")
validate(client_example, client_schema)
validate(server_example, server_schema)
validate(client_configuration_example, client_configuration_schema)
validate(client_logging_example, client_logging_schema)
def test_0(self):
"""The schema is valid"""
Draft3Validator.check_schema(self.widget.schema)
def test_schema_valid(self):
"""
The update webhook schema is valid JSON Schema Draft 3.
"""
Draft3Validator.check_schema(group_schemas.update_webhook)
def test_schema_valid(self):
"""
The schema itself is valid Draft 3 schema
"""
Draft3Validator.check_schema(group_schemas.config)
def test_schema_valid(self):
"""
The schema itself is a valid Draft 3 schema
"""
Draft3Validator.check_schema(rest_schemas.create_group_request)
class JSON(Actor):
'''**A Wishbone module which converts and validates JSON.**
This module has 2 main modes:
- Validate JSON data and convert into a Python data structure.
- Convert a Python data structure into a JSON string.
Parameters:
- name (str): The instance name when initiated.
- mode (str): Determines whether the input has to be encoded, decoded or
passed through.
Can have 3 values: "encode", "decode", "pass"
Default: pass
- schema (str): The filename of the JSON validation schema to load. When no
schema is defined no validation is done.
Default: ''
Queues:
- inbox: Incoming events.
- outbox: Outgoing events.
Data which cannot be converted or which fails the validation is purged.
The schema should be in valid JSON syntax notation. JSON validation can
only be done on Python objects so you will have to convert your any JSON
data to a Python object first.
'''
def __init__(self, name, mode="pass", schema=''):
Actor.__init__(self, name)
self.name=name
self.mode=mode
self.schema=schema
if mode == "decode":
self.convert = self.__loads
elif mode == "encode":
self.convert = self.__dumps
elif mode == "pass":
self.convert = self.__pass
else:
raise Exception ("mode should be either 'encode' or 'decode'.")
if schema != "":
self.logging.debug("Validation schema defined. Doing validation.")
schema_data = self.__loadValidationSchema(schema)
self.validate = self.__validate
self.validator=Validator(schema_data)
else:
self.logging.debug("No validation schema defined. No validation.")
self.validate = self.__noValidate
def consume(self, event):
try:
event["data"] = self.convert(event["data"])
except Exception as err:
self.logging.warn("Unable to convert incoming data. Purged. Reason: %s"%(err))
return
try:
self.validate(event["data"])
except ValidationError as err:
self.logging.warn("JSON data does not pass the validation schema. Purged. Reason: %s"%(str(err).replace("\n"," > ")))
return
try:
self.queuepool.outbox.put(event)
except QueueLocked:
self.queuepool.inbox.rescue(event)
self.queuepool.outbox.waitUntilPutAllowed()
def __loadValidationSchema(self, path):
with open(path,'r') as schema:
data = ''.join(schema.readlines())
print loads(data)
return loads(data)
def __loads(self, data):
return loads(data)
def __dumps(self, data):
return dumps(data)
def __pass(self, data):
return data
def __validate(self, data):
return self.validator.validate(data)
def __noValidate(self, data):
return True
def shutdown(self):
self.logging.info('Shutdown')
