def validate_data_against_schema(data: Dict, schema: Dict):
"""Validate dict data against given JSON Schema
Parameters
----------
data : Dict
Data set to be validated
schema : Dict
JSON Schema to validate against
Raises
------
SchemaValidationError
When schema validation fails against data set
InvalidSchemaFormatError
When JSON schema provided is invalid
"""
try:
fastjsonschema.validate(definition=schema, data=data)
except fastjsonschema.JsonSchemaException as e:
message = f"Failed schema validation. Error: {e.message}, Path: {e.path}, Data: {e.value}" # noqa: B306, E501
raise SchemaValidationError(message)
except (TypeError, AttributeError) as e:
raise InvalidSchemaFormatError(
f"Schema received: {schema}. Error: {e}")
def validate_data_against_schema(data: Union[Dict, str],
schema: Dict,
formats: Optional[Dict] = None):
"""Validate dict data against given JSON Schema
Parameters
----------
data : Dict
Data set to be validated
schema : Dict
JSON Schema to validate against
formats: Dict
Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool
Raises
------
SchemaValidationError
When schema validation fails against data set
InvalidSchemaFormatError
When JSON schema provided is invalid
"""
try:
formats = formats or {}
fastjsonschema.validate(definition=schema, data=data, formats=formats)
except (TypeError, AttributeError,
fastjsonschema.JsonSchemaDefinitionException) as e:
raise InvalidSchemaFormatError(
f"Schema received: {schema}, Formats: {formats}. Error: {e}")
except fastjsonschema.JsonSchemaException as e:
message = f"Failed schema validation. Error: {e.message}, Path: {e.path}, Data: {e.value}" # noqa: B306, E501
raise SchemaValidationError(message)
def firewall(args):
# retrieving relative path for schema.json
dirname = os.path.dirname(__file__)
if args.ingress:
schema_path = os.path.join(dirname, 'schema/fw-ingress.json')
else:
schema_path = os.path.join(dirname, 'schema/fw-egress.json')
# loading schema as JSON Object
with open(schema_path) as f:
schema = json.load(f)
data = json.loads(args.json)
# Validating input against schema
validate(schema, data)
allow_output = assembleIngressFWString(
data, "allow") if args.ingress else assembleEgressFWString(
data, "allow")
deny_output = assembleIngressFWString(
data, "deny") if args.ingress else assembleEgressFWString(
data, "deny")
# result is stored into a json object
json_obj = {
"allow_all": allow_output["all"],
"allow_tag": allow_output["tag"],
"allow_sa": allow_output["sa"],
"deny_all": deny_output["all"],
"deny_tag": deny_output["tag"],
"deny_sa": deny_output["sa"]
}
# # results are dumped to stdout
print(json.dumps(json_obj, indent=4))
def iam(args):
# retrieving relative path for schema.json
dirname = os.path.dirname(__file__)
schema_path = os.path.join(dirname, 'schema/iam.json')
# loading schema as JSON Object
with open(schema_path) as f:
schema = json.load(f)
data = json.loads(args.json)
# Validating input against schema
validate(schema, data)
# building flat map.
# Keys are seperated via |
# Emails are seperated via space
# Roles are seperated via ,
output = ""
for k in data:
output = output + k + "|"
for access in data[k]:
output = output + access + " "
output = output + ","
# result is stored into a json object
json_obj = {"iam": output}
# results are dumped to stdout
print(json.dumps(json_obj, indent=4))
def __validate_spellbook(spellbook_contents):
try:
with open(SPELLBOOK_SCHEMA_PATH, "r", encoding="utf-8") as file:
schema = json.load(file)
validate(schema, spellbook_contents)
except Exception as error:
raise Exception(f"Spellbook is invalid: {error}") from error
def main(import_id, citizen_id):
# Пытаемся парсить пришедие данные, как JSON.
if request.get_json(silent=True) is None:
# Если не удалось распарсить, выбрасываем исключение.
raise ValueError("request data can't be parsed as json")
# Получаем соответствующего жителя по двум
# основным ключам -- номеру выгрузки и id.
citizen = Citizen.query.filter_by(citizen_id=citizen_id,
dataset_id=import_id).first()
if citizen is None:
# Если не нашли такого пользователя, выбрасываем исключение.
raise ValueError(
"Citizen with such citizen_id and import_id wasn't found")
else:
citizen_obj = request.json
# Общая валидация PATCH-схемы.
validate(dataset_patch_schema, citizen_obj)
# Валидация отдельных полей. Проще сделать её вручную.
if 'birth_date' in citizen_obj:
validate_date(citizen_obj['birth_date'])
if 'relatives' in citizen_obj:
validate_id_not_in_relatives(citizen.citizen_id,
citizen_obj['relatives'])
# Валидация закончена
if 'town' in citizen_obj:
citizen.town = citizen_obj['town']
if 'street' in citizen_obj:
citizen.street = citizen_obj['street']
if 'building' in citizen_obj:
citizen.building = citizen_obj['building']
if 'apartment' in citizen_obj:
citizen.apartment = citizen_obj['apartment']
if 'name' in citizen_obj:
citizen.name = citizen_obj['name']
if 'birth_date' in citizen_obj:
citizen.birth_date = citizen_obj['birth_date']
if 'gender' in citizen_obj:
citizen.gender = citizen_obj['gender']
if 'relatives' in citizen_obj:
# Получим два сета с id пользователей
prev_relatives_list = unpack_relatives_to_int_list(
citizen.relatives)
cur_relatives_list = citizen_obj['relatives']
prev_relatives = set(prev_relatives_list)
cur_relatives = set(cur_relatives_list)
# Получим разницу в обоих случаях и
# удалим/добавим необходимые id
for relative_id in prev_relatives.difference(cur_relatives):
remove_cur_citizen_from_other_relative(relative_id, citizen_id,
import_id)
for relative_id in cur_relatives.difference(prev_relatives):
add_cur_citizen_to_other_relative(relative_id, citizen_id,
import_id)
citizen.relatives = pack_relatives_to_db_format(
citizen_obj['relatives'])
# Добавим в базу данных.
db.session.commit()
res = {"data": citizen.json_representation()}
return jsonify(res), 200
def test_search_index_schema(self):
l1block = create_l1_block()
fastjsonschema.validate(schema.l1_search_index_schema,
l1block.export_as_search_index())
tx_search_indexes = l1block.export_full_transactions_search_indexes()
for index in tx_search_indexes.keys():
fastjsonschema.validate(schema.transaction_search_index_schema,
tx_search_indexes[index])
def test_at_rest_schema(self):
l4block = create_l4_block()
fastjsonschema.validate(schema.l4_block_at_rest_schema,
l4block.export_as_at_rest())
l4block.nonce = 1
l4block.scheme = "work"
fastjsonschema.validate(schema.l4_block_at_rest_schema,
l4block.export_as_at_rest())
def test_metrics(endpoint: str):
with open(
f"""{DIR_META}/runs/{RUN_ID[endpoint]}/metrics_{RUN_ID[endpoint]}.json""", "r"
) as fread:
try:
validate(CONFIG[endpoint]["metrics_schema"], json.load(fread))
except Exception as ex:
raise Exception("Wrong metrics content") from ex
def change_citizen(self, import_id: int, citizen_id: int,
patch_citizen_data: dict) -> dict:
# check if citizen exists
if not self.citizen_exists(import_id, citizen_id):
raise DBHelperIDError
# check patch_citizen_data
try:
validate(self.CHANGE_CITIZEN_SCHEMA, patch_citizen_data)
except JsonSchemaException:
raise DBHelperJsonSchemaError
# check if patch_citizen_data have field "citizen_id" or not
if 'citizen_id' in patch_citizen_data.keys():
raise DBHelperJsonSchemaError
# get and change relatives, if patch_citizen_data contains relatives
try:
new_relatives = patch_citizen_data.pop('relatives')
except KeyError:
pass
else:
self.change_relatives(import_id, citizen_id, new_relatives)
# check birth_date and change to postgresql format, if patch_citizen_data contains birth_date
try:
patch_citizen_data[
'birth_date'] = self.json_date_to_postrgesql_date(
patch_citizen_data['birth_date'])
except KeyError:
pass
# same as validate_birth_date
except ValueError:
raise DBHelperJsonSchemaError
# if patch_citizen_data is not empty, change citizen data in database
if patch_citizen_data:
with psycopg2.connect(**self.DB_REQUISITES) as conn:
with conn.cursor() as cursor:
cursor.execute(
"SELECT id FROM citizens WHERE import_id = %s AND citizen_id = %s;",
(import_id, citizen_id))
citizen_db_id = cursor.fetchone()[0]
update_text = "UPDATE citizens SET " + \
",".join(map(lambda key: "{0}=%({0})s".format(key), patch_citizen_data.keys())) + \
"WHERE id = %(id)s;"
cursor.execute(update_text, {
**patch_citizen_data,
**{
"id": citizen_db_id
}
})
conn.commit()
return self.get_citizen(import_id, citizen_id)
def validate_json_files(filter_input_paths,
schema_file_path,
repo_path,
method='jsonschema'):
error = 0
try:
s = open(schema_file_path, 'r')
schema = json.load(s)
except JSONDecodeError as jsonerror:
print(
"--------------------------------------------------------------------------------------------------"
)
print("JSONDecodeError in SCHEMA FILE ", schema_file_path)
print(jsonerror)
raise
if method == 'jsonschema':
for input_file in filter_input_paths:
try:
f = open(os.path.join(repo_path, input_file), 'r')
example = json.load(f)
validate(instance=example, schema=schema)
except JSONDecodeError as jsonerror:
print(
"--------------------------------------------------------------------------------------------------"
)
print("JSONDecodeError in ", input_file)
print(jsonerror)
error = 1
except ValidationError as valerror:
print(
"--------------------------------------------------------------------------------------------------"
)
print("Schema ValidationError in ", input_file)
print(valerror)
error = 1
f.close()
if error == 1:
raise Exception(
'Errors Occured while validating input json files...')
elif method == 'fastjsonschema':
for input_file in filter_input_paths:
try:
f = open(os.path.join(repo_path, input_file), 'r')
example = json.load(f)
fastjsonschema.validate(schema, example)
except Exception:
print("Exception in File: ", input_file)
raise
f.close()
def validate_fields(self, fqid: str, fields: Dict[str, Any]) -> None:
model = model_registry[get_collection_from_fqid(fqid)]()
for field_name, value in fields.items():
field = model.get_field(field_name)
if isinstance(field, BaseTemplateField) and "$_" in field_name:
schema = {
"type": ["array", "null"],
"items": {
"type": "string"
},
}
else:
schema = field.get_schema()
validate(schema, value)
def validate_collection(file_name) -> json:
"""
Validates the postman collection against the postman schema (2.1.0)
:param file_name: the postman collection file path
:return: validated collection json
"""
root = os.path.dirname(os.path.abspath(__file__))
schema_filename = os.path.join(root, POSTMAN_SCHEMA_DIR, POSTMAN_JSON_SCHEMA)
json_collection = DocumentGenerator.get_json_file(file_name)
json_schema = DocumentGenerator.get_json_file(schema_filename)
validate(json_schema, json_collection)
return json_collection
def validator(schema: Dict[str, Any], obj: Dict[str, Any]) -> bool:
"""Function to validate object against a jsonschema.
Args:
schema: Jsonschema.
obj: Object to validate.
Returns:
Flag showing if the object is valid.
"""
try:
validate(schema, obj)
return True
except (JsonSchemaException, JsonSchemaDefinitionException):
return False
def wrapper(request, *args, **kwargs):
data = json.loads(request.body.decode())
try:
validated_data = validate(schema, data)
return func(request, validated_data, *args, **kwargs)
except JsonSchemaException as ex:
return HttpResponseBadRequest(ex.message)
def validate_config(config: object):
with open(path.dirname(path.abspath(__file__)) + '/config.schema.json',
encoding="utf-8") as file:
schema = json.load(file)
try:
validate(schema, config)
return True
except JsonSchemaException as e:
logging.warning("Your config is invalid: %s!", e)
logging.warning(
"NOTE: Support for legacy formats of the configuration will be dropped in future releases!"
)
logging.warning(
"Please update your configuration file according to https://github.com/layereight/MFRC522-trigger#json-schema."
)
return False
def __validate_json_with_schema(directory_client: DataLakeDirectoryClient, json_schema_file_path: str, data_dict: Dict):
file_client = directory_client.get_file_client(json_schema_file_path)
schema = __get_validation_schema(file_client)
try:
fastjsonschema.validate(schema, data_dict)
except (TypeError, fastjsonschema.JsonSchemaDefinitionException, AttributeError) as error:
raise HTTPException(
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
detail=f'Invalid schema definition: {getattr(error, "message", error)}') from error
except fastjsonschema.JsonSchemaValueException as error:
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST,
detail={
'message': f'JSON Schema validation error: {error.message}',
'name': f'{error.name}',
'rule': f'{error.rule}',
'rule_definition': f'{error.rule_definition}'
}) from error
def main():
# Пытаемся парсить пришедие данные, как JSON.
if request.get_json(silent=True) is None:
# Если не удалось распарсить, выбрасываем исключение.
raise ValueError("request data can't be parsed as json")
# Проверяем request.json на соответствие схеме валидации.
validate(import_schema, request.json)
# validate(instance=request.json, schema=import_schema)
citizens = request.json['citizens']
# Получим счётчик из БД
dataset_counter = get_dataset_counter()
# Если будет неудача, то import_id не изменится
dataset = Dataset(id=(dataset_counter.counter + 1))
for citizen_obj in citizens:
# Валидации для одного жителя.
do_single_citizen_validations(citizen_obj)
citizen = Citizen(
citizen_id=citizen_obj['citizen_id'],
town=citizen_obj['town'],
street=citizen_obj['street'],
building=citizen_obj['building'],
apartment=citizen_obj['apartment'],
name=citizen_obj['name'],
birth_date=citizen_obj['birth_date'],
gender=citizen_obj['gender'],
relatives=pack_relatives_to_db_format(citizen_obj['relatives']),
dataset=dataset)
# Валидации на возможные некорректные данные
# пользователей в поле relatives
validate_citizens_ids_intersection(dataset.citizens)
validate_relatives(dataset.citizens)
# Данные корректны, добавим в бд.
# Добавление происходит следующим образом:
# каждому пользователю сопоставляется
# конкретный dataset (выгрузка). При условии, что
# все данные пользователей корректны,
# мы добавляем данную выгрузку в БД.
db.session.add(dataset)
db.session.commit()
success_response = {
"data": {
"import_id": dataset_counter.inc()
}
}
return jsonify(success_response), 201
def validate(self, input_data):
try:
with open(self.get_json_schema(), 'r') as file:
schema = json.load(file)
return True, fastjsonschema.validate(schema, input_data)
except fastjsonschema.exceptions.JsonSchemaException as e:
return False, {'json_schema': {'valid': False, 'reason': str(e)}}
except Exception as e:
return False, {'input': {'valid': False, 'reason': str(e)}}
def imports():
if request.content_type != JSON_MIME_TYPE:
error = json.dumps({'error': 'Invalid Content Type'})
return json_response(error, 400)
data = request.json
try:
validate(citizens_schema, data)
except JsonSchemaException as e:
error = json.dumps({'error': f'{e}'})
return json_response(error, 400)
try:
json_validation(data)
except ValueError as e:
error = json.dumps({'error': f'{e}'})
return json_response(error, 400)
except KeyError:
error = json.dumps(
{'error': 'Relatives array contains nonexistent citizen_id'})
return json_response(error, 400)
max_import_id_from_table = db.session.query(func.max(
Citizen.import_id)).first()[0]
if max_import_id_from_table:
import_id = max_import_id_from_table + 1
else:
import_id = 1
for item in data['citizens']:
item.update({"import_id": import_id})
try:
db.session.bulk_insert_mappings(Citizen, data['citizens'])
db.session.commit()
except exc.SQLAlchemyError:
db.session.rollback()
error = json.dumps({'error': 'Cannot insert citizens into db'})
return json_response(error, 400)
import_id = {"data": {"import_id": import_id}}
return json_response(json.dumps(import_id))
def download(
self,
bucket: str,
path_source: str,
path_destination: str,
configuration: dict = None,
) -> None:
"""Function to download the object from a bucket to disk.
Args:
bucket: Bucket name.
path_source: Path to locate the object in bucket.
path_destination: Fs path to store the object to.
configuration: Transfer config parameters.
See: https://boto3.amazonaws.com/v1/documentation/api/1.14.2/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig
Raises:
exceptions.ObjectNotFound: Raised when the object not found.
exceptions.BucketNotFound: Raised when the bucket not found.
exceptions.DestinationPathError: Raised when cannot save object to provided location.
exceptions.DestinationPathPermissionsError: Raised when cannot save object to provided
location due to lack of permissons.
"""
if configuration:
try:
_ = validate(Client.S3_TRANSFER_SCHEMA, configuration)
except JsonSchemaException as ex:
raise exceptions.ConfigurationError(ex)
else:
configuration = {k: v["default"]
for k, v in Client.S3_TRANSFER_SCHEMA["properties"].items()
}
try:
self.client.download_file(
Filename=path_destination,
Bucket=bucket,
Key=path_source,
Config=boto3.s3.transfer.TransferConfig(**configuration),
)
except (NotADirectoryError, FileNotFoundError):
raise exceptions.DestinationPathError(
f"Cannot download file to {path_destination}"
)
except PermissionError:
raise exceptions.DestinationPathPermissionsError(
f"Cannot download file to {path_destination}"
)
except ClientError as ex:
if type(ex).__name__ == "NoSuchBucket":
raise exceptions.BucketNotFound(f"Bucket '{bucket}' not found.")
if ex.response["Error"]["Code"] == "404":
raise exceptions.ObjectNotFound(
f"Object '{path_source}' not found in bucket '{bucket}'"
)
raise Exception(ex) # pragma: no cover
def __init__(self, configuration: dict = None) -> None:
if configuration:
try:
_ = validate(Client.CLIENT_CONFIG_SCHEMA, configuration)
except JsonSchemaException as ex:
raise exceptions.ConfigurationError(ex)
else:
configuration = {}
self.client = boto3.client("s3", **configuration)
def test_export_schemas(self):
txn = create_tx()
txn_id = "an id"
txn.txn_id = txn_id
fastjsonschema.validate(schema.transaction_full_schema,
txn.export_as_full())
fastjsonschema.validate(schema.transaction_stripped_schema,
txn.export_as_stripped())
fastjsonschema.validate(schema.get_transaction_queue_task_schema(),
txn.export_as_queue_task())
fastjsonschema.validate(schema.transaction_search_index_schema,
txn.export_as_search_index())
def validate_fields(self, fqid: str, fields: Dict[str, Any]) -> None:
model = model_registry[get_collection_from_fqid(fqid)]()
for field_name, value in fields.items():
field = model.get_field(field_name)
if isinstance(field, BaseTemplateField) and field.is_template_field(
field_name
):
schema = {
"type": ["array", "null"],
"items": {"type": "string"},
}
else:
schema = field.get_schema()
try:
validate(schema, value)
except JsonSchemaException as e:
raise JsonSchemaException(
f"Invalid data for {fqid}/{field_name}: " + e.message
)
def test_seconds_high_range(self):
meta = {
"version": "latest",
"env": {
"a variable": "a value"
},
"txn_type": "an txn_type",
"id": "an id",
"image": "docker/image:1.0.0",
"status": "a status",
"cmd": "none?",
"execution_order": "serial",
"seconds": 61,
}
try:
fastjsonschema.validate(schema.smart_contract_create_schema_v1,
meta)
except fastjsonschema.JsonSchemaException as e:
self.assertEqual(
str(e), "data.seconds must be smaller than or equal to 60")
return
self.assertFail() # Force test failure if validation does not throw
def test_from_input(self):
meta = {
"version": "3",
"env": {
"a_variable": "a value"
},
"txn_type": "an txn_type",
"id": "an id",
"image": "docker/image:1.0.0",
"status": "a status",
"cmd": "none?",
"execution_order": "serial",
}
fastjsonschema.validate(schema.smart_contract_create_schema_v1, meta)
test = smart_contract_model.new_contract_from_user(meta)
for key in meta.keys():
if key != "id" and key != "status" and key != "version":
self.assertEqual(test.__dict__[key], meta[key])
meta["version"] = "1"
test = smart_contract_model.new_from_at_rest(meta)
del meta["version"]
for key in meta.keys():
self.assertEqual(test.__dict__[key], meta[key])
def assume_role(configuration: dict) -> dict:
"""Function to assume an AWS role.
Args:
configuration: Config dict:
{
"aws_access_key_id": str,
"aws_secret_access_key": str,
"region_name": str,
"role_arn": str,
}
Returns:
Dict with the temp credentials:
{
"aws_access_key_id": str,
"aws_secret_access_key": str,
"aws_session_token": str,
}
Raises:
ConnectionError: Raised when connection cannot be established,
e.g. credentials not found.
"""
try:
_ = validate(CONFIG_SCHEMA, configuration)
except JsonSchemaException as ex:
raise ConfigurationError(ex)
role_arn = configuration.pop("role_arn")
try:
client = boto3.client("sts", **configuration)
resp = client.assume_role(RoleArn=role_arn,
RoleSessionName="s3-interface")
except ClientError as ex:
if ex.response['Error']['Code'] == "InvalidClientTokenId":
raise ConnectionError("Invalid client token")
except (PartialCredentialsError, CredentialRetrievalError,
NoCredentialsError) as ex: # pragma: no cover
raise ConnectionError(ex)
if resp["ResponseMetadata"]["HTTPStatusCode"] == 200:
return {
"aws_access_key_id": resp["Credentials"]["AccessKeyId"],
"aws_secret_access_key": resp["Credentials"]["SecretAccessKey"],
"aws_session_token": resp["Credentials"]["SessionToken"],
}
return {} # pragma: no cover
def __init__(self,
config: dict = CONF_DEFAULT,
autocommit: bool = True) -> None:
try:
_ = fastjsonschema.validate(Client.SCHEMA, config)
except fastjsonschema.JsonSchemaException as ex:
raise exceptions.ConfigurationError(ex)
try:
self.conn = psycopg2.connect(**config)
except psycopg2.DatabaseError as ex:
raise exceptions.DatabaseConnectionError(ex)
self.autocommit = autocommit
self.conn.set_session(autocommit=self.autocommit)
def test_assume_role() -> None:
OUTPUT_KEYS = {
"aws_access_key_id", "aws_secret_access_key", "aws_session_token"
}
tests = [
{
"valid": True,
"in": {
"region_name": "eu-central-1",
"role_arn": "arn:aws:iam::111111111111:role/test",
},
},
{
"valid": False,
"in": {
"region_name": "eu-central-1",
"role_arn": "arn:aws:iam::11111111111:role/test",
},
},
]
for test in tests:
if test["valid"]:
temp_credentials = module.assume_role(test["in"])
missing = OUTPUT_KEYS.difference(set(temp_credentials.keys()))
if missing:
LOGGER.error(
f"""Key(s) '{"', '".join(missing)}' is(are) missing - 'assume_role' method."""
)
sys.exit(1)
try:
_ = validate(s3_client.CLIENT_CONFIG_SCHEMA, temp_credentials)
except JsonSchemaException as ex:
LOGGER.error("Temp keys validation failure.")
sys.exit(1)
else:
try:
temp_credentials = module.assume_role(test["in"])
except Exception as ex:
if type(ex).__name__ != "ConfigurationError":
LOGGER.error(
"Wrong error type to handle Config error error")
sys.exit(1)
def _validate_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""Validates the model config.
Args:
config: Model configuration.
Returns:
Config in case it's valid.
Raises:
ModelConfigError: When model config fails validation.
"""
self._logger.debug("Validate model's config with %s", str(self))
try:
return validate(self.schema(), config)
except Exception as ex:
message = f"Config failed validation, the config {config} {str(self)}.\n{str(ex)}"
raise ModelConfigError(message) from ex