def validate_data_against_schema(data: Dict, schema: Dict): """Validate dict data against given JSON Schema Parameters ---------- data : Dict Data set to be validated schema : Dict JSON Schema to validate against Raises ------ SchemaValidationError When schema validation fails against data set InvalidSchemaFormatError When JSON schema provided is invalid """ try: fastjsonschema.validate(definition=schema, data=data) except fastjsonschema.JsonSchemaException as e: message = f"Failed schema validation. Error: {e.message}, Path: {e.path}, Data: {e.value}" # noqa: B306, E501 raise SchemaValidationError(message) except (TypeError, AttributeError) as e: raise InvalidSchemaFormatError( f"Schema received: {schema}. Error: {e}")
def validate_data_against_schema(data: Union[Dict, str], schema: Dict, formats: Optional[Dict] = None): """Validate dict data against given JSON Schema Parameters ---------- data : Dict Data set to be validated schema : Dict JSON Schema to validate against formats: Dict Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool Raises ------ SchemaValidationError When schema validation fails against data set InvalidSchemaFormatError When JSON schema provided is invalid """ try: formats = formats or {} fastjsonschema.validate(definition=schema, data=data, formats=formats) except (TypeError, AttributeError, fastjsonschema.JsonSchemaDefinitionException) as e: raise InvalidSchemaFormatError( f"Schema received: {schema}, Formats: {formats}. Error: {e}") except fastjsonschema.JsonSchemaException as e: message = f"Failed schema validation. Error: {e.message}, Path: {e.path}, Data: {e.value}" # noqa: B306, E501 raise SchemaValidationError(message)
def firewall(args): # retrieving relative path for schema.json dirname = os.path.dirname(__file__) if args.ingress: schema_path = os.path.join(dirname, 'schema/fw-ingress.json') else: schema_path = os.path.join(dirname, 'schema/fw-egress.json') # loading schema as JSON Object with open(schema_path) as f: schema = json.load(f) data = json.loads(args.json) # Validating input against schema validate(schema, data) allow_output = assembleIngressFWString( data, "allow") if args.ingress else assembleEgressFWString( data, "allow") deny_output = assembleIngressFWString( data, "deny") if args.ingress else assembleEgressFWString( data, "deny") # result is stored into a json object json_obj = { "allow_all": allow_output["all"], "allow_tag": allow_output["tag"], "allow_sa": allow_output["sa"], "deny_all": deny_output["all"], "deny_tag": deny_output["tag"], "deny_sa": deny_output["sa"] } # # results are dumped to stdout print(json.dumps(json_obj, indent=4))
def iam(args): # retrieving relative path for schema.json dirname = os.path.dirname(__file__) schema_path = os.path.join(dirname, 'schema/iam.json') # loading schema as JSON Object with open(schema_path) as f: schema = json.load(f) data = json.loads(args.json) # Validating input against schema validate(schema, data) # building flat map. # Keys are seperated via | # Emails are seperated via space # Roles are seperated via , output = "" for k in data: output = output + k + "|" for access in data[k]: output = output + access + " " output = output + "," # result is stored into a json object json_obj = {"iam": output} # results are dumped to stdout print(json.dumps(json_obj, indent=4))
def __validate_spellbook(spellbook_contents): try: with open(SPELLBOOK_SCHEMA_PATH, "r", encoding="utf-8") as file: schema = json.load(file) validate(schema, spellbook_contents) except Exception as error: raise Exception(f"Spellbook is invalid: {error}") from error
def main(import_id, citizen_id): # Пытаемся парсить пришедие данные, как JSON. if request.get_json(silent=True) is None: # Если не удалось распарсить, выбрасываем исключение. raise ValueError("request data can't be parsed as json") # Получаем соответствующего жителя по двум # основным ключам -- номеру выгрузки и id. citizen = Citizen.query.filter_by(citizen_id=citizen_id, dataset_id=import_id).first() if citizen is None: # Если не нашли такого пользователя, выбрасываем исключение. raise ValueError( "Citizen with such citizen_id and import_id wasn't found") else: citizen_obj = request.json # Общая валидация PATCH-схемы. validate(dataset_patch_schema, citizen_obj) # Валидация отдельных полей. Проще сделать её вручную. if 'birth_date' in citizen_obj: validate_date(citizen_obj['birth_date']) if 'relatives' in citizen_obj: validate_id_not_in_relatives(citizen.citizen_id, citizen_obj['relatives']) # Валидация закончена if 'town' in citizen_obj: citizen.town = citizen_obj['town'] if 'street' in citizen_obj: citizen.street = citizen_obj['street'] if 'building' in citizen_obj: citizen.building = citizen_obj['building'] if 'apartment' in citizen_obj: citizen.apartment = citizen_obj['apartment'] if 'name' in citizen_obj: citizen.name = citizen_obj['name'] if 'birth_date' in citizen_obj: citizen.birth_date = citizen_obj['birth_date'] if 'gender' in citizen_obj: citizen.gender = citizen_obj['gender'] if 'relatives' in citizen_obj: # Получим два сета с id пользователей prev_relatives_list = unpack_relatives_to_int_list( citizen.relatives) cur_relatives_list = citizen_obj['relatives'] prev_relatives = set(prev_relatives_list) cur_relatives = set(cur_relatives_list) # Получим разницу в обоих случаях и # удалим/добавим необходимые id for relative_id in prev_relatives.difference(cur_relatives): remove_cur_citizen_from_other_relative(relative_id, citizen_id, import_id) for relative_id in cur_relatives.difference(prev_relatives): add_cur_citizen_to_other_relative(relative_id, citizen_id, import_id) citizen.relatives = pack_relatives_to_db_format( citizen_obj['relatives']) # Добавим в базу данных. db.session.commit() res = {"data": citizen.json_representation()} return jsonify(res), 200
def test_search_index_schema(self): l1block = create_l1_block() fastjsonschema.validate(schema.l1_search_index_schema, l1block.export_as_search_index()) tx_search_indexes = l1block.export_full_transactions_search_indexes() for index in tx_search_indexes.keys(): fastjsonschema.validate(schema.transaction_search_index_schema, tx_search_indexes[index])
def test_at_rest_schema(self): l4block = create_l4_block() fastjsonschema.validate(schema.l4_block_at_rest_schema, l4block.export_as_at_rest()) l4block.nonce = 1 l4block.scheme = "work" fastjsonschema.validate(schema.l4_block_at_rest_schema, l4block.export_as_at_rest())
def test_metrics(endpoint: str): with open( f"""{DIR_META}/runs/{RUN_ID[endpoint]}/metrics_{RUN_ID[endpoint]}.json""", "r" ) as fread: try: validate(CONFIG[endpoint]["metrics_schema"], json.load(fread)) except Exception as ex: raise Exception("Wrong metrics content") from ex
def change_citizen(self, import_id: int, citizen_id: int, patch_citizen_data: dict) -> dict: # check if citizen exists if not self.citizen_exists(import_id, citizen_id): raise DBHelperIDError # check patch_citizen_data try: validate(self.CHANGE_CITIZEN_SCHEMA, patch_citizen_data) except JsonSchemaException: raise DBHelperJsonSchemaError # check if patch_citizen_data have field "citizen_id" or not if 'citizen_id' in patch_citizen_data.keys(): raise DBHelperJsonSchemaError # get and change relatives, if patch_citizen_data contains relatives try: new_relatives = patch_citizen_data.pop('relatives') except KeyError: pass else: self.change_relatives(import_id, citizen_id, new_relatives) # check birth_date and change to postgresql format, if patch_citizen_data contains birth_date try: patch_citizen_data[ 'birth_date'] = self.json_date_to_postrgesql_date( patch_citizen_data['birth_date']) except KeyError: pass # same as validate_birth_date except ValueError: raise DBHelperJsonSchemaError # if patch_citizen_data is not empty, change citizen data in database if patch_citizen_data: with psycopg2.connect(**self.DB_REQUISITES) as conn: with conn.cursor() as cursor: cursor.execute( "SELECT id FROM citizens WHERE import_id = %s AND citizen_id = %s;", (import_id, citizen_id)) citizen_db_id = cursor.fetchone()[0] update_text = "UPDATE citizens SET " + \ ",".join(map(lambda key: "{0}=%({0})s".format(key), patch_citizen_data.keys())) + \ "WHERE id = %(id)s;" cursor.execute(update_text, { **patch_citizen_data, **{ "id": citizen_db_id } }) conn.commit() return self.get_citizen(import_id, citizen_id)
def validate_json_files(filter_input_paths, schema_file_path, repo_path, method='jsonschema'): error = 0 try: s = open(schema_file_path, 'r') schema = json.load(s) except JSONDecodeError as jsonerror: print( "--------------------------------------------------------------------------------------------------" ) print("JSONDecodeError in SCHEMA FILE ", schema_file_path) print(jsonerror) raise if method == 'jsonschema': for input_file in filter_input_paths: try: f = open(os.path.join(repo_path, input_file), 'r') example = json.load(f) validate(instance=example, schema=schema) except JSONDecodeError as jsonerror: print( "--------------------------------------------------------------------------------------------------" ) print("JSONDecodeError in ", input_file) print(jsonerror) error = 1 except ValidationError as valerror: print( "--------------------------------------------------------------------------------------------------" ) print("Schema ValidationError in ", input_file) print(valerror) error = 1 f.close() if error == 1: raise Exception( 'Errors Occured while validating input json files...') elif method == 'fastjsonschema': for input_file in filter_input_paths: try: f = open(os.path.join(repo_path, input_file), 'r') example = json.load(f) fastjsonschema.validate(schema, example) except Exception: print("Exception in File: ", input_file) raise f.close()
def validate_fields(self, fqid: str, fields: Dict[str, Any]) -> None: model = model_registry[get_collection_from_fqid(fqid)]() for field_name, value in fields.items(): field = model.get_field(field_name) if isinstance(field, BaseTemplateField) and "$_" in field_name: schema = { "type": ["array", "null"], "items": { "type": "string" }, } else: schema = field.get_schema() validate(schema, value)
def validate_collection(file_name) -> json: """ Validates the postman collection against the postman schema (2.1.0) :param file_name: the postman collection file path :return: validated collection json """ root = os.path.dirname(os.path.abspath(__file__)) schema_filename = os.path.join(root, POSTMAN_SCHEMA_DIR, POSTMAN_JSON_SCHEMA) json_collection = DocumentGenerator.get_json_file(file_name) json_schema = DocumentGenerator.get_json_file(schema_filename) validate(json_schema, json_collection) return json_collection
def validator(schema: Dict[str, Any], obj: Dict[str, Any]) -> bool: """Function to validate object against a jsonschema. Args: schema: Jsonschema. obj: Object to validate. Returns: Flag showing if the object is valid. """ try: validate(schema, obj) return True except (JsonSchemaException, JsonSchemaDefinitionException): return False
def wrapper(request, *args, **kwargs): data = json.loads(request.body.decode()) try: validated_data = validate(schema, data) return func(request, validated_data, *args, **kwargs) except JsonSchemaException as ex: return HttpResponseBadRequest(ex.message)
def validate_config(config: object): with open(path.dirname(path.abspath(__file__)) + '/config.schema.json', encoding="utf-8") as file: schema = json.load(file) try: validate(schema, config) return True except JsonSchemaException as e: logging.warning("Your config is invalid: %s!", e) logging.warning( "NOTE: Support for legacy formats of the configuration will be dropped in future releases!" ) logging.warning( "Please update your configuration file according to https://github.com/layereight/MFRC522-trigger#json-schema." ) return False
def __validate_json_with_schema(directory_client: DataLakeDirectoryClient, json_schema_file_path: str, data_dict: Dict): file_client = directory_client.get_file_client(json_schema_file_path) schema = __get_validation_schema(file_client) try: fastjsonschema.validate(schema, data_dict) except (TypeError, fastjsonschema.JsonSchemaDefinitionException, AttributeError) as error: raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=f'Invalid schema definition: {getattr(error, "message", error)}') from error except fastjsonschema.JsonSchemaValueException as error: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail={ 'message': f'JSON Schema validation error: {error.message}', 'name': f'{error.name}', 'rule': f'{error.rule}', 'rule_definition': f'{error.rule_definition}' }) from error
def main(): # Пытаемся парсить пришедие данные, как JSON. if request.get_json(silent=True) is None: # Если не удалось распарсить, выбрасываем исключение. raise ValueError("request data can't be parsed as json") # Проверяем request.json на соответствие схеме валидации. validate(import_schema, request.json) # validate(instance=request.json, schema=import_schema) citizens = request.json['citizens'] # Получим счётчик из БД dataset_counter = get_dataset_counter() # Если будет неудача, то import_id не изменится dataset = Dataset(id=(dataset_counter.counter + 1)) for citizen_obj in citizens: # Валидации для одного жителя. do_single_citizen_validations(citizen_obj) citizen = Citizen( citizen_id=citizen_obj['citizen_id'], town=citizen_obj['town'], street=citizen_obj['street'], building=citizen_obj['building'], apartment=citizen_obj['apartment'], name=citizen_obj['name'], birth_date=citizen_obj['birth_date'], gender=citizen_obj['gender'], relatives=pack_relatives_to_db_format(citizen_obj['relatives']), dataset=dataset) # Валидации на возможные некорректные данные # пользователей в поле relatives validate_citizens_ids_intersection(dataset.citizens) validate_relatives(dataset.citizens) # Данные корректны, добавим в бд. # Добавление происходит следующим образом: # каждому пользователю сопоставляется # конкретный dataset (выгрузка). При условии, что # все данные пользователей корректны, # мы добавляем данную выгрузку в БД. db.session.add(dataset) db.session.commit() success_response = { "data": { "import_id": dataset_counter.inc() } } return jsonify(success_response), 201
def validate(self, input_data): try: with open(self.get_json_schema(), 'r') as file: schema = json.load(file) return True, fastjsonschema.validate(schema, input_data) except fastjsonschema.exceptions.JsonSchemaException as e: return False, {'json_schema': {'valid': False, 'reason': str(e)}} except Exception as e: return False, {'input': {'valid': False, 'reason': str(e)}}
def imports(): if request.content_type != JSON_MIME_TYPE: error = json.dumps({'error': 'Invalid Content Type'}) return json_response(error, 400) data = request.json try: validate(citizens_schema, data) except JsonSchemaException as e: error = json.dumps({'error': f'{e}'}) return json_response(error, 400) try: json_validation(data) except ValueError as e: error = json.dumps({'error': f'{e}'}) return json_response(error, 400) except KeyError: error = json.dumps( {'error': 'Relatives array contains nonexistent citizen_id'}) return json_response(error, 400) max_import_id_from_table = db.session.query(func.max( Citizen.import_id)).first()[0] if max_import_id_from_table: import_id = max_import_id_from_table + 1 else: import_id = 1 for item in data['citizens']: item.update({"import_id": import_id}) try: db.session.bulk_insert_mappings(Citizen, data['citizens']) db.session.commit() except exc.SQLAlchemyError: db.session.rollback() error = json.dumps({'error': 'Cannot insert citizens into db'}) return json_response(error, 400) import_id = {"data": {"import_id": import_id}} return json_response(json.dumps(import_id))
def download( self, bucket: str, path_source: str, path_destination: str, configuration: dict = None, ) -> None: """Function to download the object from a bucket to disk. Args: bucket: Bucket name. path_source: Path to locate the object in bucket. path_destination: Fs path to store the object to. configuration: Transfer config parameters. See: https://boto3.amazonaws.com/v1/documentation/api/1.14.2/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig Raises: exceptions.ObjectNotFound: Raised when the object not found. exceptions.BucketNotFound: Raised when the bucket not found. exceptions.DestinationPathError: Raised when cannot save object to provided location. exceptions.DestinationPathPermissionsError: Raised when cannot save object to provided location due to lack of permissons. """ if configuration: try: _ = validate(Client.S3_TRANSFER_SCHEMA, configuration) except JsonSchemaException as ex: raise exceptions.ConfigurationError(ex) else: configuration = {k: v["default"] for k, v in Client.S3_TRANSFER_SCHEMA["properties"].items() } try: self.client.download_file( Filename=path_destination, Bucket=bucket, Key=path_source, Config=boto3.s3.transfer.TransferConfig(**configuration), ) except (NotADirectoryError, FileNotFoundError): raise exceptions.DestinationPathError( f"Cannot download file to {path_destination}" ) except PermissionError: raise exceptions.DestinationPathPermissionsError( f"Cannot download file to {path_destination}" ) except ClientError as ex: if type(ex).__name__ == "NoSuchBucket": raise exceptions.BucketNotFound(f"Bucket '{bucket}' not found.") if ex.response["Error"]["Code"] == "404": raise exceptions.ObjectNotFound( f"Object '{path_source}' not found in bucket '{bucket}'" ) raise Exception(ex) # pragma: no cover
def __init__(self, configuration: dict = None) -> None: if configuration: try: _ = validate(Client.CLIENT_CONFIG_SCHEMA, configuration) except JsonSchemaException as ex: raise exceptions.ConfigurationError(ex) else: configuration = {} self.client = boto3.client("s3", **configuration)
def test_export_schemas(self): txn = create_tx() txn_id = "an id" txn.txn_id = txn_id fastjsonschema.validate(schema.transaction_full_schema, txn.export_as_full()) fastjsonschema.validate(schema.transaction_stripped_schema, txn.export_as_stripped()) fastjsonschema.validate(schema.get_transaction_queue_task_schema(), txn.export_as_queue_task()) fastjsonschema.validate(schema.transaction_search_index_schema, txn.export_as_search_index())
def validate_fields(self, fqid: str, fields: Dict[str, Any]) -> None: model = model_registry[get_collection_from_fqid(fqid)]() for field_name, value in fields.items(): field = model.get_field(field_name) if isinstance(field, BaseTemplateField) and field.is_template_field( field_name ): schema = { "type": ["array", "null"], "items": {"type": "string"}, } else: schema = field.get_schema() try: validate(schema, value) except JsonSchemaException as e: raise JsonSchemaException( f"Invalid data for {fqid}/{field_name}: " + e.message )
def test_seconds_high_range(self): meta = { "version": "latest", "env": { "a variable": "a value" }, "txn_type": "an txn_type", "id": "an id", "image": "docker/image:1.0.0", "status": "a status", "cmd": "none?", "execution_order": "serial", "seconds": 61, } try: fastjsonschema.validate(schema.smart_contract_create_schema_v1, meta) except fastjsonschema.JsonSchemaException as e: self.assertEqual( str(e), "data.seconds must be smaller than or equal to 60") return self.assertFail() # Force test failure if validation does not throw
def test_from_input(self): meta = { "version": "3", "env": { "a_variable": "a value" }, "txn_type": "an txn_type", "id": "an id", "image": "docker/image:1.0.0", "status": "a status", "cmd": "none?", "execution_order": "serial", } fastjsonschema.validate(schema.smart_contract_create_schema_v1, meta) test = smart_contract_model.new_contract_from_user(meta) for key in meta.keys(): if key != "id" and key != "status" and key != "version": self.assertEqual(test.__dict__[key], meta[key]) meta["version"] = "1" test = smart_contract_model.new_from_at_rest(meta) del meta["version"] for key in meta.keys(): self.assertEqual(test.__dict__[key], meta[key])
def assume_role(configuration: dict) -> dict: """Function to assume an AWS role. Args: configuration: Config dict: { "aws_access_key_id": str, "aws_secret_access_key": str, "region_name": str, "role_arn": str, } Returns: Dict with the temp credentials: { "aws_access_key_id": str, "aws_secret_access_key": str, "aws_session_token": str, } Raises: ConnectionError: Raised when connection cannot be established, e.g. credentials not found. """ try: _ = validate(CONFIG_SCHEMA, configuration) except JsonSchemaException as ex: raise ConfigurationError(ex) role_arn = configuration.pop("role_arn") try: client = boto3.client("sts", **configuration) resp = client.assume_role(RoleArn=role_arn, RoleSessionName="s3-interface") except ClientError as ex: if ex.response['Error']['Code'] == "InvalidClientTokenId": raise ConnectionError("Invalid client token") except (PartialCredentialsError, CredentialRetrievalError, NoCredentialsError) as ex: # pragma: no cover raise ConnectionError(ex) if resp["ResponseMetadata"]["HTTPStatusCode"] == 200: return { "aws_access_key_id": resp["Credentials"]["AccessKeyId"], "aws_secret_access_key": resp["Credentials"]["SecretAccessKey"], "aws_session_token": resp["Credentials"]["SessionToken"], } return {} # pragma: no cover
def __init__(self, config: dict = CONF_DEFAULT, autocommit: bool = True) -> None: try: _ = fastjsonschema.validate(Client.SCHEMA, config) except fastjsonschema.JsonSchemaException as ex: raise exceptions.ConfigurationError(ex) try: self.conn = psycopg2.connect(**config) except psycopg2.DatabaseError as ex: raise exceptions.DatabaseConnectionError(ex) self.autocommit = autocommit self.conn.set_session(autocommit=self.autocommit)
def test_assume_role() -> None: OUTPUT_KEYS = { "aws_access_key_id", "aws_secret_access_key", "aws_session_token" } tests = [ { "valid": True, "in": { "region_name": "eu-central-1", "role_arn": "arn:aws:iam::111111111111:role/test", }, }, { "valid": False, "in": { "region_name": "eu-central-1", "role_arn": "arn:aws:iam::11111111111:role/test", }, }, ] for test in tests: if test["valid"]: temp_credentials = module.assume_role(test["in"]) missing = OUTPUT_KEYS.difference(set(temp_credentials.keys())) if missing: LOGGER.error( f"""Key(s) '{"', '".join(missing)}' is(are) missing - 'assume_role' method.""" ) sys.exit(1) try: _ = validate(s3_client.CLIENT_CONFIG_SCHEMA, temp_credentials) except JsonSchemaException as ex: LOGGER.error("Temp keys validation failure.") sys.exit(1) else: try: temp_credentials = module.assume_role(test["in"]) except Exception as ex: if type(ex).__name__ != "ConfigurationError": LOGGER.error( "Wrong error type to handle Config error error") sys.exit(1)
def _validate_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """Validates the model config. Args: config: Model configuration. Returns: Config in case it's valid. Raises: ModelConfigError: When model config fails validation. """ self._logger.debug("Validate model's config with %s", str(self)) try: return validate(self.schema(), config) except Exception as ex: message = f"Config failed validation, the config {config} {str(self)}.\n{str(ex)}" raise ModelConfigError(message) from ex