def validator(req, resp, resource, params, schema: Schema):
data = req.json
try:
schema.load(data)
except ValidationError as err:
print(err.messages)
raise falcon.HTTPBadRequest(title="Validator Error",
description=err.messages)
def _validate(schema: ma.Schema, blob: dict) -> None:
sn = schema.__class__.__name__
logger.debug(f"{sn} started validation")
try:
loaded_blob = schema.load(blob)
except ma.ValidationError as e:
logger.debug(f"{sn} failed validation")
errors = e.messages
# TODO for now just send back the first error
name, error = next(iter(errors.items()))
if len(error) > 1:
logger.info(f"multiple errors for param {name} but only returning first {error}")
error = error[0]
if error == "type":
raise Exceptions.invalid_parameter(name, blob[name], type=schema.fields[name].type_name)
elif error == "missing":
raise Exceptions.missing_parameter(name)
elif error not in error_messages:
# XXX @numberoverzero marshmallow doesn't have a complete list
# XXX of keys to provide for default_errors, and some (like Unknown field)
# XXX don't have a key at all. So we'll have to be do some string inspection here :(
if error == "Unknown field.":
raise Exceptions.unknown_parameter(name)
else:
logger.warning(f"unexpected error message during validation: {name} {error}")
raise Exceptions.internal_error()
else:
logger.debug(f"{sn} succeeded validation")
blob.clear()
blob.update(loaded_blob)
def load_object(schema: Schema, json: dict):
if json is None or bool(json) is False:
return {
"error": "must provide JSON for this request"
}, 400
result, errors = schema.load(json)
if errors is not None and bool(errors) is True:
return {
"error":{
"message": "The JSON schema you have sent is invalid",
"details": errors
}
}, 400
try:
session = get_db()
session.add(result)
session.commit()
except SQLAlchemyError as e:
capture_message(e)
raise e
return {
"error": "An Internal Service Error has occured"
}, 500
return {},200
def deserialize(string, object_class, format='json'):
if format in formats:
m = get_doc_map(object_class)
Schema = build_schema(m, object_class.__module__)
schema = Schema()
if isinstance(string, str):
data = formats[format].loads(string)
loaded_data = schema.load(data).data
else:
loaded_data = string
mod = importlib.import_module(object_class.__module__)
for key, value in loaded_data.items():
if isinstance(value, list):
sub_type = m[key][1:-1]
elements = []
for sub_element in value:
sub_type_class = getattr(mod, sub_type)
elements.append(deserialize(sub_element, sub_type_class))
loaded_data[key] = elements
elif isinstance(value, dict):
sub_type_class = getattr(mod, m[key])
element = deserialize(value, sub_type_class)
loaded_data[key] = element
return object_class(**loaded_data)
else:
raise Exception(
"Format '{format}' not supported".format(format=format))
def _export(self, export_type: Literal['vulns', 'assets', 'compliance'],
schema: Schema, **kwargs) -> Union[ExportsIterator, UUID]:
'''
Get the list of jobs for for the specified datatype.
API Documentation for the job listings for
:devportal:`assets <exports-assets-request-export>`,
:devportal:`compliance <io-exports-compliance-create>`, and
:devportal:`vulnerabilities <exports-vulns-request-export>` datatypes.
'''
export_uuid = kwargs.pop('uuid', None)
use_iterator = kwargs.pop('use_iterator', True)
when_done = kwargs.pop('when_done', False)
Iterator = kwargs.pop('iterator', ExportsIterator) # noqa: PLC0103
timeout = kwargs.pop('timeout', None)
payload = schema.dump(schema.load(kwargs))
if not export_uuid:
export_uuid = self._api.post(f'{export_type}/export',
json=payload,
box=True).export_uuid
self._log.debug(
f'{export_type} export job {export_uuid} initiated')
if use_iterator:
return Iterator(self._api,
type=export_type,
uuid=export_uuid,
_wait_for_complete=when_done,
timeout=timeout)
return UUID(export_uuid)
def update_or_create_object(
self,
model: Model,
model_schema: Schema,
model_pk_field: str,
existing_pks: typing.Set[typing.Union[str, int]],
data_row: typing.Dict,
) -> int:
"""Update or create a database entry, returning 1 for if
the data_row was successfully processed, 0 if skipped"""
try:
model_instance = model_schema.load(data_row)
if model_instance: # DirectionSchema returns None when given a bad route_id value
instance_pk = getattr(model_instance, model_pk_field)
if instance_pk in existing_pks:
# Update existing
instance_dict = ( # Some values must be converted from data_row format
model_instance.__dict__)
instance_dict.pop("_sa_instance_state", None)
self.db.session.query(model).filter(
getattr(model, model_pk_field) == instance_pk).update(
instance_dict)
else:
# Create new
self.db.session.add(model_instance)
return 1
return 0
except (ValidationError, KeyError) as e:
print(json.dumps(data_row, sort_keys=True, indent=4))
raise e
def load_data(data, load_schema: Schema) -> Dict or List:
try:
return load_schema.load(data)
except ValidationError as err:
raise HTTPBadRequest(
body=json.dumps(err.normalized_messages()),
content_type="application/json",
)
def test_toggle_load_instance_per_schema(self, models, Schema):
tname = "Teachy T"
source = {"full_name": tname}
# No per-instance override
load_instance_default = Schema()
result = load_instance_default.load(source)
default = load_instance_default.opts.load_instance
default_type = models.Teacher if default else dict
assert isinstance(result, default_type)
# Override the default
override = Schema(load_instance=not default)
result = override.load(source)
override_type = dict if default else models.Teacher
assert isinstance(result, override_type)
def _decode(*, src, schema: Schema = None) -> dict:
result = dict()
for key, value in src.items():
result[key] = [item.decode() for item in value]
if len(result[key]) == 1:
result[key] = result[key][0]
if schema:
return schema.load(result)
return result
def get_validated_request(deserializer: Schema) -> (dict, list, None):
json_input = request.get_json()
if json_input is None:
raise RequestProcessingError('Validation error; data is non-Json!')
data, errors = deserializer.load(json_input)
if errors:
raise RequestProcessingError(errors)
return data
def test_dataclass_deserialize(schema: Schema):
res: TestDTO = schema.load({
'array': {
'data': [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]],
'dtype': 'float64'
}
})
assert res.array.tobytes() == np.ones((3, 3)).tobytes()
assert res.array.dtype == np.dtype('float64')
def post_to_endpoint_with_patient_struct(
data_set_id: int,
json_data: json,
update_path: str,
summary_schema: marshmallow.Schema = None,
commit: bool = False,
return_patient_struct: Union[PatientStruct, PcorPatientStruct] = None
) -> Tuple[Optional[object], List[str], Union[PatientStruct,
PcorPatientStruct]]:
"""Calls the patient update endpoint, and returns a summary.
If the update struct is None, then the method returns the list of errors
that happened while trying to update this Patient data.
:param data_set_id: The ID of the data set that this is part of - used to
pass into the update endpoint so that we can debug on both sides.
:param json_data: A serialized structure that will be sent to update.
:param update_path: The path to post to.
:param summary_schema: An instantiated schema that will be used to
deserialize the expected response from the API.
:param commit: True/False on whether this is just to validate or actually
to commit the data to the DB.
:param return_patient_struct: Patient struct, returned as-is, used in async
requests to be able to map request to a patient.
:return: A tuple with the first being a SummaryStruct created by the
serializer, and if this value is None, then a list of errors that caused
the SummaryStruct to be None. Third returned value is a patient struct.
"""
#from django.conf import settings
logger = prefect.context.get("logger")
logger.info(f"Trying to post to https://app-9097.on-aptible.com")
err, content = requests_post("https://app-9097.on-aptible.com" +
update_path,
json_dict={
'json_data': json_data,
'commit': commit,
'data_set_id': data_set_id,
'api_key': "pBSBtzsb3OqTx57W"
})
if err:
return None, [err], return_patient_struct
response = json.loads(content)
success = response['success']
if not success:
return None, response['errors'], return_patient_struct
errors = None
summary = summary_schema.load(response['update_summary'])
if errors:
errors = ["%s: %s" % (key, err) for key, err in errors.items()]
return None, errors, return_patient_struct
return summary, [], return_patient_struct
async def parse_json_body(request: web.Request, schema: ma.Schema):
try:
parsed = schema.load(await request.json())
except (TypeError, json.JSONDecodeError):
raise RequestUnparsableJsonError
except ma.ValidationError as ex:
raise RequestValidationError(
message='Request contains schema-invalid json',
errors=ex.normalized_messages()
)
return parsed
def _try_load(schema: Schema, response: Response) -> Any:
if 200 <= response.status_code < 300:
try:
return schema.load(response.json())
except JSONDecodeError as e:
logger.exception(e.msg)
return None
except ValidationError as e:
logger.exception(str(e.messages))
return None
else:
return None
def get_json_body(self, schema: Schema = None) -> dict:
content_type = self.request.headers.get('Content-Type')
if not content_type:
return dict()
if not content_type.startswith('application/json'):
return dict()
try:
result = orjson.loads(self.request.body or b'{}')
except orjson.JSONDecodeError:
raise WidgetsParameterError
if schema:
return schema.load(result)
return result
def validate_data(raw_data: dict, schema: Schema) -> dict:
"""Helper to load and validate raw_data using schema.
Uses flask.abort to throw http errors 400 or 422 with appropriate messages.
:param raw_data: dict of raw data from request body
:param schema: marshmallow.Schema
:return: validated data
"""
if not raw_data:
return abort(400, {'message': 'No input data provided'})
try:
return schema.load(raw_data)
except ValidationError as e:
return abort(422, {'message': e.messages})
def accepts_logic(payload: dict = {},
temp: dict = {},
schema: Schema = None,
many=False):
schema = _get_or_create_schema(schema, many=many)
try:
payload['temp'] = temp
payload = schema.load(payload, unknown=INCLUDE)
del payload['temp']
except ValidationError as err:
err.messages = _parse_error_to_camel_key(err.messages)
raise LogicalValidationException(extra=err.messages)
return payload
async def parse_by_schema(request: Request, *,
schema: Schema) -> t.Dict[str, t.Any]:
try:
payload = await request.json()
except json.JSONDecodeError as e:
raise HTTPException(status_code=400,
detail=json.dumps({"message": str(e)}))
try:
return schema.load(payload)
except ValidationError as e:
raise HTTPException(
status_code=400,
detail=json.dumps({
"message": "validation error",
"details": e.normalized_messages()
}),
)
def assert_serde(
self,
dictionary: Dict[str, Any],
schema: marshmallow.Schema,
typ: type,
) -> None:
"""
Checks if the given dictionary when deserialized with the given schema has the passed type
and when serialised again is equal to the original dictionary.
"""
object = schema.load(dictionary)
self.assertEqual(type(object), typ)
parsed = schema.dump(object)
if hasattr(object, 'to_string'):
self.assertFalse('\n' in object.to_string())
self.assertDictEqual(dictionary, parsed)
def _base_request(request: Request, request_schema: Schema, response_schema: Schema,
method: Callable[[object, Optional[str]], object], is_paging: bool = False,
code=None, code_name: Optional[str] = None):
"""Request processing base method.
:param request: Request object
:param request_schema: RequestSchema Instance
:param response_schema: ResponseSchema Instance
:param method: method to be processed by the request
:param is_paging: paging type request
:param code: request identifier
:param code_name: parameter name of request identifier
:return:
"""
if not request:
return response.HttpResponseNotFound()
request_param = {}
if request.query_params:
request_param.update(request.query_params.dict())
if request.data:
request_param.update(request.data)
if code and code_name:
request_param[code_name] = code
request_obj, errors = request_schema.load(request_param)
if errors:
return response.HttpResponseBadRequest(errors)
try:
if is_paging:
path = _parse_path(request)
response_obj = method(request_obj, path)
else:
response_obj = method(request_obj)
except RequestParameterException:
return response.HttpResponseBadRequest()
except ResourceNotFoundException:
return response.HttpResponseNotFound()
except Exception as e:
return response.HttpResponseServerError()
data, _ = response_schema.dump(response_obj)
if request.method == 'POST':
return Response(data, status.HTTP_201_CREATED)
elif request.method == 'DELETE':
return Response({}, status.HTTP_204_NO_CONTENT)
else:
return Response(data, status.HTTP_200_OK)
def read_json_request(self, schema: Schema):
"""
:param schema:
:type schema: Schema descendant
:return:
"""
# Ensure body can be JSON decoded
try:
json_data = json.loads(self.request.body)
except JSONDecodeError as e:
self.set_status(self.STATUS_ERROR_EXTERNAL, reason=str(e))
self.write_error()
raise BaseApiError("Expected request body to be JSON. Received '{}'".format(self.request.body))
request_validation_errors = schema.validate(json_data)
if request_validation_errors:
self.error_messages = request_validation_errors
self.set_status(self.STATUS_ERROR_EXTERNAL, reason="Failed request schema validation")
self.write_error()
raise BaseApiError("Failed schema validation: {}".format(str(request_validation_errors)))
return schema.dump(schema.load(json_data))
def verify_data(self,
data: dict,
schema: Schema,
context: dict = None) -> dict:
"""
使用schema实例验证数据，有错误时抛出ValidateError
:param dict data: 需要验证的数据
:param Schema schema: schema实例
:param dict context: 传递给schema使用的额外数据，保存在schema的context属性中
:return:
"""
# 传递给schema使用的额外数据
if context:
schema.context = context
try:
data = schema.load(data)
except ValidationError as e:
# 合并多个验证器对于同一字段的相同错误
for key in e.messages.keys():
e.messages[key] = list(set(e.messages[key]))
raise ValidateError(e.messages, replace=True)
return data
def test_fields_empty_load_handle(app: Flask, pendulum_field_schema: Schema):
with app.app_context():
data = {"time": ""}
with pytest.raises(ValidationError):
pendulum_field_schema.load(data)
def _data_to_entity(self, schema: Schema,
data: Dict[str, Any]) -> ApplicantStatus:
return schema.load(data)
def test_fileds_none_load_handle(app: Flask, pendulum_field_schema: Schema):
with app.app_context():
data = {"time": None}
res = pendulum_field_schema.load(data)
assert res["time"] is None
def test_fields_load(app: Flask, pendulum_field_schema: Schema):
with app.app_context():
data = {"time": "1994-09-11 08:20:00"}
res = pendulum_field_schema.load(data)
assert str(res["time"]) == "1994-09-11T00:20:00+00:00"
def as_dict(schema: Schema, ob: dataclasses.dataclass) -> t.Dict[str, t.Any]:
d = schema.omit_none(schema.from_dataclass(ob))
d = schema.load(d) # or validate()?
return schema.dump(d)
def _validate(schema: marshmallow.Schema, data: Dict[str, Any]) -> Dict[str, Any]:
try:
return schema.load(data)
except marshmallow.ValidationError as ex:
abort(make_response(jsonify(errors=ex.normalized_messages()), 400))
def schema_from_request(self, schema: Schema, partial=False):
return schema.load(request.get_json() or request.form, partial=partial)
def rows_mm_deserialization_iterator(
rows_iterator: Iterable[Dict[str, object]],
row_schema: marshmallow.Schema,
n_rows_offset: int = 0,
max_n_rows: int = None,
fields_to_remove_names: Sequence[str] = None,
) -> Iterable[Tuple[int, Dict[str, object], Dict[str, object], dict]]:
"""
Marshmallow deserialization iterator.
Iterate over ``rows_iterator``, deserialize each row using ``row_schema``
and yield the data before and after deserialization, plus any
validation/deserialization errors.
:param rows_iterator:
:param row_schema:
Marshmallow schema for deserializing each row
:param n_rows_offset:
(optional) number of rows to skip (and not deserialize)
:param max_n_rows:
(optional) max number of rows to deserialize (raise exception
if exceeded); ``None`` means no limit
:param fields_to_remove_names:
(optional) the name of each field that must be removed (if it exists)
from the row
:returns:
yields a tuple of (``row_ix`` (1-based), ``row_data``,
``deserialized_row_data``, ``validation_errors``)
:raises MaxRowsExceeded:
number of data rows processed exceeded ``max_n_rows``
"""
if not n_rows_offset >= 0:
raise ValueError("Param 'n_rows_offset' must be an integer >= 0.")
fields_to_remove_names = fields_to_remove_names or ()
for row_ix, row_data in enumerate(rows_iterator, start=1):
if max_n_rows is not None and row_ix > max_n_rows + n_rows_offset:
raise MaxRowsExceeded(f"Exceeded 'max_n_rows' limit: {max_n_rows}.")
if row_ix <= n_rows_offset:
continue
for _field_name in fields_to_remove_names:
row_data.pop(_field_name, None)
try:
mm_result: marshmallow.UnmarshalResult = row_schema.load(row_data)
deserialized_row_data: dict = mm_result.data
raised_validation_errors: dict = {}
returned_validation_errors: dict = mm_result.errors
except marshmallow.ValidationError as exc:
deserialized_row_data = {}
raised_validation_errors = dict(exc.normalized_messages())
returned_validation_errors = {}
validation_errors = raised_validation_errors
if returned_validation_errors:
if row_schema.strict:
# 'marshmallow.schema.BaseSchema':
# > :param bool strict: If `True`, raise errors if invalid data are passed in
# > instead of failing silently and storing the errors.
logger.error(
"Marshmallow schema is 'strict' but validation errors were returned by "
"method 'load' ('UnmarshalResult.errors') instead of being raised. "
"Errors: %s",
repr(returned_validation_errors))
if raised_validation_errors:
logger.fatal(
"Programming error: either returned or raised validation errors "
"(depending on 'strict') but never both. "
"Returned errors: %s. Raised errors: %s",
repr(returned_validation_errors), repr(raised_validation_errors))
validation_errors.update(returned_validation_errors)
yield row_ix, row_data, deserialized_row_data, validation_errors
def __parse_request(
schema: Schema, request: HttpRequest
) -> t.Union[PostConfigOptions, DeleteConfigOption, PutConfigOptions]:
return schema.load(json.loads(request.body))