def validator(req, resp, resource, params, schema: Schema): data = req.json try: schema.load(data) except ValidationError as err: print(err.messages) raise falcon.HTTPBadRequest(title="Validator Error", description=err.messages)
def _validate(schema: ma.Schema, blob: dict) -> None: sn = schema.__class__.__name__ logger.debug(f"{sn} started validation") try: loaded_blob = schema.load(blob) except ma.ValidationError as e: logger.debug(f"{sn} failed validation") errors = e.messages # TODO for now just send back the first error name, error = next(iter(errors.items())) if len(error) > 1: logger.info(f"multiple errors for param {name} but only returning first {error}") error = error[0] if error == "type": raise Exceptions.invalid_parameter(name, blob[name], type=schema.fields[name].type_name) elif error == "missing": raise Exceptions.missing_parameter(name) elif error not in error_messages: # XXX @numberoverzero marshmallow doesn't have a complete list # XXX of keys to provide for default_errors, and some (like Unknown field) # XXX don't have a key at all. So we'll have to be do some string inspection here :( if error == "Unknown field.": raise Exceptions.unknown_parameter(name) else: logger.warning(f"unexpected error message during validation: {name} {error}") raise Exceptions.internal_error() else: logger.debug(f"{sn} succeeded validation") blob.clear() blob.update(loaded_blob)
def load_object(schema: Schema, json: dict): if json is None or bool(json) is False: return { "error": "must provide JSON for this request" }, 400 result, errors = schema.load(json) if errors is not None and bool(errors) is True: return { "error":{ "message": "The JSON schema you have sent is invalid", "details": errors } }, 400 try: session = get_db() session.add(result) session.commit() except SQLAlchemyError as e: capture_message(e) raise e return { "error": "An Internal Service Error has occured" }, 500 return {},200
def deserialize(string, object_class, format='json'): if format in formats: m = get_doc_map(object_class) Schema = build_schema(m, object_class.__module__) schema = Schema() if isinstance(string, str): data = formats[format].loads(string) loaded_data = schema.load(data).data else: loaded_data = string mod = importlib.import_module(object_class.__module__) for key, value in loaded_data.items(): if isinstance(value, list): sub_type = m[key][1:-1] elements = [] for sub_element in value: sub_type_class = getattr(mod, sub_type) elements.append(deserialize(sub_element, sub_type_class)) loaded_data[key] = elements elif isinstance(value, dict): sub_type_class = getattr(mod, m[key]) element = deserialize(value, sub_type_class) loaded_data[key] = element return object_class(**loaded_data) else: raise Exception( "Format '{format}' not supported".format(format=format))
def _export(self, export_type: Literal['vulns', 'assets', 'compliance'], schema: Schema, **kwargs) -> Union[ExportsIterator, UUID]: ''' Get the list of jobs for for the specified datatype. API Documentation for the job listings for :devportal:`assets <exports-assets-request-export>`, :devportal:`compliance <io-exports-compliance-create>`, and :devportal:`vulnerabilities <exports-vulns-request-export>` datatypes. ''' export_uuid = kwargs.pop('uuid', None) use_iterator = kwargs.pop('use_iterator', True) when_done = kwargs.pop('when_done', False) Iterator = kwargs.pop('iterator', ExportsIterator) # noqa: PLC0103 timeout = kwargs.pop('timeout', None) payload = schema.dump(schema.load(kwargs)) if not export_uuid: export_uuid = self._api.post(f'{export_type}/export', json=payload, box=True).export_uuid self._log.debug( f'{export_type} export job {export_uuid} initiated') if use_iterator: return Iterator(self._api, type=export_type, uuid=export_uuid, _wait_for_complete=when_done, timeout=timeout) return UUID(export_uuid)
def update_or_create_object( self, model: Model, model_schema: Schema, model_pk_field: str, existing_pks: typing.Set[typing.Union[str, int]], data_row: typing.Dict, ) -> int: """Update or create a database entry, returning 1 for if the data_row was successfully processed, 0 if skipped""" try: model_instance = model_schema.load(data_row) if model_instance: # DirectionSchema returns None when given a bad route_id value instance_pk = getattr(model_instance, model_pk_field) if instance_pk in existing_pks: # Update existing instance_dict = ( # Some values must be converted from data_row format model_instance.__dict__) instance_dict.pop("_sa_instance_state", None) self.db.session.query(model).filter( getattr(model, model_pk_field) == instance_pk).update( instance_dict) else: # Create new self.db.session.add(model_instance) return 1 return 0 except (ValidationError, KeyError) as e: print(json.dumps(data_row, sort_keys=True, indent=4)) raise e
def load_data(data, load_schema: Schema) -> Dict or List: try: return load_schema.load(data) except ValidationError as err: raise HTTPBadRequest( body=json.dumps(err.normalized_messages()), content_type="application/json", )
def test_toggle_load_instance_per_schema(self, models, Schema): tname = "Teachy T" source = {"full_name": tname} # No per-instance override load_instance_default = Schema() result = load_instance_default.load(source) default = load_instance_default.opts.load_instance default_type = models.Teacher if default else dict assert isinstance(result, default_type) # Override the default override = Schema(load_instance=not default) result = override.load(source) override_type = dict if default else models.Teacher assert isinstance(result, override_type)
def _decode(*, src, schema: Schema = None) -> dict: result = dict() for key, value in src.items(): result[key] = [item.decode() for item in value] if len(result[key]) == 1: result[key] = result[key][0] if schema: return schema.load(result) return result
def get_validated_request(deserializer: Schema) -> (dict, list, None): json_input = request.get_json() if json_input is None: raise RequestProcessingError('Validation error; data is non-Json!') data, errors = deserializer.load(json_input) if errors: raise RequestProcessingError(errors) return data
def test_dataclass_deserialize(schema: Schema): res: TestDTO = schema.load({ 'array': { 'data': [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], 'dtype': 'float64' } }) assert res.array.tobytes() == np.ones((3, 3)).tobytes() assert res.array.dtype == np.dtype('float64')
def post_to_endpoint_with_patient_struct( data_set_id: int, json_data: json, update_path: str, summary_schema: marshmallow.Schema = None, commit: bool = False, return_patient_struct: Union[PatientStruct, PcorPatientStruct] = None ) -> Tuple[Optional[object], List[str], Union[PatientStruct, PcorPatientStruct]]: """Calls the patient update endpoint, and returns a summary. If the update struct is None, then the method returns the list of errors that happened while trying to update this Patient data. :param data_set_id: The ID of the data set that this is part of - used to pass into the update endpoint so that we can debug on both sides. :param json_data: A serialized structure that will be sent to update. :param update_path: The path to post to. :param summary_schema: An instantiated schema that will be used to deserialize the expected response from the API. :param commit: True/False on whether this is just to validate or actually to commit the data to the DB. :param return_patient_struct: Patient struct, returned as-is, used in async requests to be able to map request to a patient. :return: A tuple with the first being a SummaryStruct created by the serializer, and if this value is None, then a list of errors that caused the SummaryStruct to be None. Third returned value is a patient struct. """ #from django.conf import settings logger = prefect.context.get("logger") logger.info(f"Trying to post to https://app-9097.on-aptible.com") err, content = requests_post("https://app-9097.on-aptible.com" + update_path, json_dict={ 'json_data': json_data, 'commit': commit, 'data_set_id': data_set_id, 'api_key': "pBSBtzsb3OqTx57W" }) if err: return None, [err], return_patient_struct response = json.loads(content) success = response['success'] if not success: return None, response['errors'], return_patient_struct errors = None summary = summary_schema.load(response['update_summary']) if errors: errors = ["%s: %s" % (key, err) for key, err in errors.items()] return None, errors, return_patient_struct return summary, [], return_patient_struct
async def parse_json_body(request: web.Request, schema: ma.Schema): try: parsed = schema.load(await request.json()) except (TypeError, json.JSONDecodeError): raise RequestUnparsableJsonError except ma.ValidationError as ex: raise RequestValidationError( message='Request contains schema-invalid json', errors=ex.normalized_messages() ) return parsed
def _try_load(schema: Schema, response: Response) -> Any: if 200 <= response.status_code < 300: try: return schema.load(response.json()) except JSONDecodeError as e: logger.exception(e.msg) return None except ValidationError as e: logger.exception(str(e.messages)) return None else: return None
def get_json_body(self, schema: Schema = None) -> dict: content_type = self.request.headers.get('Content-Type') if not content_type: return dict() if not content_type.startswith('application/json'): return dict() try: result = orjson.loads(self.request.body or b'{}') except orjson.JSONDecodeError: raise WidgetsParameterError if schema: return schema.load(result) return result
def validate_data(raw_data: dict, schema: Schema) -> dict: """Helper to load and validate raw_data using schema. Uses flask.abort to throw http errors 400 or 422 with appropriate messages. :param raw_data: dict of raw data from request body :param schema: marshmallow.Schema :return: validated data """ if not raw_data: return abort(400, {'message': 'No input data provided'}) try: return schema.load(raw_data) except ValidationError as e: return abort(422, {'message': e.messages})
def accepts_logic(payload: dict = {}, temp: dict = {}, schema: Schema = None, many=False): schema = _get_or_create_schema(schema, many=many) try: payload['temp'] = temp payload = schema.load(payload, unknown=INCLUDE) del payload['temp'] except ValidationError as err: err.messages = _parse_error_to_camel_key(err.messages) raise LogicalValidationException(extra=err.messages) return payload
async def parse_by_schema(request: Request, *, schema: Schema) -> t.Dict[str, t.Any]: try: payload = await request.json() except json.JSONDecodeError as e: raise HTTPException(status_code=400, detail=json.dumps({"message": str(e)})) try: return schema.load(payload) except ValidationError as e: raise HTTPException( status_code=400, detail=json.dumps({ "message": "validation error", "details": e.normalized_messages() }), )
def assert_serde( self, dictionary: Dict[str, Any], schema: marshmallow.Schema, typ: type, ) -> None: """ Checks if the given dictionary when deserialized with the given schema has the passed type and when serialised again is equal to the original dictionary. """ object = schema.load(dictionary) self.assertEqual(type(object), typ) parsed = schema.dump(object) if hasattr(object, 'to_string'): self.assertFalse('\n' in object.to_string()) self.assertDictEqual(dictionary, parsed)
def _base_request(request: Request, request_schema: Schema, response_schema: Schema, method: Callable[[object, Optional[str]], object], is_paging: bool = False, code=None, code_name: Optional[str] = None): """Request processing base method. :param request: Request object :param request_schema: RequestSchema Instance :param response_schema: ResponseSchema Instance :param method: method to be processed by the request :param is_paging: paging type request :param code: request identifier :param code_name: parameter name of request identifier :return: """ if not request: return response.HttpResponseNotFound() request_param = {} if request.query_params: request_param.update(request.query_params.dict()) if request.data: request_param.update(request.data) if code and code_name: request_param[code_name] = code request_obj, errors = request_schema.load(request_param) if errors: return response.HttpResponseBadRequest(errors) try: if is_paging: path = _parse_path(request) response_obj = method(request_obj, path) else: response_obj = method(request_obj) except RequestParameterException: return response.HttpResponseBadRequest() except ResourceNotFoundException: return response.HttpResponseNotFound() except Exception as e: return response.HttpResponseServerError() data, _ = response_schema.dump(response_obj) if request.method == 'POST': return Response(data, status.HTTP_201_CREATED) elif request.method == 'DELETE': return Response({}, status.HTTP_204_NO_CONTENT) else: return Response(data, status.HTTP_200_OK)
def read_json_request(self, schema: Schema): """ :param schema: :type schema: Schema descendant :return: """ # Ensure body can be JSON decoded try: json_data = json.loads(self.request.body) except JSONDecodeError as e: self.set_status(self.STATUS_ERROR_EXTERNAL, reason=str(e)) self.write_error() raise BaseApiError("Expected request body to be JSON. Received '{}'".format(self.request.body)) request_validation_errors = schema.validate(json_data) if request_validation_errors: self.error_messages = request_validation_errors self.set_status(self.STATUS_ERROR_EXTERNAL, reason="Failed request schema validation") self.write_error() raise BaseApiError("Failed schema validation: {}".format(str(request_validation_errors))) return schema.dump(schema.load(json_data))
def verify_data(self, data: dict, schema: Schema, context: dict = None) -> dict: """ 使用schema实例验证数据,有错误时抛出ValidateError :param dict data: 需要验证的数据 :param Schema schema: schema实例 :param dict context: 传递给schema使用的额外数据,保存在schema的context属性中 :return: """ # 传递给schema使用的额外数据 if context: schema.context = context try: data = schema.load(data) except ValidationError as e: # 合并多个验证器对于同一字段的相同错误 for key in e.messages.keys(): e.messages[key] = list(set(e.messages[key])) raise ValidateError(e.messages, replace=True) return data
def test_fields_empty_load_handle(app: Flask, pendulum_field_schema: Schema): with app.app_context(): data = {"time": ""} with pytest.raises(ValidationError): pendulum_field_schema.load(data)
def _data_to_entity(self, schema: Schema, data: Dict[str, Any]) -> ApplicantStatus: return schema.load(data)
def test_fileds_none_load_handle(app: Flask, pendulum_field_schema: Schema): with app.app_context(): data = {"time": None} res = pendulum_field_schema.load(data) assert res["time"] is None
def test_fields_load(app: Flask, pendulum_field_schema: Schema): with app.app_context(): data = {"time": "1994-09-11 08:20:00"} res = pendulum_field_schema.load(data) assert str(res["time"]) == "1994-09-11T00:20:00+00:00"
def as_dict(schema: Schema, ob: dataclasses.dataclass) -> t.Dict[str, t.Any]: d = schema.omit_none(schema.from_dataclass(ob)) d = schema.load(d) # or validate()? return schema.dump(d)
def _validate(schema: marshmallow.Schema, data: Dict[str, Any]) -> Dict[str, Any]: try: return schema.load(data) except marshmallow.ValidationError as ex: abort(make_response(jsonify(errors=ex.normalized_messages()), 400))
def schema_from_request(self, schema: Schema, partial=False): return schema.load(request.get_json() or request.form, partial=partial)
def rows_mm_deserialization_iterator( rows_iterator: Iterable[Dict[str, object]], row_schema: marshmallow.Schema, n_rows_offset: int = 0, max_n_rows: int = None, fields_to_remove_names: Sequence[str] = None, ) -> Iterable[Tuple[int, Dict[str, object], Dict[str, object], dict]]: """ Marshmallow deserialization iterator. Iterate over ``rows_iterator``, deserialize each row using ``row_schema`` and yield the data before and after deserialization, plus any validation/deserialization errors. :param rows_iterator: :param row_schema: Marshmallow schema for deserializing each row :param n_rows_offset: (optional) number of rows to skip (and not deserialize) :param max_n_rows: (optional) max number of rows to deserialize (raise exception if exceeded); ``None`` means no limit :param fields_to_remove_names: (optional) the name of each field that must be removed (if it exists) from the row :returns: yields a tuple of (``row_ix`` (1-based), ``row_data``, ``deserialized_row_data``, ``validation_errors``) :raises MaxRowsExceeded: number of data rows processed exceeded ``max_n_rows`` """ if not n_rows_offset >= 0: raise ValueError("Param 'n_rows_offset' must be an integer >= 0.") fields_to_remove_names = fields_to_remove_names or () for row_ix, row_data in enumerate(rows_iterator, start=1): if max_n_rows is not None and row_ix > max_n_rows + n_rows_offset: raise MaxRowsExceeded(f"Exceeded 'max_n_rows' limit: {max_n_rows}.") if row_ix <= n_rows_offset: continue for _field_name in fields_to_remove_names: row_data.pop(_field_name, None) try: mm_result: marshmallow.UnmarshalResult = row_schema.load(row_data) deserialized_row_data: dict = mm_result.data raised_validation_errors: dict = {} returned_validation_errors: dict = mm_result.errors except marshmallow.ValidationError as exc: deserialized_row_data = {} raised_validation_errors = dict(exc.normalized_messages()) returned_validation_errors = {} validation_errors = raised_validation_errors if returned_validation_errors: if row_schema.strict: # 'marshmallow.schema.BaseSchema': # > :param bool strict: If `True`, raise errors if invalid data are passed in # > instead of failing silently and storing the errors. logger.error( "Marshmallow schema is 'strict' but validation errors were returned by " "method 'load' ('UnmarshalResult.errors') instead of being raised. " "Errors: %s", repr(returned_validation_errors)) if raised_validation_errors: logger.fatal( "Programming error: either returned or raised validation errors " "(depending on 'strict') but never both. " "Returned errors: %s. Raised errors: %s", repr(returned_validation_errors), repr(raised_validation_errors)) validation_errors.update(returned_validation_errors) yield row_ix, row_data, deserialized_row_data, validation_errors
def __parse_request( schema: Schema, request: HttpRequest ) -> t.Union[PostConfigOptions, DeleteConfigOption, PutConfigOptions]: return schema.load(json.loads(request.body))