def test_schema_round_trip(self, mimetype):
data = DataToTest()
schema = SchemaToTest()
headers = dict()
encoded = encode_content(
data, mimetype=mimetype, headers=headers, data_schema=schema
)
print(encoded.decode())
assert isinstance(encoded, bytes)
header_content_type = headers["Content-Type"]
if mimetype is None:
assert header_content_type == MimeType.JSON.value
else:
assert header_content_type == MimeType.from_name(mimetype).value
loaded, decoded = decode_content(
encoded, mimetype=mimetype, data_schema=schema, allow_sniff=True
)
assert isinstance(loaded, DataToTest)
assert loaded == data
assert isinstance(decoded, Mapping)
def validate_media(self, mimetype: MimeTypeTolerant) -> None:
if self.media is None:
return
if self.req_data is None:
raise DataValidationError(
f"Expected: {repr(self.media)}, got {None}", response=None)
try:
self.req_data_decoded, _ = decode_content(content=self.req_data,
mimetype=mimetype,
allow_sniff=True)
except ContentTypeUnknownError:
pass
# This handles casting bson docs (which are not comparable) to dicts / lists
self.req_data_decoded = self._cast_bson_to_dict(self.req_data_decoded)
if isinstance(self.media, type(self.req_data)):
check_data = self.req_data
else:
check_data = self.req_data_decoded
# Check the media
try:
assert check_data == self.media
except AssertionError:
raise DataValidationError(
f"Expected: {repr(self.media)}, got {repr(check_data)}",
response=None)
def _validate_response_content(
response: requests.Response,
data_schema: Optional[Schema] = None) -> Optional[Any]:
if data_schema is None:
return None
try:
try:
mimetype: Optional[MimeTypeTolerant] = MimeType.from_name(
response.headers.get("Content-Type"))
except ValueError:
mimetype = None
loaded, _ = decode_content(
response.content,
mimetype=mimetype,
data_schema=data_schema,
allow_sniff=True,
)
return loaded
except (json.JSONDecodeError, InvalidBSON, InvalidDocument,
UnicodeDecodeError):
raise ContentDecodeError("Could not load response data.")
except ValidationError:
raise DataValidationError("Error validating returned data")
def test_register_mimetype_encoders(self):
from spantools import DEFAULT_ENCODERS, DEFAULT_DECODERS
def csv_encode(data: List[dict]) -> bytes:
encoded = io.StringIO()
headers = list(data[0].keys())
writer = csv.DictWriter(encoded, fieldnames=headers)
writer.writeheader()
writer.writerows(data)
return encoded.getvalue().encode()
def csv_decode(data: bytes) -> List[Dict[str, Any]]:
csv_file = io.StringIO(data.decode())
reader = csv.DictReader(csv_file)
return [row for row in reader]
custom_encoders = copy.copy(DEFAULT_ENCODERS)
custom_decoders = copy.copy(DEFAULT_DECODERS)
custom_encoders["text/csv"] = csv_encode
custom_decoders["text/csv"] = csv_decode
data = [{"key": "value1"}, {"key": "value2"}]
encoded = encode_content(data, mimetype="text/csv", encoders=custom_encoders)
assert isinstance(encoded, bytes)
loaded, decoded = decode_content(
encoded, mimetype="text/csv", decoders=custom_decoders
)
assert loaded == decoded == data
def test_bson_single_list(self):
data = [{"key": "value"}]
encoded = encode_content(content=data, mimetype=MimeType.BSON)
loaded, decoded = decode_content(encoded, mimetype=MimeType.BSON)
assert isinstance(decoded, list)
assert len(decoded) == 1
assert isinstance(decoded[0], RawBSONDocument)
assert dict(decoded[0]) == {"key": "value"}
def test_data_round_trip(self, mimetype):
data = {"key": 10}
headers = dict()
encoded = encode_content(data, mimetype=mimetype, headers=headers)
print(str(encoded))
assert isinstance(encoded, bytes)
assert headers["Content-Type"] == MimeType.from_name(mimetype).value
loaded, decoded = decode_content(encoded, mimetype=mimetype)
assert dict(decoded) == dict(loaded) == data
def test_sniff_json(self):
data = {"key": 10}
headers = dict()
encoded = encode_content(data, mimetype=None, headers=headers)
print(encoded.decode())
assert isinstance(encoded, bytes)
assert headers["Content-Type"] == MimeType.JSON.value
loaded, decoded = decode_content(encoded, allow_sniff=True)
assert decoded == loaded == data
async def media(self) -> Optional[MediaType]:
"""
Replacement for request's ``Request.media()``. Can handle bson with no special
modification, and loads through schema when desired.
"""
if self._media is not NOT_LOADED:
self._media = cast(Optional[MediaType], self._media)
return self._media
if self._load_options is LoadOptions.IGNORE:
schema = None
else:
schema = self._schema
if isinstance(schema, MimeType):
mimetype: MimeTypeTolerant = schema
schema = None
else:
mimetype = self.mimetype
content: Optional[bytes] = await self.content
if content == b"":
content = None
if content is None and self._schema is None:
return None
try:
loaded, mimetype_decoded = decode_content( # type: ignore
content=content,
mimetype=mimetype,
data_schema=schema,
allow_sniff=True,
decoders=self._decoders,
)
except ValidationError as error:
raise RequestValidationError(
message=REQ_VALIDATION_ERROR_MESSAGE, error_data=error.messages
)
except (ContentDecodeError, ContentTypeUnknownError):
raise RequestValidationError("Media could not be decoded.")
if self._load_options is LoadOptions.VALIDATE_ONLY:
loaded = mimetype_decoded
self._media = mimetype_decoded
self._media_loaded = loaded
self._media = cast(Optional[MediaType], self._media)
return self._media
def test_text_round_trip(self, mimetype):
data = "test text"
headers = dict()
encoded = encode_content(data, mimetype=mimetype, headers=headers)
print(encoded.decode())
assert isinstance(encoded, bytes)
assert headers["Content-Type"] == MimeType.TEXT.value
loaded, decoded = decode_content(encoded, mimetype=mimetype)
assert loaded == decoded == data
assert isinstance(loaded, str)
assert isinstance(decoded, str)
assert not isinstance(loaded, bytes)
assert not isinstance(decoded, bytes)
def validate_media_type(self, mimetype: MimeTypeTolerant) -> None:
if self.req_data is None:
return
try:
decoded, _ = decode_content(self.req_data, mimetype=mimetype)
except ContentTypeUnknownError:
return
except NoContentError:
# This used to return None, now we need to catch the error.
decoded = None
except BaseException:
raise DataTypeValidationError(
f"Request content was not expected type {mimetype}",
response=None)
if self.req_data_decoded is None:
self.req_data_decoded = decoded
def test_dump_bson_compatible_objects(
self, mimetype, feed_bson: bool, feed_list: bool
):
data = {
"uuid": uuid.uuid4(),
"datetime": dt_factory(),
"bytes": b"Some Bin Data",
"raw_bson": RawBSONDocument(
BSON.encode({"key": "value", "nested": {"key": "value"}})
),
"raw_bson_list": [
RawBSONDocument(
BSON.encode({"key": "value", "nested": {"key": "value"}})
),
RawBSONDocument(
BSON.encode({"key": "value", "nested": {"key": "value"}})
),
],
}
if feed_bson:
data = RawBSONDocument(BSON.encode(data))
if feed_list:
data = [data, data]
headers = dict()
encoded = encode_content(data, mimetype=mimetype, headers=headers)
try:
print("ENCODED:", encoded.decode(), "", sep="\n")
except BaseException:
print("ENCODED:", encoded, "", sep="\n")
loaded, decoded = decode_content(encoded, mimetype=mimetype)
print("DECODED:", loaded, sep="\n")
if feed_list:
loaded = loaded[0]
data = data[0]
data = dict(data)
data["raw_bson"] = dict(data["raw_bson"])
data["raw_bson"]["nested"] = dict(data["raw_bson"]["nested"])
assert str(loaded["uuid"]) == str(data["uuid"])
assert set(loaded.keys()) == set(data.keys())
def media(self) -> Optional[MediaType]:
if self._media is NOT_LOADED:
content: Optional[bytes] = self.content
if content == b"":
content = None
if content is None:
self._media = content
else:
self._media_loaded, self._media = decode_content(
content,
mimetype=self.mimetype,
allow_sniff=True,
data_schema=self.schema,
decoders=self._decoders,
)
self._media = cast(Optional[MediaType], self._media)
return self._media
def test_proto_schema_round_trip(self):
"""
Tests that we can use protobuf classes as schemas.
"""
echo = Echo(message="some message")
serialized = encode_content(echo, mimetype=MimeType.PROTO, data_schema=Echo)
assert isinstance(serialized, bytes)
deserialized, raw = decode_content(serialized, MimeType.PROTO, Echo)
assert isinstance(deserialized, Echo)
assert echo.message == "some message"
assert echo is not deserialized
assert isinstance(raw, bytes)
assert raw == serialized
def test_no_decoder_for_mimetype(self):
with pytest.raises(ContentTypeUnknownError):
decode_content(b"Some Bin Data", mimetype="application/unknown")
async def handle_response_aio(
response: ClientResponse,
valid_status_codes: Union[int, Tuple[int, ...]] = 200,
data_schema: Optional[Union[Schema, MimeType]] = None,
api_errors_additional: Optional[Dict[int, Type[APIError]]] = None,
current_data_object: Optional[ModelType] = None,
data_object_updater: Optional[Callable[[ModelType, Any], None]] = None,
decoders: DecoderIndexType = DEFAULT_DECODERS,
) -> ResponseData:
"""
Examines response from SpanReed service and raises reported errors.
:param response: from aiohttp
:param valid_status_codes: Valid return http code(s).
:param data_schema: Schema object for loading responses.
:param api_errors_additional: Code, Error Class Mapping of Additional APIError types
the response may return.
:param current_data_object: Current object which represents response payload. Will
be updated in-place with response data.
:param data_object_updater: Callable which takes args:
(current_data_object, new_data_object). Used to update current_data_object
in place of the default updater.
:return: Loaded data, raw data mapping (dict or bson record).
:raises ResponseStatusError: If status code does match.
:raises ContentTypeUnknownError: If content-type is not a type that is known.
:raises marshmallow.ValidationError: If data not consistent with schema.
"""
# Try to raise error, catch if it does not exist.
try:
raise Error.from_headers(response.headers).to_exception(api_errors_additional)
except NoErrorReturnedError:
pass
_check_status_code(
received_code=response.status,
valid_status_codes=valid_status_codes,
response=response,
)
content = await response.read()
if content or data_schema is not None:
try:
loaded_data, decoded_data = decode_content(
content=content,
mimetype=MimeType.from_headers(response.headers),
data_schema=data_schema,
allow_sniff=True,
decoders=decoders,
)
except ContentDecodeBase as error:
raise ContentDecodeError(str(error), response=response)
except ContentTypeUnknownBase as error:
raise ContentTypeUnknownError(str(error), response=response)
else:
loaded_data, decoded_data = None, None
if current_data_object is not None:
_update_data(
current_data_object=current_data_object,
new_data_object=loaded_data,
object_updater=data_object_updater,
)
loaded_data = current_data_object
return ResponseData(resp=response, loaded=loaded_data, decoded=decoded_data)
def test_sniff_content_failure(self):
with pytest.raises(ContentDecodeError):
decode_content(b"Some Bin Data", mimetype=None, allow_sniff=True)
def test_no_mimetype_disallow_sniff(self):
with pytest.raises(ContentTypeUnknownError):
decode_content(b"Some Bin Data", None)
def test_no_content_is_decode_error(self):
"""
Tests that a 'NoContentError' is a ContentDecodeError.
"""
with pytest.raises(ContentDecodeError):
decode_content(b"")
def test_no_content_error(self):
"""
Tests that a 'NoContentError' is thrown when a blank byte-string is passed.
"""
with pytest.raises(NoContentError):
decode_content(b"")
def test_json_decoder_error(self):
with pytest.raises(ContentDecodeError):
encoded = json.dumps("Some Data").encode()
decode_content(encoded, mimetype=MimeType.JSON)