def test_from_json_malformed():
with pytest.raises(ValueError):
Json.from_json(
'{"item1": {"id": "1", "value": "ok-1", "nested": {"ts": "1970-01-01T00:00:00+00:00"}', MockData)
with pytest.raises(ValueError):
Json.from_json('BAD STRING', MockData)
def test_to_json_invalid_types():
with pytest.raises(ValueError):
Json.to_json(
MockData(id=42, value='ok-value', nested=MockNested(ts=datetime.now(tz=timezone.utc)))) # type: ignore
with pytest.raises(ValueError):
Json.to_json(MockData(id='1', value='ok-value', nested=MockNested(ts="NOT A DATETIME"))) # type: ignore
async def __preprocess__(payload: None, context: EventContext,
request: PreprocessHook, *,
something_id: str) -> Union[str, FileUploadInfo]:
uploaded_files = []
save_path = Path(str(context.env['upload_something']['save_path']))
chunk_size = int(context.env['upload_something']['chunk_size'])
async for file_hook in request.files():
file_name = f"{file_hook.name}-{file_hook.file_name}"
path = save_path / file_name
logger.info(context, f"Saving {path}...")
await save_multipart_attachment(file_hook, path, chunk_size=chunk_size)
uploaded_file = UploadedFile(file_hook.name,
file_name,
save_path.as_posix(),
size=file_hook.size)
uploaded_files.append(uploaded_file)
args = await request.parsed_args()
if not all(x in args for x in ('id', 'user', 'attachment', 'object')):
request.status = 400
return "Missing required fields"
something_obj = Json.parse_form_field(args['object'], Something)
return FileUploadInfo(id=args['id'],
user=args['user'],
object=something_obj,
uploaded_files=uploaded_files)
async def __preprocess__(payload: None, context: EventContext,
request: PreprocessHook) -> MockData:
args = await request.parsed_args()
data = Json.parse_form_field(args['field2'], MockData)
return MockData(
value=f"field1:{args['field1']} field2:{data.value} file:{args['file']}"
)
def test_from_json_python_types():
assert Json.from_json('{"value": "str"}', str) == "str"
assert Json.from_json('{"value": 123}', int) == int(123)
assert Json.from_json('{"value": 123.45}', float) == float(123.45)
assert Json.from_json('{"value": true}', bool) is True
assert Json.from_json('{"custom": "str"}', str, key='custom') == "str"
assert Json.from_json('{"test": "dict"}', dict) == {'test': 'dict'}
assert Json.from_json('["test1", "test2"]', set) == {'test2', 'test1'}
assert Json.from_json('["test1", "test2"]', list) == ['test1', 'test2']
async def __preprocess__(payload: None, context: EventContext, request: PreprocessHook,
*, query_arg1: str) -> Union[str, MockData]:
fields = await request.parsed_args()
if any(x not in fields for x in ('field1', 'field2', 'attachment')):
request.set_status(400)
return "Missing required fields"
data = Json.parse_form_field(fields['field2'], MockData)
return MockData(value=f"field1={fields['field1']} field2={data.value} attachment={fields['attachment']}")
def test_to_json_dataobject():
assert Json.to_json(MockData(
id='test',
value='ok',
nested=MockNested(
ts=datetime.fromtimestamp(0, tz=timezone.utc)
)
)) == '{"id": "test", "value": "ok", "nested": {"ts": "1970-01-01T00:00:00+00:00"}}'
def _failed_response(context: Optional[EventContext],
e: Exception) -> web.Response:
if context:
logger.error(context, e)
logger.failed(context)
else:
logger.error(__name__, e)
info = ErrorInfo.from_exception(e)
return web.Response(status=500, body=Json.to_json(info))
def _ignored_response(context: Optional[EventContext], status: int,
e: BaseException) -> web.Response:
if context:
logger.error(context, e)
logger.ignored(context)
else:
logger.error(__name__, e)
info = ErrorInfo.from_exception(e)
return web.Response(status=status, body=Json.to_json(info))
def _get_runtime_simple_example(url: str):
res = RUNTIME_SIMPLE_EXAMPLE
res = res.replace("${HOST_NAME}", socket.gethostname())
res = res.replace("${PID}", str(os.getpid()))
res = res.replace("${URL}", url)
res = res.replace("${ENGINE_VERSION}", ENGINE_VERSION)
res = res.replace("${APPS_API_VERSION}", APPS_API_VERSION)
res = res.replace("${APPS_ROUTE_VERSION}", APPS_ROUTE_VERSION)
return Json.from_json(res, RuntimeApps)
async def store(self, key: str, value: DataObject):
"""
Stores value under specified key
:param key: str
:param value: DataObject, instance of dataclass annotated with @dataobject
"""
assert self._conn
payload_str = str(Json.to_json(value))
await self._conn.set(key, payload_str)
def text_from_json_dataobject():
assert Json.from_json(
'{"id": "test", "value": "ok", "nested": {"ts": 0.0}}', MockData
) == MockData(
id='test',
value='ok',
nested=MockNested(
ts=datetime.fromtimestamp(0.0)
)
)
async def store(self, key: str, value: DataObject) -> str:
"""
Stores value under specified key
:param key: str
:param value: DataObject, instance of dataclass annotated with @dataobject
:return: str, path where the object was stored
"""
payload_str = Json.to_json(value)
return await self._save_file(payload_str,
path=self.path,
file_name=key + SUFFIX)
async def get(self, key: str, *, datatype: Type[DataObject]) -> Optional[DataObject]:
"""
Retrieves value under specified key, converted to datatype
:param key: str
:param datatype: dataclass implementing @dataobject (@see DataObject)
:return: instance of datatype or None if not found
"""
assert self._conn
payload_str = await self._conn.get(key, encoding='utf-8')
if payload_str:
return Json.from_json(payload_str, datatype)
return None
async def get(self, key: str, *,
datatype: Type[DataObject]) -> Optional[DataObject]:
"""
Retrieves value under specified key, converted to datatype
:param key: str
:param datatype: dataclass implementing @dataobject (@see DataObject)
:return: instance of datatype or None if not found
"""
payload_str = await self._load_file(path=self.path,
file_name=key + SUFFIX)
if payload_str:
return Json.from_json(payload_str, datatype)
return None
async def _request_process_payload(
context: EventContext, datatype: Optional[Type[EventPayloadType]],
request: web.Request) -> Optional[EventPayloadType]:
"""
Extract payload from request.
Returns payload if parsing succeeded. Raises BadRequest if payload fails to parse
"""
try:
payload_raw = await request.read()
if (payload_raw is None) or (payload_raw == b''):
return None
payload = Json.from_json(
payload_raw, datatype) if datatype else payload_raw.decode()
return payload # type: ignore
except ValueError as e:
logger.error(context, e)
raise BadRequest(e) from e
def expected_log_entries() -> LogBatch:
return Json.from_json(
"""
{
"entries": [
{
"ts": "2021-06-02 18:01:44,290",
"msg": "START",
"app_name": "simple-example",
"app_version": "${APPS_API_VERSION}",
"event_name": "query_something",
"event": "simple_example.${APP_VERSION}.query_something",
"extra": {
"track.operation_id": "f2659a30-5ac4-4dd4-b1f7-9a00db0bf7d5",
"track.request_id": "7ee59fa7-c1e4-4a60-a79b-a25dbbd6cb82",
"track.request_ts": "2021-06-02T18:01:44.289394+00:00",
"track.caller": "test",
"track.session_id": "test"
},
"host": "host",
"pid": "17031"
},
{
"ts": "2021-06-02 18:01:44,303",
"msg": "DONE",
"app_name": "simple-example",
"app_version": "${APPS_API_VERSION}",
"event_name": "query_something",
"event": "simple_example.${APP_VERSION}.query_something",
"extra": {
"response.status": "404",
"metrics.duration": "13.057",
"track.operation_id": "f2659a30-5ac4-4dd4-b1f7-9a00db0bf7d5",
"track.request_id": "7ee59fa7-c1e4-4a60-a79b-a25dbbd6cb82",
"track.request_ts": "2021-06-02T18:01:44.289394+00:00",
"track.caller": "test",
"track.session_id": "test"
},
"host": "host",
"pid": "17031"
}
]
}
""".replace("${APPS_API_VERSION}",
APPS_API_VERSION).replace("${APP_VERSION}", APP_VERSION), LogBatch)
def test_to_json_dict_mixed():
assert json.loads(Json.to_json({
"item1": {
'item': 1,
'data': MockData(id='1', value='ok-1', nested=MockNested(ts=datetime.fromtimestamp(0, tz=timezone.utc)))
},
"item2": {
'item': 2,
'data': MockData(id='2', value='ok-2', nested=MockNested(ts=datetime.fromtimestamp(0, tz=timezone.utc)))
}
})) == {'item1': {'data': {'id': '1',
'nested': {'ts': '1970-01-01T00:00:00+00:00'},
'value': 'ok-1'},
'item': 1},
'item2': {'data': {'id': '2',
'nested': {'ts': '1970-01-01T00:00:00+00:00'},
'value': 'ok-2'},
'item': 2}}
def _deser_json_utf8(data: bytes,
datatype: Type[EventPayloadType]) -> EventPayload:
return Json.from_json(data.decode('utf-8'), datatype)
def test_from_json_missing_fields():
with pytest.raises(ValueError):
Json.from_json('{"id": "1", "value": "ok-1", "nested": {}', MockData)
with pytest.raises(ValueError):
Json.from_json('{"value": 42, "nested": {"ts": "1970-01-01T00:00:00+00:00"}}', MockData)
def test_from_json_invalid_types():
with pytest.raises(ValueError):
Json.from_json('{"id": "1", "value": "ok-1", "nested": {"ts": "INVALID DATE"}}', MockData)
with pytest.raises(ValueError):
Json.from_json('{"id": 42, "value": 42, "nested": {"ts": "1970-01-01T00:00:00+00:00"}}', MockData)
def test_to_json_dict_dataobject():
assert json.loads(Json.to_json({
'item1': MockData(id='1', value='ok-1', nested=MockNested(ts=datetime.fromtimestamp(0, tz=timezone.utc))),
'item2': MockData(id='2', value='ok-2', nested=MockNested(ts=datetime.fromtimestamp(0, tz=timezone.utc))),
})) == {"item1": {"id": "1", "value": "ok-1", "nested": {"ts": "1970-01-01T00:00:00+00:00"}},
"item2": {"id": "2", "value": "ok-2", "nested": {"ts": "1970-01-01T00:00:00+00:00"}}}
def test_from_json_dataobject_do_not_validate():
data = '{"id": "test", "value": "not-ok"}'
assert Json.from_json(data, MockDataDoNotValidate) \
== MockDataDoNotValidate(id='test', value='not-ok') # type: ignore
def _ser_json_utf8(data: EventPayload, level: int) -> bytes:
return Json.to_json(data).encode('utf-8')
def test_to_json_python_types():
assert Json.to_json('str', key=None) == '"str"'
assert Json.to_json(123, key=None) == '123'
assert Json.to_json(123.45, key=None) == '123.45'
assert Json.to_json(True, key=None) == 'true'
assert Json.to_json('str', key='test') == '{"test": "str"}'
assert Json.to_json(123, key='test') == '{"test": 123}'
assert Json.to_json(123.45, key='test') == '{"test": 123.45}'
assert Json.to_json(True, key='test') == '{"test": true}'
assert Json.to_json({'test': 'dict'}) == '{"test": "dict"}'
assert set(json.loads(Json.to_json({'test2', 'test1'}))) == {"test1", "test2"}
assert Json.to_json(['test1', 'test2']) == '["test1", "test2"]'
def test_to_json_dataobject_validate():
data = MockDataValidate(id='test', value='not-ok') # type: ignore
with pytest.raises(ValueError):
Json.to_json(data)
def _application_json_response(result: DataObject, key: str, *args,
**kwargs) -> str:
return Json.to_json(result, key=key)
def test_to_json_dataobject_do_not_validate():
data = MockDataDoNotValidate(id='test', value='not-ok') # type: ignore
assert Json.to_json(data) == '{"id": "test", "value": "not-ok"}'
def test_from_json_dataobject_validate():
data = '{"id": "test", "value": "not-ok"}'
with pytest.raises(ValueError):
Json.from_json(data, MockDataValidate)
def test_to_json_list_dataobject():
assert json.loads(Json.to_json([
MockData(id='1', value='ok-1', nested=MockNested(ts=datetime.fromtimestamp(0, tz=timezone.utc))),
MockData(id='2', value='ok-2', nested=MockNested(ts=datetime.fromtimestamp(0, tz=timezone.utc))),
])) == [{"id": "1", "value": "ok-1", "nested": {"ts": "1970-01-01T00:00:00+00:00"}},
{"id": "2", "value": "ok-2", "nested": {"ts": "1970-01-01T00:00:00+00:00"}}]