def test_from_json_malformed():
with pytest.raises(ValueError):
Payload.from_json(
'{"item1": {"id": "1", "value": "ok-1", "nested": {"ts": "1970-01-01T00:00:00+00:00"}',
MockData)
with pytest.raises(ValueError):
Payload.from_json('BAD STRING', MockData)
def test_from_json_missing_fields():
with pytest.raises(ValueError):
Payload.from_json('{"id": "1", "value": "ok-1", "nested": {}',
MockData)
with pytest.raises(ValueError):
Payload.from_json(
'{"value": 42, "nested": {"ts": "1970-01-01T00:00:00+00:00"}}',
MockData)
def test_from_json_invalid_types():
with pytest.raises(ValueError):
Payload.from_json(
'{"id": "1", "value": "ok-1", "nested": {"ts": "INVALID DATE"}}',
MockData)
with pytest.raises(ValueError):
Payload.from_json(
'{"id": 42, "value": 42, "nested": {"ts": "1970-01-01T00:00:00+00:00"}}',
MockData)
def test_to_obj_invalid_types():
with pytest.raises(ValueError):
Payload.to_obj(
MockData(id=42,
value='ok-value',
nested=MockNested(ts=datetime.now(
tz=timezone.utc)))) # type: ignore
with pytest.raises(ValueError):
Payload.to_obj(
MockData(id='1',
value='ok-value',
nested=MockNested(ts="NOT A DATETIME"))) # type: ignore
async def __preprocess__(payload: None, context: EventContext,
request: PreprocessHook, *,
something_id: str) -> Union[str, FileUploadInfo]:
uploaded_files = []
save_path = Path(str(context.env['upload_something']['save_path']))
chunk_size = int(context.env['upload_something']['chunk_size'])
async for file_hook in request.files():
file_name = f"{file_hook.name}-{file_hook.file_name}"
path = save_path / file_name
logger.info(context, f"Saving {path}...")
await save_multipart_attachment(file_hook, path, chunk_size=chunk_size)
uploaded_file = UploadedFile(file_hook.name,
file_name,
save_path.as_posix(),
size=file_hook.size)
uploaded_files.append(uploaded_file)
args = await request.parsed_args()
if not all(x in args for x in ('id', 'user', 'attachment', 'object')):
request.status = 400
return "Missing required fields"
something_obj = Payload.parse_form_field(args['object'], Something)
return FileUploadInfo(id=args['id'],
user=args['user'],
object=something_obj,
uploaded_files=uploaded_files)
async def test_buffer_object_and_flush_signal(app_config,
test_objs): # noqa: F811
test_save_path = app_config.settings["test_stream_batch_storage"]["path"]
# Buffer single object
test_obj = test_objs[0]
result = await execute_event(app_config=app_config,
event_name='test_stream_batch_storage',
payload=test_obj)
assert result is None
# Send flush partition signal to force flush single object
partition_key = test_obj.object_ts.strftime("%Y/%m/%d/%H") + '/'
signal = FlushSignal(partition_key)
result = await execute_event(app_config=app_config,
event_name='test_stream_batch_storage',
payload=signal)
assert result is None
# Load saved object and check is correct
await asyncio.sleep(1) # Allow aiofiles to save
saved_objects = {}
for file_name in glob(f'{test_save_path}/{partition_key}/*.jsonlines'):
with open(file_name) as f:
for line in f:
obj = Payload.from_json(line, datatype=MyObject)
saved_objects[obj.object_id] = obj
assert len(saved_objects) == 1
assert test_obj == saved_objects[test_obj.object_id]
def test_to_json_dataobject():
assert Payload.to_json(
MockData(
id='test',
value='ok',
nested=MockNested(ts=datetime.fromtimestamp(0, tz=timezone.utc)))
) == '{"id": "test", "value": "ok", "nested": {"ts": "1970-01-01T00:00:00+00:00"}}'
async def store(self, key: str, value: DataObject, **kwargs):
"""
Stores value under specified key
:param key: str
:param value: DataObject, instance of dataclass annotated with @dataobject
:param **kwargs: You can use arguments expected by the set method in the aioredis library i.e.:
ex sets an expire flag on key name for ex seconds.
px sets an expire flag on key name for px milliseconds.
nx if set to True, set the value at key name to value only if it does not exist.
xx if set to True, set the value at key name to value only if it already exists.
keepttl if True, retain the time to live associated with the key. (Available since Redis 6.0).
*These arguments may vary depending on the version of aioredis installed.
i.e. store object:
```
redis_store.store(key='my_key', value=my_dataobject)
```
i.e. store object with kwargs option, adding `ex=60` redis set a ttl of 60 seconds for the object:
```
redis_store.store(key='my_key', value=my_dataobject, ex=60)
```
"""
assert self._conn
payload_str = str(Payload.to_json(value))
await self._conn.set(key, payload_str, **kwargs)
def __init__(self, app_config: AppConfig, app_connection: str):
self.app_key = app_config.app_key()
self.app_conn_key = app_connection
self.app_connection = app_config.app_connections[app_connection]
settings_key = self.app_connection.settings or app_connection
self.settings = Payload.from_obj(
app_config.settings.get(settings_key, {}), AppsClientSettings)
self.event_connections = {
event_name: {
conn.event: conn
for conn in event_info.connections
if conn.app_connection == app_connection
}
for event_name, event_info in app_config.events.items()
}
self.routes = {
conn.event: self._get_route(conn.event)
for event_info in app_config.events.values()
for conn in event_info.connections
if conn.app_connection == app_connection
}
self.conn_state: Optional[AppConnectionState] = None
self.session: Optional[Any] = None
self.token: Optional[str] = None
self.token_expire: int = 0
async def __preprocess__(payload: None, context: EventContext,
request: PreprocessHook) -> MockData:
args = await request.parsed_args()
data = Payload.parse_form_field(args['field2'], MockData)
return MockData(
value=f"field1:{args['field1']} field2:{data.value} file:{args['file']}"
)
def test_from_obj_dataobject_list():
assert Payload.from_obj(
[{
"id": "test1",
"value": "ok",
"nested": {
"ts": "1970-01-01T00:00:00+00:00"
}
}, {
"id": "test2",
"value": "ok",
"nested": {
"ts": "1970-01-01T00:00:00+00:00"
}
}],
list,
item_datatype=MockData) == [
MockData(
id='test1',
value='ok',
nested=MockNested(
ts=datetime.fromtimestamp(0.0).astimezone(timezone.utc))),
MockData(
id='test2',
value='ok',
nested=MockNested(
ts=datetime.fromtimestamp(0.0).astimezone(timezone.utc)))
]
def test_to_json_dict_dataobject():
assert json.loads(
Payload.to_json({
'item1':
MockData(id='1',
value='ok-1',
nested=MockNested(
ts=datetime.fromtimestamp(0, tz=timezone.utc))),
'item2':
MockData(id='2',
value='ok-2',
nested=MockNested(
ts=datetime.fromtimestamp(0, tz=timezone.utc))),
})) == {
"item1": {
"id": "1",
"value": "ok-1",
"nested": {
"ts": "1970-01-01T00:00:00+00:00"
}
},
"item2": {
"id": "2",
"value": "ok-2",
"nested": {
"ts": "1970-01-01T00:00:00+00:00"
}
}
}
def test_from_obj_python_types():
assert Payload.from_obj({"value": "str"}, str) == "str"
assert Payload.from_obj({"value": 123}, int) == int(123)
assert Payload.from_obj({"value": 123.45}, float) == float(123.45)
assert Payload.from_obj({"value": True}, bool) is True
assert Payload.from_obj({"custom": "str"}, str, key='custom') == "str"
assert Payload.from_obj({"test": "dict"}, dict) == {'test': 'dict'}
assert Payload.from_obj(["test1", "test2"], set) == {'test2', 'test1'}
assert Payload.from_obj(["test1", "test2"], list) == ['test1', 'test2']
def test_from_json_python_types():
assert Payload.from_json('{"value": "str"}', str) == "str"
assert Payload.from_json('{"value": 123}', int) == int(123)
assert Payload.from_json('{"value": 123.45}', float) == float(123.45)
assert Payload.from_json('{"value": true}', bool) is True
assert Payload.from_json('{"custom": "str"}', str, key='custom') == "str"
assert Payload.from_json('{"test": "dict"}', dict) == {'test': 'dict'}
assert Payload.from_json('["test1", "test2"]', set) == {'test2', 'test1'}
assert Payload.from_json('["test1", "test2"]', list) == ['test1', 'test2']
def mock_client_app_config():
return AppConfig(
app=AppDescriptor(name='mock_client_app', version='test'),
engine=AppEngineConfig(import_modules=['mock_client_app']),
app_connections={
"test_app_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="hopeit.apps_client.AppsClient"),
"test_app_plugin_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="hopeit.apps_client.AppsClient",
plugin_name="test_plugin",
plugin_version=APPS_API_VERSION)
},
settings={
"test_app_connection":
Payload.to_obj(
AppsClientSettings(
connection_str="http://test-host1,http://test-host2")),
"test_app_plugin_connection":
Payload.to_obj(
AppsClientSettings(
connection_str="http://test-host1,http://test-host2",
auth_strategy=ClientAuthStrategy.FORWARD_CONTEXT))
},
events={
"mock_client_event":
EventDescriptor(
type=EventType.GET,
connections=[
EventConnection(app_connection="test_app_connection",
event="test_event_get",
type=EventConnectionType.GET),
EventConnection(app_connection="test_app_connection",
event="test_event_post",
type=EventConnectionType.POST),
EventConnection(
app_connection="test_app_plugin_connection",
event="test_event_plugin",
type=EventConnectionType.GET)
])
},
server=ServerConfig(logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/"))).setup()
def test_from_json_dataobject():
assert Payload.from_json(
'{"id": "test", "value": "ok", "nested": {"ts": "1970-01-01T00:00:00+00:00"}}',
MockData) == MockData(
id='test',
value='ok',
nested=MockNested(
ts=datetime.fromtimestamp(0.0).astimezone(timezone.utc)))
def _ignored_response(context: Optional[EventContext], status: int,
e: BaseException) -> web.Response:
if context:
logger.error(context, e)
logger.ignored(context)
else:
logger.error(__name__, e)
info = ErrorInfo.from_exception(e)
return web.Response(status=status, body=Payload.to_json(info))
def _failed_response(context: Optional[EventContext],
e: Exception) -> web.Response:
if context:
logger.error(context, e)
logger.failed(context)
else:
logger.error(__name__, e)
info = ErrorInfo.from_exception(e)
return web.Response(status=500, body=Payload.to_json(info))
def test_from_obj_list():
assert Payload.from_obj([{
"value": "ok1"
}, {
"value": "ok2"
}],
list,
key='value',
item_datatype=str) == ["ok1", "ok2"]
def test_to_obj_python_types():
assert Payload.to_obj('str', key='test') == {"test": "str"}
assert Payload.to_obj(123, key='test') == {"test": 123}
assert Payload.to_obj(123.45, key='test') == {"test": 123.45}
assert Payload.to_obj(True, key='test') == {"test": True}
assert Payload.to_obj({'test': 'dict'}) == {"test": "dict"}
assert Payload.to_obj({'test2', 'test1'}) == ["test1", "test2"]
assert Payload.to_obj(['test2', 'test1']) == ["test2", "test1"]
async def _save_partition(partition_key: str, items: List[DataObject],
context: EventContext):
settings = context.settings(datatype=FileStorageSettings)
path = Path(settings.path) / partition_key
file = path / f"{uuid.uuid4()}{SUFFIX}"
logger.info(context, f"Saving {file}...")
os.makedirs(path.resolve(), exist_ok=True)
async with aiofiles.open(file, 'w') as f:
for item in items:
await f.write(Payload.to_json(item) + "\n")
async def _parse_response(
self, response, context: EventContext,
datatype: Type[EventPayloadType],
target_event_name: str) -> List[EventPayloadType]:
"""
Parses http response from external App, catching Unathorized errors
and converting the result to the desired datatype
"""
if response.status == 200:
data = await response.json()
if isinstance(data, list):
return Payload.from_obj(data, list,
item_datatype=datatype) # type: ignore
return [Payload.from_obj(data, datatype, key=target_event_name)]
if response.status == 401:
raise Unauthorized(context.app_key)
if response.status >= 500:
raise ServerException(await response.text())
raise RuntimeError(await response.text())
def effective_events_example():
res = EFFECTIVE_EVENTS_EXAMPLE
res = res.replace("${ENGINE_VERSION}", ENGINE_VERSION)
res = res.replace("${APPS_API_VERSION}", APPS_API_VERSION)
res = res.replace("${APPS_ROUTE_VERSION}", APPS_ROUTE_VERSION)
res = json.loads(res)
result = Payload.from_obj(res,
datatype=dict,
item_datatype=EventDescriptor)
return result
async def __preprocess__(payload: None, context: EventContext,
request: PreprocessHook, *,
query_arg1: str) -> Union[str, MockData]:
fields = await request.parsed_args()
if any(x not in fields for x in ('field1', 'field2', 'attachment')):
request.set_status(400)
return "Missing required fields"
data = Payload.parse_form_field(fields['field2'], MockData)
return MockData(
value=
f"field1={fields['field1']} field2={data.value} attachment={fields['attachment']}"
)
def _get_runtime_simple_example(url: str, source: str):
with open(Path(__file__).parent / source) as f:
res = f.read()
res = res.replace("${HOST_NAME}", socket.gethostname())
res = res.replace("${PID}", str(os.getpid()))
res = res.replace("${URL}", url)
res = res.replace("${ENGINE_VERSION}", ENGINE_VERSION)
res = res.replace("${APPS_API_VERSION}", APPS_API_VERSION)
res = res.replace("${APPS_ROUTE_VERSION}", APPS_ROUTE_VERSION)
result = Payload.from_json(res, RuntimeApps)
return result
def mock_runtime(monkeypatch, effective_events):
setattr(apps, "_expire", 0.0)
monkeypatch.setattr(server_config.os, 'getenv', mock_getenv)
app_config = config('apps/examples/simple-example/config/app-config.json')
basic_auth_config = config(
'plugins/auth/basic-auth/config/plugin-config.json')
client_app_config = config(
'apps/examples/client-example/config/app-config.json')
server = MockServer(basic_auth_config, app_config, client_app_config)
for app_key, app_effective_events in effective_events.items():
server.app_engines[app_key].effective_events = Payload.from_obj(
app_effective_events, datatype=dict, item_datatype=EventDescriptor)
monkeypatch.setattr(runtime, "server", server)
async def get(self, key: str, *,
datatype: Type[DataObject]) -> Optional[DataObject]:
"""
Retrieves value under specified key, converted to datatype
:param key: str
:param datatype: dataclass implementing @dataobject (@see DataObject)
:return: instance of datatype or None if not found
"""
assert self._conn
payload_str = await self._conn.get(key)
if payload_str:
return Payload.from_json(payload_str, datatype)
return None
async def store(self, key: str, value: DataObject) -> str:
"""
Stores value under specified key
:param key: str
:param value: DataObject, instance of dataclass annotated with @dataobject
:return: str, path where the object was stored
"""
payload_str = Payload.to_json(value)
path = self.path
if self.partition_dateformat:
path = path / get_partition_key(value, self.partition_dateformat)
os.makedirs(path.resolve().as_posix(), exist_ok=True)
return await self._save_file(payload_str, path=path, file_name=key + SUFFIX)
def test_from_obj_dict():
assert Payload.from_obj(
{
"item1": {
"value": "ok1"
},
"item2": {
"value": "ok2"
}
},
dict,
key='value',
item_datatype=str) == {
"item1": "ok1",
"item2": "ok2"
}
async def _request_process_payload(
context: EventContext, datatype: Optional[Type[EventPayloadType]],
request: web.Request
) -> Tuple[Optional[EventPayloadType], Optional[bytes]]:
"""
Extract payload from request.
Returns payload if parsing succeeded. Raises BadRequest if payload fails to parse
"""
try:
payload_raw = await request.read()
if datatype is not None:
return Payload.from_json(payload_raw,
datatype), payload_raw # type: ignore
return None, payload_raw
except ValueError as e:
logger.error(context, e)
raise BadRequest(e) from e