async def store(self, key: str, value: DataObject) -> str:
"""
Stores value under specified key
:param key: str
:param value: DataObject, instance of dataclass annotated with @dataobject
:return: str, path where the object was stored
"""
payload_str = Payload.to_json(value)
path = self.path
if self.partition_dateformat:
path = path / get_partition_key(value, self.partition_dateformat)
os.makedirs(path.resolve().as_posix(), exist_ok=True)
return await self._save_file(payload_str, path=path, file_name=key + SUFFIX)
async def test_it_save_something(app_config,
something_params_example): # noqa: F811
result = await execute_event(app_config=app_config,
event_name='save_something',
payload=something_params_example,
preprocess=True,
mocks=[mock_user_agent_header])
partition_key = get_partition_key(something_params_example, "%Y/%m/%d/%H")
assert result == (
f"{app_config.env['storage']['base_path']}simple_example.{APP_VERSION}.fs_storage.path/"
f"{partition_key}{something_params_example.id}.json")
with open(str(result)) as f:
fields = json.loads(f.read())
assert fields['id'] == something_params_example.id
assert fields['user']['name'] == something_params_example.user
async def test_it_process_events(
app_config, # noqa: F811
something_submitted,
something_processed): # noqa: F811
result: SomethingStored = await execute_event(
app_config=app_config,
event_name='streams.process_events',
payload=something_submitted)
something_processed.status.ts = result.payload.status.ts
something_processed.history[-1].ts = result.payload.history[-1].ts
partition_key = get_partition_key(something_processed,
partition_dateformat="%Y/%m/%d/%H")
assert result == SomethingStored(path=(
f"{app_config.env['storage']['base_path']}simple_example.{APPS_ROUTE_VERSION}.fs_storage.path/"
f"{partition_key}{result.payload.id}.json"),
payload=something_processed)
async def buffer_item(payload: DataObject,
context: EventContext) -> Optional[FlushSignal]:
"""
Consumes any Dataobject type from stream and put it local memory buffer to be flushed later
"""
settings: FileStorageSettings = context.settings(
datatype=FileStorageSettings)
partition_key = get_partition_key(payload, settings.partition_dateformat
or "")
async with buffer_lock:
partition = buffer.get(partition_key, Partition())
buffer[partition_key] = partition
async with partition.lock:
partition.items.append(payload)
if settings.flush_max_size and len(
partition.items) >= settings.flush_max_size:
return FlushSignal(partition_key=partition_key)
return None
async def test_spawn_event(monkeypatch, app_config, # noqa: F811
something_with_status_processed_example, something_with_status_example): # noqa: F811
results, msg, response = await execute_event(
app_config=app_config,
event_name='shuffle.spawn_event',
payload=something_with_status_example,
postprocess=True)
expected = [copy_payload(something_with_status_processed_example) for _ in range(3)]
for i, result in enumerate(results):
expected[i].id = str(i)
expected[i].status.ts = result.payload.status.ts
for j in range(len(expected[i].history)):
expected[i].history[j].ts = result.payload.history[j].ts
partition_key = get_partition_key(result, "%Y/%m/%d/%H")
assert result == SomethingStored(
path=(
f"{app_config.env['storage']['base_path']}simple_example.{APP_VERSION}.fs_storage.path/"
f"{partition_key}{result.payload.id}.json"
),
payload=expected[i]
)
assert msg == f"events submitted to stream: {app_config.events['shuffle.spawn_event'].write_stream.name}"
assert response.headers.get("X-Stream-Name") == app_config.events['shuffle.spawn_event'].write_stream.name
assert len(results) == len(expected)