async def test_parallelize_event(
monkeypatch,
app_config, # noqa: F811
something_with_status_processed_example,
something_with_status_example): # noqa: F811
results, msg, response = await execute_event(
app_config=app_config,
event_name='shuffle.parallelize_event',
payload=something_with_status_example,
postprocess=True)
first = copy_payload(something_with_status_processed_example)
first.id = 'first_' + first.id
second = copy_payload(something_with_status_processed_example)
second.id = 'second_' + second.id
for i, expected in enumerate([first, second]):
expected.status.ts = results[i].payload.status.ts
for j in range(len(expected.history)):
expected.history[j].ts = results[i].payload.history[j].ts
assert results[
0].path == f"{app_config.env['fs']['data_path']}{first.id}.json"
assert results[0].payload == first
assert results[
1].path == f"{app_config.env['fs']['data_path']}{second.id}.json"
assert results[1].payload == second
assert msg == f"events submitted to stream: {app_config.events['shuffle.parallelize_event'].write_stream.name}"
assert response.headers.get("X-Stream-Name") == app_config.events[
'shuffle.parallelize_event'].write_stream.name
assert len(results) == 2
async def execute_steps(
steps: Dict[str, StepInfo], *, context: EventContext,
payload: Optional[EventPayload],
**kwargs) -> AsyncGenerator[Optional[EventPayload], None]:
"""
Invoke steps from a event.
It will try to find next step in configuration order that matches input type of the payload,
and will updated the payload and invoke next valid step.
"""
start_ts = datetime.now()
step_name, func = _find_next_step(payload, pending_steps=steps)
if step_name:
assert step_name
assert func is not None, f"Cannot find implementation for step={context.event_name}.{step_name}"
payload_copy = copy_payload(payload)
async for invoke_result in _invoke_step(payload=payload_copy,
func=func,
context=context,
disable_spawn=False,
**kwargs):
invoke_result = copy_payload(invoke_result)
sub_steps = copy(steps)
del sub_steps[step_name]
yield await _execute_sub_steps(sub_steps,
start_ts=start_ts,
context=context,
payload=invoke_result)
start_ts = datetime.now()
async def _execute_steps_recursion(
payload: Optional[EventPayload], context: EventContext,
steps: StepExecutionList, step_index: int, func: Optional[Callable],
is_spawn: bool, step_delay: float,
query_args: Dict[str,
Any]) -> AsyncGenerator[Optional[EventPayload], None]:
"""
Steps execution handler that allows processing Spawn events results using recursion,
and sequential execution of events using iteration
"""
if is_spawn:
async for invoke_result in _invoke_spawn_step(
copy_payload(payload),
func, # type: ignore
context,
**query_args):
if step_delay:
await asyncio.sleep(step_delay)
i, f, it = _find_next_step(invoke_result,
steps,
from_index=step_index + 1)
if i == -1:
yield copy_payload(invoke_result)
else:
# Recursive call for each received element
async for recursion_result in _execute_steps_recursion(
invoke_result, context, steps, i, f, it, step_delay,
{}):
yield recursion_result
else:
i, f, q, it, invoke_result = step_index, func, query_args, is_spawn, payload
while 0 <= i < MAX_STEPS:
# Recursive call if result is iterable (Spawn)
if it:
async for recursion_result in _execute_steps_recursion(
invoke_result, context, steps, i, f, it, step_delay,
{}):
yield recursion_result
break
# Single step invokation
invoke_result = await _invoke_step(
copy_payload(invoke_result),
f, # type: ignore
context,
**q)
q = {}
if step_delay:
await asyncio.sleep(step_delay)
i, f, it = _find_next_step(invoke_result, steps, from_index=i + 1)
if i == -1:
# Yields result if all steps were exhausted
yield copy_payload(invoke_result)
if i >= MAX_STEPS:
raise RuntimeError(
f"Maximun number of steps to execute exceeded (MAX_STEPS={MAX_STEPS})."
)
def test_copy_mutable_collection():
test_dict = {'test': 'value'}
result = copy_payload(test_dict)
assert result == test_dict and result is not test_dict
test_list = ['test1', 'test2']
result = copy_payload(test_list)
assert result == test_list and result is not test_list
test_set = {'test2', 'test1'}
result = copy_payload(test_set)
assert result == test_set and result is not test_set
async def _get(self, name, lock=False) -> Any:
"""
Locks and waits for a collector steps is computed and return its results.
In case name is 'payload', returns collector input without blocking.
"""
if name == 'payload':
return copy_payload(self.payload)
assert self.executed, "Collector not executed. Call collector.run(...) before accessing results."
item = self.items[name]
await item.lock.acquire()
try:
return copy_payload(item.data)
finally:
if not lock:
item.lock.release()
async def _execute_sub_steps(
steps: Dict[str,
StepInfo], *, start_ts: datetime, context: EventContext,
payload: Optional[EventPayload]) -> Optional[EventPayload]:
"""
Invoke steps from a event.
It will try to find next step in configuration order that matches input type of the payload,
and will updated the payload and invoke next valid step.
"""
curr_obj = payload
step_delay = context.event_info.config.stream.step_delay / 1000.0
steps = copy(steps)
step_name, func = _find_next_step(curr_obj, pending_steps=steps)
while step_name:
assert step_name
assert func is not None, f"Cannot find implementation for step={context.event_name}.{step_name}"
if step_delay:
await asyncio.sleep(step_delay)
async for invoke_result in _invoke_step(payload=curr_obj,
func=func,
context=context,
disable_spawn=True):
invoke_result = copy_payload(invoke_result)
curr_obj = invoke_result
del steps[step_name]
step_name, func = _find_next_step(curr_obj, pending_steps=steps)
await _throttle(context, start_ts)
return curr_obj
async def test_spawn_event(
monkeypatch,
app_config, # noqa: F811
something_with_status_processed_example,
something_with_status_example): # noqa: F811
results, msg, response = await execute_event(
app_config=app_config,
event_name='shuffle.spawn_event',
payload=something_with_status_example,
postprocess=True)
expected = [
copy_payload(something_with_status_processed_example) for _ in range(3)
]
for i, result in enumerate(results):
expected[i].id = str(i)
expected[i].status.ts = result.payload.status.ts
for j in range(len(expected[i].history)):
expected[i].history[j].ts = result.payload.history[j].ts
assert result == SomethingStored(
path=f"{app_config.env['fs']['data_path']}{result.payload.id}.json",
payload=expected[i])
assert msg == f"events submitted to stream: {app_config.events['shuffle.spawn_event'].write_stream.name}"
assert response.headers.get("X-Stream-Name") == app_config.events[
'shuffle.spawn_event'].write_stream.name
assert len(results) == len(expected)
async def invoke_single_step(func: Callable, *,
payload: Optional[EventPayload],
context: EventContext,
**kwargs) -> Optional[EventPayload]:
payload_copy = copy_payload(payload)
async for res in _invoke_step(payload=payload_copy,
func=func,
context=context,
disable_spawn=True,
**kwargs):
return res
return None
def test_copy_mutable_dataobject():
now = datetime.now()
obj = MockData('id1', 'value1', MockNested(now))
new = copy_payload(obj)
assert new == obj
assert new is not obj
def test_copy_native_immutable_values_should_return_same():
test_str, test_int, test_float, test_bool = "str", 123, 123.456, True
assert copy_payload(test_str) is test_str
assert copy_payload(test_int) is test_int
assert copy_payload(test_float) is test_float
assert copy_payload(test_bool) is test_bool
def test_copy_unsafe_dataobject_should_return_same():
now = datetime.now()
obj = MockDataUnsafe('id1', 'value1', MockNested(now))
new = copy_payload(obj)
assert new == obj
assert new is obj
def test_copy_immutable_dataobject_should_return_same():
now = datetime.now(tz=timezone.utc)
obj = MockDataImmutable('id1', 'value1', MockNested(now))
new = copy_payload(obj)
assert new == obj
assert new is obj
