async def test_parallelize_event( monkeypatch, app_config, # noqa: F811 something_with_status_processed_example, something_with_status_example): # noqa: F811 results, msg, response = await execute_event( app_config=app_config, event_name='shuffle.parallelize_event', payload=something_with_status_example, postprocess=True) first = copy_payload(something_with_status_processed_example) first.id = 'first_' + first.id second = copy_payload(something_with_status_processed_example) second.id = 'second_' + second.id for i, expected in enumerate([first, second]): expected.status.ts = results[i].payload.status.ts for j in range(len(expected.history)): expected.history[j].ts = results[i].payload.history[j].ts assert results[ 0].path == f"{app_config.env['fs']['data_path']}{first.id}.json" assert results[0].payload == first assert results[ 1].path == f"{app_config.env['fs']['data_path']}{second.id}.json" assert results[1].payload == second assert msg == f"events submitted to stream: {app_config.events['shuffle.parallelize_event'].write_stream.name}" assert response.headers.get("X-Stream-Name") == app_config.events[ 'shuffle.parallelize_event'].write_stream.name assert len(results) == 2
async def execute_steps( steps: Dict[str, StepInfo], *, context: EventContext, payload: Optional[EventPayload], **kwargs) -> AsyncGenerator[Optional[EventPayload], None]: """ Invoke steps from a event. It will try to find next step in configuration order that matches input type of the payload, and will updated the payload and invoke next valid step. """ start_ts = datetime.now() step_name, func = _find_next_step(payload, pending_steps=steps) if step_name: assert step_name assert func is not None, f"Cannot find implementation for step={context.event_name}.{step_name}" payload_copy = copy_payload(payload) async for invoke_result in _invoke_step(payload=payload_copy, func=func, context=context, disable_spawn=False, **kwargs): invoke_result = copy_payload(invoke_result) sub_steps = copy(steps) del sub_steps[step_name] yield await _execute_sub_steps(sub_steps, start_ts=start_ts, context=context, payload=invoke_result) start_ts = datetime.now()
async def _execute_steps_recursion( payload: Optional[EventPayload], context: EventContext, steps: StepExecutionList, step_index: int, func: Optional[Callable], is_spawn: bool, step_delay: float, query_args: Dict[str, Any]) -> AsyncGenerator[Optional[EventPayload], None]: """ Steps execution handler that allows processing Spawn events results using recursion, and sequential execution of events using iteration """ if is_spawn: async for invoke_result in _invoke_spawn_step( copy_payload(payload), func, # type: ignore context, **query_args): if step_delay: await asyncio.sleep(step_delay) i, f, it = _find_next_step(invoke_result, steps, from_index=step_index + 1) if i == -1: yield copy_payload(invoke_result) else: # Recursive call for each received element async for recursion_result in _execute_steps_recursion( invoke_result, context, steps, i, f, it, step_delay, {}): yield recursion_result else: i, f, q, it, invoke_result = step_index, func, query_args, is_spawn, payload while 0 <= i < MAX_STEPS: # Recursive call if result is iterable (Spawn) if it: async for recursion_result in _execute_steps_recursion( invoke_result, context, steps, i, f, it, step_delay, {}): yield recursion_result break # Single step invokation invoke_result = await _invoke_step( copy_payload(invoke_result), f, # type: ignore context, **q) q = {} if step_delay: await asyncio.sleep(step_delay) i, f, it = _find_next_step(invoke_result, steps, from_index=i + 1) if i == -1: # Yields result if all steps were exhausted yield copy_payload(invoke_result) if i >= MAX_STEPS: raise RuntimeError( f"Maximun number of steps to execute exceeded (MAX_STEPS={MAX_STEPS})." )
def test_copy_mutable_collection(): test_dict = {'test': 'value'} result = copy_payload(test_dict) assert result == test_dict and result is not test_dict test_list = ['test1', 'test2'] result = copy_payload(test_list) assert result == test_list and result is not test_list test_set = {'test2', 'test1'} result = copy_payload(test_set) assert result == test_set and result is not test_set
async def _get(self, name, lock=False) -> Any: """ Locks and waits for a collector steps is computed and return its results. In case name is 'payload', returns collector input without blocking. """ if name == 'payload': return copy_payload(self.payload) assert self.executed, "Collector not executed. Call collector.run(...) before accessing results." item = self.items[name] await item.lock.acquire() try: return copy_payload(item.data) finally: if not lock: item.lock.release()
async def _execute_sub_steps( steps: Dict[str, StepInfo], *, start_ts: datetime, context: EventContext, payload: Optional[EventPayload]) -> Optional[EventPayload]: """ Invoke steps from a event. It will try to find next step in configuration order that matches input type of the payload, and will updated the payload and invoke next valid step. """ curr_obj = payload step_delay = context.event_info.config.stream.step_delay / 1000.0 steps = copy(steps) step_name, func = _find_next_step(curr_obj, pending_steps=steps) while step_name: assert step_name assert func is not None, f"Cannot find implementation for step={context.event_name}.{step_name}" if step_delay: await asyncio.sleep(step_delay) async for invoke_result in _invoke_step(payload=curr_obj, func=func, context=context, disable_spawn=True): invoke_result = copy_payload(invoke_result) curr_obj = invoke_result del steps[step_name] step_name, func = _find_next_step(curr_obj, pending_steps=steps) await _throttle(context, start_ts) return curr_obj
async def test_spawn_event( monkeypatch, app_config, # noqa: F811 something_with_status_processed_example, something_with_status_example): # noqa: F811 results, msg, response = await execute_event( app_config=app_config, event_name='shuffle.spawn_event', payload=something_with_status_example, postprocess=True) expected = [ copy_payload(something_with_status_processed_example) for _ in range(3) ] for i, result in enumerate(results): expected[i].id = str(i) expected[i].status.ts = result.payload.status.ts for j in range(len(expected[i].history)): expected[i].history[j].ts = result.payload.history[j].ts assert result == SomethingStored( path=f"{app_config.env['fs']['data_path']}{result.payload.id}.json", payload=expected[i]) assert msg == f"events submitted to stream: {app_config.events['shuffle.spawn_event'].write_stream.name}" assert response.headers.get("X-Stream-Name") == app_config.events[ 'shuffle.spawn_event'].write_stream.name assert len(results) == len(expected)
async def invoke_single_step(func: Callable, *, payload: Optional[EventPayload], context: EventContext, **kwargs) -> Optional[EventPayload]: payload_copy = copy_payload(payload) async for res in _invoke_step(payload=payload_copy, func=func, context=context, disable_spawn=True, **kwargs): return res return None
def test_copy_mutable_dataobject(): now = datetime.now() obj = MockData('id1', 'value1', MockNested(now)) new = copy_payload(obj) assert new == obj assert new is not obj
def test_copy_native_immutable_values_should_return_same(): test_str, test_int, test_float, test_bool = "str", 123, 123.456, True assert copy_payload(test_str) is test_str assert copy_payload(test_int) is test_int assert copy_payload(test_float) is test_float assert copy_payload(test_bool) is test_bool
def test_copy_unsafe_dataobject_should_return_same(): now = datetime.now() obj = MockDataUnsafe('id1', 'value1', MockNested(now)) new = copy_payload(obj) assert new == obj assert new is obj
def test_copy_immutable_dataobject_should_return_same(): now = datetime.now(tz=timezone.utc) obj = MockDataImmutable('id1', 'value1', MockNested(now)) new = copy_payload(obj) assert new == obj assert new is obj