def clone_event(
self,
old_event: ActiveEvent,
task: ExecutableNode,
parent_id: Optional[str] = None,
) -> ActiveEvent:
if parent_id is None:
parent_id = old_event.parent_id
event = old_event.clone(task, parent_id)
update_deduplication_id(event)
# if it's an event that comes from a deduplication, we need
# to track how many of them are running. These are either new
# deduplication events running tracked in events.transition(RUNNING),
# or cloned events from the original deduplication event, and
# tracked here
if (
old_event.deduplication_id is not None
and event.deduplication_id is not None
and event.deduplication_id == old_event.deduplication_id
):
self.events.register_deduplication_event(event)
self.register_event(event)
return event
def test_transitions(self):
pe = ProcessEvents()
context = ExecutionToken(
task=ExecutableNode(id="_1", name="Test", parent_process=None),
execution_id="root",
token_id="root",
data=None,
)
event = ActiveEvent(execution_id="root",
parent_id=None,
context=context)
self.assertEqual({}, pe.events)
pe[event.token_id] = event
self.assertEqual(1, len(pe))
self.assertEqual(1, len(pe.bystate[ActiveEventState.NEW]))
self.assertEqual(0, len(pe.bystate[ActiveEventState.PROCESSING]))
self.assertEqual(event, pe.get(ActiveEventState.NEW))
self.assertIsNone(pe.get(ActiveEventState.PROCESSING))
pe.transition(event=event, state=ActiveEventState.PROCESSING)
self.assertEqual(1, len(pe))
self.assertEqual(0, len(pe.bystate[ActiveEventState.NEW]))
self.assertEqual(1, len(pe.bystate[ActiveEventState.PROCESSING]))
self.assertIsNone(pe.get(ActiveEventState.NEW))
self.assertEqual(event, pe.get(ActiveEventState.PROCESSING))
def unregister_deduplication_event(self, event: ActiveEvent):
# if the event wasn't registered, there's nothing to unregister
if not event.deduplication_registered:
return
if event.deduplication_unregistered:
return
event.deduplication_unregistered = True
if event.deduplication_id is None:
raise Exception(
"deduplication_id is none. This is an Adhesive BUG, "
"please report it.")
self._deduplicated_active_count[event.deduplication_id] = (
self._deduplicated_active_count.get(event.deduplication_id, 0) - 1)
LOG.debug(
"Unregistered deduplicated event %s. Active event count for %s: %d",
event,
event.deduplication_id,
self._deduplicated_active_count.get(event.deduplication_id, 0),
)
if self._deduplicated_active_count[event.deduplication_id] == 0:
del self._deduplicated_active_count[event.deduplication_id]
def assign_event_future(self, event: ActiveEvent, future: Future) -> None:
LOG.debug(f"Assigned {future} to {event}")
self.futures[future] = FutureMapping(
event_id=event.token_id,
description=event.context.task_name,
)
event.future = future
def update_deduplication_id(event: ActiveEvent, ) -> None:
if not isinstance(event.task, ProcessTask):
return
expression = cast(ProcessTask, event.task).deduplicate
if expression is None:
return
eval_data = addict.Dict(get_eval_data(event.context))
deduplication_id = eval(expression, {}, eval_data)
if not deduplication_id:
LOG.warning(f"Deduplication returned a falsy object for {expression}. "
f"The return was {deduplication_id}.")
event.deduplication_id = deduplication_id
def routing_event(self, event: ActiveEvent, data: Any) -> None:
try:
# Since we're in routing, we passed the actual running, so we need to update the
# context with the new execution token.
event.context = data
# we don't route, since we have live events created from the
# INITIAL loop type
if event.loop_type == ActiveLoopType.INITIAL:
self.events.transition(event=event, state=ActiveEventState.DONE)
return
if loop_controller.next_conditional_loop_iteration(event, self.clone_event):
# obviously the done checks are not needed, since we're
# still in the loop
self.events.transition(event=event, state=ActiveEventState.DONE)
return
process = self.get_process(event)
outgoing_edges = GatewayController.compute_outgoing_edges(process, event)
for outgoing_edge in outgoing_edges:
target_task = process.tasks[outgoing_edge.target_id]
if isinstance(target_task, ProcessTask) and target_task.loop:
# we start a loop by firing the loop events, and consume this event.
loop_controller.create_loop(event, self.clone_event, target_task)
else:
self.clone_event(event, target_task)
self.events.transition(
event=event,
state=ActiveEventState.DONE_CHECK,
data=OutgoingEdgesFinishMode(outgoing_edges),
)
except Exception as e:
self.handle_task_error(
TaskError(error=traceback.format_exc(), exception=e, failed_event=event)
)
def register_deduplication_event(self, event: ActiveEvent) -> None:
if event.deduplication_id is None:
raise Exception(
"deduplication_id is none. This is an Adhesive BUG, "
"please report it.")
if event.deduplication_registered:
LOG.warning(f"The event is already registered: {event}. "
f"Not counting it twice.")
return
event.deduplication_registered = True
self._deduplicated_active_count[event.deduplication_id] = (
self._deduplicated_active_count.get(event.deduplication_id, 0) + 1)
LOG.debug(
"Registered deduplicated event %s. Active event count for %s: %d",
event,
event.deduplication_id,
self._deduplicated_active_count.get(event.deduplication_id, 0),
)
def __init__(
self,
root_event: ActiveEvent,
message_event: MessageEvent,
execution_message_event: ExecutionMessageEvent,
enqueue_event,
) -> None:
self.id = str(uuid.uuid4())
self.root_event = root_event.clone(
message_event, None) # Used only to print the task name
self.execution_message_event = execution_message_event
self.enqueue_event = enqueue_event
# Future used to signal the termination of the message ingestion, so the
# process can finish.
self.future: Future = Future()
thread = Thread(target=self.run_thread_loop)
thread.setDaemon(True)
thread.start()
def execute(self, initial_data=None) -> ExecutionData:
"""
Execute the current events. This will ensure new events are
generating for forked events.
"""
process = self.adhesive_process.process
self.tasks_impl = dict()
_validate_tasks(self, process)
if adhesive.config.current.verify_mode:
self._print_task_mappings()
return ExecutionData(initial_data)
signal.signal(signal.SIGUSR1, self.print_state)
signal.signal(signal.SIGINT, self.kill_itself)
# since the workspaces are allocated by lanes, we need to ensure
# our default lane is existing.
lane_controller.ensure_default_lane(self.adhesive_process)
LOG.info(f"Adhesive version: {adhesive.version.current}")
if config.current.pool_size:
LOG.info(f"Config: Pool size: {config.current.pool_size}")
else:
LOG.info(
f"Config: Pool size: {config.current.pool_size} "
f"(defaulting to {ProcessExecutor.pool_size})"
)
LOG.info(
f"Config: Parallel processing mode: {config.current.parallel_processing}"
)
LOG.info(f"Config: stdout: {config.current.stdout}")
LOG.info(f"Config: temp_folder: {config.current.temp_polder}")
# FIXME: it's getting pretty crowded
token_id = str(uuid.uuid4())
process_context: ExecutionToken = ExecutionToken(
task=process,
execution_id=self.execution_id,
token_id=token_id,
data=initial_data,
)
fake_event = ActiveEvent(
execution_id=self.execution_id, parent_id=None, context=process_context
)
fake_event.token_id = "" # FIXME: why
root_event = self.clone_event(fake_event, process)
self.root_event = root_event
try:
self.startup_processing_pool()
self.start_message_event_listeners(root_event=root_event)
self.execute_process_event_loop()
finally:
self.shutdown_processing_pool()
return root_event.context.data