def data(self) -> ExecutionData:
result_dict = dict()
for name, ui_control in self.ui_controls.items():
if isinstance(self.ui_controls[name], npyscreen.TitleMultiSelect):
result = {self.values[name][it] for it in ui_control.value}
result_dict[name] = result
continue
if isinstance(self.ui_controls[name], npyscreen.TitleCombo):
result = (
self.values[name][ui_control.value]
if ui_control.value >= 0
else None
)
result_dict[name] = result
continue
if isinstance(self.ui_controls[name], npyscreen.TitleSelectOne):
result = (
self.values[name][ui_control.value[0]]
if len(ui_control.value) > 0
else None
)
result_dict[name] = result
continue
result_dict[name] = ui_control.value
if self.buttons is not DEFAULT_BUTTONS:
assert self.selected_button
result_dict[self.selected_button.name] = self.selected_button.value
return ExecutionData(result_dict)
def __init__(self,
*,
task: ExecutableNode,
execution_id: str,
token_id: str,
data: Optional[Dict],
workspace: Optional[Workspace] = None,
lane: Optional[ExecutionLaneId] = None) -> None:
self.task = task
self.data: T = cast(T, ExecutionData(data))
self.execution_id = execution_id
self.token_id = token_id
# we need to define it before calling token_utils.parse_name. That's
# since parse_name() will read this object using as_mapping(), that in
# turn reads the task_name
self.task_name = ""
# These are None until the task is assgined to a lane
self.workspace: Optional[Workspace] = workspace
# The lane execution id is kept in case of clonning to allow
# tracking from what lane dhis event came from.
self.lane: Optional[ExecutionLaneId] = lane
self.loop: Optional[ExecutionLoop] = None
def test_process_data_merge(self):
data1 = ExecutionData({
"list": [1, 2, 3],
"set": {"a", "b"},
"dict": {
"a": "a",
"b": "b",
},
"number": 3,
"to_remove": 1,
})
data2 = ExecutionData({
"list": [4, 5, 6],
"set": {"b", "c"},
"dict": {
"b": "b2",
"c": "c",
},
"number": 1,
"to_remove": None,
})
result = ExecutionData.merge(data1, data2)
self.assertEqual(
{
"list": [4, 5, 6], # same type, gets overwritten
"set": {"a", "b", "c"},
"dict": {
"a": "a",
"b": "b2", # keys are taken first from the 2nd object
"c": "c",
},
"number": 1,
},
result.as_dict(),
)
def run_on_curses(x):
try:
# build the UI components on ncurses:
ui.form = ConsoleUserTaskForm(ui_builder=ui, name=event.task.name)
for ncurses_call in ui.ncurses_calls:
ncurses_call()
# call the actual UI
ui.form.edit()
# put the values in the data.
context.data = ExecutionData.merge(context.data, ui.data)
event.future.set_result(context)
except Exception as e:
event.future.set_exception(e)
def done_check_event(self, event: ActiveEvent, finish_mode: TaskFinishMode) -> None:
"""
Runs the handling for an event that is considered an end
event in its parent.
:param _event:
:return:
"""
# even if there might be other edges getting out from the task,
# this event might need to register the next event in a serial
# loop execution.
if event.loop_type == ActiveLoopType.COLLECTION_SERIAL:
if event._next_event:
event._next_event.context.data = ExecutionData.merge(
event._next_event.context.data, event.context.data
)
self.register_event(event._next_event)
if (
isinstance(finish_mode, OutgoingEdgesFinishMode)
and finish_mode.outgoing_edges
or isinstance(finish_mode, CancelTaskFinishModeException)
and not finish_mode.root_node
):
self.events.transition(
event=event,
state=ActiveEventState.DONE,
)
return None
# if the event is a deduplication ID, we should get the next waiting event
# (if it exists) and continue it.
if event.deduplication_id:
if not self.events.get_running_deduplication_event_count(event=event):
ev = self.events.get_waiting_deduplication(event=event)
# if we have another
if ev:
# FIXME, there is a whole procedure to unmark things
self.events.clear_waiting_deduplication(event=ev)
self.events.transition(
event=ev,
state=ActiveEventState.RUNNING,
reason="last event for same deduplication_id was done",
)
self.events.transition(
event=event,
state=ActiveEventState.DONE,
reason="deduplication_id previous last event cleanup",
)
# if our deduplication node is an end state, we need to merge its
# content into the parent
if (
isinstance(finish_mode, OutgoingEdgesFinishMode)
and not finish_mode.outgoing_edges
):
assert event.parent_id
# since this happens in done, we don't need to update the task_name anymore
self.events[event.parent_id].context.data = ExecutionData.merge(
self.events[event.parent_id].context.data,
event.context.data,
)
return None # ==> we're done
# we should check all the WAITING processes if they finished.
event_count: Dict[ExecutableNode, int] = dict()
waiting_events: List[ActiveEvent] = list()
process = self.get_process(event)
for id, self_event in self.events.events.items():
if self_event.state in DONE_STATES:
continue
if self_event.task.process_id != process.id:
continue
event_count[self_event.task] = event_count.get(self_event.task, 0) + 1
# deduplication waiting events need to be woken up only when there's no other
# event downstream running.
if (
self_event.state == ActiveEventState.WAITING
and not is_deduplication_event(self_event)
):
waiting_events.append(self_event)
for waiting_event in waiting_events:
if event_count[waiting_event.task] > 1:
continue
if waiting_event.task.process_id != process.id:
continue
potential_predecessors = list(
map(
lambda e: e.task,
filter(
lambda e: is_potential_predecessor(self, waiting_event, e),
self.events.events.values(),
),
)
)
# FIXME: it seems that WAITING events get invoked even if they shouldn't be
# since they are deduplication events that should still be waiting
# until the count of deduplication_id events is 0. Probably the check
# for the deduplication events should be separate.
if not process.are_predecessors(waiting_event.task, potential_predecessors):
self.events.transition(
event=waiting_event,
state=ActiveEventState.RUNNING,
reason="no more predecessors for waiting event",
)
# check sub-process termination
is_active_event_same_parent = False
for not_done_event in self.events.excluding(ActiveEventState.DONE):
if not_done_event.parent_id == event.parent_id and not_done_event != event:
is_active_event_same_parent = True
break
# we merge into the parent event if it's an end state.
if event.parent_id is not None and not isinstance(
finish_mode, CancelTaskFinishModeException
):
# since this happens in done, we don't need to update the task_name anymore
self.events[event.parent_id].context.data = ExecutionData.merge(
self.events[event.parent_id].context.data, event.context.data
)
if (
not is_active_event_same_parent
and event.parent_id == self.root_event.token_id
and self.are_active_futures
):
self.events.transition(event=event, state=ActiveEventState.DONE)
return None # ==> if we still have running futures, we don't kill the main process
if not is_active_event_same_parent:
parent_event = self.events[event.parent_id]
self.events.transition(
event=parent_event,
state=ActiveEventState.ROUTING,
data=parent_event.context,
)
self.events.transition(event=event, state=ActiveEventState.DONE)
def waiting_event(self, event: ActiveEvent, data: Any) -> None:
# if we have deduplication, we might be waiting, even without predecessors,
# since we need to check for events after in the graph with the same
# deduplication id
if self.cleanup_clustered_deduplication_events(
event, search_state=ActiveEventState.WAITING
):
return
if (
isinstance(event.task, ProcessTask)
and cast(ProcessTask, event.task).deduplicate is not None
):
self.events.set_waiting_deduplication(event=event)
# if we don't have events downstream running, we're done, we need to
# start executing.
if not self.events.get_running_deduplication_event_count(event=event):
self.events.clear_waiting_deduplication(event=event)
self.events.transition(
event=event,
state=ActiveEventState.RUNNING,
reason="no deduplication downstream event running",
)
return
# is another waiting task already present?
other_waiting, tasks_waiting_count = self.events.get_other_task_waiting(event)
# FIXME: deduplication events even if they land
if other_waiting:
new_data = ExecutionData.merge(
other_waiting.context.data, event.context.data
)
other_waiting.context.data = new_data
self.events.transition(
event=event,
state=ActiveEventState.DONE,
reason="merged to other event waiting to same task",
)
# return # this event is done
# FIXME: implement a search map for the graph with the events
potential_predecessors = list(
map(
lambda e: e.task,
filter(
lambda e: is_potential_predecessor(self, event, e),
self.events.iterate(ACTIVE_STATES),
),
)
)
# this should return for initial loops
process = self.get_process(event)
# if we have predecessors, we stay in waiting
predecessor_id = process.are_predecessors(event.task, potential_predecessors)
if predecessor_id:
LOG.debug(f"Predecessor found for {event}. Waiting for {predecessor_id}.")
return None
if not other_waiting:
self.events.transition(
event=event,
state=ActiveEventState.RUNNING,
reason="no other event is running",
)
return
if other_waiting.state == ActiveEventState.WAITING and tasks_waiting_count == 1:
# FIXME: why no data mereg is happening here?
self.events.transition(
event=other_waiting,
state=ActiveEventState.RUNNING,
reason="transitioned by another waiting task",
)
return
LOG.debug(f"Waiting for none, yet staying in WAITING? {event}")
return None
def execute(self, initial_data=None) -> ExecutionData:
"""
Execute the current events. This will ensure new events are
generating for forked events.
"""
process = self.adhesive_process.process
self.tasks_impl = dict()
_validate_tasks(self, process)
if adhesive.config.current.verify_mode:
self._print_task_mappings()
return ExecutionData(initial_data)
signal.signal(signal.SIGUSR1, self.print_state)
signal.signal(signal.SIGINT, self.kill_itself)
# since the workspaces are allocated by lanes, we need to ensure
# our default lane is existing.
lane_controller.ensure_default_lane(self.adhesive_process)
LOG.info(f"Adhesive version: {adhesive.version.current}")
if config.current.pool_size:
LOG.info(f"Config: Pool size: {config.current.pool_size}")
else:
LOG.info(
f"Config: Pool size: {config.current.pool_size} "
f"(defaulting to {ProcessExecutor.pool_size})"
)
LOG.info(
f"Config: Parallel processing mode: {config.current.parallel_processing}"
)
LOG.info(f"Config: stdout: {config.current.stdout}")
LOG.info(f"Config: temp_folder: {config.current.temp_polder}")
# FIXME: it's getting pretty crowded
token_id = str(uuid.uuid4())
process_context: ExecutionToken = ExecutionToken(
task=process,
execution_id=self.execution_id,
token_id=token_id,
data=initial_data,
)
fake_event = ActiveEvent(
execution_id=self.execution_id, parent_id=None, context=process_context
)
fake_event.token_id = "" # FIXME: why
root_event = self.clone_event(fake_event, process)
self.root_event = root_event
try:
self.startup_processing_pool()
self.start_message_event_listeners(root_event=root_event)
self.execute_process_event_loop()
finally:
self.shutdown_processing_pool()
return root_event.context.data