예제 #1
0
    def clone_event(
        self,
        old_event: ActiveEvent,
        task: ExecutableNode,
        parent_id: Optional[str] = None,
    ) -> ActiveEvent:

        if parent_id is None:
            parent_id = old_event.parent_id

        event = old_event.clone(task, parent_id)
        update_deduplication_id(event)

        # if it's an event that comes from a deduplication, we need
        # to track how many of them are running. These are either new
        # deduplication events running tracked in events.transition(RUNNING),
        # or cloned events from the original deduplication event, and
        # tracked here
        if (
            old_event.deduplication_id is not None
            and event.deduplication_id is not None
            and event.deduplication_id == old_event.deduplication_id
        ):
            self.events.register_deduplication_event(event)

        self.register_event(event)

        return event
예제 #2
0
    def test_transitions(self):
        pe = ProcessEvents()

        context = ExecutionToken(
            task=ExecutableNode(id="_1", name="Test", parent_process=None),
            execution_id="root",
            token_id="root",
            data=None,
        )
        event = ActiveEvent(execution_id="root",
                            parent_id=None,
                            context=context)

        self.assertEqual({}, pe.events)

        pe[event.token_id] = event

        self.assertEqual(1, len(pe))
        self.assertEqual(1, len(pe.bystate[ActiveEventState.NEW]))
        self.assertEqual(0, len(pe.bystate[ActiveEventState.PROCESSING]))
        self.assertEqual(event, pe.get(ActiveEventState.NEW))
        self.assertIsNone(pe.get(ActiveEventState.PROCESSING))

        pe.transition(event=event, state=ActiveEventState.PROCESSING)

        self.assertEqual(1, len(pe))
        self.assertEqual(0, len(pe.bystate[ActiveEventState.NEW]))
        self.assertEqual(1, len(pe.bystate[ActiveEventState.PROCESSING]))
        self.assertIsNone(pe.get(ActiveEventState.NEW))
        self.assertEqual(event, pe.get(ActiveEventState.PROCESSING))
예제 #3
0
    def unregister_deduplication_event(self, event: ActiveEvent):
        # if the event wasn't registered, there's nothing to unregister
        if not event.deduplication_registered:
            return

        if event.deduplication_unregistered:
            return

        event.deduplication_unregistered = True

        if event.deduplication_id is None:
            raise Exception(
                "deduplication_id is none. This is an Adhesive BUG, "
                "please report it.")

        self._deduplicated_active_count[event.deduplication_id] = (
            self._deduplicated_active_count.get(event.deduplication_id, 0) - 1)

        LOG.debug(
            "Unregistered deduplicated event %s. Active event count for %s: %d",
            event,
            event.deduplication_id,
            self._deduplicated_active_count.get(event.deduplication_id, 0),
        )

        if self._deduplicated_active_count[event.deduplication_id] == 0:
            del self._deduplicated_active_count[event.deduplication_id]
예제 #4
0
    def assign_event_future(self, event: ActiveEvent, future: Future) -> None:
        LOG.debug(f"Assigned {future} to {event}")

        self.futures[future] = FutureMapping(
            event_id=event.token_id,
            description=event.context.task_name,
        )
        event.future = future
예제 #5
0
def update_deduplication_id(event: ActiveEvent, ) -> None:
    if not isinstance(event.task, ProcessTask):
        return

    expression = cast(ProcessTask, event.task).deduplicate

    if expression is None:
        return

    eval_data = addict.Dict(get_eval_data(event.context))
    deduplication_id = eval(expression, {}, eval_data)

    if not deduplication_id:
        LOG.warning(f"Deduplication returned a falsy object for {expression}. "
                    f"The return was {deduplication_id}.")

    event.deduplication_id = deduplication_id
예제 #6
0
    def routing_event(self, event: ActiveEvent, data: Any) -> None:

        try:
            # Since we're in routing, we passed the actual running, so we need to update the
            # context with the new execution token.
            event.context = data

            # we don't route, since we have live events created from the
            # INITIAL loop type
            if event.loop_type == ActiveLoopType.INITIAL:
                self.events.transition(event=event, state=ActiveEventState.DONE)
                return

            if loop_controller.next_conditional_loop_iteration(event, self.clone_event):
                # obviously the done checks are not needed, since we're
                # still in the loop
                self.events.transition(event=event, state=ActiveEventState.DONE)

                return

            process = self.get_process(event)

            outgoing_edges = GatewayController.compute_outgoing_edges(process, event)

            for outgoing_edge in outgoing_edges:
                target_task = process.tasks[outgoing_edge.target_id]
                if isinstance(target_task, ProcessTask) and target_task.loop:
                    # we start a loop by firing the loop events, and consume this event.
                    loop_controller.create_loop(event, self.clone_event, target_task)
                else:
                    self.clone_event(event, target_task)

            self.events.transition(
                event=event,
                state=ActiveEventState.DONE_CHECK,
                data=OutgoingEdgesFinishMode(outgoing_edges),
            )
        except Exception as e:
            self.handle_task_error(
                TaskError(error=traceback.format_exc(), exception=e, failed_event=event)
            )
예제 #7
0
    def register_deduplication_event(self, event: ActiveEvent) -> None:
        if event.deduplication_id is None:
            raise Exception(
                "deduplication_id is none. This is an Adhesive BUG, "
                "please report it.")

        if event.deduplication_registered:
            LOG.warning(f"The event is already registered: {event}. "
                        f"Not counting it twice.")
            return

        event.deduplication_registered = True
        self._deduplicated_active_count[event.deduplication_id] = (
            self._deduplicated_active_count.get(event.deduplication_id, 0) + 1)

        LOG.debug(
            "Registered deduplicated event %s. Active event count for %s: %d",
            event,
            event.deduplication_id,
            self._deduplicated_active_count.get(event.deduplication_id, 0),
        )
예제 #8
0
    def __init__(
        self,
        root_event: ActiveEvent,
        message_event: MessageEvent,
        execution_message_event: ExecutionMessageEvent,
        enqueue_event,
    ) -> None:
        self.id = str(uuid.uuid4())

        self.root_event = root_event.clone(
            message_event, None)  # Used only to print the task name
        self.execution_message_event = execution_message_event
        self.enqueue_event = enqueue_event

        # Future used to signal the termination of the message ingestion, so the
        # process can finish.
        self.future: Future = Future()

        thread = Thread(target=self.run_thread_loop)
        thread.setDaemon(True)

        thread.start()
예제 #9
0
    def execute(self, initial_data=None) -> ExecutionData:
        """
        Execute the current events. This will ensure new events are
        generating for forked events.
        """
        process = self.adhesive_process.process
        self.tasks_impl = dict()

        _validate_tasks(self, process)

        if adhesive.config.current.verify_mode:
            self._print_task_mappings()
            return ExecutionData(initial_data)

        signal.signal(signal.SIGUSR1, self.print_state)
        signal.signal(signal.SIGINT, self.kill_itself)

        # since the workspaces are allocated by lanes, we need to ensure
        # our default lane is existing.
        lane_controller.ensure_default_lane(self.adhesive_process)

        LOG.info(f"Adhesive version: {adhesive.version.current}")
        if config.current.pool_size:
            LOG.info(f"Config: Pool size: {config.current.pool_size}")
        else:
            LOG.info(
                f"Config: Pool size: {config.current.pool_size} "
                f"(defaulting to {ProcessExecutor.pool_size})"
            )
        LOG.info(
            f"Config: Parallel processing mode: {config.current.parallel_processing}"
        )
        LOG.info(f"Config: stdout: {config.current.stdout}")
        LOG.info(f"Config: temp_folder: {config.current.temp_polder}")

        # FIXME: it's getting pretty crowded
        token_id = str(uuid.uuid4())
        process_context: ExecutionToken = ExecutionToken(
            task=process,
            execution_id=self.execution_id,
            token_id=token_id,
            data=initial_data,
        )

        fake_event = ActiveEvent(
            execution_id=self.execution_id, parent_id=None, context=process_context
        )
        fake_event.token_id = ""  # FIXME: why

        root_event = self.clone_event(fake_event, process)
        self.root_event = root_event

        try:
            self.startup_processing_pool()

            self.start_message_event_listeners(root_event=root_event)
            self.execute_process_event_loop()
        finally:
            self.shutdown_processing_pool()

        return root_event.context.data