def _some_failed(upstream_states: Set["state.State"]) -> bool: """ The underlying trigger function. Args: - upstream_states (set[State]): the set of all upstream states Returns: - bool: whether the trigger thresolds were met """ if not upstream_states: return True # scale conversions num_failed = len([s for s in upstream_states if s.is_failed()]) num_states = len(upstream_states) if at_least is not None: min_num = (num_states * at_least) if at_least < 1 else at_least else: min_num = 0 if at_most is not None: max_num = (num_states * at_most) if at_most < 1 else at_most else: max_num = num_states if not (min_num <= num_failed <= max_num): raise signals.TRIGGERFAIL( 'Trigger was "some_failed" but thresholds were not met.') return True
def check_task_trigger( self, state: State, upstream_states: Dict[Edge, State] ) -> State: """ Checks if the task's trigger function passes. Args: - state (State): the current state of this task - upstream_states (Dict[Edge, Union[State, List[State]]]): the upstream states Returns: - State: the state of the task after running the check Raises: - ENDRUN: if the trigger raises an error """ all_states = set() # type: Set[State] for upstream_state in upstream_states.values(): if isinstance(upstream_state, Mapped): all_states.update(upstream_state.map_states) else: all_states.add(upstream_state) try: if not self.task.trigger(all_states): raise signals.TRIGGERFAIL(message="Trigger failed") except signals.PrefectStateSignal as exc: self.logger.debug( "Task '{name}': {signal} signal raised during execution.".format( name=prefect.context.get("task_full_name", self.task.name), signal=type(exc).__name__, ) ) if prefect.context.get("raise_on_exception"): raise exc raise ENDRUN(exc.state) # Exceptions are trapped and turned into TriggerFailed states except Exception as exc: self.logger.exception( "Task '{name}': unexpected error while evaluating task trigger: {exc}".format( exc=repr(exc), name=prefect.context.get("task_full_name", self.task.name), ) ) if prefect.context.get("raise_on_exception"): raise exc raise ENDRUN( TriggerFailed( "Unexpected error while checking task trigger: {}".format( repr(exc) ), result=exc, ) ) return state
def _some_successful( upstream_states: Dict["core.Edge", "state.State"]) -> bool: """ The underlying trigger function. Args: - upstream_states (dict[Edge, State]): the set of all upstream states Returns: - bool: whether the trigger thresolds were met """ if not upstream_states: return True # scale conversions num_success = len([ s for s in _get_all_states_as_set(upstream_states) if s.is_successful() ]) num_states = len(_get_all_states_as_set(upstream_states)) if at_least is not None: min_num = (num_states * at_least) if at_least < 1 else at_least else: min_num = 0 if at_most is not None: max_num = (num_states * at_most) if at_most < 1 else at_most else: max_num = num_states if not (min_num <= num_success <= max_num): raise signals.TRIGGERFAIL( 'Trigger was "some_successful" but thresholds were not met.') return True
def all_finished(upstream_states: Set["state.State"]) -> bool: """ This task will run no matter what the upstream states are, as long as they are finished. Args: - upstream_states (set[State]): the set of all upstream states """ if not all(s.is_finished() for s in upstream_states): raise signals.TRIGGERFAIL( 'Trigger was "all_finished" but some of the upstream tasks were not finished.' ) return True
def any_failed(upstream_states: Set["state.State"]) -> bool: """ Runs if any tasks failed. Note that `SKIPPED` tasks are considered successes and `TRIGGER_FAILED` tasks are considered failures. Args: - upstream_states (set[State]): the set of all upstream states """ if upstream_states and not any(s.is_failed() for s in upstream_states): raise signals.TRIGGERFAIL( 'Trigger was "any_failed" but none of the upstream tasks failed.') return True
def all_finished(upstream_states: Dict["core.Edge", "state.State"]) -> bool: """ This task will run no matter what the upstream states are, as long as they are finished. Args: - upstream_states (dict[Edge, State]): the set of all upstream states """ if not all(s.is_finished() for s in _get_all_states_as_set(upstream_states)): raise signals.TRIGGERFAIL( 'Trigger was "all_finished" but some of the upstream tasks were not finished.' ) return True
def all_failed(upstream_states: Dict["core.Edge", "state.State"]) -> bool: """ Runs if all upstream tasks failed. Note that `SKIPPED` tasks are considered successes and `TRIGGER_FAILED` tasks are considered failures. Args: - upstream_states (dict[Edge, State]): the set of all upstream states """ if not all(s.is_failed() for s in _get_all_states_as_set(upstream_states)): raise signals.TRIGGERFAIL( 'Trigger was "all_failed" but some of the upstream tasks succeeded.' ) return True
def not_all_skipped(upstream_states: Set["state.State"]) -> bool: """ Runs if all upstream tasks were successful and were not all skipped. Args: - upstream_states (set[State]): the set of all upstream states """ if all(state.is_skipped() for state in upstream_states): raise signals.SKIP("All upstreams were skipped", result=None) elif not all(state.is_successful() for state in upstream_states): raise signals.TRIGGERFAIL( 'Trigger was "not_all_skipped" but some of the upstream tasks failed.' ) return True
def resource_cleanup_trigger(upstream_states: Dict[Edge, State]) -> bool: """Run the cleanup task, provided the following hold: - All upstream tasks have finished - The resource init task succeeded and wasn't skipped - The resource setup task succeeded and wasn't skipped """ for edge, state in upstream_states.items(): if not state.is_finished(): raise signals.TRIGGERFAIL( "Trigger was 'resource_cleanup_trigger' but some of the " "upstream tasks were not finished.") if edge.key == "mgr": if state.is_skipped(): raise signals.SKIP("Resource manager init skipped") elif not state.is_successful(): raise signals.SKIP("Resource manager init failed") elif edge.key == "resource": if state.is_skipped(): raise signals.SKIP("Resource manager setup skipped") elif not state.is_successful(): raise signals.SKIP("Resource manager setup failed") return True