Beispiel #1
0
class TaskState(object):
    """Task status and utilities."""

    # Memory optimization - constrain possible attributes to this list.
    __slots__ = [
        "identity", "status", "hold_swap", "_is_satisfied",
        "_suicide_is_satisfied", "prerequisites", "suicide_prerequisites",
        "external_triggers", "outputs", "kill_failed", "time_updated",
        "confirming_with_poll"
    ]

    def __init__(self, tdef, point, status, hold_swap):
        self.identity = TaskID.get(tdef.name, str(point))
        self.status = status
        self.hold_swap = hold_swap
        self.time_updated = None

        self._is_satisfied = None
        self._suicide_is_satisfied = None

        # Prerequisites.
        self.prerequisites = []
        self.suicide_prerequisites = []
        self._add_prerequisites(point, tdef)

        # External Triggers.
        self.external_triggers = {}
        for ext in tdef.external_triggers:
            # Allow cycle-point-specific external triggers - GitHub #1893.
            if '$CYLC_TASK_CYCLE_POINT' in ext:
                ext = ext.replace('$CYLC_TASK_CYCLE_POINT', str(point))
            # set unsatisfied
            self.external_triggers[ext] = False

        # Message outputs.
        self.outputs = TaskOutputs(tdef)

        # Standard outputs.
        self.outputs.add(TASK_OUTPUT_SUBMITTED)
        self.outputs.add(TASK_OUTPUT_STARTED)
        self.outputs.add(TASK_OUTPUT_SUCCEEDED)

        self.kill_failed = False
        self.confirming_with_poll = False

    def satisfy_me(self, all_task_outputs):
        """Attempt to get my prerequisites satisfied."""
        for prereqs in [self.prerequisites, self.suicide_prerequisites]:
            for prereq in prereqs:
                if prereq.satisfy_me(all_task_outputs):
                    self._is_satisfied = None
                    self._suicide_is_satisfied = None

    def prerequisites_are_all_satisfied(self):
        """Return True if (non-suicide) prerequisites are fully satisfied."""
        if self._is_satisfied is None:
            self._is_satisfied = all(preq.is_satisfied()
                                     for preq in self.prerequisites)
        return self._is_satisfied

    def prerequisites_are_not_all_satisfied(self):
        """Return True if (any) prerequisites are not fully satisfied."""
        return (not self.prerequisites_are_all_satisfied()
                or not self.suicide_prerequisites_are_all_satisfied())

    def suicide_prerequisites_are_all_satisfied(self):
        """Return True if all suicide prerequisites are satisfied."""
        if self._suicide_is_satisfied is None:
            self._suicide_is_satisfied = all(
                preq.is_satisfied() for preq in self.suicide_prerequisites)
        return self._suicide_is_satisfied

    def prerequisites_get_target_points(self):
        """Return a list of cycle points targeted by each prerequisite."""
        return set(point for prerequisite in self.prerequisites
                   for point in prerequisite.get_target_points())

    def prerequisites_eval_all(self):
        """Set all prerequisites to satisfied."""
        # (Validation: will abort on illegal trigger expressions.)
        for preqs in [self.prerequisites, self.suicide_prerequisites]:
            for preq in preqs:
                preq.is_satisfied()

    def set_prerequisites_all_satisfied(self):
        """Set prerequisites to all satisfied."""
        for prereq in self.prerequisites:
            prereq.set_satisfied()
        self._is_satisfied = None

    def set_prerequisites_not_satisfied(self):
        """Reset prerequisites."""
        for prereq in self.prerequisites:
            prereq.set_not_satisfied()
        self._is_satisfied = None

    def prerequisites_dump(self, list_prereqs=False):
        """Dump prerequisites."""
        if list_prereqs:
            return [
                Prerequisite.MESSAGE_TEMPLATE % msg
                for prereq in self.prerequisites
                for msg in sorted(prereq.satisfied)
            ]
        else:
            return [x for prereq in self.prerequisites for x in prereq.dump()]

    def get_resolved_dependencies(self):
        """Return a list of dependencies which have been met for this task.

        E.G: ['foo.1', 'bar.2']

        The returned list is sorted to allow comparison with reference run
        task with lots of near-simultaneous triggers.

        """
        return list(
            sorted(dep for prereq in self.prerequisites
                   for dep in prereq.get_resolved_dependencies()))

    def unset_special_outputs(self):
        """Remove special outputs added for triggering purposes.

        (Otherwise they appear as incomplete outputs when the task finishes).

        """
        self.kill_failed = False
        self.outputs.remove(TASK_OUTPUT_EXPIRED)
        self.outputs.remove(TASK_OUTPUT_SUBMIT_FAILED)
        self.outputs.remove(TASK_OUTPUT_FAILED)

    def set_held(self):
        """Set state to TASK_STATUS_HELD, if possible.

        If state can be held, set hold_swap to current state.
        If state is active, set hold_swap to TASK_STATUS_HELD.
        If state cannot be held, do nothing.
        """
        if self.status in TASK_STATUSES_ACTIVE:
            self.hold_swap = TASK_STATUS_HELD
            return
        elif self.status in [
                TASK_STATUS_WAITING, TASK_STATUS_QUEUED,
                TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RETRYING
        ]:
            return self._set_state(TASK_STATUS_HELD)

    def unset_held(self):
        """Reset to my pre-held state, if not beyond the stop point."""
        if self.status != TASK_STATUS_HELD:
            return
        elif self.hold_swap is None:
            self.reset_state(TASK_STATUS_WAITING)
        elif self.hold_swap == TASK_STATUS_HELD:
            self.hold_swap = None
        else:
            self.reset_state(self.hold_swap)

    def reset_state(self, status):
        """Reset status of task."""
        if status == TASK_STATUS_EXPIRED:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
            self.outputs.add(TASK_OUTPUT_EXPIRED, is_completed=True)
        elif status == TASK_STATUS_WAITING:
            self.set_prerequisites_not_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_READY:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_SUBMITTED:
            self.set_prerequisites_all_satisfied()
            self.outputs.set_completion(TASK_OUTPUT_SUBMITTED, True)
            # In case of manual reset, set final outputs incomplete (but assume
            # completed message outputs remain completed).
            self.outputs.set_completion(TASK_OUTPUT_SUCCEEDED, False)
            self.outputs.set_completion(TASK_OUTPUT_FAILED, False)
        elif status == TASK_STATUS_RUNNING:
            self.set_prerequisites_all_satisfied()
            self.outputs.set_completion(TASK_OUTPUT_SUBMITTED, True)
            self.outputs.set_completion(TASK_OUTPUT_STARTED, True)
            # In case of manual reset, set final outputs incomplete (but assume
            # completed message outputs remain completed).
            self.outputs.set_completion(TASK_OUTPUT_SUCCEEDED, False)
            self.outputs.set_completion(TASK_OUTPUT_FAILED, False)
        elif status == TASK_STATUS_SUBMIT_RETRYING:
            self.set_prerequisites_all_satisfied()
            self.outputs.remove(TASK_OUTPUT_SUBMITTED)
        elif status == TASK_STATUS_SUBMIT_FAILED:
            self.set_prerequisites_all_satisfied()
            self.outputs.remove(TASK_OUTPUT_SUBMITTED)
            self.outputs.add(TASK_OUTPUT_SUBMIT_FAILED, is_completed=True)
        elif status == TASK_STATUS_SUCCEEDED:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_completion(TASK_OUTPUT_SUBMITTED, True)
            self.outputs.set_completion(TASK_OUTPUT_STARTED, True)
            self.outputs.set_completion(TASK_OUTPUT_SUCCEEDED, True)
        elif status == TASK_STATUS_RETRYING:
            self.set_prerequisites_all_satisfied()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_FAILED:
            self.set_prerequisites_all_satisfied()
            self.outputs.set_all_incomplete()
            # Set a new failed output just as if a failure message came in
            self.outputs.add(TASK_OUTPUT_FAILED, is_completed=True)

        return self._set_state(status)

    def _set_state(self, status):
        """Set, log and record task status (normal change, not forced - don't
        update task_events table)."""
        if self.status == self.hold_swap:
            self.hold_swap = None
        if status == self.status and self.hold_swap is None:
            return
        o_status, o_hold_swap = self.status, self.hold_swap
        if status == TASK_STATUS_HELD:
            self.hold_swap = self.status
        elif status in TASK_STATUSES_ACTIVE:
            if self.status == TASK_STATUS_HELD:
                self.hold_swap = TASK_STATUS_HELD
        elif (self.hold_swap == TASK_STATUS_HELD
              and status not in TASK_STATUSES_FINAL):
            self.hold_swap = status
            status = TASK_STATUS_HELD
        elif self.hold_swap:
            self.hold_swap = None
        self.status = status
        self.time_updated = get_current_time_string()
        flags.iflag = True
        # Log
        message = str(o_status)
        if o_hold_swap:
            message += " (%s)" % o_hold_swap
        message += " => %s" % self.status
        if self.hold_swap:
            message += " (%s)" % self.hold_swap
        LOG.debug(message, itask=self.identity)

    def is_greater_than(self, status):
        """"Return True if self.status > status."""
        return (TASK_STATUSES_ORDERED.index(self.status) >
                TASK_STATUSES_ORDERED.index(status))

    def _add_prerequisites(self, point, tdef):
        """Add task prerequisites."""
        # Triggers for sequence_i only used if my cycle point is a
        # valid member of sequence_i's sequence of cycle points.
        self._is_satisfied = None
        self._suicide_is_satisfied = None

        for sequence, dependencies in tdef.dependencies.items():
            if not sequence.is_valid(point):
                continue
            for dependency in dependencies:
                cpre = dependency.get_prerequisite(point, tdef)
                if dependency.suicide:
                    self.suicide_prerequisites.append(cpre)
                else:
                    self.prerequisites.append(cpre)

        if tdef.sequential:
            # Add a previous-instance succeeded prerequisite.
            p_prev = None
            adjusted = []
            for seq in tdef.sequences:
                prv = seq.get_nearest_prev_point(point)
                if prv:
                    # None if out of sequence bounds.
                    adjusted.append(prv)
            if adjusted:
                p_prev = max(adjusted)
                cpre = Prerequisite(point, tdef.start_point)
                cpre.add(tdef.name, p_prev, TASK_STATUS_SUCCEEDED,
                         p_prev < tdef.start_point)
                cpre.set_condition(tdef.name)
                self.prerequisites.append(cpre)
Beispiel #2
0
class TaskState(object):
    """Task status and utilities."""

    # Associate status names with other properties.
    _STATUS_MAP = {
        TASK_STATUS_RUNAHEAD: {
            "gtk_label": "r_unahead",  # GTK widget labels.
            "ascii_ctrl": "\033[1;37;44m"  # Terminal color control codes.
        },
        TASK_STATUS_WAITING: {
            "gtk_label": "_waiting",
            "ascii_ctrl": "\033[1;36m"
        },
        TASK_STATUS_HELD: {
            "gtk_label": "_held",
            "ascii_ctrl": "\033[1;37;43m"
        },
        TASK_STATUS_QUEUED: {
            "gtk_label": "_queued",
            "ascii_ctrl": "\033[1;38;44m"
        },
        TASK_STATUS_READY: {
            "gtk_label": "rea_dy",
            "ascii_ctrl": "\033[1;32m"
        },
        TASK_STATUS_EXPIRED: {
            "gtk_label": "e_xpired",
            "ascii_ctrl": "\033[1;37;40m"
        },
        TASK_STATUS_SUBMITTED: {
            "gtk_label": "sub_mitted",
            "ascii_ctrl": "\033[1;33m"
        },
        TASK_STATUS_SUBMIT_FAILED: {
            "gtk_label": "submit-f_ailed",
            "ascii_ctrl": "\033[1;34m"
        },
        TASK_STATUS_SUBMIT_RETRYING: {
            "gtk_label": "submit-retryin_g",
            "ascii_ctrl": "\033[1;31m"
        },
        TASK_STATUS_RUNNING: {
            "gtk_label": "_running",
            "ascii_ctrl": "\033[1;37;42m"
        },
        TASK_STATUS_SUCCEEDED: {
            "gtk_label": "_succeeded",
            "ascii_ctrl": "\033[0m"
        },
        TASK_STATUS_FAILED: {
            "gtk_label": "_failed",
            "ascii_ctrl": "\033[1;37;41m"
        },
        TASK_STATUS_RETRYING: {
            "gtk_label": "retr_ying",
            "ascii_ctrl": "\033[1;35m"
        }
    }

    @classmethod
    def get_status_prop(cls, status, key, subst=None):
        """Return property for a task status."""
        if key == "ascii_ctrl":
            if subst is not None:
                return "%s%s\033[0m" % (cls._STATUS_MAP[status][key], subst)
            else:
                return "%s%s\033[0m" % (cls._STATUS_MAP[status][key], status)
        try:
            return cls._STATUS_MAP[status][key]
        except KeyError:
            raise TaskStateError("Bad task status (%s, %s)" % (status, key))

    def __init__(self, status, point, identity, tdef, db_events_insert,
                 db_update_status, log):

        self.status = status
        self.identity = identity
        self.db_events_insert = db_events_insert
        self.db_update_status = db_update_status
        self.log = log

        self._recalc_satisfied = True
        self._is_satisfied = False
        self._suicide_is_satisfied = False

        # Prerequisites.
        self.prerequisites = []
        self.suicide_prerequisites = []
        self._add_prerequisites(point, identity, tdef)

        # External Triggers.
        self.external_triggers = {}
        for ext in tdef.external_triggers:
            # Allow cycle-point-specific external triggers - GitHub #1893.
            if '$CYLC_TASK_CYCLE_POINT' in ext:
                ext = ext.replace('$CYLC_TASK_CYCLE_POINT', str(point))
            # set unsatisfied
            self.external_triggers[ext] = False

        # Message outputs.
        self.outputs = TaskOutputs(identity)
        for outp in tdef.outputs:
            self.outputs.add(outp.get_string(point))

        # Standard outputs.
        self.outputs.add(TASK_OUTPUT_SUBMITTED)
        self.outputs.add(TASK_OUTPUT_STARTED)
        self.outputs.add(TASK_OUTPUT_SUCCEEDED)

        self.kill_failed = False
        self.hold_on_retry = False
        self._state_pre_hold = None
        self.run_mode = tdef.run_mode

        # TODO - these are here because current use in reset_state(); should be
        # disentangled and put in the task_proxy module.
        self.submission_timer_timeout = None
        self.execution_timer_timeout = None

    def satisfy_me(self, task_output_msgs, task_outputs):
        """Attempt to get my prerequisites satisfied."""
        self._recalc_satisfied = False
        for preqs in [self.prerequisites, self.suicide_prerequisites]:
            for preq in preqs:
                if preq.satisfy_me(task_output_msgs, task_outputs):
                    self._recalc_satisfied = True

    def prerequisites_are_all_satisfied(self):
        """Return True if (non-suicide) prerequisites are fully satisfied."""
        if self._recalc_satisfied:
            self._is_satisfied = all(
                preq.is_satisfied() for preq in self.prerequisites)
        return self._is_satisfied

    def prerequisites_are_not_all_satisfied(self):
        """Return True if (any) prerequisites are not fully satisfied."""
        if self._recalc_satisfied:
            return (not self.prerequisites_are_all_satisfied() or
                    not self.suicide_prerequisites_are_all_satisfied())
        return (not self._is_satisfied or not self._suicide_is_satisfied)

    def suicide_prerequisites_are_all_satisfied(self):
        """Return True if all suicide prerequisites are satisfied."""
        if self._recalc_satisfied:
            self._suicide_is_satisfied = all(
                preq.is_satisfied() for preq in self.suicide_prerequisites)
        return self._suicide_is_satisfied

    def prerequisites_get_target_points(self):
        """Return a list of cycle points targetted by each prerequisite."""
        points = []
        for preq in self.prerequisites:
            points += preq.get_target_points()
        return points

    def prerequisites_eval_all(self):
        """Set all prerequisites to satisfied."""
        # (Validation: will abort on illegal trigger expressions.)
        for preqs in [self.prerequisites, self.suicide_prerequisites]:
            for preq in preqs:
                preq.is_satisfied()

    def set_prerequisites_all_satisfied(self):
        """Set prerequisites to all satisfied."""
        for prereq in self.prerequisites:
            prereq.set_satisfied()
        self._recalc_satisfied = True

    def set_prerequisites_not_satisfied(self):
        """Reset prerequisites."""
        for prereq in self.prerequisites:
            prereq.set_not_satisfied()
        self._recalc_satisfied = True

    def prerequisites_dump(self):
        """Dump prerequisites."""
        res = []
        for preq in self.prerequisites:
            res += preq.dump()
        return res

    def get_resolved_dependencies(self):
        """Report who I triggered off."""
        satby = {}
        for req in self.prerequisites:
            satby.update(req.satisfied_by)
        dep = satby.values()
        # order does not matter here; sort to allow comparison with
        # reference run task with lots of near-simultaneous triggers.
        dep.sort()
        return dep

    def unset_special_outputs(self):
        """Remove special outputs added for triggering purposes.

        (Otherwise they appear as incomplete outputs when the task finishes).

        """
        self.hold_on_retry = False
        self.kill_failed = False
        self.outputs.remove(TASK_OUTPUT_EXPIRED)
        self.outputs.remove(TASK_OUTPUT_SUBMIT_FAILED)
        self.outputs.remove(TASK_OUTPUT_FAILED)

    def release(self):
        """Reset to my pre-held state, if not beyond the stop point."""
        self.hold_on_retry = False
        if not self.status == TASK_STATUS_HELD:
            return
        if self._state_pre_hold is None:
            self.reset_state(TASK_STATUS_WAITING)
            return
        old_status = self._state_pre_hold
        self._state_pre_hold = None
        self.log(INFO, 'held => %s' % (old_status))

        # Turn off submission and execution timeouts.
        self.submission_timer_timeout = None
        self.execution_timer_timeout = None
        self.set_state(old_status)

    def set_state(self, status):
        """Set, log and record task status (normal change, not forced - don't
        update task_events table)."""
        if status != self.status:
            flags.iflag = True
            self.log(DEBUG, '(setting: %s)' % status)
            self.status = status
            self.db_update_status()

    def reset_state(self, status):
        """Reset status of task."""
        if status == TASK_STATUS_HELD:
            if self.status in TASK_STATUSES_ACTIVE:
                self.hold_on_retry = True
                return
            if self.status not in [
                    TASK_STATUS_WAITING, TASK_STATUS_QUEUED,
                    TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RETRYING]:
                return
            self._state_pre_hold = self.status
            self.log(INFO, '%s => held' % self._state_pre_hold)

        # Turn off submission and execution timeouts.
        self.submission_timer_timeout = None
        self.execution_timer_timeout = None

        self.set_state(status)
        if status == TASK_STATUS_EXPIRED:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
            self.outputs.add(TASK_OUTPUT_EXPIRED, True)
        elif status == TASK_STATUS_WAITING:
            self.set_prerequisites_not_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_READY:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_SUCCEEDED:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            # TODO - for message outputs this should be optional (see #1551):
            self.outputs.set_all_completed()
        elif status == TASK_STATUS_FAILED:
            self.set_prerequisites_all_satisfied()
            self.hold_on_retry = False
            self.outputs.set_all_incomplete()
            # Set a new failed output just as if a failure message came in
            self.outputs.add(TASK_OUTPUT_FAILED, True)
        # TODO - handle other state resets here too, such as retrying...?

    def is_ready_to_run(self, retry_delay_done, start_time_reached):
        """With current status, is the task ready to run?"""
        return (
            (
                (
                    self.status == TASK_STATUS_WAITING and
                    self.prerequisites_are_all_satisfied() and
                    all(self.external_triggers.values())
                ) or
                (
                    self.status in [TASK_STATUS_SUBMIT_RETRYING,
                                    TASK_STATUS_RETRYING] and
                    retry_delay_done
                )
            ) and start_time_reached
        )

    def is_greater_than(self, status):
        """"Return True if self.status > status."""
        return (TASK_STATUSES_ORDERED.index(self.status) >
                TASK_STATUSES_ORDERED.index(status))

    def set_expired(self):
        """Manipulate state for task expired."""
        self.reset_state(TASK_STATUS_EXPIRED)

    def set_ready_to_submit(self):
        """Manipulate state just prior to job submission."""
        self.set_state(TASK_STATUS_READY)

    def set_submit_failed(self):
        """Manipulate state after job submission failure."""
        self.set_state(TASK_STATUS_SUBMIT_FAILED)
        self.outputs.remove(TASK_OUTPUT_SUBMITTED)
        self.outputs.add(TASK_OUTPUT_SUBMIT_FAILED, True)

    def set_submit_retry(self):
        """Manipulate state for job submission retry."""
        self.outputs.remove(TASK_OUTPUT_SUBMITTED)
        self.set_state(TASK_STATUS_SUBMIT_RETRYING)
        self.set_prerequisites_all_satisfied()
        if self.hold_on_retry:
            self.reset_state(TASK_STATUS_HELD)

    def set_submit_succeeded(self):
        """Set status to submitted."""
        if not self.outputs.is_completed(TASK_OUTPUT_SUBMITTED):
            self.outputs.set_completed(TASK_OUTPUT_SUBMITTED)
            # Allow submitted tasks to spawn even if nothing else is happening.
            flags.pflag = True
        if self.status == TASK_STATUS_READY:
            # In rare occassions, the submit command of a batch system has sent
            # the job to its server, and the server has started the job before
            # the job submit command returns.
            self.set_state(TASK_STATUS_SUBMITTED)
            return True
        else:
            return False

    def set_executing(self):
        """Manipulate state for job execution."""
        self.set_state(TASK_STATUS_RUNNING)
        if self.run_mode == 'simulation':
            self.outputs.set_completed(TASK_OUTPUT_STARTED)

    def set_execution_succeeded(self, msg_was_polled):
        """Manipulate state for job execution success."""
        self.set_state(TASK_STATUS_SUCCEEDED)
        if not self.outputs.all_completed():
            err = "Succeeded with unreported outputs:"
            for key in self.outputs.not_completed:
                err += "\n  " + key
            self.log(WARNING, err)
            if msg_was_polled:
                # Assume all outputs complete (e.g. poll at restart).
                # TODO - just poll for outputs in the job status file.
                self.log(WARNING, "Assuming ALL outputs completed.")
                self.outputs.set_all_completed()
            else:
                # A succeeded task MUST have submitted and started.
                # TODO - just poll for outputs in the job status file?
                for output in [TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED]:
                    if not self.outputs.is_completed(output):
                        self.log(WARNING,
                                 "Assuming output completed:  \n %s" % output)
                        self.outputs.set_completed(output)

    def set_execution_failed(self):
        """Manipulate state for job execution failure."""
        self.reset_state(TASK_STATUS_FAILED)

    def set_execution_retry(self):
        """Manipulate state for job execution retry."""
        self.set_state(TASK_STATUS_RETRYING)
        self.set_prerequisites_all_satisfied()
        if self.hold_on_retry:
            self.reset_state(TASK_STATUS_HELD)

    def record_output(self, msg, msg_was_polled):
        """Record a completed output."""
        if self.outputs.exists(msg):
            if not self.outputs.is_completed(msg):
                flags.pflag = True
                self.outputs.set_completed(msg)
                self.db_events_insert(event="output completed", message=msg)
            elif not msg_was_polled:
                # This output has already been reported complete. Not an error
                # condition - maybe the network was down for a bit. Ok for
                # polling as multiple polls *should* produce the same result.
                self.log(WARNING,
                         "Unexpected output (already completed):\n  " + msg)

    def _add_prerequisites(self, point, identity, tdef):
        """Add task prerequisites."""
        # self.triggers[sequence] = [triggers for sequence]
        # Triggers for sequence_i only used if my cycle point is a
        # valid member of sequence_i's sequence of cycle points.
        self._recalc_satisfied = True

        for sequence, exps in tdef.triggers.items():
            for ctrig, exp in exps:
                key = ctrig.keys()[0]
                if not sequence.is_valid(point):
                    # This trigger is not valid for current cycle (see NOTE
                    # just above)
                    continue

                cpre = Prerequisite(identity, point, tdef.start_point)

                for label in ctrig:
                    trig = ctrig[label]
                    if trig.graph_offset_string is not None:
                        prereq_offset_point = get_point_relative(
                            trig.graph_offset_string, point)
                        if prereq_offset_point > point:
                            prereq_offset = prereq_offset_point - point
                            if (tdef.max_future_prereq_offset is None or
                                    (prereq_offset >
                                     tdef.max_future_prereq_offset)):
                                tdef.max_future_prereq_offset = (
                                    prereq_offset)
                        cpre.add(trig.get_prereq(point), label,
                                 ((prereq_offset_point < tdef.start_point) &
                                  (point >= tdef.start_point)))
                    else:
                        cpre.add(trig.get_prereq(point), label)
                cpre.set_condition(exp)
                if ctrig[key].suicide:
                    self.suicide_prerequisites.append(cpre)
                else:
                    self.prerequisites.append(cpre)

        if tdef.sequential:
            # Add a previous-instance succeeded prerequisite.
            p_prev = None
            adjusted = []
            for seq in tdef.sequences:
                prv = seq.get_nearest_prev_point(point)
                if prv:
                    # None if out of sequence bounds.
                    adjusted.append(prv)
            if adjusted:
                p_prev = max(adjusted)
                cpre = Prerequisite(identity, point, tdef.start_point)
                prereq = "%s %s" % (TaskID.get(tdef.name, p_prev),
                                    TASK_STATUS_SUCCEEDED)
                label = tdef.name
                cpre.add(prereq, label, p_prev < tdef.start_point)
                cpre.set_condition(label)
                self.prerequisites.append(cpre)
Beispiel #3
0
class TaskState(object):
    """Task status and utilities."""

    # Memory optimization - constrain possible attributes to this list.
    __slots__ = ["identity", "status", "hold_swap",
                 "_is_satisfied", "_suicide_is_satisfied", "prerequisites",
                 "suicide_prerequisites", "external_triggers", "outputs",
                 "kill_failed", "time_updated"]

    def __init__(self, tdef, point, status, hold_swap):
        self.identity = TaskID.get(tdef.name, str(point))
        self.status = status
        self.hold_swap = hold_swap
        self.time_updated = None

        self._is_satisfied = None
        self._suicide_is_satisfied = None

        # Prerequisites.
        self.prerequisites = []
        self.suicide_prerequisites = []
        self._add_prerequisites(point, tdef)

        # External Triggers.
        self.external_triggers = {}
        for ext in tdef.external_triggers:
            # Allow cycle-point-specific external triggers - GitHub #1893.
            if '$CYLC_TASK_CYCLE_POINT' in ext:
                ext = ext.replace('$CYLC_TASK_CYCLE_POINT', str(point))
            # set unsatisfied
            self.external_triggers[ext] = False

        # Message outputs.
        self.outputs = TaskOutputs(tdef, point)

        # Standard outputs.
        self.outputs.add(TASK_OUTPUT_SUBMITTED)
        self.outputs.add(TASK_OUTPUT_STARTED)
        self.outputs.add(TASK_OUTPUT_SUCCEEDED)

        self.kill_failed = False

    def satisfy_me(self, task_output_msgs, task_outputs):
        """Attempt to get my prerequisites satisfied."""
        for preqs in [self.prerequisites, self.suicide_prerequisites]:
            for preq in preqs:
                if preq.satisfy_me(task_output_msgs, task_outputs):
                    self._is_satisfied = None
                    self._suicide_is_satisfied = None

    def prerequisites_are_all_satisfied(self):
        """Return True if (non-suicide) prerequisites are fully satisfied."""
        if self._is_satisfied is None:
            self._is_satisfied = all(
                preq.is_satisfied() for preq in self.prerequisites)
        return self._is_satisfied

    def prerequisites_are_not_all_satisfied(self):
        """Return True if (any) prerequisites are not fully satisfied."""
        return (not self.prerequisites_are_all_satisfied() or
                not self.suicide_prerequisites_are_all_satisfied())

    def suicide_prerequisites_are_all_satisfied(self):
        """Return True if all suicide prerequisites are satisfied."""
        if self._suicide_is_satisfied is None:
            self._suicide_is_satisfied = all(
                preq.is_satisfied() for preq in self.suicide_prerequisites)
        return self._suicide_is_satisfied

    def prerequisites_get_target_points(self):
        """Return a list of cycle points targeted by each prerequisite."""
        points = []
        for preq in self.prerequisites:
            points += preq.get_target_points()
        return points

    def prerequisites_eval_all(self):
        """Set all prerequisites to satisfied."""
        # (Validation: will abort on illegal trigger expressions.)
        for preqs in [self.prerequisites, self.suicide_prerequisites]:
            for preq in preqs:
                preq.is_satisfied()

    def set_prerequisites_all_satisfied(self):
        """Set prerequisites to all satisfied."""
        for prereq in self.prerequisites:
            prereq.set_satisfied()
        self._is_satisfied = None

    def set_prerequisites_not_satisfied(self):
        """Reset prerequisites."""
        for prereq in self.prerequisites:
            prereq.set_not_satisfied()
        self._is_satisfied = None

    def prerequisites_dump(self, list_prereqs=False):
        """Dump prerequisites."""
        res = []
        if list_prereqs:
            for prereq in self.prerequisites:
                for task in sorted(prereq.messages):
                    res.append(task)
        else:
            for preq in self.prerequisites:
                res += preq.dump()
        return res

    def get_resolved_dependencies(self):
        """Report who I triggered off."""
        satby = {}
        for req in self.prerequisites:
            satby.update(req.satisfied_by)
        dep = satby.values()
        # order does not matter here; sort to allow comparison with
        # reference run task with lots of near-simultaneous triggers.
        dep.sort()
        return dep

    def unset_special_outputs(self):
        """Remove special outputs added for triggering purposes.

        (Otherwise they appear as incomplete outputs when the task finishes).

        """
        self.kill_failed = False
        self.outputs.remove(TASK_OUTPUT_EXPIRED)
        self.outputs.remove(TASK_OUTPUT_SUBMIT_FAILED)
        self.outputs.remove(TASK_OUTPUT_FAILED)

    def set_held(self):
        """Set state to TASK_STATUS_HELD, if possible.

        If state can be held, set hold_swap to current state.
        If state is active, set hold_swap to TASK_STATUS_HELD.
        If state cannot be held, do nothing.
        """
        if self.status in TASK_STATUSES_ACTIVE:
            self.hold_swap = TASK_STATUS_HELD
            return
        elif self.status in [
                TASK_STATUS_WAITING, TASK_STATUS_QUEUED,
                TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RETRYING]:
            return self._set_state(TASK_STATUS_HELD)

    def unset_held(self):
        """Reset to my pre-held state, if not beyond the stop point."""
        if self.status != TASK_STATUS_HELD:
            return
        elif self.hold_swap is None:
            self.reset_state(TASK_STATUS_WAITING)
        elif self.hold_swap == TASK_STATUS_HELD:
            self.hold_swap = None
        else:
            self.reset_state(self.hold_swap)

    def reset_state(self, status):
        """Reset status of task."""
        if status == TASK_STATUS_EXPIRED:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
            self.outputs.add(TASK_OUTPUT_EXPIRED, is_completed=True)
        elif status == TASK_STATUS_WAITING:
            self.set_prerequisites_not_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_READY:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_SUBMITTED:
            self.set_prerequisites_all_satisfied()
            self.outputs.set_completed(TASK_OUTPUT_SUBMITTED)
        elif status == TASK_STATUS_SUBMIT_RETRYING:
            self.set_prerequisites_all_satisfied()
            self.outputs.remove(TASK_OUTPUT_SUBMITTED)
        elif status == TASK_STATUS_SUBMIT_FAILED:
            self.set_prerequisites_all_satisfied()
            self.outputs.remove(TASK_OUTPUT_SUBMITTED)
            self.outputs.add(TASK_OUTPUT_SUBMIT_FAILED, is_completed=True)
        elif status == TASK_STATUS_SUCCEEDED:
            self.set_prerequisites_all_satisfied()
            self.unset_special_outputs()
            self.outputs.set_completed(TASK_OUTPUT_SUBMITTED)
            self.outputs.set_completed(TASK_OUTPUT_STARTED)
            self.outputs.set_completed(TASK_OUTPUT_SUCCEEDED)
        elif status == TASK_STATUS_RETRYING:
            self.set_prerequisites_all_satisfied()
            self.outputs.set_all_incomplete()
        elif status == TASK_STATUS_FAILED:
            self.set_prerequisites_all_satisfied()
            self.outputs.set_all_incomplete()
            # Set a new failed output just as if a failure message came in
            self.outputs.add(TASK_OUTPUT_FAILED, is_completed=True)

        return self._set_state(status)

    def _set_state(self, status):
        """Set, log and record task status (normal change, not forced - don't
        update task_events table)."""
        if self.status == self.hold_swap:
            self.hold_swap = None
        if status == self.status and self.hold_swap is None:
            return
        o_status, o_hold_swap = self.status, self.hold_swap
        if status == TASK_STATUS_HELD:
            self.hold_swap = self.status
        elif (self.hold_swap == TASK_STATUS_HELD and
                status not in TASK_STATUSES_FINAL):
            self.hold_swap = status
            status = TASK_STATUS_HELD
        elif self.hold_swap:
            self.hold_swap = None
        self.status = status
        self.time_updated = get_current_time_string()
        flags.iflag = True
        # Log
        message = str(o_status)
        if o_hold_swap:
            message += " (%s)" % o_hold_swap
        message += " => %s" % self.status
        if self.hold_swap:
            message += " (%s)" % self.hold_swap
        LOG.debug(message, itask=self.identity)

    def is_greater_than(self, status):
        """"Return True if self.status > status."""
        return (TASK_STATUSES_ORDERED.index(self.status) >
                TASK_STATUSES_ORDERED.index(status))

    def _add_prerequisites(self, point, tdef):
        """Add task prerequisites."""
        # Triggers for sequence_i only used if my cycle point is a
        # valid member of sequence_i's sequence of cycle points.
        self._is_satisfied = None
        self._suicide_is_satisfied = None
        identity = TaskID.get(tdef.name, str(point))

        for sequence, dependencies in tdef.dependencies.items():
            if not sequence.is_valid(point):
                continue
            for dependency in dependencies:
                cpre = dependency.get_prerequisite(point, tdef)
                if dependency.suicide:
                    self.suicide_prerequisites.append(cpre)
                else:
                    self.prerequisites.append(cpre)

        if tdef.sequential:
            # Add a previous-instance succeeded prerequisite.
            p_prev = None
            adjusted = []
            for seq in tdef.sequences:
                prv = seq.get_nearest_prev_point(point)
                if prv:
                    # None if out of sequence bounds.
                    adjusted.append(prv)
            if adjusted:
                p_prev = max(adjusted)
                cpre = Prerequisite(point, tdef.start_point)
                prereq = "%s %s" % (TaskID.get(tdef.name, p_prev),
                                    TASK_STATUS_SUCCEEDED)
                cpre.add(prereq, p_prev < tdef.start_point)
                cpre.set_condition(tdef.name)
                self.prerequisites.append(cpre)