def __init__(self, tdef: 'TaskDef', start_point: 'PointBase', flow_label: Optional[str], status: str = TASK_STATUS_WAITING, is_held: bool = False, submit_num: int = 0, is_late: bool = False, reflow: bool = True) -> None: self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs: List[str] = [] self.flow_label = flow_label self.reflow = reflow self.point = start_point self.identity: str = TaskID.get(self.tdef.name, self.point) self.reload_successor: Optional['TaskProxy'] = None self.point_as_seconds: Optional[int] = None self.is_manual_submit = False self.summary: Dict[str, Any] = { 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'platforms_used': {}, 'execution_time_limit': None, 'job_runner_name': None, 'submit_method_id': None, 'flow_label': None } self.local_job_file_path: Optional[str] = None self.platform = get_platform() self.job_vacated = False self.poll_timer: Optional['TaskActionTimer'] = None self.timeout: Optional[float] = None self.try_timers: Dict[str, 'TaskActionTimer'] = {} self.non_unique_events = Counter() # type: ignore # TODO: figure out self.clock_trigger_time: Optional[float] = None self.expire_time: Optional[float] = None self.late_time: Optional[float] = None self.is_late = is_late self.waiting_on_job_prep = True self.state = TaskState(tdef, self.point, status, is_held) # Determine graph children of this task (for spawning). self.graph_children = generate_graph_children(tdef, self.point)
def test_reset_outputs(before, after, outputs): """Test that outputs are reset correctly on state changes.""" tdef = TaskDef('foo', {}, 'live', '123', '123') orig_status, orig_is_held = before new_status, new_is_held = after tstate = TaskState(tdef, '123', orig_status, orig_is_held) assert tstate.outputs.get_completed() == [] tstate.reset(status=new_status, is_held=new_is_held) assert tstate.outputs.get_completed() == outputs
def test_reset(state, is_held, should_reset): """Test that tasks do or don't have their state changed.""" tdef = TaskDef('foo', {}, 'live', '123', '123') # create task state: # * status: waiting # * is_held: true tstate = TaskState(tdef, '123', TASK_STATUS_WAITING, True) assert tstate.reset(state, is_held) == should_reset if is_held is not None: assert tstate.is_held == is_held if state is not None: assert tstate.status == state
def test_reset_state(self): """Test instantiation and simple resets.""" point = ISO8601Point('2020') taskdef = TaskDef('who-cares', {}, 'live', point, False) taskstate = TaskState(taskdef, point, TASK_STATUS_WAITING, None) self.assertIsNone( taskstate.reset_state(TASK_STATUS_WAITING), 'same status returns None', ) self.assertEqual( taskstate.reset_state(TASK_STATUS_SUCCEEDED), (TASK_STATUS_WAITING, None), 'different status returns previous (status, hold_swap)', ) self.assertEqual( (taskstate.status, taskstate.hold_swap), (TASK_STATUS_SUCCEEDED, None), 'reset status OK', )
def test_reset_state_respect_hold_swap(self): point = ISO8601Point('2020') taskdef = TaskDef('who-cares', {}, 'live', point, False) taskstate = TaskState( taskdef, point, TASK_STATUS_HELD, TASK_STATUS_RETRYING) self.assertIsNone( taskstate.reset_state( TASK_STATUS_RETRYING, respect_hold_swap=True), 'same status returns None', ) self.assertEqual( taskstate.reset_state( TASK_STATUS_SUCCEEDED, respect_hold_swap=True), (TASK_STATUS_HELD, TASK_STATUS_RETRYING), 'different status returns previous (status, hold_swap)', ) self.assertEqual( (taskstate.status, taskstate.hold_swap), (TASK_STATUS_SUCCEEDED, None), 'reset status OK', )
def test_reset_state_respect_hold_swap(self): point = ISO8601Point('2020') taskdef = TaskDef('who-cares', {}, 'live', point, False) taskstate = TaskState(taskdef, point, TASK_STATUS_HELD, TASK_STATUS_RETRYING) self.assertIsNone( taskstate.reset_state(TASK_STATUS_RETRYING, respect_hold_swap=True), 'same status returns None', ) self.assertEqual( taskstate.reset_state(TASK_STATUS_SUCCEEDED, respect_hold_swap=True), (TASK_STATUS_HELD, TASK_STATUS_RETRYING), 'different status returns previous (status, hold_swap)', ) self.assertEqual( (taskstate.status, taskstate.hold_swap), (TASK_STATUS_SUCCEEDED, None), 'reset status OK', )
def test_state_comparison(state, is_held): """Test the __call__ method.""" tdef = TaskDef('foo', {}, 'live', '123', '123') tstate = TaskState(tdef, '123', state, is_held) assert tstate(state, is_held=is_held) assert tstate(state) assert tstate(is_held=is_held) assert tstate(state, 'of', 'flux') assert tstate(state, 'of', 'flux', is_held=is_held) assert not tstate(state + 'x', is_held=not is_held) assert not tstate(state, is_held=not is_held) assert not tstate(state + 'x', is_held=is_held) assert not tstate(state + 'x') assert not tstate(is_held=not is_held) assert not tstate(state + 'x', 'of', 'flux')
class TaskProxy(object): """Represent an instance of a cycling task in a running suite. Attributes: .cleanup_cutoff (cylc.flow.cycling.PointBase): Cycle point beyond which this task can be removed from the pool. .clock_trigger_time (float): Clock trigger time in seconds since epoch. .expire_time (float): Time in seconds since epoch when this task is considered expired. .has_spawned (boolean): Has this task spawned its successor in the sequence? .identity (str): Task ID in NAME.POINT syntax. .is_late (boolean): Is the task late? .is_manual_submit (boolean): Is the latest job submission due to a manual trigger? .job_vacated (boolean): Is the latest job pre-empted (or vacated)? .local_job_file_path (str): Path on suite host to the latest job script for running the task. .late_time (float): Time in seconds since epoch, beyond which the task is considered late if it is never active. .manual_trigger (boolean): Has this task received a manual trigger command? This flag is reset on trigger. .non_unique_events (dict): Count non-unique events (e.g. critical, warning, custom). .point (cylc.flowcycling.PointBase): Cycle point of the task. .point_as_seconds (int): Cycle point as seconds since epoch. .poll_timer (cylc.flow.task_action_timer.TaskActionTimer): Schedule for polling submitted or running jobs. .reload_successor (cylc.flow.task_proxy.TaskProxy): The task proxy object that replaces the current instance on reload. This attribute provides a useful link to the latest replacement instance while the current object may still be referenced by a job manipulation command. .stop_point (cylc.flow.cycling.PointBase): Do not spawn successor beyond this point. .submit_num (int): Number of times the task has attempted job submission. .summary (dict): batch_sys_name (str): Name of batch system where latest job is submitted. description (str): Same as the .tdef.rtconfig['meta']['description'] attribute. execution_time_limit (float): Execution time limit of latest job. finished_time (float): Latest job exit time. finished_time_string (str): Latest job exit time as string. job_hosts (dict): Jobs' owner@host by submit number. label (str): The .point attribute as string. latest_message (str): Latest job or event message. logfiles (list): List of names of (extra) known job log files. name (str): Same as the .tdef.name attribute. started_time (float): Latest job execution start time. started_time_string (str): Latest job execution start time as string. submit_method_id (str): Latest ID of job in batch system. submit_num (int): Same as the .submit_num attribute. submitted_time (float): Latest job submission time. submitted_time_string (str): Latest job submission time as string. title (str): Same as the .tdef.rtconfig['meta']['title'] attribute. .state (cylc.flow.task_state.TaskState): Object representing the state of this task. .task_host (str) Name of host where latest job is submitted. .task_owner (str) Name of user (at task_host) where latest job is submitted. .tdef (cylc.flow.taskdef.TaskDef): The definition object of this task. .timeout (float): Timeout value in seconds since epoch for latest job submission/execution. .try_timers (dict) Retry schedules as cylc.flow.task_action_timer.TaskActionTimer objects. Arguments: tdef (cylc.flow.taskdef.TaskDef): The definition object of this task. start_point (cylc.flow.cycling.PointBase): Start point to calculate the task's cycle point on start up or the cycle point for subsequent tasks. status (str): Task state string. hold_swap (str): Original task state string, if task is held. has_spawned (boolean): Has this task spawned its successor in the sequence. stop_point (cylc.flow.cycling.PointBase): Do not spawn successor beyond this point. is_startup (boolean): Is this on start up? submit_num (int): Number of times the task has attempted job submission. late_time (float): Time in seconds since epoch, beyond which the task is considered late if it is never active. """ # Memory optimization - constrain possible attributes to this list. __slots__ = [ 'cleanup_cutoff', 'clock_trigger_time', 'expire_time', 'has_spawned', 'identity', 'is_late', 'is_manual_submit', 'job_vacated', 'late_time', 'local_job_file_path', 'manual_trigger', 'non_unique_events', 'point', 'point_as_seconds', 'poll_timer', 'reload_successor', 'submit_num', 'tdef', 'state', 'stop_point', 'summary', 'task_host', 'task_owner', 'timeout', 'try_timers', ] def __init__( self, tdef, start_point, status=TASK_STATUS_WAITING, hold_swap=None, has_spawned=False, stop_point=None, is_startup=False, submit_num=0, is_late=False): self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num if is_startup: # adjust up to the first on-sequence cycle point adjusted = [] for seq in self.tdef.sequences: adj = seq.get_first_point(start_point) if adj: # may be None if out of sequence bounds adjusted.append(adj) if not adjusted: # This task is out of sequence bounds raise TaskProxySequenceBoundsError(self.tdef.name) self.point = min(adjusted) self.late_time = None else: self.point = start_point self.cleanup_cutoff = self.tdef.get_cleanup_cutoff_point(self.point) self.identity = TaskID.get(self.tdef.name, self.point) self.has_spawned = has_spawned self.reload_successor = None self.point_as_seconds = None # Manually inserted tasks may have a final cycle point set. self.stop_point = stop_point self.manual_trigger = False self.is_manual_submit = False self.summary = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'job_hosts': {}, 'execution_time_limit': None, 'batch_sys_name': None, 'submit_method_id': None } self.local_job_file_path = None self.task_host = 'localhost' self.task_owner = None self.job_vacated = False self.poll_timer = None self.timeout = None self.try_timers = {} # Use dict here for Python 2.6 compat. # Should use collections.Counter in Python 2.7+ self.non_unique_events = {} self.clock_trigger_time = None self.expire_time = None self.late_time = None self.is_late = is_late self.state = TaskState(tdef, self.point, status, hold_swap) if tdef.sequential: # Adjust clean-up cutoff. p_next = None adjusted = [] for seq in tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) if (self.cleanup_cutoff is not None and self.cleanup_cutoff < p_next): self.cleanup_cutoff = p_next def __str__(self): """Stringify using "self.identity".""" return self.identity def copy_to_reload_successor(self, reload_successor): """Copy attributes to successor on reload of this task proxy.""" self.reload_successor = reload_successor reload_successor.submit_num = self.submit_num reload_successor.has_spawned = self.has_spawned reload_successor.manual_trigger = self.manual_trigger reload_successor.is_manual_submit = self.is_manual_submit reload_successor.summary = self.summary reload_successor.local_job_file_path = self.local_job_file_path reload_successor.try_timers = self.try_timers reload_successor.task_host = self.task_host reload_successor.task_owner = self.task_owner reload_successor.job_vacated = self.job_vacated reload_successor.poll_timer = self.poll_timer reload_successor.timeout = self.timeout reload_successor.state.outputs = self.state.outputs reload_successor.state.is_updated = self.state.is_updated @staticmethod def get_offset_as_seconds(offset): """Return an ISO interval as seconds.""" iso_offset = cylc.flow.cycling.iso8601.interval_parse(str(offset)) return int(iso_offset.get_seconds()) def get_late_time(self): """Compute and store late time as seconds since epoch.""" if self.late_time is None: if self.tdef.rtconfig['events']['late offset']: self.late_time = ( self.get_point_as_seconds() + self.tdef.rtconfig['events']['late offset']) else: # Not used, but allow skip of the above "is None" test self.late_time = 0 return self.late_time def get_point_as_seconds(self): """Compute and store my cycle point as seconds since epoch.""" if self.point_as_seconds is None: iso_timepoint = cylc.flow.cycling.iso8601.point_parse( str(self.point)) self.point_as_seconds = int(iso_timepoint.get( 'seconds_since_unix_epoch')) if iso_timepoint.time_zone.unknown: utc_offset_hours, utc_offset_minutes = ( get_local_time_zone()) utc_offset_in_seconds = ( 3600 * utc_offset_hours + 60 * utc_offset_minutes) self.point_as_seconds += utc_offset_in_seconds return self.point_as_seconds def get_state_summary(self): """Return a dict containing the state summary of this task proxy.""" ret = self.summary.copy() ret['name'] = self.tdef.name ret['description'] = self.tdef.rtconfig['meta']['description'] ret['title'] = self.tdef.rtconfig['meta']['title'] ret['label'] = str(self.point) ret['submit_num'] = self.submit_num ret['state'] = self.state.status ret['spawned'] = str(self.has_spawned) ntimes = len(self.tdef.elapsed_times) if ntimes: ret['mean_elapsed_time'] = ( float(sum(self.tdef.elapsed_times)) / ntimes) elif ret['execution_time_limit']: ret['mean_elapsed_time'] = float( ret['execution_time_limit']) else: ret['mean_elapsed_time'] = None return ret def get_try_num(self): """Return the number of automatic tries (try number).""" try: return self.try_timers[TASK_STATUS_RETRYING].num + 1 except (AttributeError, KeyError): return 0 def next_point(self): """Return the next cycle point.""" p_next = None adjusted = [] for seq in self.tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) return p_next def is_ready(self, now): """Am I in a pre-run state but ready to run? Queued tasks are not counted as they've already been deemed ready. """ if self.manual_trigger: return True waiting_retry = self.is_waiting_retry(now) if waiting_retry is not None: return not waiting_retry if self.state.status != TASK_STATUS_WAITING: return False return not (self.is_waiting_clock(now) or self.is_waiting_prereqs()) def reset_manual_trigger(self): """This is called immediately after manual trigger flag used.""" if self.manual_trigger: self.manual_trigger = False self.is_manual_submit = True # unset any retry delay timers for timer in self.try_timers.values(): timer.timeout = None def set_summary_message(self, message): """Set `.summary['latest_message']` if necessary. Set `.state.is_updated` to `True` if message is updated. """ if self.summary['latest_message'] != message: self.summary['latest_message'] = message self.state.is_updated = True def set_summary_time(self, event_key, time_str=None): """Set an event time in self.summary Set values of both event_key + "_time" and event_key + "_time_string". """ if time_str is None: self.summary[event_key + '_time'] = None else: self.summary[event_key + '_time'] = float(str2time(time_str)) self.summary[event_key + '_time_string'] = time_str def is_waiting_clock(self, now): """Is this task waiting for its clock trigger time?""" if self.tdef.clocktrigger_offset is None: return None if self.clock_trigger_time is None: self.clock_trigger_time = ( self.get_point_as_seconds() + self.get_offset_as_seconds(self.tdef.clocktrigger_offset)) return self.clock_trigger_time > now def is_waiting_prereqs(self): """Is this task waiting for its prerequisites?""" return ( any(not pre.is_satisfied() for pre in self.state.prerequisites) or any(not tri for tri in self.state.external_triggers.values()) or not self.state.xtriggers_all_satisfied() ) def is_waiting_retry(self, now): """Is this task waiting for its latest (submission) retry delay time? Return True if waiting for next retry delay time, False if not. Return None if no retry lined up. """ try: return not self.try_timers[self.state.status].is_delay_done(now) except KeyError: return None
def __init__( self, tdef, start_point, status=TASK_STATUS_WAITING, hold_swap=None, has_spawned=False, stop_point=None, is_startup=False, submit_num=0, is_late=False): self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num if is_startup: # adjust up to the first on-sequence cycle point adjusted = [] for seq in self.tdef.sequences: adj = seq.get_first_point(start_point) if adj: # may be None if out of sequence bounds adjusted.append(adj) if not adjusted: # This task is out of sequence bounds raise TaskProxySequenceBoundsError(self.tdef.name) self.point = min(adjusted) self.late_time = None else: self.point = start_point self.cleanup_cutoff = self.tdef.get_cleanup_cutoff_point(self.point) self.identity = TaskID.get(self.tdef.name, self.point) self.has_spawned = has_spawned self.reload_successor = None self.point_as_seconds = None # Manually inserted tasks may have a final cycle point set. self.stop_point = stop_point self.manual_trigger = False self.is_manual_submit = False self.summary = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'job_hosts': {}, 'execution_time_limit': None, 'batch_sys_name': None, 'submit_method_id': None } self.local_job_file_path = None self.task_host = 'localhost' self.task_owner = None self.job_vacated = False self.poll_timer = None self.timeout = None self.try_timers = {} # Use dict here for Python 2.6 compat. # Should use collections.Counter in Python 2.7+ self.non_unique_events = {} self.clock_trigger_time = None self.expire_time = None self.late_time = None self.is_late = is_late self.state = TaskState(tdef, self.point, status, hold_swap) if tdef.sequential: # Adjust clean-up cutoff. p_next = None adjusted = [] for seq in tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) if (self.cleanup_cutoff is not None and self.cleanup_cutoff < p_next): self.cleanup_cutoff = p_next
class TaskProxy: """Represent an instance of a cycling task in a running workflow. Attributes: .clock_trigger_time: Clock trigger time in seconds since epoch. (Used for both old-style clock triggers and wall_clock xtrigger). .expire_time: Time in seconds since epoch when this task is considered expired. .identity: Task ID in POINT/NAME syntax. .tokens: Task ID tokens. .is_late: Is the task late? .is_manual_submit: Is the latest job submission due to a manual trigger? .job_vacated: Is the latest job pre-empted (or vacated)? .jobs: A list of job ids associated with the task proxy. .local_job_file_path: Path on workflow host to the latest job script for the task. .late_time: Time in seconds since epoch, beyond which the task is considered late if it is never active. .non_unique_events (collections.Counter): Count non-unique events (e.g. critical, warning, custom). .point: Cycle point of the task. .point_as_seconds: Cycle point as seconds since epoch. .poll_timer: Schedule for polling submitted or running jobs. .reload_successor: The task proxy object that replaces the current instance on reload. This attribute provides a useful link to the latest replacement instance while the current object may still be referenced by a job manipulation command. .submit_num: Number of times the task has attempted job submission. .summary (dict): job_runner_name (str): Name of job runner where latest job is submitted. description (str): Same as the .tdef.rtconfig['meta']['description'] attribute. execution_time_limit (float): Execution time limit of latest job. finished_time (float): Latest job exit time. finished_time_string (str): Latest job exit time as string. platforms_used (dict): Jobs' platform by submit number. label (str): The .point attribute as string. logfiles (list): List of names of (extra) known job log files. name (str): Same as the .tdef.name attribute. started_time (float): Latest job execution start time. started_time_string (str): Latest job execution start time as string. submit_method_id (str): Latest ID of job in job runner. submit_num (int): Same as the .submit_num attribute. submitted_time (float): Latest job submission time. submitted_time_string (str): Latest job submission time as string. title (str): Same as the .tdef.rtconfig['meta']['title'] attribute. .state: Object representing the state of this task. .platform: Dict containing info for platform where latest job is submitted. .tdef: The definition object of this task. .timeout: Timeout value in seconds since epoch for latest job submission/execution. .try_timers: Retry schedules as cylc.flow.task_action_timer.TaskActionTimer objects. .graph_children (dict) graph children: {msg: [(name, point), ...]} .flow_nums: flow_nums .waiting_on_job_prep: task waiting on job prep Args: tdef: The definition object of this task. start_point: Start point to calculate the task's cycle point on start-up or the cycle point for subsequent tasks. flow_nums: Which flow within the scheduler this task belongs to. status: Task state string. is_held: True if the task is held, else False. submit_num: Number of times the task has attempted job submission. is_late: Is the task late? """ # Memory optimization - constrain possible attributes to this list. __slots__ = [ 'clock_trigger_time', 'expire_time', 'identity', 'is_late', 'is_manual_submit', 'job_vacated', 'jobs', 'late_time', 'local_job_file_path', 'non_unique_events', 'point', 'point_as_seconds', 'poll_timer', 'reload_successor', 'submit_num', 'tdef', 'state', 'summary', 'flow_nums', 'graph_children', 'platform', 'timeout', 'tokens', 'try_timers', 'waiting_on_job_prep', ] def __init__( self, tdef: 'TaskDef', start_point: 'PointBase', flow_nums: Optional[Set[int]] = None, status: str = TASK_STATUS_WAITING, is_held: bool = False, submit_num: int = 0, is_late: bool = False, ) -> None: self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs: List[dict] = [] if flow_nums is None: self.flow_nums = set() else: self.flow_nums = flow_nums self.point = start_point self.tokens = Tokens( # TODO: make these absolute? cycle=str(self.point), task=self.tdef.name, ) self.identity = self.tokens.relative_id self.reload_successor: Optional['TaskProxy'] = None self.point_as_seconds: Optional[int] = None self.is_manual_submit = False self.summary: Dict[str, Any] = { 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'platforms_used': {}, 'execution_time_limit': None, 'job_runner_name': None, 'submit_method_id': None, 'flow_nums': set() } self.local_job_file_path: Optional[str] = None self.platform = get_platform() self.job_vacated = False self.poll_timer: Optional['TaskActionTimer'] = None self.timeout: Optional[float] = None self.try_timers: Dict[str, 'TaskActionTimer'] = {} self.non_unique_events = Counter() # type: ignore # TODO: figure out self.clock_trigger_time: Optional[float] = None self.expire_time: Optional[float] = None self.late_time: Optional[float] = None self.is_late = is_late self.waiting_on_job_prep = True self.state = TaskState(tdef, self.point, status, is_held) # Determine graph children of this task (for spawning). self.graph_children = generate_graph_children(tdef, self.point) def __repr__(self) -> str: return f"<{self.__class__.__name__} '{self.tokens}'>" def __str__(self) -> str: """Stringify with tokens, state, submit_num, and flow_nums.""" return (f"{self.identity} " f"{self.state} " f"job:{self.submit_num:02d}" f" flows:{','.join(str(i) for i in self.flow_nums) or 'none'}") def copy_to_reload_successor(self, reload_successor): """Copy attributes to successor on reload of this task proxy.""" self.reload_successor = reload_successor reload_successor.submit_num = self.submit_num reload_successor.is_manual_submit = self.is_manual_submit reload_successor.summary = self.summary reload_successor.local_job_file_path = self.local_job_file_path reload_successor.try_timers = self.try_timers reload_successor.platform = self.platform reload_successor.job_vacated = self.job_vacated reload_successor.poll_timer = self.poll_timer reload_successor.timeout = self.timeout reload_successor.state.outputs = self.state.outputs reload_successor.state.is_held = self.state.is_held reload_successor.state.is_runahead = self.state.is_runahead reload_successor.state.is_updated = self.state.is_updated reload_successor.state.prerequisites = self.state.prerequisites reload_successor.jobs = self.jobs @staticmethod def get_offset_as_seconds(offset): """Return an ISO interval as seconds.""" iso_offset = interval_parse(str(offset)) return int(iso_offset.get_seconds()) def get_late_time(self): """Compute and store late time as seconds since epoch.""" if self.late_time is None: if self.tdef.rtconfig['events']['late offset']: self.late_time = (self.get_point_as_seconds() + self.tdef.rtconfig['events']['late offset']) else: # Not used, but allow skip of the above "is None" test self.late_time = 0 return self.late_time def get_point_as_seconds(self): """Compute and store my cycle point as seconds since epoch.""" if self.point_as_seconds is None: iso_timepoint = point_parse(str(self.point)) self.point_as_seconds = int( iso_timepoint.get('seconds_since_unix_epoch')) if iso_timepoint.time_zone.unknown: utc_offset_hours, utc_offset_minutes = (get_local_time_zone()) utc_offset_in_seconds = (3600 * utc_offset_hours + 60 * utc_offset_minutes) self.point_as_seconds += utc_offset_in_seconds return self.point_as_seconds def get_clock_trigger_time(self, offset_str): """Compute, cache, and return trigger time relative to cycle point. Args: offset_str: ISO8601Interval string, e.g. "PT2M". Can be None for zero offset. Returns: Absolute trigger time in seconds since Unix epoch. """ if self.clock_trigger_time is None: if offset_str is None: trigger_time = self.point else: trigger_time = self.point + ISO8601Interval(offset_str) self.clock_trigger_time = int( point_parse(str(trigger_time)).get('seconds_since_unix_epoch')) return self.clock_trigger_time def get_try_num(self): """Return the number of automatic tries (try number).""" try: return self.try_timers[TimerFlags.EXECUTION_RETRY].num + 1 except (AttributeError, KeyError): return 0 def next_point(self): """Return the next cycle point.""" p_next = None adjusted = [] for seq in self.tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) return p_next def is_ready_to_run(self) -> Tuple[bool, ...]: """Is this task ready to run? Takes account of all dependence: on other tasks, xtriggers, and old-style ext- and clock-triggers. Or, manual triggering. """ if self.is_manual_submit: # Manually triggered, ignore unsatisfied prerequisites. return (True, ) if self.state.is_held: # A held task is not ready to run. return (False, ) if self.state.status in self.try_timers: # A try timer is still active. return (self.try_timers[self.state.status].is_delay_done(), ) return (self.state(TASK_STATUS_WAITING), self.is_waiting_clock_done(), self.is_waiting_prereqs_done()) def set_summary_time(self, event_key, time_str=None): """Set an event time in self.summary Set values of both event_key + "_time" and event_key + "_time_string". """ if time_str is None: self.summary[event_key + '_time'] = None else: self.summary[event_key + '_time'] = float(str2time(time_str)) self.summary[event_key + '_time_string'] = time_str def is_waiting_clock_done(self): """Is this task done waiting for its old-style clock trigger time? Return True if there is no clock trigger or when clock trigger is done. """ if self.tdef.clocktrigger_offset is None: return True return (time() > self.get_clock_trigger_time( str(self.tdef.clocktrigger_offset))) def is_task_prereqs_not_done(self): """Are some task prerequisites not satisfied?""" return (not all(pre.is_satisfied() for pre in self.state.prerequisites)) def is_waiting_prereqs_done(self): """Are ALL prerequisites satisfied?""" return (all(pre.is_satisfied() for pre in self.state.prerequisites) and all(tri for tri in self.state.external_triggers.values()) and self.state.xtriggers_all_satisfied()) def reset_try_timers(self): # unset any retry delay timers for timer in self.try_timers.values(): timer.timeout = None def point_match(self, point: Optional[str]) -> bool: """Return whether a string/glob matches the task's point. None is treated as '*'. """ if point is None: return True with suppress(PointParsingError): # point_str may be a glob point = standardise_point_string(point) return fnmatchcase(str(self.point), point) def status_match(self, status: Optional[str]) -> bool: """Return whether a string matches the task's status. None/an empty string is treated as a match. """ return (not status) or self.state.status == status def name_match(self, name: str) -> bool: """Return whether a string/glob matches the task's name.""" if fnmatchcase(self.tdef.name, name): return True return any( fnmatchcase(ns, name) for ns in self.tdef.namespace_hierarchy) def merge_flows(self, flow_nums: Set) -> None: """Merge another set of flow_nums with mine.""" self.flow_nums.update(flow_nums) def state_reset(self, status=None, is_held=None, is_queued=None, is_runahead=None) -> bool: """Set new state and log the change. Return whether it changed.""" before = str(self) if self.state.reset(status, is_held, is_queued, is_runahead): LOG.info(f"[{before}] => {self.state}") return True return False
def __init__(self, tdef, start_point, flow_label, status=TASK_STATUS_WAITING, is_held=False, submit_num=0, is_late=False, reflow=True): self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs = [] self.flow_label = flow_label self.reflow = reflow self.point = start_point self.identity = TaskID.get(self.tdef.name, self.point) self.reload_successor = None self.point_as_seconds = None self.manual_trigger = False self.is_manual_submit = False self.summary = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'platforms_used': {}, 'execution_time_limit': None, 'job_runner_name': None, 'submit_method_id': None, 'flow_label': None } self.local_job_file_path = None self.platform = get_platform() self.job_vacated = False self.poll_timer = None self.timeout = None self.try_timers = {} self.non_unique_events = Counter() self.clock_trigger_time = None self.expire_time = None self.late_time = None self.is_late = is_late self.waiting_on_job_prep = True self.state = TaskState(tdef, self.point, status, is_held) # Determine graph children of this task (for spawning). self.graph_children = generate_graph_children(tdef, self.point) if TASK_OUTPUT_SUCCEEDED in self.graph_children: self.state.outputs.add(TASK_OUTPUT_SUCCEEDED) if TASK_OUTPUT_FAILED in self.graph_children: self.failure_handled = True else: self.failure_handled = False
class TaskProxy(object): """Represent an instance of a cycling task in a running suite. Attributes: .cleanup_cutoff (cylc.flow.cycling.PointBase): Cycle point beyond which this task can be removed from the pool. .clock_trigger_time (float): Clock trigger time in seconds since epoch. .expire_time (float): Time in seconds since epoch when this task is considered expired. .has_spawned (boolean): Has this task spawned its successor in the sequence? .identity (str): Task ID in NAME.POINT syntax. .is_late (boolean): Is the task late? .is_manual_submit (boolean): Is the latest job submission due to a manual trigger? .job_vacated (boolean): Is the latest job pre-empted (or vacated)? .jobs (list): A list of job ids associated with the task proxy. .local_job_file_path (str): Path on suite host to the latest job script for running the task. .late_time (float): Time in seconds since epoch, beyond which the task is considered late if it is never active. .manual_trigger (boolean): Has this task received a manual trigger command? This flag is reset on trigger. .non_unique_events (dict): Count non-unique events (e.g. critical, warning, custom). .point (cylc.flowcycling.PointBase): Cycle point of the task. .point_as_seconds (int): Cycle point as seconds since epoch. .poll_timer (cylc.flow.task_action_timer.TaskActionTimer): Schedule for polling submitted or running jobs. .reload_successor (cylc.flow.task_proxy.TaskProxy): The task proxy object that replaces the current instance on reload. This attribute provides a useful link to the latest replacement instance while the current object may still be referenced by a job manipulation command. .stop_point (cylc.flow.cycling.PointBase): Do not spawn successor beyond this point. .submit_num (int): Number of times the task has attempted job submission. .summary (dict): batch_sys_name (str): Name of batch system where latest job is submitted. description (str): Same as the .tdef.rtconfig['meta']['description'] attribute. execution_time_limit (float): Execution time limit of latest job. finished_time (float): Latest job exit time. finished_time_string (str): Latest job exit time as string. job_hosts (dict): Jobs' owner@host by submit number. label (str): The .point attribute as string. latest_message (str): Latest job or event message. logfiles (list): List of names of (extra) known job log files. name (str): Same as the .tdef.name attribute. started_time (float): Latest job execution start time. started_time_string (str): Latest job execution start time as string. submit_method_id (str): Latest ID of job in batch system. submit_num (int): Same as the .submit_num attribute. submitted_time (float): Latest job submission time. submitted_time_string (str): Latest job submission time as string. title (str): Same as the .tdef.rtconfig['meta']['title'] attribute. .state (cylc.flow.task_state.TaskState): Object representing the state of this task. .task_host (str) Name of host where latest job is submitted. .task_owner (str) Name of user (at task_host) where latest job is submitted. .tdef (cylc.flow.taskdef.TaskDef): The definition object of this task. .timeout (float): Timeout value in seconds since epoch for latest job submission/execution. .try_timers (dict) Retry schedules as cylc.flow.task_action_timer.TaskActionTimer objects. Arguments: tdef (cylc.flow.taskdef.TaskDef): The definition object of this task. start_point (cylc.flow.cycling.PointBase): Start point to calculate the task's cycle point on start up or the cycle point for subsequent tasks. status (str): Task state string. is_held (bool): True if the task is held, else False. has_spawned (boolean): Has this task spawned its successor in the sequence. stop_point (cylc.flow.cycling.PointBase): Do not spawn successor beyond this point. is_startup (boolean): Is this on start up? submit_num (int): Number of times the task has attempted job submission. late_time (float): Time in seconds since epoch, beyond which the task is considered late if it is never active. """ # Memory optimization - constrain possible attributes to this list. __slots__ = [ 'cleanup_cutoff', 'clock_trigger_time', 'expire_time', 'has_spawned', 'identity', 'is_late', 'is_manual_submit', 'job_vacated', 'jobs', 'late_time', 'local_job_file_path', 'manual_trigger', 'non_unique_events', 'point', 'point_as_seconds', 'poll_timer', 'reload_successor', 'submit_num', 'tdef', 'state', 'stop_point', 'summary', 'task_host', 'task_owner', 'timeout', 'try_timers', ] def __init__(self, tdef, start_point, status=TASK_STATUS_WAITING, is_held=False, has_spawned=False, stop_point=None, is_startup=False, submit_num=0, is_late=False): self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs = [] if is_startup: # adjust up to the first on-sequence cycle point adjusted = [] for seq in self.tdef.sequences: adj = seq.get_first_point(start_point) if adj: # may be None if out of sequence bounds adjusted.append(adj) if not adjusted: # This task is out of sequence bounds raise TaskProxySequenceBoundsError(self.tdef.name) self.point = min(adjusted) self.late_time = None else: self.point = start_point self.cleanup_cutoff = self.tdef.get_cleanup_cutoff_point(self.point) self.identity = TaskID.get(self.tdef.name, self.point) self.has_spawned = has_spawned self.reload_successor = None self.point_as_seconds = None # Manually inserted tasks may have a final cycle point set. self.stop_point = stop_point self.manual_trigger = False self.is_manual_submit = False self.summary = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'job_hosts': {}, 'execution_time_limit': None, 'batch_sys_name': None, 'submit_method_id': None } self.local_job_file_path = None self.task_host = 'localhost' self.task_owner = None self.job_vacated = False self.poll_timer = None self.timeout = None self.try_timers = {} # Use dict here for Python 2.6 compat. # Should use collections.Counter in Python 2.7+ self.non_unique_events = {} self.clock_trigger_time = None self.expire_time = None self.late_time = None self.is_late = is_late self.state = TaskState(tdef, self.point, status, is_held) if tdef.sequential: # Adjust clean-up cutoff. p_next = None adjusted = [] for seq in tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) if (self.cleanup_cutoff is not None and self.cleanup_cutoff < p_next): self.cleanup_cutoff = p_next def __str__(self): """Stringify using "self.identity".""" return self.identity def copy_to_reload_successor(self, reload_successor): """Copy attributes to successor on reload of this task proxy.""" self.reload_successor = reload_successor reload_successor.submit_num = self.submit_num reload_successor.has_spawned = self.has_spawned reload_successor.manual_trigger = self.manual_trigger reload_successor.is_manual_submit = self.is_manual_submit reload_successor.summary = self.summary reload_successor.local_job_file_path = self.local_job_file_path reload_successor.try_timers = self.try_timers reload_successor.task_host = self.task_host reload_successor.task_owner = self.task_owner reload_successor.job_vacated = self.job_vacated reload_successor.poll_timer = self.poll_timer reload_successor.timeout = self.timeout reload_successor.state.outputs = self.state.outputs reload_successor.state.is_updated = self.state.is_updated @staticmethod def get_offset_as_seconds(offset): """Return an ISO interval as seconds.""" iso_offset = cylc.flow.cycling.iso8601.interval_parse(str(offset)) return int(iso_offset.get_seconds()) def get_late_time(self): """Compute and store late time as seconds since epoch.""" if self.late_time is None: if self.tdef.rtconfig['events']['late offset']: self.late_time = (self.get_point_as_seconds() + self.tdef.rtconfig['events']['late offset']) else: # Not used, but allow skip of the above "is None" test self.late_time = 0 return self.late_time def get_point_as_seconds(self): """Compute and store my cycle point as seconds since epoch.""" if self.point_as_seconds is None: iso_timepoint = cylc.flow.cycling.iso8601.point_parse( str(self.point)) self.point_as_seconds = int( iso_timepoint.get('seconds_since_unix_epoch')) if iso_timepoint.time_zone.unknown: utc_offset_hours, utc_offset_minutes = (get_local_time_zone()) utc_offset_in_seconds = (3600 * utc_offset_hours + 60 * utc_offset_minutes) self.point_as_seconds += utc_offset_in_seconds return self.point_as_seconds def get_state_summary(self): """Return a dict containing the state summary of this task proxy.""" ret = self.summary.copy() ret['name'] = self.tdef.name ret['description'] = self.tdef.rtconfig['meta']['description'] ret['title'] = self.tdef.rtconfig['meta']['title'] ret['label'] = str(self.point) ret['submit_num'] = self.submit_num ret['state'] = self.state.status ret['is_held'] = self.state.is_held ret['spawned'] = str(self.has_spawned) ntimes = len(self.tdef.elapsed_times) if ntimes: ret['mean_elapsed_time'] = (float(sum(self.tdef.elapsed_times)) / ntimes) elif ret['execution_time_limit']: ret['mean_elapsed_time'] = float(ret['execution_time_limit']) else: ret['mean_elapsed_time'] = None return ret def get_try_num(self): """Return the number of automatic tries (try number).""" try: return self.try_timers[TASK_STATUS_RETRYING].num + 1 except (AttributeError, KeyError): return 0 def next_point(self): """Return the next cycle point.""" p_next = None adjusted = [] for seq in self.tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) return p_next def is_ready(self, now): """Am I in a pre-run state but ready to run? Queued tasks are not counted as they've already been deemed ready. """ if self.manual_trigger: return True waiting_retry = self.is_waiting_retry(now) if waiting_retry is not None: return not waiting_retry if not self.state(TASK_STATUS_WAITING, is_held=False): return False return not (self.is_waiting_clock(now) or self.is_waiting_prereqs()) def reset_manual_trigger(self): """This is called immediately after manual trigger flag used.""" if self.manual_trigger: self.manual_trigger = False self.is_manual_submit = True # unset any retry delay timers for timer in self.try_timers.values(): timer.timeout = None def set_summary_message(self, message): """Set `.summary['latest_message']` if necessary. Set `.state.is_updated` to `True` if message is updated. """ if self.summary['latest_message'] != message: self.summary['latest_message'] = message self.state.is_updated = True def set_summary_time(self, event_key, time_str=None): """Set an event time in self.summary Set values of both event_key + "_time" and event_key + "_time_string". """ if time_str is None: self.summary[event_key + '_time'] = None else: self.summary[event_key + '_time'] = float(str2time(time_str)) self.summary[event_key + '_time_string'] = time_str def is_waiting_clock(self, now): """Is this task waiting for its clock trigger time?""" if self.tdef.clocktrigger_offset is None: return None if self.clock_trigger_time is None: self.clock_trigger_time = ( self.get_point_as_seconds() + self.get_offset_as_seconds(self.tdef.clocktrigger_offset)) return self.clock_trigger_time > now def is_waiting_prereqs(self): """Is this task waiting for its prerequisites?""" return (any(not pre.is_satisfied() for pre in self.state.prerequisites) or any(not tri for tri in self.state.external_triggers.values()) or not self.state.xtriggers_all_satisfied()) def is_waiting_retry(self, now): """Is this task waiting for its latest (submission) retry delay time? Return True if waiting for next retry delay time, False if not. Return None if no retry lined up. """ try: return not self.try_timers[self.state.status].is_delay_done(now) except KeyError: return None
def __init__(self, tdef, start_point, status=TASK_STATUS_WAITING, is_held=False, has_spawned=False, stop_point=None, is_startup=False, submit_num=0, is_late=False): self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs = [] if is_startup: # adjust up to the first on-sequence cycle point adjusted = [] for seq in self.tdef.sequences: adj = seq.get_first_point(start_point) if adj: # may be None if out of sequence bounds adjusted.append(adj) if not adjusted: # This task is out of sequence bounds raise TaskProxySequenceBoundsError(self.tdef.name) self.point = min(adjusted) self.late_time = None else: self.point = start_point self.cleanup_cutoff = self.tdef.get_cleanup_cutoff_point(self.point) self.identity = TaskID.get(self.tdef.name, self.point) self.has_spawned = has_spawned self.reload_successor = None self.point_as_seconds = None # Manually inserted tasks may have a final cycle point set. self.stop_point = stop_point self.manual_trigger = False self.is_manual_submit = False self.summary = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'job_hosts': {}, 'execution_time_limit': None, 'batch_sys_name': None, 'submit_method_id': None } self.local_job_file_path = None self.task_host = 'localhost' self.task_owner = None self.job_vacated = False self.poll_timer = None self.timeout = None self.try_timers = {} # Use dict here for Python 2.6 compat. # Should use collections.Counter in Python 2.7+ self.non_unique_events = {} self.clock_trigger_time = None self.expire_time = None self.late_time = None self.is_late = is_late self.state = TaskState(tdef, self.point, status, is_held) if tdef.sequential: # Adjust clean-up cutoff. p_next = None adjusted = [] for seq in tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) if (self.cleanup_cutoff is not None and self.cleanup_cutoff < p_next): self.cleanup_cutoff = p_next
def __init__(self, tdef, start_point, flow_label, status=TASK_STATUS_WAITING, is_held=False, submit_num=0, is_late=False, reflow=True): self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs = [] self.flow_label = flow_label self.reflow = reflow self.point = start_point self.identity = TaskID.get(self.tdef.name, self.point) self.reload_successor = None self.point_as_seconds = None self.manual_trigger = False self.is_manual_submit = False self.summary = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'platforms_used': {}, 'execution_time_limit': None, 'batch_sys_name': None, 'submit_method_id': None, 'flow_label': None } self.local_job_file_path = None self.platform = get_platform() self.task_owner = None self.job_vacated = False self.poll_timer = None self.timeout = None self.try_timers = {} # Use dict here for Python 2.6 compat. # Should use collections.Counter in Python 2.7+ self.non_unique_events = {} self.clock_trigger_time = None self.expire_time = None self.late_time = None self.is_late = is_late self.state = TaskState(tdef, self.point, status, is_held) # Determine graph children of this task (for spawning). self.graph_children = {} for seq, dout in tdef.graph_children.items(): for output, downs in dout.items(): if output not in self.graph_children: self.graph_children[output] = [] for name, trigger in downs: child_point = trigger.get_child_point(self.point, seq) is_abs = (trigger.offset_is_absolute or trigger.offset_is_from_icp) if is_abs: if trigger.get_parent_point(self.point) != self.point: # If 'foo[^] => bar' only spawn off of '^'. continue if seq.is_on_sequence(child_point): # E.g.: foo should trigger only on T06: # PT6H = "waz" # T06 = "waz[-PT6H] => foo" self.graph_children[output].append( (name, child_point, is_abs)) if tdef.sequential: # Add next-instance child. nexts = [] for seq in tdef.sequences: nxt = seq.get_next_point(self.point) if nxt is not None: # Within sequence bounds. nexts.append(nxt) if nexts: if TASK_OUTPUT_SUCCEEDED not in self.graph_children: self.graph_children[TASK_OUTPUT_SUCCEEDED] = [] self.state.outputs.add(TASK_OUTPUT_SUCCEEDED) self.graph_children[TASK_OUTPUT_SUCCEEDED].append( (tdef.name, min(nexts), False)) if TASK_OUTPUT_FAILED in self.graph_children: self.failure_handled = True else: self.failure_handled = False
class TaskProxy: """Represent an instance of a cycling task in a running suite. Attributes: .clock_trigger_time (float): Clock trigger time in seconds since epoch. .expire_time (float): Time in seconds since epoch when this task is considered expired. .identity (str): Task ID in NAME.POINT syntax. .is_late (boolean): Is the task late? .is_manual_submit (boolean): Is the latest job submission due to a manual trigger? .job_vacated (boolean): Is the latest job pre-empted (or vacated)? .jobs (list): A list of job ids associated with the task proxy. .local_job_file_path (str): Path on suite host to the latest job script for running the task. .late_time (float): Time in seconds since epoch, beyond which the task is considered late if it is never active. .manual_trigger (boolean): Has this task received a manual trigger command? This flag is reset on trigger. .non_unique_events (dict): Count non-unique events (e.g. critical, warning, custom). .point (cylc.flow.cycling.PointBase): Cycle point of the task. .point_as_seconds (int): Cycle point as seconds since epoch. .poll_timer (cylc.flow.task_action_timer.TaskActionTimer): Schedule for polling submitted or running jobs. .reload_successor (cylc.flow.task_proxy.TaskProxy): The task proxy object that replaces the current instance on reload. This attribute provides a useful link to the latest replacement instance while the current object may still be referenced by a job manipulation command. .submit_num (int): Number of times the task has attempted job submission. .summary (dict): batch_sys_name (str): Name of batch system where latest job is submitted. description (str): Same as the .tdef.rtconfig['meta']['description'] attribute. execution_time_limit (float): Execution time limit of latest job. finished_time (float): Latest job exit time. finished_time_string (str): Latest job exit time as string. platforms_used (dict): Jobs' platform by submit number. label (str): The .point attribute as string. latest_message (str): Latest job or event message. logfiles (list): List of names of (extra) known job log files. name (str): Same as the .tdef.name attribute. started_time (float): Latest job execution start time. started_time_string (str): Latest job execution start time as string. submit_method_id (str): Latest ID of job in batch system. submit_num (int): Same as the .submit_num attribute. submitted_time (float): Latest job submission time. submitted_time_string (str): Latest job submission time as string. title (str): Same as the .tdef.rtconfig['meta']['title'] attribute. .state (cylc.flow.task_state.TaskState): Object representing the state of this task. .platform (dict) Dict containing info for platform where latest job is submitted. .tdef (cylc.flow.taskdef.TaskDef): The definition object of this task. .timeout (float): Timeout value in seconds since epoch for latest job submission/execution. .try_timers (dict) Retry schedules as cylc.flow.task_action_timer.TaskActionTimer objects. .graph_children (dict) graph children: {msg: [(name, point), ...]} .failure_handled (bool) task failure is handled (by children) .flow_label (str) flow label .reflow (bool) flow on from outputs Arguments: tdef (cylc.flow.taskdef.TaskDef): The definition object of this task. start_point (cylc.flow.cycling.PointBase): Start point to calculate the task's cycle point on start up or the cycle point for subsequent tasks. status (str): Task state string. is_held (bool): True if the task is held, else False. submit_num (int): Number of times the task has attempted job submission. late_time (float): Time in seconds since epoch, beyond which the task is considered late if it is never active. """ # Memory optimization - constrain possible attributes to this list. __slots__ = [ 'clock_trigger_time', 'expire_time', 'identity', 'is_late', 'is_manual_submit', 'job_vacated', 'jobs', 'late_time', 'local_job_file_path', 'manual_trigger', 'non_unique_events', 'point', 'point_as_seconds', 'poll_timer', 'reload_successor', 'submit_num', 'tdef', 'state', 'summary', 'platform', 'task_owner', 'timeout', 'try_timers', 'graph_children', 'failure_handled', 'flow_label', 'reflow', ] def __init__(self, tdef, start_point, flow_label, status=TASK_STATUS_WAITING, is_held=False, submit_num=0, is_late=False, reflow=True): self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs = [] self.flow_label = flow_label self.reflow = reflow self.point = start_point self.identity = TaskID.get(self.tdef.name, self.point) self.reload_successor = None self.point_as_seconds = None self.manual_trigger = False self.is_manual_submit = False self.summary = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'platforms_used': {}, 'execution_time_limit': None, 'batch_sys_name': None, 'submit_method_id': None, 'flow_label': None } self.local_job_file_path = None self.platform = get_platform() self.task_owner = None self.job_vacated = False self.poll_timer = None self.timeout = None self.try_timers = {} # Use dict here for Python 2.6 compat. # Should use collections.Counter in Python 2.7+ self.non_unique_events = {} self.clock_trigger_time = None self.expire_time = None self.late_time = None self.is_late = is_late self.state = TaskState(tdef, self.point, status, is_held) # Determine graph children of this task (for spawning). self.graph_children = {} for seq, dout in tdef.graph_children.items(): for output, downs in dout.items(): if output not in self.graph_children: self.graph_children[output] = [] for name, trigger in downs: child_point = trigger.get_child_point(self.point, seq) is_abs = (trigger.offset_is_absolute or trigger.offset_is_from_icp) if is_abs: if trigger.get_parent_point(self.point) != self.point: # If 'foo[^] => bar' only spawn off of '^'. continue if seq.is_on_sequence(child_point): # E.g.: foo should trigger only on T06: # PT6H = "waz" # T06 = "waz[-PT6H] => foo" self.graph_children[output].append( (name, child_point, is_abs)) if tdef.sequential: # Add next-instance child. nexts = [] for seq in tdef.sequences: nxt = seq.get_next_point(self.point) if nxt is not None: # Within sequence bounds. nexts.append(nxt) if nexts: if TASK_OUTPUT_SUCCEEDED not in self.graph_children: self.graph_children[TASK_OUTPUT_SUCCEEDED] = [] self.state.outputs.add(TASK_OUTPUT_SUCCEEDED) self.graph_children[TASK_OUTPUT_SUCCEEDED].append( (tdef.name, min(nexts), False)) if TASK_OUTPUT_FAILED in self.graph_children: self.failure_handled = True else: self.failure_handled = False def __str__(self): """Stringify using "self.identity".""" return self.identity def copy_to_reload_successor(self, reload_successor): """Copy attributes to successor on reload of this task proxy.""" self.reload_successor = reload_successor reload_successor.submit_num = self.submit_num reload_successor.manual_trigger = self.manual_trigger reload_successor.is_manual_submit = self.is_manual_submit reload_successor.summary = self.summary reload_successor.local_job_file_path = self.local_job_file_path reload_successor.try_timers = self.try_timers reload_successor.platform = self.platform reload_successor.task_owner = self.task_owner reload_successor.job_vacated = self.job_vacated reload_successor.poll_timer = self.poll_timer reload_successor.timeout = self.timeout reload_successor.state.outputs = self.state.outputs reload_successor.state.is_held = self.state.is_held reload_successor.state.is_updated = self.state.is_updated reload_successor.state.prerequisites = self.state.prerequisites reload_successor.graph_children = self.graph_children @staticmethod def get_offset_as_seconds(offset): """Return an ISO interval as seconds.""" iso_offset = cylc.flow.cycling.iso8601.interval_parse(str(offset)) return int(iso_offset.get_seconds()) def get_late_time(self): """Compute and store late time as seconds since epoch.""" if self.late_time is None: if self.tdef.rtconfig['events']['late offset']: self.late_time = (self.get_point_as_seconds() + self.tdef.rtconfig['events']['late offset']) else: # Not used, but allow skip of the above "is None" test self.late_time = 0 return self.late_time def get_point_as_seconds(self): """Compute and store my cycle point as seconds since epoch.""" if self.point_as_seconds is None: iso_timepoint = cylc.flow.cycling.iso8601.point_parse( str(self.point)) self.point_as_seconds = int( iso_timepoint.get('seconds_since_unix_epoch')) if iso_timepoint.time_zone.unknown: utc_offset_hours, utc_offset_minutes = (get_local_time_zone()) utc_offset_in_seconds = (3600 * utc_offset_hours + 60 * utc_offset_minutes) self.point_as_seconds += utc_offset_in_seconds return self.point_as_seconds def get_try_num(self): """Return the number of automatic tries (try number).""" try: return self.try_timers[TimerFlags.EXECUTION_RETRY].num + 1 except (AttributeError, KeyError): return 0 def next_point(self): """Return the next cycle point.""" p_next = None adjusted = [] for seq in self.tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) return p_next def is_ready(self): """Am I in a pre-run state but ready to run? Queued tasks are not counted as they've already been deemed ready. """ if self.manual_trigger: return True if self.state.is_held: return False if self.state.status in self.try_timers: return self.try_timers[self.state.status].is_delay_done() return (self.state(TASK_STATUS_WAITING) and self.is_waiting_clock_done() and self.is_waiting_prereqs_done()) def reset_manual_trigger(self): """This is called immediately after manual trigger flag used.""" if self.manual_trigger: self.manual_trigger = False self.is_manual_submit = True # unset any retry delay timers for timer in self.try_timers.values(): timer.timeout = None def set_summary_message(self, message): """Set `.summary['latest_message']` if necessary. Set `.state.is_updated` to `True` if message is updated. """ if self.summary['latest_message'] != message: self.summary['latest_message'] = message self.state.is_updated = True def set_summary_time(self, event_key, time_str=None): """Set an event time in self.summary Set values of both event_key + "_time" and event_key + "_time_string". """ if time_str is None: self.summary[event_key + '_time'] = None else: self.summary[event_key + '_time'] = float(str2time(time_str)) self.summary[event_key + '_time_string'] = time_str def is_waiting_clock_done(self): """Is this task done waiting for its clock trigger time? Return True if there is no clock trigger or when clock trigger is done. """ if self.tdef.clocktrigger_offset is None: return True if self.clock_trigger_time is None: self.clock_trigger_time = ( self.get_point_as_seconds() + self.get_offset_as_seconds(self.tdef.clocktrigger_offset)) return time() >= self.clock_trigger_time def is_task_prereqs_not_done(self): """Is this task waiting on other-task prerequisites?""" return (len(self.state.prerequisites) > 0 and not all(pre.is_satisfied() for pre in self.state.prerequisites)) def is_waiting_prereqs_done(self): """Is this task waiting for its prerequisites?""" return (all(pre.is_satisfied() for pre in self.state.prerequisites) and all(tri for tri in self.state.external_triggers.values()) and self.state.xtriggers_all_satisfied())
class TaskProxy: """Represent an instance of a cycling task in a running workflow. Attributes: .clock_trigger_time: Clock trigger time in seconds since epoch. .expire_time: Time in seconds since epoch when this task is considered expired. .identity: Task ID in NAME.POINT syntax. .is_late: Is the task late? .is_manual_submit: Is the latest job submission due to a manual trigger? .job_vacated: Is the latest job pre-empted (or vacated)? .jobs: A list of job ids associated with the task proxy. .local_job_file_path: Path on workflow host to the latest job script for the task. .late_time: Time in seconds since epoch, beyond which the task is considered late if it is never active. .non_unique_events (collections.Counter): Count non-unique events (e.g. critical, warning, custom). .point: Cycle point of the task. .point_as_seconds: Cycle point as seconds since epoch. .poll_timer: Schedule for polling submitted or running jobs. .reload_successor: The task proxy object that replaces the current instance on reload. This attribute provides a useful link to the latest replacement instance while the current object may still be referenced by a job manipulation command. .submit_num: Number of times the task has attempted job submission. .summary (dict): job_runner_name (str): Name of job runner where latest job is submitted. description (str): Same as the .tdef.rtconfig['meta']['description'] attribute. execution_time_limit (float): Execution time limit of latest job. finished_time (float): Latest job exit time. finished_time_string (str): Latest job exit time as string. platforms_used (dict): Jobs' platform by submit number. label (str): The .point attribute as string. latest_message (str): Latest job or event message. logfiles (list): List of names of (extra) known job log files. name (str): Same as the .tdef.name attribute. started_time (float): Latest job execution start time. started_time_string (str): Latest job execution start time as string. submit_method_id (str): Latest ID of job in job runner. submit_num (int): Same as the .submit_num attribute. submitted_time (float): Latest job submission time. submitted_time_string (str): Latest job submission time as string. title (str): Same as the .tdef.rtconfig['meta']['title'] attribute. .state: Object representing the state of this task. .platform: Dict containing info for platform where latest job is submitted. .tdef: The definition object of this task. .timeout: Timeout value in seconds since epoch for latest job submission/execution. .try_timers: Retry schedules as cylc.flow.task_action_timer.TaskActionTimer objects. .graph_children (dict) graph children: {msg: [(name, point), ...]} .failure_handled: task failure is handled (by children) .flow_label: flow label .reflow: flow on from outputs .waiting_on_job_prep: task waiting on job prep Args: tdef: The definition object of this task. start_point: Start point to calculate the task's cycle point on start-up or the cycle point for subsequent tasks. flow_label: Which flow within the scheduler this task belongs to. status: Task state string. is_held: True if the task is held, else False. submit_num: Number of times the task has attempted job submission. is_late: Is the task late? reflow: Flow on from outputs. TODO: better description for arg? """ # Memory optimization - constrain possible attributes to this list. __slots__ = [ 'clock_trigger_time', 'expire_time', 'identity', 'is_late', 'is_manual_submit', 'job_vacated', 'jobs', 'late_time', 'local_job_file_path', 'non_unique_events', 'point', 'point_as_seconds', 'poll_timer', 'reload_successor', 'submit_num', 'tdef', 'state', 'summary', 'platform', 'timeout', 'try_timers', 'graph_children', 'failure_handled', 'flow_label', 'reflow', 'waiting_on_job_prep', ] def __init__(self, tdef: 'TaskDef', start_point: 'PointBase', flow_label: Optional[str], status: str = TASK_STATUS_WAITING, is_held: bool = False, submit_num: int = 0, is_late: bool = False, reflow: bool = True) -> None: self.tdef = tdef if submit_num is None: submit_num = 0 self.submit_num = submit_num self.jobs: List[str] = [] self.flow_label = flow_label self.reflow = reflow self.point = start_point self.identity: str = TaskID.get(self.tdef.name, self.point) self.reload_successor: Optional['TaskProxy'] = None self.point_as_seconds: Optional[int] = None self.is_manual_submit = False self.summary: Dict[str, Any] = { 'latest_message': '', 'submitted_time': None, 'submitted_time_string': None, 'started_time': None, 'started_time_string': None, 'finished_time': None, 'finished_time_string': None, 'logfiles': [], 'platforms_used': {}, 'execution_time_limit': None, 'job_runner_name': None, 'submit_method_id': None, 'flow_label': None } self.local_job_file_path: Optional[str] = None self.platform = get_platform() self.job_vacated = False self.poll_timer: Optional['TaskActionTimer'] = None self.timeout: Optional[float] = None self.try_timers: Dict[str, 'TaskActionTimer'] = {} self.non_unique_events = Counter() # type: ignore # TODO: figure out self.clock_trigger_time: Optional[float] = None self.expire_time: Optional[float] = None self.late_time: Optional[float] = None self.is_late = is_late self.waiting_on_job_prep = True self.state = TaskState(tdef, self.point, status, is_held) # Determine graph children of this task (for spawning). self.graph_children = generate_graph_children(tdef, self.point) if TASK_OUTPUT_SUCCEEDED in self.graph_children: self.state.outputs.add(TASK_OUTPUT_SUCCEEDED) self.failure_handled: bool = TASK_OUTPUT_FAILED in self.graph_children def __str__(self): """Stringify using "self.identity".""" return self.identity def copy_to_reload_successor(self, reload_successor): """Copy attributes to successor on reload of this task proxy.""" self.reload_successor = reload_successor reload_successor.submit_num = self.submit_num reload_successor.is_manual_submit = self.is_manual_submit reload_successor.summary = self.summary reload_successor.local_job_file_path = self.local_job_file_path reload_successor.try_timers = self.try_timers reload_successor.platform = self.platform reload_successor.job_vacated = self.job_vacated reload_successor.poll_timer = self.poll_timer reload_successor.timeout = self.timeout reload_successor.state.outputs = self.state.outputs reload_successor.state.is_held = self.state.is_held reload_successor.state.is_runahead = self.state.is_runahead reload_successor.state.is_updated = self.state.is_updated reload_successor.state.prerequisites = self.state.prerequisites reload_successor.graph_children = self.graph_children @staticmethod def get_offset_as_seconds(offset): """Return an ISO interval as seconds.""" iso_offset = cylc.flow.cycling.iso8601.interval_parse(str(offset)) return int(iso_offset.get_seconds()) def get_late_time(self): """Compute and store late time as seconds since epoch.""" if self.late_time is None: if self.tdef.rtconfig['events']['late offset']: self.late_time = (self.get_point_as_seconds() + self.tdef.rtconfig['events']['late offset']) else: # Not used, but allow skip of the above "is None" test self.late_time = 0 return self.late_time def get_point_as_seconds(self): """Compute and store my cycle point as seconds since epoch.""" if self.point_as_seconds is None: iso_timepoint = cylc.flow.cycling.iso8601.point_parse( str(self.point)) self.point_as_seconds = int( iso_timepoint.get('seconds_since_unix_epoch')) if iso_timepoint.time_zone.unknown: utc_offset_hours, utc_offset_minutes = (get_local_time_zone()) utc_offset_in_seconds = (3600 * utc_offset_hours + 60 * utc_offset_minutes) self.point_as_seconds += utc_offset_in_seconds return self.point_as_seconds def get_try_num(self): """Return the number of automatic tries (try number).""" try: return self.try_timers[TimerFlags.EXECUTION_RETRY].num + 1 except (AttributeError, KeyError): return 0 def next_point(self): """Return the next cycle point.""" p_next = None adjusted = [] for seq in self.tdef.sequences: nxt = seq.get_next_point(self.point) if nxt: # may be None if beyond the sequence bounds adjusted.append(nxt) if adjusted: p_next = min(adjusted) return p_next def is_ready_to_run(self) -> Tuple[bool, ...]: """Is this task ready to run? Takes account of all dependence: on other tasks, xtriggers, and old-style ext- and clock-triggers. Or, manual triggering. """ if self.is_manual_submit: # Manually triggered, ignore unsatisified prerequisites. return (True, ) if self.state.is_held: # A held task is not ready to run. return (False, ) if self.state.status in self.try_timers: # A try timer is still active. return (self.try_timers[self.state.status].is_delay_done(), ) return (self.state(TASK_STATUS_WAITING), self.is_waiting_clock_done(), self.is_waiting_prereqs_done()) def set_summary_message(self, message): """Set `.summary['latest_message']` if necessary. Set `.state.is_updated` to `True` if message is updated. """ if self.summary['latest_message'] != message: self.summary['latest_message'] = message self.state.is_updated = True def set_summary_time(self, event_key, time_str=None): """Set an event time in self.summary Set values of both event_key + "_time" and event_key + "_time_string". """ if time_str is None: self.summary[event_key + '_time'] = None else: self.summary[event_key + '_time'] = float(str2time(time_str)) self.summary[event_key + '_time_string'] = time_str def is_waiting_clock_done(self): """Is this task done waiting for its clock trigger time? Return True if there is no clock trigger or when clock trigger is done. """ if self.tdef.clocktrigger_offset is None: return True if self.clock_trigger_time is None: self.clock_trigger_time = ( self.get_point_as_seconds() + self.get_offset_as_seconds(self.tdef.clocktrigger_offset)) return time() >= self.clock_trigger_time def is_task_prereqs_not_done(self): """Are some task prerequisites not satisfied?""" return (not all(pre.is_satisfied() for pre in self.state.prerequisites)) def is_waiting_prereqs_done(self): """Are ALL prerequisites satisfied?""" return (all(pre.is_satisfied() for pre in self.state.prerequisites) and all(tri for tri in self.state.external_triggers.values()) and self.state.xtriggers_all_satisfied()) def reset_try_timers(self): # unset any retry delay timers for timer in self.try_timers.values(): timer.timeout = None