def _check_timeout(self, itask, now): """Check/handle submission/execution timeouts.""" if itask.state.status == TASK_STATUS_RUNNING: timer = itask.poll_timers.get(self.KEY_EXECUTE_TIME_LIMIT) if timer is not None: if not timer.is_timeout_set(): timer.next() if not timer.is_delay_done(): # Don't poll return False if timer.next() is not None: # Poll now, and more retries lined up return True # No more retry lined up, can issue execution timeout event if itask.state.status in itask.timeout_timers: timeout = itask.timeout_timers[itask.state.status] if timeout is None or now <= timeout: return False itask.timeout_timers[itask.state.status] = None if (itask.state.status == TASK_STATUS_RUNNING and itask.summary['started_time'] is not None): msg = 'job started %s ago, but has not finished' % ( get_seconds_as_interval_string( timeout - itask.summary['started_time'])) event = 'execution timeout' elif (itask.state.status == TASK_STATUS_SUBMITTED and itask.summary['submitted_time'] is not None): msg = 'job submitted %s ago, but has not started' % ( get_seconds_as_interval_string( timeout - itask.summary['submitted_time'])) event = 'submission timeout' else: return False LOG.warning(msg, itask=itask) self.task_events_mgr.setup_event_handlers(itask, event, msg) return True
def delay_timeout_as_str(self): """Return a string in the form "delay (after timeout)".""" return r"%s (after %s)" % (get_seconds_as_interval_string( self.delay), get_time_string_from_unix_time(self.timeout))
def __str__(self): return get_seconds_as_interval_string(self)
def delay_as_seconds(self): """Return the delay as PTnS, where n is number of seconds.""" return get_seconds_as_interval_string(self.delay)
def __str__(self): if SyntaxVersion.VERSION == VERSION_PREV: return float.__str__(self) else: return get_seconds_as_interval_string(self)
def update(self): """Try and connect and do an update.""" if self._no_update_event.is_set(): return False if not self.connect_schd.ready(): self.info_bar.set_update_time( None, get_seconds_as_interval_string(round( self.connect_schd.dt_next))) return False if cylc.flags.debug: print >> sys.stderr, "UPDATE %s" % get_current_time_string() if not self.connected: # Only reconnect via self.reconnect(). self.reconnect() if not self.connected: self.set_stopped() if cylc.flags.debug: print >> sys.stderr, "(not connected)" return False if cylc.flags.debug: print >> sys.stderr, "(connected)" try: err_log_changed = self.retrieve_err_log() summaries_changed = self.retrieve_summary_update_time() if summaries_changed: self.retrieve_state_summaries() except SuiteStillInitialisingError: # Connection achieved but state summary data not available yet. if cylc.flags.debug: print >> sys.stderr, " connected, suite initializing ..." self.set_status(SUITE_STATUS_INITIALISING) if self.info_bar.prog_bar_can_start(): gobject.idle_add(self.info_bar.prog_bar_start, SUITE_STATUS_INITIALISING) self.info_bar.set_state([]) return False except Pyro.errors.NamingError as exc: if self.daemon_version is not None: # Back compat <= 6.4.0 the state summary object was not # connected to Pyro until initialisation was completed. if cylc.flags.debug: print >> sys.stderr, ( " daemon <= 6.4.0, suite initializing ...") self.set_status(SUITE_STATUS_INITIALISING) if self.info_bar.prog_bar_can_start(): gobject.idle_add(self.info_bar.prog_bar_start, SUITE_STATUS_INITIALISING) self.info_bar.set_state([]) # Reconnect till we get the suite state object. self.reconnect() return False else: if cylc.flags.debug: print >> sys.stderr, " CONNECTION LOST", str(exc) self.set_stopped() if self.info_bar.prog_bar_active(): gobject.idle_add(self.info_bar.prog_bar_stop) self.reconnect() return False except Exception as exc: if self.status == SUITE_STATUS_STOPPING: # Expected stop: prevent the reconnection warning dialog. self.connect_fail_warned = True if cylc.flags.debug: print >> sys.stderr, " CONNECTION LOST", str(exc) self.set_stopped() if self.info_bar.prog_bar_active(): gobject.idle_add(self.info_bar.prog_bar_stop) self.reconnect() return False else: # Got suite data. self.version_mismatch_warned = False if (self.status == SUITE_STATUS_STOPPING and self.info_bar.prog_bar_can_start()): gobject.idle_add(self.info_bar.prog_bar_start, self.status) if (self.is_reloading and self.info_bar.prog_bar_can_start()): gobject.idle_add(self.info_bar.prog_bar_start, "reloading") if (self.info_bar.prog_bar_active() and not self.is_reloading and self.status not in [SUITE_STATUS_STOPPING, SUITE_STATUS_INITIALISING]): gobject.idle_add(self.info_bar.prog_bar_stop) if summaries_changed or err_log_changed: return True else: return False