def _check_timeout(self, itask, now):
     """Check/handle submission/execution timeouts."""
     if itask.state.status == TASK_STATUS_RUNNING:
         timer = itask.poll_timers.get(self.KEY_EXECUTE_TIME_LIMIT)
         if timer is not None:
             if not timer.is_timeout_set():
                 timer.next()
             if not timer.is_delay_done():
                 # Don't poll
                 return False
             if timer.next() is not None:
                 # Poll now, and more retries lined up
                 return True
             # No more retry lined up, can issue execution timeout event
     if itask.state.status in itask.timeout_timers:
         timeout = itask.timeout_timers[itask.state.status]
         if timeout is None or now <= timeout:
             return False
         itask.timeout_timers[itask.state.status] = None
         if (itask.state.status == TASK_STATUS_RUNNING
                 and itask.summary['started_time'] is not None):
             msg = 'job started %s ago, but has not finished' % (
                 get_seconds_as_interval_string(
                     timeout - itask.summary['started_time']))
             event = 'execution timeout'
         elif (itask.state.status == TASK_STATUS_SUBMITTED
               and itask.summary['submitted_time'] is not None):
             msg = 'job submitted %s ago, but has not started' % (
                 get_seconds_as_interval_string(
                     timeout - itask.summary['submitted_time']))
             event = 'submission timeout'
         else:
             return False
         LOG.warning(msg, itask=itask)
         self.task_events_mgr.setup_event_handlers(itask, event, msg)
         return True
Exemple #2
0
 def delay_timeout_as_str(self):
     """Return a string in the form "delay (after timeout)"."""
     return r"%s (after %s)" % (get_seconds_as_interval_string(
         self.delay), get_time_string_from_unix_time(self.timeout))
 def __str__(self):
     return get_seconds_as_interval_string(self)
Exemple #4
0
 def delay_as_seconds(self):
     """Return the delay as PTnS, where n is number of seconds."""
     return get_seconds_as_interval_string(self.delay)
Exemple #5
0
 def __str__(self):
     if SyntaxVersion.VERSION == VERSION_PREV:
         return float.__str__(self)
     else:
         return get_seconds_as_interval_string(self)
Exemple #6
0
 def update(self):
     """Try and connect and do an update."""
     if self._no_update_event.is_set():
         return False
     if not self.connect_schd.ready():
         self.info_bar.set_update_time(
             None,
             get_seconds_as_interval_string(round(
                 self.connect_schd.dt_next)))
         return False
     if cylc.flags.debug:
         print >> sys.stderr, "UPDATE %s" % get_current_time_string()
     if not self.connected:
         # Only reconnect via self.reconnect().
         self.reconnect()
     if not self.connected:
         self.set_stopped()
         if cylc.flags.debug:
             print >> sys.stderr, "(not connected)"
         return False
     if cylc.flags.debug:
         print >> sys.stderr, "(connected)"
     try:
         err_log_changed = self.retrieve_err_log()
         summaries_changed = self.retrieve_summary_update_time()
         if summaries_changed:
             self.retrieve_state_summaries()
     except SuiteStillInitialisingError:
         # Connection achieved but state summary data not available yet.
         if cylc.flags.debug:
             print >> sys.stderr, "  connected, suite initializing ..."
         self.set_status(SUITE_STATUS_INITIALISING)
         if self.info_bar.prog_bar_can_start():
             gobject.idle_add(self.info_bar.prog_bar_start,
                              SUITE_STATUS_INITIALISING)
             self.info_bar.set_state([])
         return False
     except Pyro.errors.NamingError as exc:
         if self.daemon_version is not None:
             # Back compat <= 6.4.0 the state summary object was not
             # connected to Pyro until initialisation was completed.
             if cylc.flags.debug:
                 print >> sys.stderr, (
                     "  daemon <= 6.4.0, suite initializing ...")
             self.set_status(SUITE_STATUS_INITIALISING)
             if self.info_bar.prog_bar_can_start():
                 gobject.idle_add(self.info_bar.prog_bar_start,
                                  SUITE_STATUS_INITIALISING)
                 self.info_bar.set_state([])
             # Reconnect till we get the suite state object.
             self.reconnect()
             return False
         else:
             if cylc.flags.debug:
                 print >> sys.stderr, "  CONNECTION LOST", str(exc)
             self.set_stopped()
             if self.info_bar.prog_bar_active():
                 gobject.idle_add(self.info_bar.prog_bar_stop)
             self.reconnect()
             return False
     except Exception as exc:
         if self.status == SUITE_STATUS_STOPPING:
             # Expected stop: prevent the reconnection warning dialog.
             self.connect_fail_warned = True
         if cylc.flags.debug:
             print >> sys.stderr, "  CONNECTION LOST", str(exc)
         self.set_stopped()
         if self.info_bar.prog_bar_active():
             gobject.idle_add(self.info_bar.prog_bar_stop)
         self.reconnect()
         return False
     else:
         # Got suite data.
         self.version_mismatch_warned = False
         if (self.status == SUITE_STATUS_STOPPING
                 and self.info_bar.prog_bar_can_start()):
             gobject.idle_add(self.info_bar.prog_bar_start, self.status)
         if (self.is_reloading and self.info_bar.prog_bar_can_start()):
             gobject.idle_add(self.info_bar.prog_bar_start, "reloading")
         if (self.info_bar.prog_bar_active() and not self.is_reloading
                 and self.status
                 not in [SUITE_STATUS_STOPPING, SUITE_STATUS_INITIALISING]):
             gobject.idle_add(self.info_bar.prog_bar_stop)
         if summaries_changed or err_log_changed:
             return True
         else:
             return False