def _check_timeout(self, itask, now):
"""Check/handle submission/execution timeouts."""
if itask.state.status == TASK_STATUS_RUNNING:
timer = itask.poll_timers.get(self.KEY_EXECUTE_TIME_LIMIT)
if timer is not None:
if not timer.is_timeout_set():
timer.next()
if not timer.is_delay_done():
# Don't poll
return False
if timer.next() is not None:
# Poll now, and more retries lined up
return True
# No more retry lined up, can issue execution timeout event
if itask.state.status in itask.timeout_timers:
timeout = itask.timeout_timers[itask.state.status]
if timeout is None or now <= timeout:
return False
itask.timeout_timers[itask.state.status] = None
if (itask.state.status == TASK_STATUS_RUNNING
and itask.summary['started_time'] is not None):
msg = 'job started %s ago, but has not finished' % (
get_seconds_as_interval_string(
timeout - itask.summary['started_time']))
event = 'execution timeout'
elif (itask.state.status == TASK_STATUS_SUBMITTED
and itask.summary['submitted_time'] is not None):
msg = 'job submitted %s ago, but has not started' % (
get_seconds_as_interval_string(
timeout - itask.summary['submitted_time']))
event = 'submission timeout'
else:
return False
LOG.warning(msg, itask=itask)
self.task_events_mgr.setup_event_handlers(itask, event, msg)
return True
def delay_timeout_as_str(self):
"""Return a string in the form "delay (after timeout)"."""
return r"%s (after %s)" % (get_seconds_as_interval_string(
self.delay), get_time_string_from_unix_time(self.timeout))
def __str__(self):
return get_seconds_as_interval_string(self)
def delay_as_seconds(self):
"""Return the delay as PTnS, where n is number of seconds."""
return get_seconds_as_interval_string(self.delay)
def __str__(self):
if SyntaxVersion.VERSION == VERSION_PREV:
return float.__str__(self)
else:
return get_seconds_as_interval_string(self)
def update(self):
"""Try and connect and do an update."""
if self._no_update_event.is_set():
return False
if not self.connect_schd.ready():
self.info_bar.set_update_time(
None,
get_seconds_as_interval_string(round(
self.connect_schd.dt_next)))
return False
if cylc.flags.debug:
print >> sys.stderr, "UPDATE %s" % get_current_time_string()
if not self.connected:
# Only reconnect via self.reconnect().
self.reconnect()
if not self.connected:
self.set_stopped()
if cylc.flags.debug:
print >> sys.stderr, "(not connected)"
return False
if cylc.flags.debug:
print >> sys.stderr, "(connected)"
try:
err_log_changed = self.retrieve_err_log()
summaries_changed = self.retrieve_summary_update_time()
if summaries_changed:
self.retrieve_state_summaries()
except SuiteStillInitialisingError:
# Connection achieved but state summary data not available yet.
if cylc.flags.debug:
print >> sys.stderr, " connected, suite initializing ..."
self.set_status(SUITE_STATUS_INITIALISING)
if self.info_bar.prog_bar_can_start():
gobject.idle_add(self.info_bar.prog_bar_start,
SUITE_STATUS_INITIALISING)
self.info_bar.set_state([])
return False
except Pyro.errors.NamingError as exc:
if self.daemon_version is not None:
# Back compat <= 6.4.0 the state summary object was not
# connected to Pyro until initialisation was completed.
if cylc.flags.debug:
print >> sys.stderr, (
" daemon <= 6.4.0, suite initializing ...")
self.set_status(SUITE_STATUS_INITIALISING)
if self.info_bar.prog_bar_can_start():
gobject.idle_add(self.info_bar.prog_bar_start,
SUITE_STATUS_INITIALISING)
self.info_bar.set_state([])
# Reconnect till we get the suite state object.
self.reconnect()
return False
else:
if cylc.flags.debug:
print >> sys.stderr, " CONNECTION LOST", str(exc)
self.set_stopped()
if self.info_bar.prog_bar_active():
gobject.idle_add(self.info_bar.prog_bar_stop)
self.reconnect()
return False
except Exception as exc:
if self.status == SUITE_STATUS_STOPPING:
# Expected stop: prevent the reconnection warning dialog.
self.connect_fail_warned = True
if cylc.flags.debug:
print >> sys.stderr, " CONNECTION LOST", str(exc)
self.set_stopped()
if self.info_bar.prog_bar_active():
gobject.idle_add(self.info_bar.prog_bar_stop)
self.reconnect()
return False
else:
# Got suite data.
self.version_mismatch_warned = False
if (self.status == SUITE_STATUS_STOPPING
and self.info_bar.prog_bar_can_start()):
gobject.idle_add(self.info_bar.prog_bar_start, self.status)
if (self.is_reloading and self.info_bar.prog_bar_can_start()):
gobject.idle_add(self.info_bar.prog_bar_start, "reloading")
if (self.info_bar.prog_bar_active() and not self.is_reloading
and self.status
not in [SUITE_STATUS_STOPPING, SUITE_STATUS_INITIALISING]):
gobject.idle_add(self.info_bar.prog_bar_stop)
if summaries_changed or err_log_changed:
return True
else:
return False
