def run(self, run):
"""Execute the specified run
A run consists of a very specific set of interfaces which allow us to
execute a command on this remote machine and return results.
"""
log.info("Running %s for %s on %s", run.command, run.id, self.hostname)
# When this run completes, for good or bad, we'll inform the caller by
# calling 'succeed' or 'fail' on the run Since the definined interface
# is on these specific callbacks, we won't bother returning the
# deferred here. This allows the caller to not really care about
# twisted specific stuff at all, all it needs to know is that one of
# those functions will eventually be called back
if run.id in self.run_states:
raise Error("Run %s already running !?!", run.id)
if self.idle_timer.active():
self.idle_timer.cancel()
self.run_states[run.id] = RunState(run)
# TODO: have this return a runner instead of number
fudge_factor = determine_jitter(len(self.run_states), self.node_settings)
if fudge_factor == 0.0:
self._do_run(run)
else:
log.info("Delaying execution of %s for %.2f secs", run.id, fudge_factor)
eventloop.call_later(fudge_factor, self._do_run, run)
# We return the deferred here, but really we're trying to keep the rest
# of the world from getting too involved with twisted.
return self.run_states[run.id].deferred
def run(self, run):
"""Execute the specified run
A run consists of a very specific set of interfaces which allow us to
execute a command on this remote machine and return results.
"""
log.info("Running %s for %s on %s", run.command, run.id, self.hostname)
# When this run completes, for good or bad, we'll inform the caller by
# calling 'succeed' or 'fail' on the run Since the definined interface
# is on these specific callbacks, we won't bother returning the
# deferred here. This allows the caller to not really care about
# twisted specific stuff at all, all it needs to know is that one of
# those functions will eventually be called back
if run.id in self.run_states:
log.warning("Run %s(%r) already running !?!", run.id, self.run_states[run.id])
if self.idle_timer.active():
self.idle_timer.cancel()
self.run_states[run.id] = RunState(run)
# TODO: have this return a runner instead of number
fudge_factor = determine_jitter(len(self.run_states), self.node_settings)
if fudge_factor == 0.0:
self._do_run(run)
else:
log.info("Delaying execution of %s for %.2f secs", run.id, fudge_factor)
eventloop.call_later(fudge_factor, self._do_run, run)
# We return the deferred here, but really we're trying to keep the rest
# of the world from getting too involved with twisted.
return self.run_states[run.id].deferred
def run_queue_schedule(self):
# TODO: this should only start runs on the same node if this is an
# all_nodes job, but that is currently not possible
queued_run = self.job.runs.get_first_queued()
if queued_run:
eventloop.call_later(0, self.run_job, queued_run, run_queued=True)
# Attempt to schedule a new run. This will only schedule a run if the
# previous run was cancelled from a scheduled state, or if the job
# scheduler is `schedule_on_complete`.
self.schedule()
def run_queue_schedule(self):
# TODO: this should only start runs on the same node if this is an
# all_nodes job, but that is currently not possible
queued_run = self.job.runs.get_first_queued()
if queued_run:
eventloop.call_later(0, self.run_job, queued_run, run_queued=True)
# Attempt to schedule a new run. This will only schedule a run if the
# previous run was cancelled from a scheduled state, or if the job
# scheduler is `schedule_on_complete`.
self.schedule()
def _set_callback(self, job_run):
"""Set a callback for JobRun to fire at the appropriate time."""
seconds = job_run.seconds_until_run_time()
human_time = humanize.naturaltime(datetime.timedelta(seconds=seconds))
log.info(
"Scheduling next Jobrun for %s about %s from now (%d seconds)",
self.job.name,
human_time,
seconds,
)
eventloop.call_later(seconds, self.run_job, job_run)
def handle_job_events(self, _observable, event):
"""Handle notifications from observables. If a JobRun has completed
look for queued JobRuns that may need to start now.
"""
if event != Job.NOTIFY_RUN_DONE:
return
# TODO: this should only start runs on the same node if this is an
# all_nodes job, but that is currently not possible
queued_run = self.job.runs.get_first_queued()
if queued_run:
eventloop.call_later(0, self.run_job, queued_run, run_queued=True)
# Attempt to schedule a new run. This will only schedule a run if the
# previous run was cancelled from a scheduled state, or if the job
# scheduler is `schedule_on_complete`.
self.schedule()
def handle_job_events(self, _observable, event):
"""Handle notifications from observables. If a JobRun has completed
look for queued JobRuns that may need to start now.
"""
if event != Job.NOTIFY_RUN_DONE:
return
# TODO: this should only start runs on the same node if this is an
# all_nodes job, but that is currently not possible
queued_run = self.job.runs.get_first_queued()
if queued_run:
eventloop.call_later(0, self.run_job, queued_run, run_queued=True)
# Attempt to schedule a new run. This will only schedule a run if the
# previous run was cancelled from a scheduled state, or if the job
# scheduler is `schedule_on_complete`.
self.schedule()
def _cleanup(self, run):
# TODO: why set to None before deleting it?
self.run_states[run.id].channel = None
del self.run_states[run.id]
if not self.run_states:
self.idle_timer = eventloop.call_later(
self.node_settings.idle_connection_timeout,
self._connection_idle_timeout)
def _cleanup(self, run):
# TODO: why set to None before deleting it?
self.run_states[run.id].channel = None
del self.run_states[run.id]
if not self.run_states:
self.idle_timer = eventloop.call_later(
self.node_settings.idle_connection_timeout,
self._connection_idle_timeout)
def schedule_termination(self, job_run):
if self.job.max_runtime:
seconds = timeutils.delta_total_seconds(self.job.max_runtime)
eventloop.call_later(seconds, job_run.stop)
def _set_callback(self, job_run):
"""Set a callback for JobRun to fire at the appropriate time."""
log.info("Scheduling next Jobrun for %s", self.job.name)
seconds = job_run.seconds_until_run_time()
eventloop.call_later(seconds, self.run_job, job_run)
def schedule_termination(self, job_run):
if self.job.max_runtime:
seconds = timeutils.delta_total_seconds(self.job.max_runtime)
eventloop.call_later(seconds, job_run.stop)
def _set_callback(self, job_run):
"""Set a callback for JobRun to fire at the appropriate time."""
log.info("Scheduling next Jobrun for %s", self.job.name)
seconds = job_run.seconds_until_run_time()
eventloop.call_later(seconds, self.run_job, job_run)
