def _event_popen_fail(self, os_error):
assert not self._done
self._done = True
return JobRunner.Event(
JobRunner.EVENT_FAILURE,
jobs.make_error_run_log(jobs.ERROR_START_PROCESS,
command=self._command_path,
exception=str(os_error)))
def _event_heartbeat_timeout(self):
assert self._running and not self._done
if self._auto_heartbeat:
return JobRunner.Event(JobRunner.EVENT_PROGRESS, '')
else:
return JobRunner.Event(
JobRunner.EVENT_FAILURE,
jobs.make_error_run_log(jobs.ERROR_HEARTBEAT,
stderr=self._kill_and_grab_stderr()))
def _event_subprocess_exit(self, stderr_log):
error_code = self._subprocess.returncode
assert error_code is not None
self._finalise()
if error_code == 0:
return JobRunner.Event(
JobRunner.EVENT_SUCCESS,
jobs.make_success_run_log(stderr=stderr_log))
elif error_code < 0:
return JobRunner.Event(
JobRunner.EVENT_FAILURE,
jobs.make_error_run_log(jobs.ERROR_SIGNAL, signal=-error_code,
stderr=stderr_log))
else:
return JobRunner.Event(
JobRunner.EVENT_FAILURE,
jobs.make_error_run_log(jobs.ERROR_NONZERO_EXIT,
returncode=error_code,
stderr=stderr_log))
def monitor_greenlet(job_queue, check_interval, timeout):
log.info('monitor: Greenlet up.')
while True:
try:
gevent.sleep(check_interval)
job_id = job_queue.monitor_inprogress()
snapshot = job_queue.fetch_snapshot(job_id)
update_secs_ago = job_queue.timestamp() - snapshot.time_updated
if update_secs_ago > timeout:
job_queue.fail(job_id,
jobs.make_error_run_log(jobs.ERROR_ORPHANED))
log.info('monitor: Reported orphaned job with ID %s.', job_id)
except gevent.GreenletExit:
break
log.info('monitor: Greenlet down.')
def _event_overall_timeout(self):
assert self._running and not self._done
return JobRunner.Event(
JobRunner.EVENT_FAILURE,
jobs.make_error_run_log(jobs.ERROR_TIMEOUT,
stderr=self._kill_and_grab_stderr()))
