def _event_popen_fail(self, os_error): assert not self._done self._done = True return JobRunner.Event( JobRunner.EVENT_FAILURE, jobs.make_error_run_log(jobs.ERROR_START_PROCESS, command=self._command_path, exception=str(os_error)))
def _event_heartbeat_timeout(self): assert self._running and not self._done if self._auto_heartbeat: return JobRunner.Event(JobRunner.EVENT_PROGRESS, '') else: return JobRunner.Event( JobRunner.EVENT_FAILURE, jobs.make_error_run_log(jobs.ERROR_HEARTBEAT, stderr=self._kill_and_grab_stderr()))
def _event_subprocess_exit(self, stderr_log): error_code = self._subprocess.returncode assert error_code is not None self._finalise() if error_code == 0: return JobRunner.Event( JobRunner.EVENT_SUCCESS, jobs.make_success_run_log(stderr=stderr_log)) elif error_code < 0: return JobRunner.Event( JobRunner.EVENT_FAILURE, jobs.make_error_run_log(jobs.ERROR_SIGNAL, signal=-error_code, stderr=stderr_log)) else: return JobRunner.Event( JobRunner.EVENT_FAILURE, jobs.make_error_run_log(jobs.ERROR_NONZERO_EXIT, returncode=error_code, stderr=stderr_log))
def monitor_greenlet(job_queue, check_interval, timeout): log.info('monitor: Greenlet up.') while True: try: gevent.sleep(check_interval) job_id = job_queue.monitor_inprogress() snapshot = job_queue.fetch_snapshot(job_id) update_secs_ago = job_queue.timestamp() - snapshot.time_updated if update_secs_ago > timeout: job_queue.fail(job_id, jobs.make_error_run_log(jobs.ERROR_ORPHANED)) log.info('monitor: Reported orphaned job with ID %s.', job_id) except gevent.GreenletExit: break log.info('monitor: Greenlet down.')
def _event_overall_timeout(self): assert self._running and not self._done return JobRunner.Event( JobRunner.EVENT_FAILURE, jobs.make_error_run_log(jobs.ERROR_TIMEOUT, stderr=self._kill_and_grab_stderr()))