def _execute_job(self, job_id: str, canceled: _LockedVar[bool]):
_LOGGER.info(f"Starting job {job_id} in JobWorkerManager")
state = _JobExecutionState()
job = None
try:
_LOGGER.debug(f"Getting job {job_id} from DB")
job = Job.get(Job.job_id == job_id)
if self._check_cancel_job(job, canceled, state):
_LOGGER.debug(
f"Job {job_id} cancel requested before starting, canceling"
)
raise JobCancelError()
self._update_job_started(job)
_LOGGER.debug(f"Creating worker for job {job_id}")
worker = JobWorkerRegistry.create_worker(job) # type: JobWorker
_LOGGER.debug(f"Starting worker run for job {job_id}")
for progress in worker.run():
if self._check_cancel_job(job, canceled, state):
_LOGGER.debug(f"Job {job_id} cancel requested, canceling")
raise JobCancelError()
self._update_job_progress(job, state, progress)
self._update_job_completed(job)
except JobCancelError:
if not job:
raise RuntimeError("job is None after JobCancelError")
self._update_job_canceled(job)
_LOGGER.info(f"Job {job_id} canceled in JobWorkerManager")
except Exception as err:
# try to update the job in the DB in case the job doesn't exist
# or the job was deleted from the DB
try:
self._update_job_error(job, err)
except Exception as save_err:
_LOGGER.warning(
f"Could not update job state in db to errored "
f"for job {job_id}: {save_err}: for error {err}"
)
def _worker_thread(self):
_LOGGER.debug(
"job worker for job_id {} and project_id {} thead init".format(
self._worker.job_id, self._worker.project_id
)
)
with database.connection_context():
with self._lock:
job = Job.get(Job.job_id == self._worker.job_id)
self._report_started(job)
canceled = False
error = None
try:
# initial check to see if job was canceled before it started
if self._should_cancel():
raise JobCancelError()
for progress in self._worker.run():
with self._lock:
if self._should_report_progress():
self._report_progress(job, progress)
if self._should_cancel():
raise JobCancelError()
except JobCancelError:
canceled = True
_LOGGER.debug(
"cancel job worker for job_id {} and project_id {} received".format(
self._worker.job_id, self._worker.project_id
)
)
except Exception as err:
_LOGGER.info(
(
"job worker for job_id {} and project_id {} "
"encountered error: {}"
).format(self._worker.job_id, self._worker.project_id, err)
)
error = err
with self._lock:
self._start_completed = True
if canceled:
self._report_canceled(job)
_LOGGER.info(
"canceled job worker for job_id {} and project_id {}".format(
self._worker.job_id, self._worker.project_id
)
)
elif error is not None:
self._report_error(job, str(error))
_LOGGER.info(
"errored job worker for job_id {} and project_id {}".format(
self._worker.job_id, self._worker.project_id
)
)
else:
self._report_completed(job)
_LOGGER.info(
"completed job worker for job_id {} and project_id {}".format(
self._worker.job_id, self._worker.project_id
)
)
self._done_callback()