def _set_api_result(self, result):
with self._condition:
if result.state in FAILED:
try:
err_msg = str(result['error'])
except:
err_msg = str(result)
self._set_api_exception(exc=CivisJobFailure(err_msg, result),
result=result)
elif result.state in DONE:
self.set_result(result)
self.cleanup()
def _check_result(self):
"""Return the job result from Civis. Once the job completes, store the
result and never poll again."""
# If we haven't started the polling thread, do it now.
if self._self_polling_executor is None and self._result is None:
# Start a single thread continuously polling. It will stop once the
# job completes.
self._self_polling_executor = futures.ThreadPoolExecutor(1)
self._self_polling_executor.submit(self._wait_for_completion)
with self._condition:
if self._result is not None:
# If the job is already completed, just return the stored
# result.
return self._result
# Check to see if the job has finished, but don't poll more
# frequently than the requested polling frequency.
now = time.time()
if not self._last_polled or self._poll_wait_elapsed(now):
# Poll for a new result
self._last_polled = now
try:
self._last_result = self._poller(*self._poller_args)
except Exception as e:
# The _poller can raise API exceptions
# Set those directly as this Future's exception
self._result = Response({"state": FAILED[0]})
self._last_result = self._result
self.set_exception(e)
else:
# If the job has finished, then register completion and
# store the results. Because of the `if self._result` check
# up top, we will never get here twice.
if self._last_result.state in FAILED:
try:
err_msg = str(self._last_result['error'])
except:
err_msg = str(self._last_result)
self.set_exception(
CivisJobFailure(err_msg, self._last_result))
self._result = self._last_result
elif self._last_result.state in DONE:
self.set_result(self._last_result)
return self._last_result
def _set_api_result(self, result):
with self._condition:
if result.state in FAILED:
try:
err_msg = str(result['error'])
except: # NOQA
err_msg = str(result)
job_id = getattr(self, "job_id", None)
run_id = getattr(self, "run_id", None)
self._set_api_exception(
exc=CivisJobFailure(err_msg, result, job_id, run_id),
result=result,
)
elif result.state in DONE:
self.set_result(result)
self.cleanup()
def test_result_exception_no_result(m_sleep):
# If the job errored but didn't write an output, we should get
# a generic TransportableException back.
callback = mock.MagicMock()
mock_client = create_client_mock_for_container_tests(1,
2,
state='failed',
run_outputs=[])
fut = ContainerFuture(1, 2, client=mock_client)
res = civis.parallel._CivisBackendResult(fut, callback)
fut._set_api_exception(CivisJobFailure(Response({'state': 'failed'})))
with pytest.raises(TransportableException) as exc:
res.get()
assert "{'state': 'failed'}" in str(exc.value)
assert callback.call_count == 0
def _exception_from_logs(self, exc, nlog=15):
"""Create an exception if the log has a recognizable error
Search "error" emits in the last ``n_log`` lines.
This function presently recognizes the following errors:
- MemoryError
"""
# Traceback in platform logs may be delayed for a few seconds.
time.sleep(15)
logs = self.client.jobs.list_runs_logs(self.job_id,
self.run_id,
limit=nlog)
# Reverse order as logs come back in reverse chronological order.
logs = logs[::-1]
# Check for memory errors
msgs = [x['message'] for x in logs if x['level'] == 'error']
mem_err = [m for m in msgs if m.startswith('Process ran out of its')]
if mem_err:
err_msg = _err_msg_with_job_run_ids(mem_err[0], self.job_id,
self.run_id)
exc = MemoryError(err_msg)
else:
# Unknown error; return logs to the user as a sort of traceback
all_logs = '\n'.join([x['message'] for x in logs])
if isinstance(exc, CivisJobFailure):
err_msg = _err_msg_with_job_run_ids(
all_logs + '\n' + exc.error_message, self.job_id,
self.run_id)
exc.error_message = err_msg
else:
err_msg = _err_msg_with_job_run_ids(all_logs, self.job_id,
self.run_id)
exc = CivisJobFailure("",
job_id=self.job_id,
run_id=self.run_id)
exc.error_message = err_msg
return exc
def _exception_from_logs(exc, job_id, run_id, client, nlog=15):
"""Create an exception if the log has a recognizable error
Search "error" emits in the last ``n_log`` lines.
This function presently recognizes the following errors:
- MemoryError
"""
logs = client.scripts.list_containers_runs_logs(job_id, run_id, limit=nlog)
# Check for memory errors
msgs = [l['message'] for l in logs if l['level'] == 'error']
mem_err = [m for m in msgs if m.startswith('Process ran out of its')]
if mem_err:
exc = MemoryError(mem_err[0])
else:
# Unknown error; return logs to the user as a sort of traceback
all_logs = '\n'.join([l['message'] for l in logs])
if isinstance(exc, CivisJobFailure):
exc.error_message = all_logs + '\n' + exc.error_message
else:
exc = CivisJobFailure(all_logs)
return exc