def _set_api_exception(self, exc, result=None):
# Catch attempts to set an exception. If there's retries
# remaining, retry the run instead of erroring.
with self._condition:
if self._max_n_retries > 0:
# Start a new run of the script and update
# the run ID used by the poller.
self.cleanup()
self._last_result = self.client.jobs.post_runs(self.job_id)
orig_run_id = self.run_id
self.poller_args[1] = run_id = self._last_result.id
self._max_n_retries -= 1
self._last_polled = time.time()
# Threads can only be started once, and the last thread
# stopped in cleanup. Start a new polling thread.
# Note that it's possible to have a race condition if
# you shut down the old thread too soon after starting it.
# In practice this only happens when testing retries
# with extremely short polling intervals.
self._polling_thread = _ResultPollingThread(
self._check_result, (), self.polling_interval)
self._polling_thread.start()
if hasattr(self, '_pubnub'):
# Subscribe to the new run's notifications endpoint
self._pubnub = self._subscribe(*self._pubnub_config())
log.debug(
'Job ID %d / Run ID %d failed. Retrying '
'with run %d. %d retries remaining.', self.job_id,
orig_run_id, run_id, self._max_n_retries)
else:
super()._set_api_exception(exc=exc, result=result)
def test_poller_returns_none(self):
poller = mock.Mock(
side_effect=[None, None,
Response({'state': 'success'})])
polling_thread = _ResultPollingThread(poller, (),
polling_interval=0.01)
polling_thread.run()
assert poller.call_count == 3
def __setstate__(self, state):
self.__dict__ = state
self._condition = threading.Condition()
self.client = APIClient(resources='all')
if getattr(self, '_pubnub', None) is True:
# Re-subscribe to notifications channel
self._pubnub = self._subscribe(*self._pubnub_config())
self._polling_thread = _ResultPollingThread(self._check_result, (),
self.polling_interval)
self.poller = self.client.scripts.get_containers_runs
self.add_done_callback(self._set_model_exception)