def status(self, throw_except=True, internal_storage=None):
"""
Return the status returned by the call.
If the call raised an exception, this method will raise the same exception
If the future is cancelled before completing then CancelledError will be raised.
:param check_only: Return None immediately if job is not complete. Default False.
:param throw_except: Reraise exception if call raised. Default true.
:param storage_handler: Storage handler to poll cloud storage. Default None.
:return: Result of the call.
:raises CancelledError: If the job is cancelled before completed.
:raises TimeoutError: If job is not complete after `timeout` seconds.
"""
if self._state == ResponseFuture.State.New:
raise ValueError("task not yet invoked")
if self._state in [
ResponseFuture.State.Ready, ResponseFuture.State.Success
]:
return self._call_status
if internal_storage is None:
internal_storage = InternalStorage(self._storage_config)
if self._call_status is None:
check_storage_path(internal_storage.get_storage_config(),
self._storage_path)
self._call_status = internal_storage.get_call_status(
self.executor_id, self.job_id, self.call_id)
self._status_query_count += 1
while self._call_status is None:
time.sleep(self.GET_RESULT_SLEEP_SECS)
self._call_status = internal_storage.get_call_status(
self.executor_id, self.job_id, self.call_id)
self._status_query_count += 1
self.stats['host_status_done_tstamp'] = time.time()
self.stats['host_status_query_count'] = self._status_query_count
self.activation_id = self._call_status.pop('activation_id', None)
if self._call_status['type'] == '__init__':
self._set_state(ResponseFuture.State.Running)
return self._call_status
if self._call_status['exception']:
self._set_state(ResponseFuture.State.Error)
self._exception = pickle.loads(eval(self._call_status['exc_info']))
msg1 = (
'ExecutorID {} | JobID {} - There was an exception - Activation '
'ID: {}'.format(self.executor_id, self.job_id,
self.activation_id))
if not self._call_status.get('exc_pickle_fail', False):
fn_exctype = self._exception[0]
fn_exc = self._exception[1]
if fn_exc.args and fn_exc.args[0] == "HANDLER":
self._handler_exception = True
try:
del fn_exc.errno
except Exception:
pass
fn_exc.args = (fn_exc.args[1], )
else:
fn_exctype = Exception
fn_exc = Exception(self._exception['exc_value'])
self._exception = (fn_exctype, fn_exc,
self._exception['exc_traceback'])
def exception_hook(exctype, exc, trcbck):
if exctype == fn_exctype and str(exc) == str(fn_exc):
logger.warning(msg1)
if self._handler_exception:
msg2 = 'Exception: {} - {}'.format(
fn_exctype.__name__, fn_exc)
logger.warning(msg2)
else:
traceback.print_exception(*self._exception)
else:
sys.excepthook = sys.__excepthook__
traceback.print_exception(exctype, exc, trcbck)
if throw_except:
sys.excepthook = exception_hook
time.sleep(1)
reraise(*self._exception)
else:
logger.warning(msg1)
msg2 = 'Exception: {} - {}'.format(self._exception[0].__name__,
self._exception[1])
logger.warning(msg2)
return None
for key in self._call_status:
if any(ss in key for ss in ['time', 'tstamp', 'count', 'size']):
self.stats[key] = self._call_status[key]
self.stats['worker_exec_time'] = round(
self.stats['worker_end_tstamp'] -
self.stats['worker_start_tstamp'], 8)
total_time = format(round(self.stats['worker_exec_time'], 2), '.2f')
log_msg = (
'ExecutorID {} | JobID {} - Got status from call {} - Activation '
'ID: {} - Time: {} seconds'.format(self.executor_id, self.job_id,
self.call_id,
self.activation_id,
str(total_time)))
logger.debug(log_msg)
self._set_state(ResponseFuture.State.Ready)
if not self._call_status['result']:
self._produce_output = False
if not self._produce_output:
self._set_state(ResponseFuture.State.Success)
if 'new_futures' in self._call_status:
self.result(throw_except=throw_except,
internal_storage=internal_storage)
return self._call_status
def status(self,
throw_except=True,
internal_storage=None,
check_only=False):
"""
Return the status returned by the call.
If the call raised an exception, this method will raise the same exception
If the future is cancelled before completing then CancelledError will be raised.
:param check_only: Return None immediately if job is not complete. Default False.
:param throw_except: Reraise exception if call raised. Default true.
:param storage_handler: Storage handler to poll cloud storage. Default None.
:return: Result of the call.
:raises CancelledError: If the job is cancelled before completed.
:raises TimeoutError: If job is not complete after `timeout` seconds.
"""
if self._state == ResponseFuture.State.New:
raise ValueError("task not yet invoked")
if self.success or self.done:
return self._call_status
if self.ready and self._new_futures:
self._set_state(ResponseFuture.State.Done)
return self._call_status
if self._call_status is None or self._call_status['type'] == '__init__':
if internal_storage is None:
internal_storage = InternalStorage(self._storage_config)
check_storage_path(internal_storage.get_storage_config(),
self._storage_path)
self._call_status = internal_storage.get_call_status(
self.executor_id, self.job_id, self.call_id)
self._status_query_count += 1
if check_only:
return self._call_status
while self._call_status is None:
time.sleep(self.GET_RESULT_SLEEP_SECS)
self._call_status = internal_storage.get_call_status(
self.executor_id, self.job_id, self.call_id)
self._status_query_count += 1
self._host_status_done_tstamp = time.time()
self.stats[
'host_status_done_tstamp'] = self._host_status_done_tstamp or time.time(
)
self.stats['host_status_query_count'] = self._status_query_count
self.activation_id = self._call_status['activation_id']
if 'logs' in self._call_status:
self.logs = zlib.decompress(
base64.b64decode(self._call_status['logs'].encode())).decode()
job_key = create_job_key(self.executor_id, self.job_id)
log_file = os.path.join(LOGS_DIR, job_key + '.log')
header = "Activation: '{}' ({})\n[\n".format(
self.runtime_name, self.activation_id)
tail = ']\n\n'
output = self.logs.replace('\r',
'').replace('\n', '\n ',
self.logs.count('\n') - 1)
with open(log_file, 'a') as lf:
lf.write(header + ' ' + output + tail)
with open(FN_LOG_FILE, 'a') as lf:
lf.write(header + ' ' + output + tail)
if self._call_status['exception']:
self._set_state(ResponseFuture.State.Error)
self._exception = pickle.loads(eval(self._call_status['exc_info']))
msg1 = (
'ExecutorID {} | JobID {} - There was an exception - Activation '
'ID: {}'.format(self.executor_id, self.job_id,
self.activation_id))
if not self._call_status.get('exc_pickle_fail', False):
fn_exctype = self._exception[0]
fn_exc = self._exception[1]
if fn_exc.args and fn_exc.args[0] == "HANDLER":
self._handler_exception = True
try:
del fn_exc.errno
except Exception:
pass
fn_exc.args = (fn_exc.args[1], )
else:
fn_exctype = Exception
fn_exc = Exception(self._exception['exc_value'])
self._exception = (fn_exctype, fn_exc,
self._exception['exc_traceback'])
def exception_hook(exctype, exc, trcbck):
if exctype == fn_exctype and str(exc) == str(fn_exc):
logger.warning(msg1)
if self._handler_exception:
msg2 = 'Exception: {} - {}'.format(
fn_exctype.__name__, fn_exc)
logger.warning(msg2)
else:
traceback.print_exception(*self._exception)
else:
sys.excepthook = sys.__excepthook__
traceback.print_exception(exctype, exc, trcbck)
if throw_except:
sys.excepthook = exception_hook
reraise(*self._exception)
else:
logger.warning(msg1)
msg2 = 'Exception: {} - {}'.format(self._exception[0].__name__,
self._exception[1])
logger.warning(msg2)
return None
for key in self._call_status:
if any(ss in key
for ss in ['time', 'tstamp', 'count', 'size', 'container']):
self.stats[key] = self._call_status[key]
self.stats['worker_exec_time'] = round(
self.stats['worker_end_tstamp'] -
self.stats['worker_start_tstamp'], 8)
total_time = format(round(self.stats['worker_exec_time'], 2), '.2f')
logger.debug(
'ExecutorID {} | JobID {} - Got status from call {} - Activation '
'ID: {} - Time: {} seconds'.format(self.executor_id, self.job_id,
self.call_id,
self.activation_id,
str(total_time)))
self._set_state(ResponseFuture.State.Success)
if not self._call_status['result']:
self._produce_output = False
if not self._produce_output:
self._set_state(ResponseFuture.State.Done)
if 'new_futures' in self._call_status and not self._new_futures:
new_futures = pickle.loads(eval(self._call_status['new_futures']))
self._new_futures = [
new_futures
] if type(new_futures) == ResponseFuture else new_futures
self._set_state(ResponseFuture.State.Futures)
return self._call_status