def async_termination(e):
"""Actual termination runs in a thread."""
with e.termination_lock:
try:
terminate_execution(e)
except ZoeException as ex:
log.error('Error in termination thread: {}'.format(ex))
return
self.trigger()
log.debug('Execution {} terminated successfully'.format(e.id))
def execution_terminate(scheduler: ZoeBaseScheduler, execution: Execution):
"""Remove an execution form the scheduler."""
if execution.is_running or execution.status == execution.SCHEDULED_STATUS:
execution.set_cleaning_up()
scheduler.terminate(execution)
elif execution.status == execution.SUBMIT_STATUS or execution.status == execution.STARTING_STATUS:
return # It is unsafe to terminate executions in these statuses
elif execution.status == execution.ERROR_STATUS or execution.status == execution.CLEANING_UP_STATUS:
terminate_execution(execution)
elif execution.status == execution.TERMINATED_STATUS:
return
def execution_terminate(scheduler: ZoeBaseScheduler, execution: Execution,
reason: str):
"""Remove an execution from the scheduler."""
if execution.is_running or execution.status == execution.QUEUED_STATUS:
execution.set_cleaning_up()
execution.set_error_message(reason)
scheduler.terminate(execution)
elif execution.status == execution.CLEANING_UP_STATUS:
scheduler.terminate(execution)
elif execution.status == execution.SUBMIT_STATUS:
execution.set_terminated(reason)
elif execution.status == execution.STARTING_STATUS:
return # It is unsafe to terminate executions in these statuses
elif execution.status == execution.ERROR_STATUS:
terminate_execution(execution, reason)
elif execution.status == execution.TERMINATED_STATUS:
return
def _terminate_executions(self):
while len(self.queue_termination) > 0:
execution = self.queue_termination.pop(0)
try:
self.queue.remove(execution)
except ValueError:
try:
self.queue_running.remove(execution)
except ValueError:
log.warning('Execution {} is not in any queue, attempting termination anyway'.format(execution.id))
try:
del self.additional_exec_state[execution.id]
except KeyError:
pass
terminate_execution(execution)
log.info('Execution {} terminated successfully'.format(execution.id))
def async_termination():
"""Actual termination run in a thread."""
terminate_execution(execution)
self.trigger()