def sleep_with_cancel_support(ctx, use_legacy_cancel, **kwargs):
node_instance = get_instance(ctx)
node_instance.execute_operation(
'test_interface.operation',
kwargs={'key': 'before-sleep',
'value': None})
node_instance.set_state('asleep')
is_cancelled = False
for i in range(10):
if api.has_cancel_request():
is_cancelled = True
break
time.sleep(1)
if is_cancelled:
if use_legacy_cancel:
return api.EXECUTION_CANCELLED_RESULT
else:
raise api.ExecutionCancelled()
node_instance.execute_operation(
'test_interface.operation',
kwargs={'key': 'after-sleep',
'value': None})
def sleep_with_cancel_support(ctx, use_legacy_cancel, **kwargs):
node_instance = get_instance(ctx)
node_instance.execute_operation('test_interface.operation',
kwargs={
'key': 'before-sleep',
'value': None
})
node_instance.set_state('asleep')
is_cancelled = False
for i in range(10):
if api.has_cancel_request():
is_cancelled = True
break
time.sleep(1)
if is_cancelled:
if use_legacy_cancel:
return api.EXECUTION_CANCELLED_RESULT
else:
raise api.ExecutionCancelled()
node_instance.execute_operation('test_interface.operation',
kwargs={
'key': 'after-sleep',
'value': None
})
def get(self, retry_on_failure=True):
"""Get the task result. Will block until the task execution ends.
:return: The task result
"""
done = threading.Event()
api.cancel_callbacks.add(done.set)
self.on_result(lambda _result: done.set())
done.wait()
api.cancel_callbacks.discard(done.set)
if api.has_cancel_request():
if self._result is self._NOT_SET:
self.result = api.ExecutionCancelled()
raise self.result
ctx = self.task.workflow_context
if not ctx.internal.graph_mode:
ctx.internal.task_graph.remove_task(self.task)
if self.task.get_state() in (TASK_FAILED, TASK_RESCHEDULED):
handler_result = self.task.handle_task_terminated()
if handler_result.retried_task and retry_on_failure:
handler_result.retried_task.apply_async()
return handler_result.retried_task.async_result.get()
else:
raise self.result
return self._result
def _wait_for_sent_tasks(ctx, graph):
"""Wait for tasks that are in the SENT state to return"""
for task in graph.tasks_iter():
# Check type.
ctx.logger.debug(
'Parallel task to failed task: {0}. State: {1}'.format(
task.id, task.get_state()))
try:
deadline = time.time() + ctx.wait_after_fail
except AttributeError:
deadline = time.time() + 1800
while deadline > time.time():
try:
cancelled = api.has_cancel_request()
except AttributeError:
cancelled = graph._is_execution_cancelled()
if cancelled:
raise api.ExecutionCancelled()
try:
finished_tasks = graph._finished_tasks()
except AttributeError:
finished_tasks = graph._terminated_tasks()
for task in finished_tasks:
try:
graph._handle_terminated_task(task)
except RuntimeError:
ctx.logger.error('Unhandled Failed task: {0}'.format(task))
if not any(task.get_state() == tasks.TASK_SENT
for task in graph.tasks_iter()):
break
else:
time.sleep(0.1)
def run_jobs(**kwargs): # pylint: disable=W0613
""" Workflow to execute long running batch operations """
success = True
root_nodes, job_instances_map = build_graph(ctx.nodes)
monitor = Monitor(job_instances_map, ctx.logger)
new_exec_nodes = root_nodes
# Monitoring and next executions loop
while new_exec_nodes or monitor.is_something_executing(
) and not api.has_cancel_request():
# perform new executions
jobs_result_list = []
for new_node in new_exec_nodes:
monitor.add_node(new_node)
if new_node.is_job:
jobs_result_list += new_node.launch_all_instances()
wait_jobs_to_finish(jobs_result_list)
# Monitor the infrastructure
monitor.update_status()
exec_nodes_finished = []
new_exec_nodes = []
for node_name, exec_node in monitor.get_executions_iterator():
if exec_node.check_status():
if exec_node.completed:
exec_node.clean_all_instances()
exec_nodes_finished.append(node_name)
new_nodes_to_execute = exec_node.get_children_ready()
for new_node in new_nodes_to_execute:
new_exec_nodes.append(new_node)
else:
# Something went wrong in the node, cancel execution
cancel_all(monitor.get_executions_iterator())
return
# remove finished nodes
for node_name in exec_nodes_finished:
monitor.finish_node(node_name)
wait_jobs_to_finish(jobs_result_list)
if monitor.is_something_executing():
ctx.logger.info("Cancelling jobs...")
cancel_all(monitor.get_executions_iterator())
success = False
deleted_reservations = []
for instance_name in job_instances_map:
instance = job_instances_map[instance_name]
if instance.reservation not in deleted_reservations and instance.reservation:
instance.delete_reservation()
deleted_reservations.append(instance.reservation)
if not success:
raise api.ExecutionCancelled()
ctx.logger.info(
"------------------Workflow Finished-----------------------")
return
def run_jobs(**kwargs): # pylint: disable=W0613
""" Workflow to execute long running batch operations """
root_nodes, job_instances_map = build_graph(ctx.nodes)
monitor = Monitor(job_instances_map, ctx.logger)
ctx.logger.info('WORKFLOWS.PY::RUN_JOBS() L390')
# Execution of first job instances
tasks_list = []
for root in root_nodes:
tasks_list += root.queue_all_instances()
monitor.add_node(root)
wait_tasks_to_finish(tasks_list)
prev_status = str()
# Monitoring and next executions loop
while monitor.is_something_executing() and not api.has_cancel_request():
# Monitor the infrastructure
monitor.update_status()
exec_nodes_finished = []
new_exec_nodes = []
for node_name, exec_node in monitor.get_executions_iterator():
if exec_node.check_status():
if exec_node.status != prev_status:
ctx.logger.info('SPIROS TEMP exec_node.status = ' + str(exec_node.status))
prev_status = exec_node.status
if exec_node.completed:
exec_node.clean_all_instances()
exec_nodes_finished.append(node_name)
new_nodes_to_execute = exec_node.get_children_ready()
for new_node in new_nodes_to_execute:
new_exec_nodes.append(new_node)
else:
# Something went wrong in the node, cancel execution
cancel_all(monitor.get_executions_iterator())
return
# remove finished nodes
for node_name in exec_nodes_finished:
monitor.finish_node(node_name)
# perform new executions
tasks_list = []
for new_node in new_exec_nodes:
tasks_list += new_node.queue_all_instances()
monitor.add_node(new_node)
wait_tasks_to_finish(tasks_list)
if monitor.is_something_executing():
cancel_all(monitor.get_executions_iterator())
ctx.logger.info(
"------------------Workflow Finished-----------------------")
return
def _inner(*args, **kwargs):
task = args[0]
if api.has_cancel_request():
return task.async_result
if task.execute_after and task.execute_after > time.time():
t = threading.Timer(task.execute_after - time.time(),
_inner,
args=args,
kwargs=kwargs)
t.daemon = True
t.start()
return task.async_result
with current_workflow_ctx.push(task.workflow_context):
return f(*args, **kwargs)
def _is_finished(self):
if api.has_cancel_request():
self._error = api.ExecutionCancelled()
return True
if not self._tasks:
return True
if self._error:
if not self._waiting_for:
return True
deadline = self._error_time + self.ctx.wait_after_fail
if time.time() > deadline:
return True
else:
self._wake_after_fail = threading.Timer(
deadline - time.time(), self._tasks_wait.set)
self._wake_after_fail.daemon = True
self._wake_after_fail.start()
return False
def _check_execution_cancelled():
if api.has_cancel_request():
raise api.ExecutionCancelled()
def _check_execution_cancelled():
if api.has_cancel_request():
raise api.ExecutionCancelled()
def _is_execution_cancelled(self):
return api.has_cancel_request()
def _is_execution_cancelled():
return api.has_cancel_request()
