def decide(self, decision_response):
"""
Delegate the decision to the executor.
:param decision_response: an object wrapping the PollForDecisionTask response
:type decision_response: swf.responses.Response
:returns: the decisions
:rtype: list[swf.models.decision.base.Decision]
"""
history = decision_response.history
self._workflow_name = history[0].workflow_type['name']
workflow_executor = self._workflows[self._workflow_name]
try:
decisions = workflow_executor.replay(decision_response)
if isinstance(
decisions,
tuple) and len(decisions) == 2: # (decisions, context)
decisions = decisions[0]
except Exception as err:
import traceback
details = traceback.format_exc()
message = "workflow decision failed: {}".format(err)
logger.error(message)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(reason=swf.format.reason(message),
details=swf.format.details(details))
decisions = [decision]
return decisions
def fail(self, reason, details=None):
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason='Workflow execution failed: {}'.format(reason),
details=details)
self._decisions.append(decision)
raise exceptions.ExecutionBlocked('workflow execution failed')
def fail(self, reason, details=None):
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason='Workflow execution failed: {}'.format(reason),
details=details,
)
self._decisions_and_context.append_decision(decision)
raise exceptions.ExecutionBlocked('workflow execution failed')
def fail(self, reason, details=None):
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(
'Workflow execution failed: {}'.format(reason)),
details=swf.format.details(details),
)
self._decisions.append(decision)
raise exceptions.ExecutionBlocked('workflow execution failed')
def replay(self, history):
"""Executes the workflow from the start until it blocks.
"""
self.reset()
self._history = History(history)
self._history.parse()
workflow_started_event = history[0]
args = ()
kwargs = {}
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
try:
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
return self._decisions, {}
except exceptions.TaskException, err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.exception(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
return [decision], {}
def decide(self, decision_response, task_list):
"""
Delegate the decision to the executor, loading it if needed.
:param decision_response: an object wrapping the PollForDecisionTask response.
:type decision_response: swf.responses.Response
:param task_list:
:type task_list: Optional[str]
:returns: the decisions.
:rtype: list[swf.models.decision.base.Decision]
"""
history = decision_response.history
workflow_name = history[0].workflow_type['name']
workflow_executor = self._workflow_executors.get(workflow_name)
if not workflow_executor:
# Child workflow from another module
from . import helpers
workflow_executor = helpers.load_workflow_executor(
self._domain,
workflow_name,
task_list=task_list,
)
self._workflow_executors[workflow_name] = workflow_executor
try:
decisions = workflow_executor.replay(decision_response)
if isinstance(decisions, tuple) and len(
decisions) == 2: # (decisions, obsolete context)
decisions = decisions[0]
except Exception as err:
import traceback
details = traceback.format_exc()
message = "workflow decision failed: {}".format(err)
logger.exception(message)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(reason=swf.format.reason(message),
details=swf.format.details(details))
decisions = [decision]
return decisions
def decide(self, decision_response, task_list):
"""
Delegate the decision to the executor, loading it if needed.
:param decision_response: an object wrapping the PollForDecisionTask response.
:type decision_response: swf.responses.Response
:param task_list:
:type task_list: Optional[str]
:returns: the decisions.
:rtype: list[swf.models.decision.base.Decision]
"""
history = decision_response.history
workflow_name = history[0].workflow_type['name']
workflow_executor = self._workflow_executors.get(workflow_name)
if not workflow_executor:
# Child workflow from another module
from . import helpers
workflow_executor = helpers.load_workflow_executor(
self._domain,
workflow_name,
task_list=task_list,
)
self._workflow_executors[workflow_name] = workflow_executor
try:
decisions = workflow_executor.replay(decision_response)
except Exception as err:
import traceback
details = traceback.format_exc()
message = "workflow decision failed: {}".format(err)
logger.exception(message)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(reason=message, details=details)
decisions = [decision]
return decisions
def replay(self, decision_response, decref_workflow=True):
"""Replay the workflow from the start until it blocks.
Called by the DeciderWorker.
:param decision_response: an object wrapping the PollForDecisionTask response
:type decision_response: swf.responses.Response
:param decref_workflow : Decref workflow once replay is done (to save memory)
:type decref_workflow : boolean
:returns: a list of decision and a context dict (obsolete, empty)
:rtype: ([swf.models.decision.base.Decision], dict)
"""
self.reset()
history = decision_response.history
self._history = History(history)
self._history.parse()
self.build_execution_context(decision_response)
self._execution = decision_response.execution
workflow_started_event = history[0]
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
self.before_replay()
try:
self.propagate_signals()
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
self.after_replay()
if decref_workflow:
self.decref_workflow()
if self._append_timer:
self._add_start_timer_decision('_simpleflow_wake_up_timer')
return self._decisions, {}
except exceptions.TaskException as err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.exception(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return [decision], {}
except Exception as err:
reason = 'Cannot replay the workflow: {}({})'.format(
err.__class__.__name__,
err,
)
tb = traceback.format_exc()
details = 'Traceback:\n{}'.format(tb)
logger.exception(reason + '\n' + details)
self.on_failure(reason)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return [decision], {}
self.after_replay()
decision = swf.models.decision.WorkflowExecutionDecision()
decision.complete(result=swf.format.result(json_dumps(result)))
self.on_completed()
self.after_closed()
if decref_workflow:
self.decref_workflow()
return [decision], {}
def replay(self, history):
"""Executes the workflow from the start until it blocks.
"""
self.reset()
self._history = History(history)
self._history.parse()
workflow_started_event = history[0]
args = ()
kwargs = {}
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
# check if there is a workflow cancellation request
if self._history.is_cancel_requested:
# list all the running activities
cancellable_activities_id = self._history.list_cancellable_activities()
if len(cancellable_activities_id) == 0:
# nothing to cancel, completing the workflow as cancelled
cancel_decision = swf.models.decision.WorkflowExecutionDecision()
cancel_decision.cancel()
logger.info('Sucessfully canceled the workflow.')
return [cancel_decision], {}
cancel_activities_decisions = []
for activity_id in cancellable_activities_id:
# send cancel request to each of them
decision = swf.models.decision.ActivityTaskDecision(
'request_cancel',
activity_id=activity_id,
)
cancel_activities_decisions.append(decision)
return cancel_activities_decisions, {}
# handle workflow on start delay
if self._workflow.delayed_start_timer > 0:
if 'delayed_start_timer' not in self._history._timers:
logger.info('Scheduling delayed start decision.')
timer = swf.models.decision.TimerDecision(
'start',
id='delayed_start_timer',
start_to_fire_timeout=str(self._workflow.delayed_start_timer))
self._decisions.append(timer)
return self._decisions, {}
elif self._history._timers['delayed_start_timer']['state'] != 'fired':
# wait for the timer event, no-op
logger.info('Timer has not fired yet.')
return [], {}
if self._history.is_workflow_started:
# the workflow has just started
self.on_start(args, kwargs)
# workflow not cancelled
try:
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
return self._decisions, {}
except exceptions.TaskException, err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.info(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details, args, kwargs)
decision = swf.models.decision.WorkflowExecutionDecision()
if self._workflow.is_daemon:
# do not fail daemon workflow
logger.info('Task failed. Re-running continue_as_new for the daemon workflow.')
decision.continue_as_new(
input=input,
task_list={ 'name': self.task_list },
task_timeout=str(self._workflow.decision_tasks_timeout),
execution_timeout=str(self._workflow.execution_timeout),
workflow_type_version=str(self._workflow.version))
else:
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
return [decision], {}
def replay(self, decision_response):
"""Executes the workflow from the start until it blocks.
:param decision_response: an object wrapping the PollForDecisionTask response
:type decision_response: swf.responses.Response
:returns: a list of decision and a context dict
:rtype: ([swf.models.decision.base.Decision], dict)
"""
self.reset()
history = decision_response.history
self._history = History(history)
self._history.parse()
workflow_started_event = history[0]
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
self.before_replay()
try:
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
self.after_replay()
return self._decisions, {}
except exceptions.TaskException as err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.exception(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
return [decision], {}
except Exception as err:
reason = 'Cannot replay the workflow: {}({})'.format(
err.__class__.__name__,
err,
)
tb = traceback.format_exc()
details = 'Traceback:\n{}'.format(tb)
logger.exception(reason + '\n' + details)
self.on_failure(reason)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
return [decision], {}
self.after_replay()
decision = swf.models.decision.WorkflowExecutionDecision()
decision.complete(result=swf.format.result(json.dumps(result)))
self.on_completed()
self.after_closed()
return [decision], {}
def replay(self, decision_response, decref_workflow=True):
# type: (swf.responses.Response, bool) -> DecisionsAndContext
"""Replay the workflow from the start until it blocks.
Called by the DeciderWorker.
:param decision_response: an object wrapping the PollForDecisionTask response
:param decref_workflow : Decref workflow once replay is done (to save memory)
:returns: a list of decision with an optional context
"""
self.reset()
# noinspection PyUnresolvedReferences
history = decision_response.history
self._history = History(history)
self._history.parse()
self.build_run_context(decision_response)
# noinspection PyUnresolvedReferences
self._execution = decision_response.execution
workflow_started_event = history[0]
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
self.before_replay()
try:
if self._history.cancel_requested:
decisions = self.handle_cancel_requested()
if decisions is not None:
self.after_replay()
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext(decisions)
self.propagate_signals()
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions_and_context.decisions),
))
self.after_replay()
if decref_workflow:
self.decref_workflow()
if self._append_timer:
self._add_start_timer_decision('_simpleflow_wake_up_timer')
if not self._decisions_and_context.execution_context:
self.maybe_clear_execution_context()
return self._decisions_and_context
except exceptions.TaskException as err:
def _extract_reason(err):
if hasattr(err.exception, 'reason'):
raw = err.exception.reason
# don't parse eventual json object here, since we will cast
# the result to a string anyway, better keep a json representation
return format.decode(raw, parse_json=False, use_proxy=False)
return repr(err.exception)
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
_extract_reason(err))
logger.exception('%s', reason) # Don't let logger try to interpolate the message
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=reason,
details=details,
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext([decision])
except Exception as err:
reason = 'Cannot replay the workflow: {}({})'.format(
err.__class__.__name__,
err,
)
tb = traceback.format_exc()
details = 'Traceback:\n{}'.format(tb)
logger.exception('%s', reason + '\n' + details) # Don't let logger try to interpolate the message
self.on_failure(reason)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=reason,
details=details,
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext([decision])
self.after_replay()
decision = swf.models.decision.WorkflowExecutionDecision()
decision.complete(result=result)
self.on_completed()
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext([decision])
class Executor(executor.Executor):
"""
Manage a workflow's execution with Amazon SWF. It replays the workflow's
definition from the start until it blocks (i.e. raises
:py:class:`exceptions.ExecutionBlocked`).
SWF stores the history of all events that occurred in the workflow and
passes it to the executor. Only one executor handles a workflow at a time.
It means the history is consistent and there is no concurrent modifications
on the execution of the workflow.
"""
def __init__(self, domain, workflow, task_list=None):
super(Executor, self).__init__(workflow)
self._tasks = TaskRegistry()
self.domain = domain
self.task_list = task_list
def reset(self):
"""
Clears the state of the execution.
It is required to ensure the id of the tasks are assigned the same way
on each replay.
"""
self._open_activity_count = 0
self._decisions = []
self._tasks = TaskRegistry()
def _make_task_id(self, task):
"""
Assign a new ID to *task*.
:returns:
String with at most 256 characters.
"""
index = self._tasks.add(task)
task_id = '{name}-{idx}'.format(name=task.name, idx=index)
return task_id
def _get_future_from_activity_event(self, event):
"""Maps an activity event to a Future with the corresponding state.
:param event: workflow event.
:type event: swf.event.Event.
"""
future = futures.Future() # state is PENDING.
state = event['state']
if state == 'scheduled':
future._state = futures.PENDING
elif state == 'schedule_failed':
if event['cause'] == 'ACTIVITY_TYPE_DOES_NOT_EXIST':
activity_type = swf.models.ActivityType(
self.domain,
name=event['activity_type']['name'],
version=event['activity_type']['version'])
logger.info('Creating activity type {} in domain {}'.format(
activity_type.name,
self.domain.name))
try:
activity_type.save()
except swf.exceptions.AlreadyExistsError:
logger.info(
'Activity type {} in domain {} already exists'.format(
activity_type.name,
self.domain.name))
return None
logger.info('failed to schedule {}: {}'.format(
event['activity_type']['name'],
event['cause'],
))
return None
elif state == 'started':
future._state = futures.RUNNING
elif state == 'completed':
future._state = futures.FINISHED
result = event['result']
future._result = json.loads(result) if result else None
elif state == 'canceled':
future._state = futures.CANCELLED
elif state == 'failed':
future._state = futures.FINISHED
future._exception = exceptions.TaskFailed(
name=event['id'],
reason=event['reason'],
details=event.get('details'),
)
elif state == 'timed_out':
future._state = futures.FINISHED
future._exception = exceptions.TimeoutError(
event['timeout_type'],
event['timeout_value'])
return future
def _get_future_from_child_workflow_event(self, event):
"""Maps a child workflow event to a Future with the corresponding
state.
"""
future = futures.Future()
state = event['state']
if state == 'start_initiated':
future._state = futures.PENDING
elif state == 'started':
future._state = futures.RUNNING
elif state == 'completed':
future._state = futures.FINISHED
future._result = json.loads(event['result'])
return future
def find_activity_event(self, task, history):
activity = history._activities.get(task.id)
return activity
def find_child_workflow_event(self, task, history):
return history._child_workflows.get(task.id)
def find_event(self, task, history):
if isinstance(task, ActivityTask):
return self.find_activity_event(task, history)
elif isinstance(task, WorkflowTask):
return self.find_child_workflow_event(task, history)
else:
return TypeError('invalid type {} for task {}'.format(
type(task), task))
return None
def make_activity_task(self, func, *args, **kwargs):
return ActivityTask(func, *args, **kwargs)
def make_workflow_task(self, func, *args, **kwargs):
return WorkflowTask(func, *args, **kwargs)
def resume_activity(self, task, event):
future = self._get_future_from_activity_event(event)
if not future: # Task in history does not count.
return None
if not future.finished: # Still pending or running...
return future
if future.exception is None: # Result available!
return future
if event.get('retry', 0) == task.activity.retry: # No more retry!
if task.activity.raises_on_failure:
raise exceptions.TaskException(task, future.exception)
return future # with future.exception set.
# Otherwise retry the task by scheduling it again.
return None # means the is not in SWF.
def resume_child_workflow(self, task, event):
return self._get_future_from_child_workflow_event(event)
def schedule_task(self, task, task_list=None):
logger.debug('executor is scheduling task {} on task_list {}'.format(
task.name,
task_list,
))
decisions = task.schedule(self.domain, task_list)
# ``decisions`` contains a single decision.
self._decisions.extend(decisions)
self._open_activity_count += 1
if len(self._decisions) == constants.MAX_DECISIONS - 1:
# We add a timer to wake up the workflow immediately after
# completing these decisions.
timer = swf.models.decision.TimerDecision(
'start',
id='resume-after-{}'.format(task.id),
start_to_fire_timeout='0')
self._decisions.append(timer)
raise exceptions.ExecutionBlocked()
def resume(self, task, *args, **kwargs):
"""Resume the execution of a task.
If the task was scheduled, returns a future that wraps its state,
otherwise schedules it.
"""
task.id = self._make_task_id(task)
event = self.find_event(task, self._history)
future = None
if event:
if event['type'] == 'activity':
future = self.resume_activity(task, event)
if future and future._state in (futures.PENDING, futures.RUNNING):
self._open_activity_count += 1
elif event['type'] == 'child_workflow':
future = self.resume_child_workflow(task, event)
if not future:
self.schedule_task(task, task_list=self.task_list)
future = futures.Future() # return a pending future.
if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT:
logger.warning('limit of {} open activities reached'.format(
constants.MAX_OPEN_ACTIVITY_COUNT))
raise exceptions.ExecutionBlocked
return future
def submit(self, func, *args, **kwargs):
"""Register a function and its arguments for asynchronous execution.
``*args`` and ``**kwargs`` must be serializable in JSON.
"""
errors = []
arguments = []
keyword_arguments = {}
result = None
try:
for arg in args:
if isinstance(arg, futures.Future) and arg.failed:
exc = arg._exception
if isinstance(exc, exceptions.MultipleExceptions):
errors.extend(exc.exceptions)
else:
errors.append(exc)
else:
arguments.append(executor.get_actual_value(arg))
for key, val in kwargs.iteritems():
if isinstance(val, futures.Future) and val.failed:
exc = val._exception
if isinstance(exc, exceptions.MultipleExceptions):
errors.extend(exc.exceptions)
else:
errors.append(val._exception)
else:
keyword_arguments[key] = executor.get_actual_value(val)
except exceptions.ExecutionBlocked:
result = futures.Future()
finally:
if errors:
result = futures.Future()
result._state = futures.FINISHED
result._exception = exceptions.MultipleExceptions(
'futures failed',
errors,
)
if result is not None:
return result
try:
if isinstance(func, Activity):
make_task = self.make_activity_task
elif issubclass(func, Workflow):
make_task = self.make_workflow_task
else:
raise TypeError
task = make_task(func, *arguments, **keyword_arguments)
except TypeError:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
return self.resume(task, *arguments, **keyword_arguments)
def map(self, callable, iterable):
"""Submit *callable* with each of the items in ``*iterables``.
All items in ``*iterables`` must be serializable in JSON.
"""
iterable = executor.get_actual_value(iterable)
return super(Executor, self).map(callable, iterable)
def starmap(self, callable, iterable):
iterable = executor.get_actual_value(iterable)
return super(Executor, self).starmap(callable, iterable)
def replay(self, history):
"""Executes the workflow from the start until it blocks.
"""
self.reset()
self._history = History(history)
self._history.parse()
workflow_started_event = history[0]
args = ()
kwargs = {}
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
try:
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
return self._decisions, {}
except exceptions.TaskException, err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.exception(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
return [decision], {}
except Exception, err:
reason = 'Cannot replay the workflow: {}({})'.format(
err.__class__.__name__,
err,
)
tb = traceback.format_exc()
details = 'Traceback:\n{}'.format(tb)
logger.exception(reason + '\n' + details)
self.on_failure(reason)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
return [decision], {}
