def decide(self, decision_response): """ Delegate the decision to the executor. :param decision_response: an object wrapping the PollForDecisionTask response :type decision_response: swf.responses.Response :returns: the decisions :rtype: list[swf.models.decision.base.Decision] """ history = decision_response.history self._workflow_name = history[0].workflow_type['name'] workflow_executor = self._workflows[self._workflow_name] try: decisions = workflow_executor.replay(decision_response) if isinstance( decisions, tuple) and len(decisions) == 2: # (decisions, context) decisions = decisions[0] except Exception as err: import traceback details = traceback.format_exc() message = "workflow decision failed: {}".format(err) logger.error(message) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail(reason=swf.format.reason(message), details=swf.format.details(details)) decisions = [decision] return decisions
def fail(self, reason, details=None): self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason='Workflow execution failed: {}'.format(reason), details=details) self._decisions.append(decision) raise exceptions.ExecutionBlocked('workflow execution failed')
def fail(self, reason, details=None): self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason='Workflow execution failed: {}'.format(reason), details=details, ) self._decisions_and_context.append_decision(decision) raise exceptions.ExecutionBlocked('workflow execution failed')
def fail(self, reason, details=None): self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason( 'Workflow execution failed: {}'.format(reason)), details=swf.format.details(details), ) self._decisions.append(decision) raise exceptions.ExecutionBlocked('workflow execution failed')
def replay(self, history): """Executes the workflow from the start until it blocks. """ self.reset() self._history = History(history) self._history.parse() workflow_started_event = history[0] args = () kwargs = {} input = workflow_started_event.input if input is None: input = {} args = input.get('args', ()) kwargs = input.get('kwargs', {}) try: result = self.run_workflow(*args, **kwargs) except exceptions.ExecutionBlocked: logger.info('{} open activities ({} decisions)'.format( self._open_activity_count, len(self._decisions), )) return self._decisions, {} except exceptions.TaskException, err: reason = 'Workflow execution error in task {}: "{}"'.format( err.task.name, getattr(err.exception, 'reason', repr(err.exception))) logger.exception(reason) details = getattr(err.exception, 'details', None) self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) return [decision], {}
def decide(self, decision_response, task_list): """ Delegate the decision to the executor, loading it if needed. :param decision_response: an object wrapping the PollForDecisionTask response. :type decision_response: swf.responses.Response :param task_list: :type task_list: Optional[str] :returns: the decisions. :rtype: list[swf.models.decision.base.Decision] """ history = decision_response.history workflow_name = history[0].workflow_type['name'] workflow_executor = self._workflow_executors.get(workflow_name) if not workflow_executor: # Child workflow from another module from . import helpers workflow_executor = helpers.load_workflow_executor( self._domain, workflow_name, task_list=task_list, ) self._workflow_executors[workflow_name] = workflow_executor try: decisions = workflow_executor.replay(decision_response) if isinstance(decisions, tuple) and len( decisions) == 2: # (decisions, obsolete context) decisions = decisions[0] except Exception as err: import traceback details = traceback.format_exc() message = "workflow decision failed: {}".format(err) logger.exception(message) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail(reason=swf.format.reason(message), details=swf.format.details(details)) decisions = [decision] return decisions
def decide(self, decision_response, task_list): """ Delegate the decision to the executor, loading it if needed. :param decision_response: an object wrapping the PollForDecisionTask response. :type decision_response: swf.responses.Response :param task_list: :type task_list: Optional[str] :returns: the decisions. :rtype: list[swf.models.decision.base.Decision] """ history = decision_response.history workflow_name = history[0].workflow_type['name'] workflow_executor = self._workflow_executors.get(workflow_name) if not workflow_executor: # Child workflow from another module from . import helpers workflow_executor = helpers.load_workflow_executor( self._domain, workflow_name, task_list=task_list, ) self._workflow_executors[workflow_name] = workflow_executor try: decisions = workflow_executor.replay(decision_response) except Exception as err: import traceback details = traceback.format_exc() message = "workflow decision failed: {}".format(err) logger.exception(message) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail(reason=message, details=details) decisions = [decision] return decisions
def replay(self, decision_response, decref_workflow=True): """Replay the workflow from the start until it blocks. Called by the DeciderWorker. :param decision_response: an object wrapping the PollForDecisionTask response :type decision_response: swf.responses.Response :param decref_workflow : Decref workflow once replay is done (to save memory) :type decref_workflow : boolean :returns: a list of decision and a context dict (obsolete, empty) :rtype: ([swf.models.decision.base.Decision], dict) """ self.reset() history = decision_response.history self._history = History(history) self._history.parse() self.build_execution_context(decision_response) self._execution = decision_response.execution workflow_started_event = history[0] input = workflow_started_event.input if input is None: input = {} args = input.get('args', ()) kwargs = input.get('kwargs', {}) self.before_replay() try: self.propagate_signals() result = self.run_workflow(*args, **kwargs) except exceptions.ExecutionBlocked: logger.info('{} open activities ({} decisions)'.format( self._open_activity_count, len(self._decisions), )) self.after_replay() if decref_workflow: self.decref_workflow() if self._append_timer: self._add_start_timer_decision('_simpleflow_wake_up_timer') return self._decisions, {} except exceptions.TaskException as err: reason = 'Workflow execution error in task {}: "{}"'.format( err.task.name, getattr(err.exception, 'reason', repr(err.exception))) logger.exception(reason) details = getattr(err.exception, 'details', None) self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) self.after_closed() if decref_workflow: self.decref_workflow() return [decision], {} except Exception as err: reason = 'Cannot replay the workflow: {}({})'.format( err.__class__.__name__, err, ) tb = traceback.format_exc() details = 'Traceback:\n{}'.format(tb) logger.exception(reason + '\n' + details) self.on_failure(reason) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) self.after_closed() if decref_workflow: self.decref_workflow() return [decision], {} self.after_replay() decision = swf.models.decision.WorkflowExecutionDecision() decision.complete(result=swf.format.result(json_dumps(result))) self.on_completed() self.after_closed() if decref_workflow: self.decref_workflow() return [decision], {}
def replay(self, history): """Executes the workflow from the start until it blocks. """ self.reset() self._history = History(history) self._history.parse() workflow_started_event = history[0] args = () kwargs = {} input = workflow_started_event.input if input is None: input = {} args = input.get('args', ()) kwargs = input.get('kwargs', {}) # check if there is a workflow cancellation request if self._history.is_cancel_requested: # list all the running activities cancellable_activities_id = self._history.list_cancellable_activities() if len(cancellable_activities_id) == 0: # nothing to cancel, completing the workflow as cancelled cancel_decision = swf.models.decision.WorkflowExecutionDecision() cancel_decision.cancel() logger.info('Sucessfully canceled the workflow.') return [cancel_decision], {} cancel_activities_decisions = [] for activity_id in cancellable_activities_id: # send cancel request to each of them decision = swf.models.decision.ActivityTaskDecision( 'request_cancel', activity_id=activity_id, ) cancel_activities_decisions.append(decision) return cancel_activities_decisions, {} # handle workflow on start delay if self._workflow.delayed_start_timer > 0: if 'delayed_start_timer' not in self._history._timers: logger.info('Scheduling delayed start decision.') timer = swf.models.decision.TimerDecision( 'start', id='delayed_start_timer', start_to_fire_timeout=str(self._workflow.delayed_start_timer)) self._decisions.append(timer) return self._decisions, {} elif self._history._timers['delayed_start_timer']['state'] != 'fired': # wait for the timer event, no-op logger.info('Timer has not fired yet.') return [], {} if self._history.is_workflow_started: # the workflow has just started self.on_start(args, kwargs) # workflow not cancelled try: result = self.run_workflow(*args, **kwargs) except exceptions.ExecutionBlocked: logger.info('{} open activities ({} decisions)'.format( self._open_activity_count, len(self._decisions), )) return self._decisions, {} except exceptions.TaskException, err: reason = 'Workflow execution error in task {}: "{}"'.format( err.task.name, getattr(err.exception, 'reason', repr(err.exception))) logger.info(reason) details = getattr(err.exception, 'details', None) self.on_failure(reason, details, args, kwargs) decision = swf.models.decision.WorkflowExecutionDecision() if self._workflow.is_daemon: # do not fail daemon workflow logger.info('Task failed. Re-running continue_as_new for the daemon workflow.') decision.continue_as_new( input=input, task_list={ 'name': self.task_list }, task_timeout=str(self._workflow.decision_tasks_timeout), execution_timeout=str(self._workflow.execution_timeout), workflow_type_version=str(self._workflow.version)) else: decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) return [decision], {}
def replay(self, decision_response): """Executes the workflow from the start until it blocks. :param decision_response: an object wrapping the PollForDecisionTask response :type decision_response: swf.responses.Response :returns: a list of decision and a context dict :rtype: ([swf.models.decision.base.Decision], dict) """ self.reset() history = decision_response.history self._history = History(history) self._history.parse() workflow_started_event = history[0] input = workflow_started_event.input if input is None: input = {} args = input.get('args', ()) kwargs = input.get('kwargs', {}) self.before_replay() try: result = self.run_workflow(*args, **kwargs) except exceptions.ExecutionBlocked: logger.info('{} open activities ({} decisions)'.format( self._open_activity_count, len(self._decisions), )) self.after_replay() return self._decisions, {} except exceptions.TaskException as err: reason = 'Workflow execution error in task {}: "{}"'.format( err.task.name, getattr(err.exception, 'reason', repr(err.exception))) logger.exception(reason) details = getattr(err.exception, 'details', None) self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) self.after_closed() return [decision], {} except Exception as err: reason = 'Cannot replay the workflow: {}({})'.format( err.__class__.__name__, err, ) tb = traceback.format_exc() details = 'Traceback:\n{}'.format(tb) logger.exception(reason + '\n' + details) self.on_failure(reason) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) self.after_closed() return [decision], {} self.after_replay() decision = swf.models.decision.WorkflowExecutionDecision() decision.complete(result=swf.format.result(json.dumps(result))) self.on_completed() self.after_closed() return [decision], {}
def replay(self, decision_response, decref_workflow=True): # type: (swf.responses.Response, bool) -> DecisionsAndContext """Replay the workflow from the start until it blocks. Called by the DeciderWorker. :param decision_response: an object wrapping the PollForDecisionTask response :param decref_workflow : Decref workflow once replay is done (to save memory) :returns: a list of decision with an optional context """ self.reset() # noinspection PyUnresolvedReferences history = decision_response.history self._history = History(history) self._history.parse() self.build_run_context(decision_response) # noinspection PyUnresolvedReferences self._execution = decision_response.execution workflow_started_event = history[0] input = workflow_started_event.input if input is None: input = {} args = input.get('args', ()) kwargs = input.get('kwargs', {}) self.before_replay() try: if self._history.cancel_requested: decisions = self.handle_cancel_requested() if decisions is not None: self.after_replay() self.after_closed() if decref_workflow: self.decref_workflow() return DecisionsAndContext(decisions) self.propagate_signals() result = self.run_workflow(*args, **kwargs) except exceptions.ExecutionBlocked: logger.info('{} open activities ({} decisions)'.format( self._open_activity_count, len(self._decisions_and_context.decisions), )) self.after_replay() if decref_workflow: self.decref_workflow() if self._append_timer: self._add_start_timer_decision('_simpleflow_wake_up_timer') if not self._decisions_and_context.execution_context: self.maybe_clear_execution_context() return self._decisions_and_context except exceptions.TaskException as err: def _extract_reason(err): if hasattr(err.exception, 'reason'): raw = err.exception.reason # don't parse eventual json object here, since we will cast # the result to a string anyway, better keep a json representation return format.decode(raw, parse_json=False, use_proxy=False) return repr(err.exception) reason = 'Workflow execution error in task {}: "{}"'.format( err.task.name, _extract_reason(err)) logger.exception('%s', reason) # Don't let logger try to interpolate the message details = getattr(err.exception, 'details', None) self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=reason, details=details, ) self.after_closed() if decref_workflow: self.decref_workflow() return DecisionsAndContext([decision]) except Exception as err: reason = 'Cannot replay the workflow: {}({})'.format( err.__class__.__name__, err, ) tb = traceback.format_exc() details = 'Traceback:\n{}'.format(tb) logger.exception('%s', reason + '\n' + details) # Don't let logger try to interpolate the message self.on_failure(reason) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=reason, details=details, ) self.after_closed() if decref_workflow: self.decref_workflow() return DecisionsAndContext([decision]) self.after_replay() decision = swf.models.decision.WorkflowExecutionDecision() decision.complete(result=result) self.on_completed() self.after_closed() if decref_workflow: self.decref_workflow() return DecisionsAndContext([decision])
class Executor(executor.Executor): """ Manage a workflow's execution with Amazon SWF. It replays the workflow's definition from the start until it blocks (i.e. raises :py:class:`exceptions.ExecutionBlocked`). SWF stores the history of all events that occurred in the workflow and passes it to the executor. Only one executor handles a workflow at a time. It means the history is consistent and there is no concurrent modifications on the execution of the workflow. """ def __init__(self, domain, workflow, task_list=None): super(Executor, self).__init__(workflow) self._tasks = TaskRegistry() self.domain = domain self.task_list = task_list def reset(self): """ Clears the state of the execution. It is required to ensure the id of the tasks are assigned the same way on each replay. """ self._open_activity_count = 0 self._decisions = [] self._tasks = TaskRegistry() def _make_task_id(self, task): """ Assign a new ID to *task*. :returns: String with at most 256 characters. """ index = self._tasks.add(task) task_id = '{name}-{idx}'.format(name=task.name, idx=index) return task_id def _get_future_from_activity_event(self, event): """Maps an activity event to a Future with the corresponding state. :param event: workflow event. :type event: swf.event.Event. """ future = futures.Future() # state is PENDING. state = event['state'] if state == 'scheduled': future._state = futures.PENDING elif state == 'schedule_failed': if event['cause'] == 'ACTIVITY_TYPE_DOES_NOT_EXIST': activity_type = swf.models.ActivityType( self.domain, name=event['activity_type']['name'], version=event['activity_type']['version']) logger.info('Creating activity type {} in domain {}'.format( activity_type.name, self.domain.name)) try: activity_type.save() except swf.exceptions.AlreadyExistsError: logger.info( 'Activity type {} in domain {} already exists'.format( activity_type.name, self.domain.name)) return None logger.info('failed to schedule {}: {}'.format( event['activity_type']['name'], event['cause'], )) return None elif state == 'started': future._state = futures.RUNNING elif state == 'completed': future._state = futures.FINISHED result = event['result'] future._result = json.loads(result) if result else None elif state == 'canceled': future._state = futures.CANCELLED elif state == 'failed': future._state = futures.FINISHED future._exception = exceptions.TaskFailed( name=event['id'], reason=event['reason'], details=event.get('details'), ) elif state == 'timed_out': future._state = futures.FINISHED future._exception = exceptions.TimeoutError( event['timeout_type'], event['timeout_value']) return future def _get_future_from_child_workflow_event(self, event): """Maps a child workflow event to a Future with the corresponding state. """ future = futures.Future() state = event['state'] if state == 'start_initiated': future._state = futures.PENDING elif state == 'started': future._state = futures.RUNNING elif state == 'completed': future._state = futures.FINISHED future._result = json.loads(event['result']) return future def find_activity_event(self, task, history): activity = history._activities.get(task.id) return activity def find_child_workflow_event(self, task, history): return history._child_workflows.get(task.id) def find_event(self, task, history): if isinstance(task, ActivityTask): return self.find_activity_event(task, history) elif isinstance(task, WorkflowTask): return self.find_child_workflow_event(task, history) else: return TypeError('invalid type {} for task {}'.format( type(task), task)) return None def make_activity_task(self, func, *args, **kwargs): return ActivityTask(func, *args, **kwargs) def make_workflow_task(self, func, *args, **kwargs): return WorkflowTask(func, *args, **kwargs) def resume_activity(self, task, event): future = self._get_future_from_activity_event(event) if not future: # Task in history does not count. return None if not future.finished: # Still pending or running... return future if future.exception is None: # Result available! return future if event.get('retry', 0) == task.activity.retry: # No more retry! if task.activity.raises_on_failure: raise exceptions.TaskException(task, future.exception) return future # with future.exception set. # Otherwise retry the task by scheduling it again. return None # means the is not in SWF. def resume_child_workflow(self, task, event): return self._get_future_from_child_workflow_event(event) def schedule_task(self, task, task_list=None): logger.debug('executor is scheduling task {} on task_list {}'.format( task.name, task_list, )) decisions = task.schedule(self.domain, task_list) # ``decisions`` contains a single decision. self._decisions.extend(decisions) self._open_activity_count += 1 if len(self._decisions) == constants.MAX_DECISIONS - 1: # We add a timer to wake up the workflow immediately after # completing these decisions. timer = swf.models.decision.TimerDecision( 'start', id='resume-after-{}'.format(task.id), start_to_fire_timeout='0') self._decisions.append(timer) raise exceptions.ExecutionBlocked() def resume(self, task, *args, **kwargs): """Resume the execution of a task. If the task was scheduled, returns a future that wraps its state, otherwise schedules it. """ task.id = self._make_task_id(task) event = self.find_event(task, self._history) future = None if event: if event['type'] == 'activity': future = self.resume_activity(task, event) if future and future._state in (futures.PENDING, futures.RUNNING): self._open_activity_count += 1 elif event['type'] == 'child_workflow': future = self.resume_child_workflow(task, event) if not future: self.schedule_task(task, task_list=self.task_list) future = futures.Future() # return a pending future. if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT: logger.warning('limit of {} open activities reached'.format( constants.MAX_OPEN_ACTIVITY_COUNT)) raise exceptions.ExecutionBlocked return future def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. """ errors = [] arguments = [] keyword_arguments = {} result = None try: for arg in args: if isinstance(arg, futures.Future) and arg.failed: exc = arg._exception if isinstance(exc, exceptions.MultipleExceptions): errors.extend(exc.exceptions) else: errors.append(exc) else: arguments.append(executor.get_actual_value(arg)) for key, val in kwargs.iteritems(): if isinstance(val, futures.Future) and val.failed: exc = val._exception if isinstance(exc, exceptions.MultipleExceptions): errors.extend(exc.exceptions) else: errors.append(val._exception) else: keyword_arguments[key] = executor.get_actual_value(val) except exceptions.ExecutionBlocked: result = futures.Future() finally: if errors: result = futures.Future() result._state = futures.FINISHED result._exception = exceptions.MultipleExceptions( 'futures failed', errors, ) if result is not None: return result try: if isinstance(func, Activity): make_task = self.make_activity_task elif issubclass(func, Workflow): make_task = self.make_workflow_task else: raise TypeError task = make_task(func, *arguments, **keyword_arguments) except TypeError: raise TypeError('invalid type {} for {}'.format( type(func), func)) return self.resume(task, *arguments, **keyword_arguments) def map(self, callable, iterable): """Submit *callable* with each of the items in ``*iterables``. All items in ``*iterables`` must be serializable in JSON. """ iterable = executor.get_actual_value(iterable) return super(Executor, self).map(callable, iterable) def starmap(self, callable, iterable): iterable = executor.get_actual_value(iterable) return super(Executor, self).starmap(callable, iterable) def replay(self, history): """Executes the workflow from the start until it blocks. """ self.reset() self._history = History(history) self._history.parse() workflow_started_event = history[0] args = () kwargs = {} input = workflow_started_event.input if input is None: input = {} args = input.get('args', ()) kwargs = input.get('kwargs', {}) try: result = self.run_workflow(*args, **kwargs) except exceptions.ExecutionBlocked: logger.info('{} open activities ({} decisions)'.format( self._open_activity_count, len(self._decisions), )) return self._decisions, {} except exceptions.TaskException, err: reason = 'Workflow execution error in task {}: "{}"'.format( err.task.name, getattr(err.exception, 'reason', repr(err.exception))) logger.exception(reason) details = getattr(err.exception, 'details', None) self.on_failure(reason, details) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) return [decision], {} except Exception, err: reason = 'Cannot replay the workflow: {}({})'.format( err.__class__.__name__, err, ) tb = traceback.format_exc() details = 'Traceback:\n{}'.format(tb) logger.exception(reason + '\n' + details) self.on_failure(reason) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail( reason=swf.format.reason(reason), details=swf.format.details(details), ) return [decision], {}