def resume(self, a_task, *args, **kwargs): """Resume the execution of a task. If the task was scheduled, returns a future that wraps its state, otherwise schedules it. """ a_task.id = self._make_task_id(a_task, *args, **kwargs) event = self.find_event(a_task, self._history) future = None # check if we absolutely want to execute this task in repair mode force_execution = self.force_activities and \ self.force_activities.search(a_task.id) # try to fill in the blanks with the workflow we're trying to repair if any # TODO: maybe only do that for idempotent tasks?? if not event and self.repair_with and not force_execution: # try to find a former event matching this task former_event = self.find_event(a_task, self.repair_with) # ... but only keep the event if the task was successful if former_event and former_event['state'] == 'completed': logger.info( 'faking task completed successfully in previous ' \ 'workflow: {}'.format(former_event['id']) ) json_hash = hashlib.md5(json_dumps(former_event)).hexdigest() fake_task_list = "FAKE-" + json_hash # schedule task on a fake task list self.schedule_task(a_task, task_list=fake_task_list) future = futures.Future() # start a dedicated process to handle the fake activity run_fake_task_worker(self.domain.name, fake_task_list, former_event) # back to normal execution flow if event: if event['type'] == 'activity': future = self.resume_activity(a_task, event) if future and future._state in (futures.PENDING, futures.RUNNING): self._open_activity_count += 1 elif event['type'] == 'child_workflow': future = self.resume_child_workflow(a_task, event) if not future: self.schedule_task(a_task, task_list=self.task_list) future = futures.Future() # return a pending future. if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT: logger.warning('limit of {} open activities reached'.format( constants.MAX_OPEN_ACTIVITY_COUNT)) raise exceptions.ExecutionBlocked return future
def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. """ errors = [] arguments = [] keyword_arguments = {} result = None try: for arg in args: if isinstance(arg, futures.Future) and arg.failed: exc = arg._exception if isinstance(exc, exceptions.MultipleExceptions): errors.extend(exc.exceptions) else: errors.append(exc) else: arguments.append(executor.get_actual_value(arg)) for key, val in kwargs.iteritems(): if isinstance(val, futures.Future) and val.failed: exc = val._exception if isinstance(exc, exceptions.MultipleExceptions): errors.extend(exc.exceptions) else: errors.append(val._exception) else: keyword_arguments[key] = executor.get_actual_value(val) except exceptions.ExecutionBlocked: result = futures.Future() finally: if errors: result = futures.Future() result._state = futures.FINISHED result._exception = exceptions.MultipleExceptions( 'futures failed', errors, ) if result is not None: return result try: if isinstance(func, Activity): make_task = self.make_activity_task elif issubclass(func, Workflow): make_task = self.make_workflow_task else: raise TypeError task = make_task(func, *arguments, **keyword_arguments) except TypeError: raise TypeError('invalid type {} for {}'.format( type(func), func)) return self.resume(task, *arguments, **keyword_arguments)
def _get_future_from_timer_event(self, a_task, event): """ Maps a timer event to a Future with the corresponding state. :param a_task: Timer task; unused. :type a_task: TimerTask :param event: Timer event :type event: dict[str, Any] :return: :rtype: futures.Future """ future = futures.Future() if not event: return future state = event['state'] if state == 'started': future.set_running() elif state == 'fired': future.set_finished(None) elif state == 'canceled': future.set_cancelled() elif state in ('start_failed', 'cancel_failed'): future.set_exception(exceptions.TaskFailed( name=event['timer_id'], reason=event['cause'], )) return future
def get_future_from_external_workflow_event(self, a_task, event): """Maps an external workflow event to a Future with the corresponding state. :param a_task: currently unused :type a_task: :param event: external workflow event :type event: dict[str, Any] :rtype: futures.Future """ future = futures.Future() if not event: return future state = event['state'] if state == 'signal_execution_initiated': # Don't re-initiate signal sending future.set_running() elif state == 'execution_signaled': future.set_finished(event['input']) elif state == 'signal_execution_failed': future.set_exception( exceptions.TaskFailed( name=event['name'], reason=event['cause'], )) return future
def resume(self, task, *args, **kwargs): """Resume the execution of a task. If the task was scheduled, returns a future that wraps its state, otherwise schedules it. """ task.id = self._make_task_id(task) event = self.find_event(task, self._history) future = None if event: if event['type'] == 'activity': future = self.resume_activity(task, event) if future and future._state in (futures.PENDING, futures.RUNNING): self._open_activity_count += 1 elif event['type'] == 'child_workflow': future = self.resume_child_workflow(task, event) if not future: self.schedule_task(task, task_list=self.task_list) future = futures.Future() # return a pending future. if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT: logger.warning('limit of {} open activities reached'.format( constants.MAX_OPEN_ACTIVITY_COUNT)) raise exceptions.ExecutionBlocked return future
def _get_future_from_child_workflow_event(self, event): """Maps a child workflow event to a Future with the corresponding state. :param event: child workflow event :type event: dict[str, Any] """ future = futures.Future() state = event['state'] if state == 'start_initiated': pass # future._state = futures.PENDING elif state == 'start_failed': if event['cause'] == 'WORKFLOW_TYPE_DOES_NOT_EXIST': workflow_type = swf.models.WorkflowType( self.domain, name=event['name'], version=event['version'], ) logger.info('Creating workflow type {} in domain {}'.format( workflow_type.name, self.domain.name, )) try: workflow_type.save() except swf.exceptions.AlreadyExistsError: # Could have be created by a concurrent workflow execution. pass return None future.set_exception( exceptions.TaskFailed( name=event['id'], reason=event['cause'], details=event.get('details'), )) elif state == 'started': future.set_running() elif state == 'completed': future.set_finished(json_loads_or_raw(event['result'])) elif state == 'failed': future.set_exception( exceptions.TaskFailed( name=event['id'], reason=event['reason'], details=event.get('details'), )) elif state == 'timed_out': future.set_exception( exceptions.TimeoutError( event['timeout_type'], None, )) elif state == 'canceled': future.set_exception( exceptions.TaskCanceled(event.get('details'), )) elif state == 'terminated': future.set_exception(exceptions.TaskTerminated()) return future
def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. :type func: simpleflow.base.Submittable | Activity | Workflow """ # NB: we don't set self.current_priority here directly, because we need # to extract it from the underlying Activity() if it's not passed to # self.submit() ; we DO need to pop the "__priority" kwarg though, so it # doesn't pollute the rest of the code. priority_set_on_submit = kwargs.pop("__priority", PRIORITY_NOT_SET) # casts simpleflow.task.*Task to their equivalent in simpleflow.swf.task if not isinstance(func, SwfTask): if isinstance(func, base_task.ActivityTask): func = ActivityTask.from_generic_task(func) elif isinstance(func, base_task.WorkflowTask): func = WorkflowTask.from_generic_task(func) elif isinstance(func, base_task.SignalTask): func = SignalTask.from_generic_task(func, self._workflow_id, self._run_id, None, None) elif isinstance(func, base_task.MarkerTask): func = MarkerTask.from_generic_task(func) try: # do not use directly "Submittable" here because we want to catch if # we don't have an instance from a class known to work under simpleflow.swf if isinstance( func, (ActivityTask, WorkflowTask, SignalTask, MarkerTask)): # no need to wrap it, already wrapped in the correct format a_task = func elif isinstance(func, Activity): a_task = ActivityTask(func, *args, **kwargs) elif issubclass_(func, Workflow): a_task = WorkflowTask(self, func, *args, **kwargs) elif isinstance(func, WaitForSignal): future = self.get_future_from_signal(func.signal_name) logger.debug( 'submitted WaitForSignalTask({}): future={}'.format( func.signal_name, future)) return future elif isinstance(func, Submittable): raise TypeError( 'invalid type Submittable {} for {} (you probably wanted a simpleflow.swf.task.*Task)' .format(type(func), func)) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) except exceptions.ExecutionBlocked: return futures.Future() # extract priority now that we have a *Task self.current_priority = self._compute_priority(priority_set_on_submit, a_task) # finally resume task return self.resume(a_task, *a_task.args, **a_task.kwargs)
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_execution_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) try: future._result = task.execute() state = 'completed' except Exception as err: future._exception = err logger.info('rescuing exception: {}'.format(err)) if isinstance(func, Activity) and func.raises_on_failure: message = err.args[0] if err.args else '' raise exceptions.TaskFailed(func.name, message) state = 'failed' finally: future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={'args': args, 'kwargs': kwargs}, result=future._result) return future
def _get_future_from_activity_event(self, event): """Maps an activity event to a Future with the corresponding state. :param event: workflow event. :type event: swf.event.Event. """ future = futures.Future() # state is PENDING. state = event['state'] if state == 'scheduled': future._state = futures.PENDING elif state == 'schedule_failed': if event['cause'] == 'ACTIVITY_TYPE_DOES_NOT_EXIST': activity_type = swf.models.ActivityType( self.domain, name=event['activity_type']['name'], version=event['activity_type']['version']) logger.info('Creating activity type {} in domain {}'.format( activity_type.name, self.domain.name)) try: activity_type.save() except swf.exceptions.AlreadyExistsError: logger.info( 'Activity type {} in domain {} already exists'.format( activity_type.name, self.domain.name)) return None logger.info('failed to schedule {}: {}'.format( event['activity_type']['name'], event['cause'], )) return None elif state == 'started': future._state = futures.RUNNING elif state == 'completed': future._state = futures.FINISHED result = event['result'] future._result = json.loads(result) if result else None elif state == 'canceled': future._state = futures.CANCELLED elif state == 'failed': future._state = futures.FINISHED future._exception = exceptions.TaskFailed( name=event['id'], reason=event['reason'], details=event.get('details'), ) elif state == 'timed_out': future._state = futures.FINISHED future._exception = exceptions.TimeoutError( event['timeout_type'], event['timeout_value']) return future
def _get_future_from_activity_event(self, event): """Maps an activity event to a Future with the corresponding state. :param event: activity event :type event: dict[str, Any] :rtype: futures.Future """ future = futures.Future() # state is PENDING. state = event['state'] if state == 'scheduled': pass elif state == 'schedule_failed': if event['cause'] == 'ACTIVITY_TYPE_DOES_NOT_EXIST': activity_type = swf.models.ActivityType( self.domain, name=event['activity_type']['name'], version=event['activity_type']['version']) logger.info('creating activity type {} in domain {}'.format( activity_type.name, self.domain.name)) try: activity_type.save() except swf.exceptions.AlreadyExistsError: logger.info( 'oops: Activity type {} in domain {} already exists, creation failed, continuing...'.format( activity_type.name, self.domain.name)) return None logger.info('failed to schedule {}: {}'.format( event['activity_type']['name'], event['cause'], )) return None elif state == 'started': future.set_running() elif state == 'completed': result = event['result'] future.set_finished(json_loads_or_raw(result)) elif state == 'canceled': future.set_cancelled() elif state == 'failed': exception = exceptions.TaskFailed( name=event['id'], reason=event['reason'], details=event.get('details')) future.set_exception(exception) elif state == 'timed_out': exception = exceptions.TimeoutError( event['timeout_type'], event['timeout_value']) future.set_exception(exception) return future
def _get_future_from_child_workflow_event(event): """Maps a child workflow event to a Future with the corresponding state. """ future = futures.Future() state = event['state'] if state == 'start_initiated': future._state = futures.PENDING elif state == 'started': future._state = futures.RUNNING elif state == 'completed': future._state = futures.FINISHED future._result = json.loads(event['result']) return future
def get_future_from_signal_event(self, a_task, event): """Maps a signal event to a Future with the corresponding state. :param a_task: currently unused :type a_task: Optional[SignalTask] :param event: signal event :type event: dict[str, Any] """ future = futures.Future() if not event: return future state = event['state'] if state == 'signaled': future.set_finished(event['input']) return future
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() task = ActivityTask(func, *args, **kwargs) try: future._result = task.execute() except Exception as err: future._exception = err if func.raises_on_failure: raise exceptions.TaskFailed(func.name, err.message) finally: future._state = futures.FINISHED return future
def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. """ try: if isinstance(func, Activity): a_task = ActivityTask(func, *args, **kwargs) elif issubclass_(func, Workflow): a_task = WorkflowTask(func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) except exceptions.ExecutionBlocked: return futures.Future() return self.resume(a_task, *a_task.args, **a_task.kwargs)
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) args = [executor.get_actual_value(arg) for arg in args] kwargs = { key: executor.get_actual_value(val) for key, val in kwargs.iteritems() } future = futures.Future() try: future._result = func._callable(*args, **kwargs) except Exception as err: future._exception = err raise finally: future._state = futures.FINISHED return future
def _get_future_from_marker_event(self, a_task, event): """Maps a marker event to a Future with the corresponding state. :param a_task: currently unused :type a_task: :param event: marker event :type event: dict[str, Any] :rtype: futures.Future """ future = futures.Future() if not event: return future state = event['state'] if state == 'recorded': future.set_finished(event['details']) elif state == 'failed': future.set_exception(exceptions.TaskFailed( name=event['name'], reason=event['cause'], )) return future
def resume(self, a_task, *args, **kwargs): """Resume the execution of a task. Called by `submit`. If the task was scheduled, returns a future that wraps its state, otherwise schedules it. If in repair mode, we may fake the task to repair from the previous history. :param a_task: :type a_task: ActivityTask | WorkflowTask | SignalTask :param args: :param args: list :type kwargs: :type kwargs: dict :rtype: futures.Future :raise: exceptions.ExecutionBlocked if open activities limit reached """ if not a_task.id: # Can be already set (WorkflowTask) a_task.id = self._make_task_id(a_task, *args, **kwargs) event = self.find_event(a_task, self._history) logger.debug('executor: resume {}, event={}'.format(a_task, event)) future = None # in repair mode, check if we absolutely want to re-execute this task force_execution = (self.force_activities and self.force_activities.search(a_task.id)) # try to fill in the blanks with the workflow we're trying to repair if any # TODO: maybe only do that for idempotent tasks?? (not enough information to decide?) if not event and self.repair_with and not force_execution: # try to find a former event matching this task former_event = self.find_event(a_task, self.repair_with) # ... but only keep the event if the task was successful if former_event and former_event['state'] == 'completed': logger.info('faking task completed successfully in previous ' 'workflow: {}'.format(former_event['id'])) json_hash = hashlib.md5( json_dumps(former_event).encode('utf-8')).hexdigest() fake_task_list = "FAKE-" + json_hash # schedule task on a fake task list self.schedule_task(a_task, task_list=fake_task_list) future = futures.Future() # start a dedicated process to handle the fake activity run_fake_task_worker(self.domain.name, fake_task_list, former_event) # back to normal execution flow if event: ttf = self.EVENT_TYPE_TO_FUTURE.get(event['type']) if ttf: future = ttf(self, a_task, event) if event['type'] == 'activity': if future and future.state in (futures.PENDING, futures.RUNNING): self._open_activity_count += 1 if not future: self.schedule_task(a_task, task_list=self.task_list) future = futures.Future() # return a pending future. if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT: logger.warning('limit of {} open activities reached'.format( constants.MAX_OPEN_ACTIVITY_COUNT)) raise exceptions.ExecutionBlocked return future
def submit(self, func, *args, **kwargs): logger.info("executing task {}(args={}, kwargs={})".format(func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( "wait_signal({}) before signal was sent: unsupported by the local executor".format( signal_name ) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append( Marker(func.name, func.details) ) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, "activity", None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError("invalid type {} for {}".format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, "post_execute"): task.post_execute() state = "completed" except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) task_failed = exceptions.TaskFailed( name=getattr(task, "name", "unknown"), reason=format_exc(exc_value), details=json_dumps( { "error": exc_type.__name__, "error_type": format_exc_type(exc_type), "message": str(exc_value), "traceback": tb, }, default=repr, ), ) future.set_exception(task_failed) logger.exception("rescuing exception: {}".format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, "raises_on_failure", None ): raise task_failed state = "failed" finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={"args": args, "kwargs": kwargs}, result=future.result, ) return future
def submit(self, func, *args, **kwargs): if hasattr(func, "activity") and func.activity == MyTask: f = futures.Future() f.set_running() return f return super(CustomExecutor, self).submit(func, *args, **kwargs)
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor' .format(signal_name)) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, 'post_execute'): task.post_execute() state = 'completed' except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() future._exception = exc_value logger.exception('rescuing exception: {}'.format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, 'raises_on_failure', None): tb = traceback.format_tb(exc_traceback) message = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr) raise exceptions.TaskFailed( func.name, message, details, ) state = 'failed' finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task(func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={ 'args': args, 'kwargs': kwargs }, result=future.result) return future
def submit(self, func, *args, **kwargs): if func == running_task: f = futures.Future() f._state = futures.RUNNING return f return super(CustomExecutor, self).submit(func, *args, **kwargs)