def submit(self, activity, *args, **kwargs): """ Submit a function for asynchronous execution. :param activity: callable registered as an task. :type activity: Activity | task.ActivityTask | task.WorkflowTask | canvas.Group | canvas.Chain | Workflow :param args: arguments passed to the task. :type args: Sequence. :param kwargs: keyword-arguments passed to the task. :type kwargs: Mapping (dict). :returns: :rtype: Future. """ # If the activity is a child workflow, call directly # the executor if issubclass_(activity, Workflow): return self._executor.submit(activity, *args, **kwargs) elif isinstance(activity, (task.ActivityTask, task.WorkflowTask)): return self._executor.submit(activity.activity, *activity.args, **activity.kwargs) elif isinstance(activity, Activity): return self._executor.submit(activity, *args, **kwargs) elif isinstance(activity, canvas.Group): return activity.submit(self._executor) else: raise TypeError('Bad type for {} activity ({})'.format( activity, type(activity)))
def append(self, submittable, *args, **kwargs): from simpleflow import Workflow if isinstance(submittable, (Submittable, SubmittableContainer)): if args or kwargs: raise ValueError( "args, kwargs not supported for Submittable or SubmittableContainer" ) if isinstance(submittable, WorkflowTask): self.workflow_tasks.append(submittable) elif isinstance(submittable, Activity): submittable = ActivityTask(submittable, *args, **kwargs) elif issubclass_(submittable, Workflow): # We can't set the executor yet, so use None and remember it. submittable = WorkflowTask(None, submittable, *args, **kwargs) self.workflow_tasks.append(submittable) else: raise ValueError( "{} should be a Submittable, Group, or Activity".format( submittable)) if self.raises_on_failure is not None: submittable.propagate_attribute("raises_on_failure", self.raises_on_failure) self.activities.append(submittable)
def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. :type func: simpleflow.base.Submittable | Activity | Workflow """ # NB: we don't set self.current_priority here directly, because we need # to extract it from the underlying Activity() if it's not passed to # self.submit() ; we DO need to pop the "__priority" kwarg though, so it # doesn't pollute the rest of the code. priority_set_on_submit = kwargs.pop("__priority", PRIORITY_NOT_SET) # casts simpleflow.task.*Task to their equivalent in simpleflow.swf.task if not isinstance(func, SwfTask): if isinstance(func, base_task.ActivityTask): func = ActivityTask.from_generic_task(func) elif isinstance(func, base_task.WorkflowTask): func = WorkflowTask.from_generic_task(func) elif isinstance(func, base_task.SignalTask): func = SignalTask.from_generic_task(func, self._workflow_id, self._run_id, None, None) elif isinstance(func, base_task.MarkerTask): func = MarkerTask.from_generic_task(func) try: # do not use directly "Submittable" here because we want to catch if # we don't have an instance from a class known to work under simpleflow.swf if isinstance( func, (ActivityTask, WorkflowTask, SignalTask, MarkerTask)): # no need to wrap it, already wrapped in the correct format a_task = func elif isinstance(func, Activity): a_task = ActivityTask(func, *args, **kwargs) elif issubclass_(func, Workflow): a_task = WorkflowTask(self, func, *args, **kwargs) elif isinstance(func, WaitForSignal): future = self.get_future_from_signal(func.signal_name) logger.debug( 'submitted WaitForSignalTask({}): future={}'.format( func.signal_name, future)) return future elif isinstance(func, Submittable): raise TypeError( 'invalid type Submittable {} for {} (you probably wanted a simpleflow.swf.task.*Task)' .format(type(func), func)) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) except exceptions.ExecutionBlocked: return futures.Future() # extract priority now that we have a *Task self.current_priority = self._compute_priority(priority_set_on_submit, a_task) # finally resume task return self.resume(a_task, *a_task.args, **a_task.kwargs)
def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. """ try: if isinstance(func, Activity): a_task = ActivityTask(func, *args, **kwargs) elif issubclass_(func, Workflow): a_task = WorkflowTask(func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) except exceptions.ExecutionBlocked: return futures.Future() return self.resume(a_task, *a_task.args, **a_task.kwargs)
def append(self, submittable, *args, **kwargs): from simpleflow import Workflow if isinstance(submittable, (Submittable, SubmittableContainer)): if args or kwargs: raise ValueError('args, kwargs not supported for Submittable or SubmittableContainer') if isinstance(submittable, WorkflowTask): self.workflow_tasks.append(submittable) elif isinstance(submittable, Activity): submittable = ActivityTask(submittable, *args, **kwargs) elif issubclass_(submittable, Workflow): # We can't set the executor yet, so use None and remember it. submittable = WorkflowTask(None, submittable, *args, **kwargs) self.workflow_tasks.append(submittable) else: raise ValueError('{} should be a Submittable, Group, or Activity'.format(submittable)) if self.raises_on_failure is not None: submittable.propagate_attribute('raises_on_failure', self.raises_on_failure) self.activities.append(submittable)
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor' .format(signal_name)) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, 'post_execute'): task.post_execute() state = 'completed' except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() future._exception = exc_value logger.exception('rescuing exception: {}'.format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, 'raises_on_failure', None): tb = traceback.format_tb(exc_traceback) message = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr) raise exceptions.TaskFailed( func.name, message, details, ) state = 'failed' finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task(func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={ 'args': args, 'kwargs': kwargs }, result=future.result) return future
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, 'post_execute'): task.post_execute() state = 'completed' except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() future._exception = exc_value logger.exception('rescuing exception: {}'.format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(func, 'raises_on_failure', None): tb = traceback.format_tb(exc_traceback) message = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr ) raise exceptions.TaskFailed( func.name, message, details, ) state = 'failed' finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={'args': args, 'kwargs': kwargs}, result=future.result) return future
def submit(self, func, *args, **kwargs): logger.info("executing task {}(args={}, kwargs={})".format(func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( "wait_signal({}) before signal was sent: unsupported by the local executor".format( signal_name ) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append( Marker(func.name, func.details) ) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, "activity", None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError("invalid type {} for {}".format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, "post_execute"): task.post_execute() state = "completed" except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) task_failed = exceptions.TaskFailed( name=getattr(task, "name", "unknown"), reason=format_exc(exc_value), details=json_dumps( { "error": exc_type.__name__, "error_type": format_exc_type(exc_type), "message": str(exc_value), "traceback": tb, }, default=repr, ), ) future.set_exception(task_failed) logger.exception("rescuing exception: {}".format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, "raises_on_failure", None ): raise task_failed state = "failed" finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={"args": args, "kwargs": kwargs}, result=future.result, ) return future