def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. :type func: simpleflow.base.Submittable | Activity | Workflow """ # NB: we don't set self.current_priority here directly, because we need # to extract it from the underlying Activity() if it's not passed to # self.submit() ; we DO need to pop the "__priority" kwarg though, so it # doesn't pollute the rest of the code. priority_set_on_submit = kwargs.pop("__priority", PRIORITY_NOT_SET) # casts simpleflow.task.*Task to their equivalent in simpleflow.swf.task if not isinstance(func, SwfTask): if isinstance(func, base_task.ActivityTask): func = ActivityTask.from_generic_task(func) elif isinstance(func, base_task.WorkflowTask): func = WorkflowTask.from_generic_task(func) elif isinstance(func, base_task.SignalTask): func = SignalTask.from_generic_task(func, self._workflow_id, self._run_id, None, None) elif isinstance(func, base_task.MarkerTask): func = MarkerTask.from_generic_task(func) try: # do not use directly "Submittable" here because we want to catch if # we don't have an instance from a class known to work under simpleflow.swf if isinstance( func, (ActivityTask, WorkflowTask, SignalTask, MarkerTask)): # no need to wrap it, already wrapped in the correct format a_task = func elif isinstance(func, Activity): a_task = ActivityTask(func, *args, **kwargs) elif issubclass_(func, Workflow): a_task = WorkflowTask(self, func, *args, **kwargs) elif isinstance(func, WaitForSignal): future = self.get_future_from_signal(func.signal_name) logger.debug( 'submitted WaitForSignalTask({}): future={}'.format( func.signal_name, future)) return future elif isinstance(func, Submittable): raise TypeError( 'invalid type Submittable {} for {} (you probably wanted a simpleflow.swf.task.*Task)' .format(type(func), func)) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) except exceptions.ExecutionBlocked: return futures.Future() # extract priority now that we have a *Task self.current_priority = self._compute_priority(priority_set_on_submit, a_task) # finally resume task return self.resume(a_task, *a_task.args, **a_task.kwargs)
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id, activity_id): # handle params if not activity_id and not scheduled_id: logger.error("Please supply --scheduled-id or --activity-id.") sys.exit(1) input_override = None if input: input_override = format.decode(input) # find workflow execution try: wfe = helpers.get_workflow_execution(domain, workflow_id, run_id) except (swf.exceptions.DoesNotExistError, IndexError): logger.error("Couldn't find execution, exiting.") sys.exit(1) logger.info("Found execution: workflowId={} runId={}".format( wfe.workflow_id, wfe.run_id)) # now rerun the specified activity history = History(wfe.history()) history.parse() task, args, kwargs, meta, params = helpers.find_activity( history, scheduled_id=scheduled_id, activity_id=activity_id, input=input_override, ) kwargs["context"].update({ "workflow_id": wfe.workflow_id, "run_id": wfe.run_id, }) logger.debug("Found activity. Last execution:") for line in json_dumps(params, pretty=True).split("\n"): logger.debug(line) if input_override: logger.info("NB: input will be overriden with the passed one!") logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format( task, args, kwargs, meta)) # download binaries if needed download_binaries(meta.get("binaries", {})) # execute the activity task with the correct arguments instance = ActivityTask(task, *args, **kwargs) result = instance.execute() if hasattr(instance, "post_execute"): instance.post_execute() logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def process(self, poller, token, task): """ :param poller: :type poller: ActivityPoller :param token: :type token: str :param task: :type task: swf.models.ActivityTask """ logger.debug('ActivityWorker.process() pid={}'.format(os.getpid())) activity = self.dispatch(task) input = json.loads(task.input) args = input.get('args', ()) kwargs = input.get('kwargs', {}) context = sanitize_activity_context(task.context) try: result = ActivityTask(activity, *args, context=context, **kwargs).execute() except Exception as err: logger.exception("process error: {}".format(str(err))) tb = traceback.format_exc() return poller.fail(token, task, reason=str(err), details=tb) try: poller._complete(token, json_dumps(result)) except Exception as err: logger.exception("complete error") reason = 'cannot complete task {}: {}'.format( task.activity_id, err, ) poller.fail(token, task, reason)
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id, activity_id): # handle params if not activity_id and not scheduled_id: logger.error("Please supply --scheduled-id or --activity-id.") sys.exit(1) input_override = None if input: input_override = format.decode(input) # find workflow execution try: wfe = helpers.get_workflow_execution(domain, workflow_id, run_id) except (swf.exceptions.DoesNotExistError, IndexError): logger.error("Couldn't find execution, exiting.") sys.exit(1) logger.info("Found execution: workflowId={} runId={}".format(wfe.workflow_id, wfe.run_id)) # now rerun the specified activity history = History(wfe.history()) history.parse() task, args, kwargs, meta, params = helpers.find_activity( history, scheduled_id=scheduled_id, activity_id=activity_id, input=input_override, ) logger.debug("Found activity. Last execution:") for line in json_dumps(params, pretty=True).split("\n"): logger.debug(line) if input_override: logger.info("NB: input will be overriden with the passed one!") logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format(task, args, kwargs, meta)) # download binaries if needed download_binaries(meta.get("binaries", {})) # execute the activity task with the correct arguments instance = ActivityTask(task, *args, **kwargs) result = instance.execute() if hasattr(instance, 'post_execute'): instance.post_execute() logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def process(self, poller, token, task): """ :param poller: :type poller: ActivityPoller :param token: :type token: str :param task: :type task: swf.models.ActivityTask """ logger.debug('ActivityWorker.process() pid={}'.format(os.getpid())) try: activity = self.dispatch(task) input = format.decode(task.input) args = input.get('args', ()) kwargs = input.get('kwargs', {}) context = sanitize_activity_context(task.context) context['domain_name'] = poller.domain.name if input.get('meta', {}).get('binaries'): download_binaries(input['meta']['binaries']) result = ActivityTask(activity, *args, context=context, **kwargs).execute() except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() logger.exception("process error: {}".format(str(exc_value))) if isinstance(exc_value, ExecutionError) and len(exc_value.args): details = exc_value.args[0] reason = format_exc(exc_value) # FIXME json.loads and rebuild? else: tb = traceback.format_tb(exc_traceback) reason = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr ) return poller.fail_with_retry( token, task, reason=reason, details=details ) try: logger.info('completing activity') poller.complete_with_retry(token, result) except Exception as err: logger.exception("complete error") reason = 'cannot complete task {}: {} {}'.format( task.activity_id, err.__class__.__name__, err, ) poller.fail_with_retry(token, task, reason)
def submit(self, func, *args, **kwargs): """Register a function and its arguments for asynchronous execution. ``*args`` and ``**kwargs`` must be serializable in JSON. """ try: if isinstance(func, Activity): a_task = ActivityTask(func, *args, **kwargs) elif issubclass_(func, Workflow): a_task = WorkflowTask(func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) except exceptions.ExecutionBlocked: return futures.Future() return self.resume(a_task, *a_task.args, **a_task.kwargs)
def process(self, poller, token, task): logger.debug('ActivityWorker.process() pid={}'.format(os.getpid())) activity = self.dispatch(task) input = json.loads(task.input) args = input.get('args', ()) kwargs = input.get('kwargs', {}) try: result = ActivityTask(activity, *args, **kwargs).execute() except Exception as err: logger.exception("process error") tb = traceback.format_exc() return poller.fail(token, task, reason=str(err), details=tb) try: poller._complete(token, json.dumps(result)) except Exception as err: logger.exception("complete error") reason = 'cannot complete task {}: {}'.format( task.activity_id, err, ) poller.fail(token, task, reason)
def test_task_attaches_context_to_object_instances(): ctx = {'foo': 'bar'} expect(ActivityTask(ShowContextCls, context=ctx).execute()).to.equal(ctx) expect(ShowContextCls.context).to.be.none
def test_task_attaches_context_to_functions(): ctx = {'foo': 'bar'} expect(ActivityTask(show_context_func, context=ctx).execute()).to.equal(ctx) expect(show_context_func.context).to.equal(ctx)
def test_task_has_an_empty_context_by_default(): expect(ActivityTask(show_context_func).execute()).to.be.none expect(ActivityTask(ShowContextCls).execute()).to.be.none
def make_activity_task(self, func, *args, **kwargs): return ActivityTask(func, *args, **kwargs)