def submit(self, func, *args, **kwargs):
"""Register a function and its arguments for asynchronous execution.
``*args`` and ``**kwargs`` must be serializable in JSON.
:type func: simpleflow.base.Submittable | Activity | Workflow
"""
# NB: we don't set self.current_priority here directly, because we need
# to extract it from the underlying Activity() if it's not passed to
# self.submit() ; we DO need to pop the "__priority" kwarg though, so it
# doesn't pollute the rest of the code.
priority_set_on_submit = kwargs.pop("__priority", PRIORITY_NOT_SET)
# casts simpleflow.task.*Task to their equivalent in simpleflow.swf.task
if not isinstance(func, SwfTask):
if isinstance(func, base_task.ActivityTask):
func = ActivityTask.from_generic_task(func)
elif isinstance(func, base_task.WorkflowTask):
func = WorkflowTask.from_generic_task(func)
elif isinstance(func, base_task.SignalTask):
func = SignalTask.from_generic_task(func, self._workflow_id,
self._run_id, None, None)
elif isinstance(func, base_task.MarkerTask):
func = MarkerTask.from_generic_task(func)
try:
# do not use directly "Submittable" here because we want to catch if
# we don't have an instance from a class known to work under simpleflow.swf
if isinstance(
func,
(ActivityTask, WorkflowTask, SignalTask, MarkerTask)):
# no need to wrap it, already wrapped in the correct format
a_task = func
elif isinstance(func, Activity):
a_task = ActivityTask(func, *args, **kwargs)
elif issubclass_(func, Workflow):
a_task = WorkflowTask(self, func, *args, **kwargs)
elif isinstance(func, WaitForSignal):
future = self.get_future_from_signal(func.signal_name)
logger.debug(
'submitted WaitForSignalTask({}): future={}'.format(
func.signal_name, future))
return future
elif isinstance(func, Submittable):
raise TypeError(
'invalid type Submittable {} for {} (you probably wanted a simpleflow.swf.task.*Task)'
.format(type(func), func))
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
except exceptions.ExecutionBlocked:
return futures.Future()
# extract priority now that we have a *Task
self.current_priority = self._compute_priority(priority_set_on_submit,
a_task)
# finally resume task
return self.resume(a_task, *a_task.args, **a_task.kwargs)
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id,
activity_id):
# handle params
if not activity_id and not scheduled_id:
logger.error("Please supply --scheduled-id or --activity-id.")
sys.exit(1)
input_override = None
if input:
input_override = format.decode(input)
# find workflow execution
try:
wfe = helpers.get_workflow_execution(domain, workflow_id, run_id)
except (swf.exceptions.DoesNotExistError, IndexError):
logger.error("Couldn't find execution, exiting.")
sys.exit(1)
logger.info("Found execution: workflowId={} runId={}".format(
wfe.workflow_id, wfe.run_id))
# now rerun the specified activity
history = History(wfe.history())
history.parse()
task, args, kwargs, meta, params = helpers.find_activity(
history,
scheduled_id=scheduled_id,
activity_id=activity_id,
input=input_override,
)
kwargs["context"].update({
"workflow_id": wfe.workflow_id,
"run_id": wfe.run_id,
})
logger.debug("Found activity. Last execution:")
for line in json_dumps(params, pretty=True).split("\n"):
logger.debug(line)
if input_override:
logger.info("NB: input will be overriden with the passed one!")
logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format(
task, args, kwargs, meta))
# download binaries if needed
download_binaries(meta.get("binaries", {}))
# execute the activity task with the correct arguments
instance = ActivityTask(task, *args, **kwargs)
result = instance.execute()
if hasattr(instance, "post_execute"):
instance.post_execute()
logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def process(self, poller, token, task):
"""
:param poller:
:type poller: ActivityPoller
:param token:
:type token: str
:param task:
:type task: swf.models.ActivityTask
"""
logger.debug('ActivityWorker.process() pid={}'.format(os.getpid()))
activity = self.dispatch(task)
input = json.loads(task.input)
args = input.get('args', ())
kwargs = input.get('kwargs', {})
context = sanitize_activity_context(task.context)
try:
result = ActivityTask(activity, *args, context=context,
**kwargs).execute()
except Exception as err:
logger.exception("process error: {}".format(str(err)))
tb = traceback.format_exc()
return poller.fail(token, task, reason=str(err), details=tb)
try:
poller._complete(token, json_dumps(result))
except Exception as err:
logger.exception("complete error")
reason = 'cannot complete task {}: {}'.format(
task.activity_id,
err,
)
poller.fail(token, task, reason)
def activity_rerun(domain,
workflow_id,
run_id,
input,
scheduled_id,
activity_id):
# handle params
if not activity_id and not scheduled_id:
logger.error("Please supply --scheduled-id or --activity-id.")
sys.exit(1)
input_override = None
if input:
input_override = format.decode(input)
# find workflow execution
try:
wfe = helpers.get_workflow_execution(domain, workflow_id, run_id)
except (swf.exceptions.DoesNotExistError, IndexError):
logger.error("Couldn't find execution, exiting.")
sys.exit(1)
logger.info("Found execution: workflowId={} runId={}".format(wfe.workflow_id, wfe.run_id))
# now rerun the specified activity
history = History(wfe.history())
history.parse()
task, args, kwargs, meta, params = helpers.find_activity(
history, scheduled_id=scheduled_id, activity_id=activity_id, input=input_override,
)
logger.debug("Found activity. Last execution:")
for line in json_dumps(params, pretty=True).split("\n"):
logger.debug(line)
if input_override:
logger.info("NB: input will be overriden with the passed one!")
logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format(task, args, kwargs, meta))
# download binaries if needed
download_binaries(meta.get("binaries", {}))
# execute the activity task with the correct arguments
instance = ActivityTask(task, *args, **kwargs)
result = instance.execute()
if hasattr(instance, 'post_execute'):
instance.post_execute()
logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def process(self, poller, token, task):
"""
:param poller:
:type poller: ActivityPoller
:param token:
:type token: str
:param task:
:type task: swf.models.ActivityTask
"""
logger.debug('ActivityWorker.process() pid={}'.format(os.getpid()))
try:
activity = self.dispatch(task)
input = format.decode(task.input)
args = input.get('args', ())
kwargs = input.get('kwargs', {})
context = sanitize_activity_context(task.context)
context['domain_name'] = poller.domain.name
if input.get('meta', {}).get('binaries'):
download_binaries(input['meta']['binaries'])
result = ActivityTask(activity, *args, context=context, **kwargs).execute()
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
logger.exception("process error: {}".format(str(exc_value)))
if isinstance(exc_value, ExecutionError) and len(exc_value.args):
details = exc_value.args[0]
reason = format_exc(exc_value) # FIXME json.loads and rebuild?
else:
tb = traceback.format_tb(exc_traceback)
reason = format_exc(exc_value)
details = json_dumps(
{
'error': exc_type.__name__,
'message': str(exc_value),
'traceback': tb,
},
default=repr
)
return poller.fail_with_retry(
token,
task,
reason=reason,
details=details
)
try:
logger.info('completing activity')
poller.complete_with_retry(token, result)
except Exception as err:
logger.exception("complete error")
reason = 'cannot complete task {}: {} {}'.format(
task.activity_id,
err.__class__.__name__,
err,
)
poller.fail_with_retry(token, task, reason)
def submit(self, func, *args, **kwargs):
"""Register a function and its arguments for asynchronous execution.
``*args`` and ``**kwargs`` must be serializable in JSON.
"""
try:
if isinstance(func, Activity):
a_task = ActivityTask(func, *args, **kwargs)
elif issubclass_(func, Workflow):
a_task = WorkflowTask(func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
except exceptions.ExecutionBlocked:
return futures.Future()
return self.resume(a_task, *a_task.args, **a_task.kwargs)
def process(self, poller, token, task):
logger.debug('ActivityWorker.process() pid={}'.format(os.getpid()))
activity = self.dispatch(task)
input = json.loads(task.input)
args = input.get('args', ())
kwargs = input.get('kwargs', {})
try:
result = ActivityTask(activity, *args, **kwargs).execute()
except Exception as err:
logger.exception("process error")
tb = traceback.format_exc()
return poller.fail(token, task, reason=str(err), details=tb)
try:
poller._complete(token, json.dumps(result))
except Exception as err:
logger.exception("complete error")
reason = 'cannot complete task {}: {}'.format(
task.activity_id,
err,
)
poller.fail(token, task, reason)
def test_task_attaches_context_to_object_instances():
ctx = {'foo': 'bar'}
expect(ActivityTask(ShowContextCls, context=ctx).execute()).to.equal(ctx)
expect(ShowContextCls.context).to.be.none
def test_task_attaches_context_to_functions():
ctx = {'foo': 'bar'}
expect(ActivityTask(show_context_func, context=ctx).execute()).to.equal(ctx)
expect(show_context_func.context).to.equal(ctx)
def test_task_has_an_empty_context_by_default():
expect(ActivityTask(show_context_func).execute()).to.be.none
expect(ActivityTask(ShowContextCls).execute()).to.be.none
def make_activity_task(self, func, *args, **kwargs):
return ActivityTask(func, *args, **kwargs)
