def resume(self, a_task, *args, **kwargs):
"""Resume the execution of a task.
If the task was scheduled, returns a future that wraps its state,
otherwise schedules it.
"""
a_task.id = self._make_task_id(a_task, *args, **kwargs)
event = self.find_event(a_task, self._history)
future = None
# check if we absolutely want to execute this task in repair mode
force_execution = self.force_activities and \
self.force_activities.search(a_task.id)
# try to fill in the blanks with the workflow we're trying to repair if any
# TODO: maybe only do that for idempotent tasks??
if not event and self.repair_with and not force_execution:
# try to find a former event matching this task
former_event = self.find_event(a_task, self.repair_with)
# ... but only keep the event if the task was successful
if former_event and former_event['state'] == 'completed':
logger.info(
'faking task completed successfully in previous ' \
'workflow: {}'.format(former_event['id'])
)
json_hash = hashlib.md5(json_dumps(former_event)).hexdigest()
fake_task_list = "FAKE-" + json_hash
# schedule task on a fake task list
self.schedule_task(a_task, task_list=fake_task_list)
future = futures.Future()
# start a dedicated process to handle the fake activity
run_fake_task_worker(self.domain.name, fake_task_list,
former_event)
# back to normal execution flow
if event:
if event['type'] == 'activity':
future = self.resume_activity(a_task, event)
if future and future._state in (futures.PENDING,
futures.RUNNING):
self._open_activity_count += 1
elif event['type'] == 'child_workflow':
future = self.resume_child_workflow(a_task, event)
if not future:
self.schedule_task(a_task, task_list=self.task_list)
future = futures.Future() # return a pending future.
if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT:
logger.warning('limit of {} open activities reached'.format(
constants.MAX_OPEN_ACTIVITY_COUNT))
raise exceptions.ExecutionBlocked
return future
def submit(self, func, *args, **kwargs):
"""Register a function and its arguments for asynchronous execution.
``*args`` and ``**kwargs`` must be serializable in JSON.
"""
errors = []
arguments = []
keyword_arguments = {}
result = None
try:
for arg in args:
if isinstance(arg, futures.Future) and arg.failed:
exc = arg._exception
if isinstance(exc, exceptions.MultipleExceptions):
errors.extend(exc.exceptions)
else:
errors.append(exc)
else:
arguments.append(executor.get_actual_value(arg))
for key, val in kwargs.iteritems():
if isinstance(val, futures.Future) and val.failed:
exc = val._exception
if isinstance(exc, exceptions.MultipleExceptions):
errors.extend(exc.exceptions)
else:
errors.append(val._exception)
else:
keyword_arguments[key] = executor.get_actual_value(val)
except exceptions.ExecutionBlocked:
result = futures.Future()
finally:
if errors:
result = futures.Future()
result._state = futures.FINISHED
result._exception = exceptions.MultipleExceptions(
'futures failed',
errors,
)
if result is not None:
return result
try:
if isinstance(func, Activity):
make_task = self.make_activity_task
elif issubclass(func, Workflow):
make_task = self.make_workflow_task
else:
raise TypeError
task = make_task(func, *arguments, **keyword_arguments)
except TypeError:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
return self.resume(task, *arguments, **keyword_arguments)
def _get_future_from_timer_event(self, a_task, event):
"""
Maps a timer event to a Future with the corresponding state.
:param a_task: Timer task; unused.
:type a_task: TimerTask
:param event: Timer event
:type event: dict[str, Any]
:return:
:rtype: futures.Future
"""
future = futures.Future()
if not event:
return future
state = event['state']
if state == 'started':
future.set_running()
elif state == 'fired':
future.set_finished(None)
elif state == 'canceled':
future.set_cancelled()
elif state in ('start_failed', 'cancel_failed'):
future.set_exception(exceptions.TaskFailed(
name=event['timer_id'],
reason=event['cause'],
))
return future
def get_future_from_external_workflow_event(self, a_task, event):
"""Maps an external workflow event to a Future with the corresponding
state.
:param a_task: currently unused
:type a_task:
:param event: external workflow event
:type event: dict[str, Any]
:rtype: futures.Future
"""
future = futures.Future()
if not event:
return future
state = event['state']
if state == 'signal_execution_initiated':
# Don't re-initiate signal sending
future.set_running()
elif state == 'execution_signaled':
future.set_finished(event['input'])
elif state == 'signal_execution_failed':
future.set_exception(
exceptions.TaskFailed(
name=event['name'],
reason=event['cause'],
))
return future
def resume(self, task, *args, **kwargs):
"""Resume the execution of a task.
If the task was scheduled, returns a future that wraps its state,
otherwise schedules it.
"""
task.id = self._make_task_id(task)
event = self.find_event(task, self._history)
future = None
if event:
if event['type'] == 'activity':
future = self.resume_activity(task, event)
if future and future._state in (futures.PENDING, futures.RUNNING):
self._open_activity_count += 1
elif event['type'] == 'child_workflow':
future = self.resume_child_workflow(task, event)
if not future:
self.schedule_task(task, task_list=self.task_list)
future = futures.Future() # return a pending future.
if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT:
logger.warning('limit of {} open activities reached'.format(
constants.MAX_OPEN_ACTIVITY_COUNT))
raise exceptions.ExecutionBlocked
return future
def _get_future_from_child_workflow_event(self, event):
"""Maps a child workflow event to a Future with the corresponding
state.
:param event: child workflow event
:type event: dict[str, Any]
"""
future = futures.Future()
state = event['state']
if state == 'start_initiated':
pass # future._state = futures.PENDING
elif state == 'start_failed':
if event['cause'] == 'WORKFLOW_TYPE_DOES_NOT_EXIST':
workflow_type = swf.models.WorkflowType(
self.domain,
name=event['name'],
version=event['version'],
)
logger.info('Creating workflow type {} in domain {}'.format(
workflow_type.name,
self.domain.name,
))
try:
workflow_type.save()
except swf.exceptions.AlreadyExistsError:
# Could have be created by a concurrent workflow execution.
pass
return None
future.set_exception(
exceptions.TaskFailed(
name=event['id'],
reason=event['cause'],
details=event.get('details'),
))
elif state == 'started':
future.set_running()
elif state == 'completed':
future.set_finished(json_loads_or_raw(event['result']))
elif state == 'failed':
future.set_exception(
exceptions.TaskFailed(
name=event['id'],
reason=event['reason'],
details=event.get('details'),
))
elif state == 'timed_out':
future.set_exception(
exceptions.TimeoutError(
event['timeout_type'],
None,
))
elif state == 'canceled':
future.set_exception(
exceptions.TaskCanceled(event.get('details'), ))
elif state == 'terminated':
future.set_exception(exceptions.TaskTerminated())
return future
def submit(self, func, *args, **kwargs):
"""Register a function and its arguments for asynchronous execution.
``*args`` and ``**kwargs`` must be serializable in JSON.
:type func: simpleflow.base.Submittable | Activity | Workflow
"""
# NB: we don't set self.current_priority here directly, because we need
# to extract it from the underlying Activity() if it's not passed to
# self.submit() ; we DO need to pop the "__priority" kwarg though, so it
# doesn't pollute the rest of the code.
priority_set_on_submit = kwargs.pop("__priority", PRIORITY_NOT_SET)
# casts simpleflow.task.*Task to their equivalent in simpleflow.swf.task
if not isinstance(func, SwfTask):
if isinstance(func, base_task.ActivityTask):
func = ActivityTask.from_generic_task(func)
elif isinstance(func, base_task.WorkflowTask):
func = WorkflowTask.from_generic_task(func)
elif isinstance(func, base_task.SignalTask):
func = SignalTask.from_generic_task(func, self._workflow_id,
self._run_id, None, None)
elif isinstance(func, base_task.MarkerTask):
func = MarkerTask.from_generic_task(func)
try:
# do not use directly "Submittable" here because we want to catch if
# we don't have an instance from a class known to work under simpleflow.swf
if isinstance(
func,
(ActivityTask, WorkflowTask, SignalTask, MarkerTask)):
# no need to wrap it, already wrapped in the correct format
a_task = func
elif isinstance(func, Activity):
a_task = ActivityTask(func, *args, **kwargs)
elif issubclass_(func, Workflow):
a_task = WorkflowTask(self, func, *args, **kwargs)
elif isinstance(func, WaitForSignal):
future = self.get_future_from_signal(func.signal_name)
logger.debug(
'submitted WaitForSignalTask({}): future={}'.format(
func.signal_name, future))
return future
elif isinstance(func, Submittable):
raise TypeError(
'invalid type Submittable {} for {} (you probably wanted a simpleflow.swf.task.*Task)'
.format(type(func), func))
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
except exceptions.ExecutionBlocked:
return futures.Future()
# extract priority now that we have a *Task
self.current_priority = self._compute_priority(priority_set_on_submit,
a_task)
# finally resume task
return self.resume(a_task, *a_task.args, **a_task.kwargs)
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
context = self.get_execution_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name)
)
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name, []).append(Marker(func.name, func.details))
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, 'activity', None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
try:
future._result = task.execute()
state = 'completed'
except Exception as err:
future._exception = err
logger.info('rescuing exception: {}'.format(err))
if isinstance(func, Activity) and func.raises_on_failure:
message = err.args[0] if err.args else ''
raise exceptions.TaskFailed(func.name, message)
state = 'failed'
finally:
future._state = futures.FINISHED
if func:
self._history.add_activity_task(
func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={'args': args, 'kwargs': kwargs},
result=future._result)
return future
def _get_future_from_activity_event(self, event):
"""Maps an activity event to a Future with the corresponding state.
:param event: workflow event.
:type event: swf.event.Event.
"""
future = futures.Future() # state is PENDING.
state = event['state']
if state == 'scheduled':
future._state = futures.PENDING
elif state == 'schedule_failed':
if event['cause'] == 'ACTIVITY_TYPE_DOES_NOT_EXIST':
activity_type = swf.models.ActivityType(
self.domain,
name=event['activity_type']['name'],
version=event['activity_type']['version'])
logger.info('Creating activity type {} in domain {}'.format(
activity_type.name,
self.domain.name))
try:
activity_type.save()
except swf.exceptions.AlreadyExistsError:
logger.info(
'Activity type {} in domain {} already exists'.format(
activity_type.name,
self.domain.name))
return None
logger.info('failed to schedule {}: {}'.format(
event['activity_type']['name'],
event['cause'],
))
return None
elif state == 'started':
future._state = futures.RUNNING
elif state == 'completed':
future._state = futures.FINISHED
result = event['result']
future._result = json.loads(result) if result else None
elif state == 'canceled':
future._state = futures.CANCELLED
elif state == 'failed':
future._state = futures.FINISHED
future._exception = exceptions.TaskFailed(
name=event['id'],
reason=event['reason'],
details=event.get('details'),
)
elif state == 'timed_out':
future._state = futures.FINISHED
future._exception = exceptions.TimeoutError(
event['timeout_type'],
event['timeout_value'])
return future
def _get_future_from_activity_event(self, event):
"""Maps an activity event to a Future with the corresponding state.
:param event: activity event
:type event: dict[str, Any]
:rtype: futures.Future
"""
future = futures.Future() # state is PENDING.
state = event['state']
if state == 'scheduled':
pass
elif state == 'schedule_failed':
if event['cause'] == 'ACTIVITY_TYPE_DOES_NOT_EXIST':
activity_type = swf.models.ActivityType(
self.domain,
name=event['activity_type']['name'],
version=event['activity_type']['version'])
logger.info('creating activity type {} in domain {}'.format(
activity_type.name,
self.domain.name))
try:
activity_type.save()
except swf.exceptions.AlreadyExistsError:
logger.info(
'oops: Activity type {} in domain {} already exists, creation failed, continuing...'.format(
activity_type.name,
self.domain.name))
return None
logger.info('failed to schedule {}: {}'.format(
event['activity_type']['name'],
event['cause'],
))
return None
elif state == 'started':
future.set_running()
elif state == 'completed':
result = event['result']
future.set_finished(json_loads_or_raw(result))
elif state == 'canceled':
future.set_cancelled()
elif state == 'failed':
exception = exceptions.TaskFailed(
name=event['id'],
reason=event['reason'],
details=event.get('details'))
future.set_exception(exception)
elif state == 'timed_out':
exception = exceptions.TimeoutError(
event['timeout_type'],
event['timeout_value'])
future.set_exception(exception)
return future
def _get_future_from_child_workflow_event(event):
"""Maps a child workflow event to a Future with the corresponding
state.
"""
future = futures.Future()
state = event['state']
if state == 'start_initiated':
future._state = futures.PENDING
elif state == 'started':
future._state = futures.RUNNING
elif state == 'completed':
future._state = futures.FINISHED
future._result = json.loads(event['result'])
return future
def get_future_from_signal_event(self, a_task, event):
"""Maps a signal event to a Future with the corresponding
state.
:param a_task: currently unused
:type a_task: Optional[SignalTask]
:param event: signal event
:type event: dict[str, Any]
"""
future = futures.Future()
if not event:
return future
state = event['state']
if state == 'signaled':
future.set_finished(event['input'])
return future
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
task = ActivityTask(func, *args, **kwargs)
try:
future._result = task.execute()
except Exception as err:
future._exception = err
if func.raises_on_failure:
raise exceptions.TaskFailed(func.name, err.message)
finally:
future._state = futures.FINISHED
return future
def submit(self, func, *args, **kwargs):
"""Register a function and its arguments for asynchronous execution.
``*args`` and ``**kwargs`` must be serializable in JSON.
"""
try:
if isinstance(func, Activity):
a_task = ActivityTask(func, *args, **kwargs)
elif issubclass_(func, Workflow):
a_task = WorkflowTask(func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
except exceptions.ExecutionBlocked:
return futures.Future()
return self.resume(a_task, *a_task.args, **a_task.kwargs)
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
args = [executor.get_actual_value(arg) for arg in args]
kwargs = {
key: executor.get_actual_value(val)
for key, val in kwargs.iteritems()
}
future = futures.Future()
try:
future._result = func._callable(*args, **kwargs)
except Exception as err:
future._exception = err
raise
finally:
future._state = futures.FINISHED
return future
def _get_future_from_marker_event(self, a_task, event):
"""Maps a marker event to a Future with the corresponding
state.
:param a_task: currently unused
:type a_task:
:param event: marker event
:type event: dict[str, Any]
:rtype: futures.Future
"""
future = futures.Future()
if not event:
return future
state = event['state']
if state == 'recorded':
future.set_finished(event['details'])
elif state == 'failed':
future.set_exception(exceptions.TaskFailed(
name=event['name'],
reason=event['cause'],
))
return future
def resume(self, a_task, *args, **kwargs):
"""Resume the execution of a task.
Called by `submit`.
If the task was scheduled, returns a future that wraps its state,
otherwise schedules it.
If in repair mode, we may fake the task to repair from the previous history.
:param a_task:
:type a_task: ActivityTask | WorkflowTask | SignalTask
:param args:
:param args: list
:type kwargs:
:type kwargs: dict
:rtype: futures.Future
:raise: exceptions.ExecutionBlocked if open activities limit reached
"""
if not a_task.id: # Can be already set (WorkflowTask)
a_task.id = self._make_task_id(a_task, *args, **kwargs)
event = self.find_event(a_task, self._history)
logger.debug('executor: resume {}, event={}'.format(a_task, event))
future = None
# in repair mode, check if we absolutely want to re-execute this task
force_execution = (self.force_activities
and self.force_activities.search(a_task.id))
# try to fill in the blanks with the workflow we're trying to repair if any
# TODO: maybe only do that for idempotent tasks?? (not enough information to decide?)
if not event and self.repair_with and not force_execution:
# try to find a former event matching this task
former_event = self.find_event(a_task, self.repair_with)
# ... but only keep the event if the task was successful
if former_event and former_event['state'] == 'completed':
logger.info('faking task completed successfully in previous '
'workflow: {}'.format(former_event['id']))
json_hash = hashlib.md5(
json_dumps(former_event).encode('utf-8')).hexdigest()
fake_task_list = "FAKE-" + json_hash
# schedule task on a fake task list
self.schedule_task(a_task, task_list=fake_task_list)
future = futures.Future()
# start a dedicated process to handle the fake activity
run_fake_task_worker(self.domain.name, fake_task_list,
former_event)
# back to normal execution flow
if event:
ttf = self.EVENT_TYPE_TO_FUTURE.get(event['type'])
if ttf:
future = ttf(self, a_task, event)
if event['type'] == 'activity':
if future and future.state in (futures.PENDING,
futures.RUNNING):
self._open_activity_count += 1
if not future:
self.schedule_task(a_task, task_list=self.task_list)
future = futures.Future() # return a pending future.
if self._open_activity_count == constants.MAX_OPEN_ACTIVITY_COUNT:
logger.warning('limit of {} open activities reached'.format(
constants.MAX_OPEN_ACTIVITY_COUNT))
raise exceptions.ExecutionBlocked
return future
def submit(self, func, *args, **kwargs):
logger.info("executing task {}(args={}, kwargs={})".format(func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
"wait_signal({}) before signal was sent: unsupported by the local executor".format(
signal_name
)
)
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name, []).append(
Marker(func.name, func.details)
)
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, "activity", None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError("invalid type {} for {}".format(type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, "post_execute"):
task.post_execute()
state = "completed"
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.format_tb(exc_traceback)
task_failed = exceptions.TaskFailed(
name=getattr(task, "name", "unknown"),
reason=format_exc(exc_value),
details=json_dumps(
{
"error": exc_type.__name__,
"error_type": format_exc_type(exc_type),
"message": str(exc_value),
"traceback": tb,
},
default=repr,
),
)
future.set_exception(task_failed)
logger.exception("rescuing exception: {}".format(exc_value))
if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(
func, "raises_on_failure", None
):
raise task_failed
state = "failed"
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(
func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={"args": args, "kwargs": kwargs},
result=future.result,
)
return future
def submit(self, func, *args, **kwargs):
if hasattr(func, "activity") and func.activity == MyTask:
f = futures.Future()
f.set_running()
return f
return super(CustomExecutor, self).submit(func, *args, **kwargs)
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
'wait_signal({}) before signal was sent: unsupported by the local executor'
.format(signal_name))
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name,
[]).append(Marker(func.name,
func.details))
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, 'activity', None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, 'post_execute'):
task.post_execute()
state = 'completed'
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
future._exception = exc_value
logger.exception('rescuing exception: {}'.format(exc_value))
if (isinstance(func, Activity)
or issubclass_(func, Workflow)) and getattr(
func, 'raises_on_failure', None):
tb = traceback.format_tb(exc_traceback)
message = format_exc(exc_value)
details = json_dumps(
{
'error': exc_type.__name__,
'message': str(exc_value),
'traceback': tb,
},
default=repr)
raise exceptions.TaskFailed(
func.name,
message,
details,
)
state = 'failed'
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={
'args': args,
'kwargs': kwargs
},
result=future.result)
return future
def submit(self, func, *args, **kwargs):
if func == running_task:
f = futures.Future()
f._state = futures.RUNNING
return f
return super(CustomExecutor, self).submit(func, *args, **kwargs)
