def process(self, poller, token, task):
"""
:param poller:
:type poller: ActivityPoller
:param token:
:type token: str
:param task:
:type task: swf.models.ActivityTask
"""
logger.debug('ActivityWorker.process() pid={}'.format(os.getpid()))
try:
activity = self.dispatch(task)
input = format.decode(task.input)
args = input.get('args', ())
kwargs = input.get('kwargs', {})
context = sanitize_activity_context(task.context)
context['domain_name'] = poller.domain.name
if input.get('meta', {}).get('binaries'):
download_binaries(input['meta']['binaries'])
result = ActivityTask(activity, *args, context=context, **kwargs).execute()
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
logger.exception("process error: {}".format(str(exc_value)))
if isinstance(exc_value, ExecutionError) and len(exc_value.args):
details = exc_value.args[0]
reason = format_exc(exc_value) # FIXME json.loads and rebuild?
else:
tb = traceback.format_tb(exc_traceback)
reason = format_exc(exc_value)
details = json_dumps(
{
'error': exc_type.__name__,
'message': str(exc_value),
'traceback': tb,
},
default=repr
)
return poller.fail_with_retry(
token,
task,
reason=reason,
details=details
)
try:
logger.info('completing activity')
poller.complete_with_retry(token, result)
except Exception as err:
logger.exception("complete error")
reason = 'cannot complete task {}: {} {}'.format(
task.activity_id,
err.__class__.__name__,
err,
)
poller.fail_with_retry(token, task, reason)
def process(self, response):
"""
Process a swf.actors.ActivityWorker poll response..
:param response:
:type response: swf.responses.Response
"""
token = response.task_token
task = response.activity_task
if self.process_mode == "kubernetes":
try:
spawn_kubernetes_job(self, response.raw_response)
except Exception as err:
logger.exception("spawn_kubernetes_job error")
reason = "cannot spawn kubernetes job for task {}: {} {}".format(
task.activity_id, err.__class__.__name__, err,
)
self.fail_with_retry(token, task, reason)
else:
spawn(self, token, task, self._heartbeat)
def process(self, response):
"""
Process a swf.actors.ActivityWorker poll response..
:param response:
:type response: swf.responses.Response
"""
token = response.task_token
task = response.activity_task
if self.process_mode == "kubernetes":
try:
spawn_kubernetes_job(self, response.raw_response)
except Exception as err:
logger.exception("spawn_kubernetes_job error")
reason = 'cannot spawn kubernetes job for task {}: {} {}'.format(
task.activity_id,
err.__class__.__name__,
err,
)
self.fail_with_retry(token, task, reason)
else:
spawn(self, token, task, self._heartbeat)
def complete_with_retry(self, token, response):
"""
Complete with retry.
:param token:
:type token: str
:param response: response: decision list, JSON result, ...
:type response: Any
:return:
:rtype:
"""
try:
complete = utils.retry.with_delay(
nb_times=self.nb_retries,
delay=utils.retry.exponential,
log_with=logger.exception,
except_on=swf.exceptions.DoesNotExistError,
)(self.complete) # Exponential backoff on errors.
complete(token, response)
except Exception as err:
# This is embarrassing because the decider cannot notify SWF of the
# task completion. As it will not try again, the task will
# timeout (start_to_complete).
logger.exception("cannot complete task: %s", str(err))
def complete_with_retry(self, token, response):
"""
Complete with retry.
:param token:
:type token: str
:param response: response: decision list, JSON result, ...
:type response: Any
:return:
:rtype:
"""
try:
complete = utils.retry.with_delay(
nb_times=self.nb_retries,
delay=utils.retry.exponential,
log_with=logger.exception,
except_on=swf.exceptions.DoesNotExistError,
)(self.complete) # Exponential backoff on errors.
complete(token, response)
except Exception as err:
# This is embarrassing because the decider cannot notify SWF of the
# task completion. As it will not try again, the task will
# timeout (start_to_complete).
logger.exception("cannot complete task: %s", str(err))
def decide(self, decision_response, task_list):
"""
Delegate the decision to the executor, loading it if needed.
:param decision_response: an object wrapping the PollForDecisionTask response.
:type decision_response: swf.responses.Response
:param task_list:
:type task_list: Optional[str]
:returns: the decisions.
:rtype: list[swf.models.decision.base.Decision]
"""
history = decision_response.history
workflow_name = history[0].workflow_type["name"]
workflow_executor = self._workflow_executors.get(workflow_name)
if not workflow_executor:
# Child workflow from another module
from . import helpers
workflow_executor = helpers.load_workflow_executor(
self._domain, workflow_name, task_list=task_list,
)
self._workflow_executors[workflow_name] = workflow_executor
try:
decisions = workflow_executor.replay(decision_response)
except Exception as err:
import traceback
details = traceback.format_exc()
message = "workflow decision failed: {}".format(err)
logger.exception(message)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(reason=message, details=details)
decisions = [decision]
return decisions
def decide(self, decision_response, task_list):
"""
Delegate the decision to the executor, loading it if needed.
:param decision_response: an object wrapping the PollForDecisionTask response.
:type decision_response: swf.responses.Response
:param task_list:
:type task_list: Optional[str]
:returns: the decisions.
:rtype: list[swf.models.decision.base.Decision]
"""
history = decision_response.history
workflow_name = history[0].workflow_type['name']
workflow_executor = self._workflow_executors.get(workflow_name)
if not workflow_executor:
# Child workflow from another module
from . import helpers
workflow_executor = helpers.load_workflow_executor(
self._domain,
workflow_name,
task_list=task_list,
)
self._workflow_executors[workflow_name] = workflow_executor
try:
decisions = workflow_executor.replay(decision_response)
except Exception as err:
import traceback
details = traceback.format_exc()
message = "workflow decision failed: {}".format(err)
logger.exception(message)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(reason=message, details=details)
decisions = [decision]
return decisions
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
'wait_signal({}) before signal was sent: unsupported by the local executor'
.format(signal_name))
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name,
[]).append(Marker(func.name,
func.details))
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, 'activity', None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, 'post_execute'):
task.post_execute()
state = 'completed'
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
future._exception = exc_value
logger.exception('rescuing exception: {}'.format(exc_value))
if (isinstance(func, Activity)
or issubclass_(func, Workflow)) and getattr(
func, 'raises_on_failure', None):
tb = traceback.format_tb(exc_traceback)
message = format_exc(exc_value)
details = json_dumps(
{
'error': exc_type.__name__,
'message': str(exc_value),
'traceback': tb,
},
default=repr)
raise exceptions.TaskFailed(
func.name,
message,
details,
)
state = 'failed'
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={
'args': args,
'kwargs': kwargs
},
result=future.result)
return future
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name)
)
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name, []).append(Marker(func.name, func.details))
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, 'activity', None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, 'post_execute'):
task.post_execute()
state = 'completed'
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
future._exception = exc_value
logger.exception('rescuing exception: {}'.format(exc_value))
if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(func, 'raises_on_failure', None):
tb = traceback.format_tb(exc_traceback)
message = format_exc(exc_value)
details = json_dumps(
{
'error': exc_type.__name__,
'message': str(exc_value),
'traceback': tb,
},
default=repr
)
raise exceptions.TaskFailed(
func.name,
message,
details,
)
state = 'failed'
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(
func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={'args': args, 'kwargs': kwargs},
result=future.result)
return future
def submit(self, func, *args, **kwargs):
logger.info("executing task {}(args={}, kwargs={})".format(func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
"wait_signal({}) before signal was sent: unsupported by the local executor".format(
signal_name
)
)
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name, []).append(
Marker(func.name, func.details)
)
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, "activity", None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError("invalid type {} for {}".format(type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, "post_execute"):
task.post_execute()
state = "completed"
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.format_tb(exc_traceback)
task_failed = exceptions.TaskFailed(
name=getattr(task, "name", "unknown"),
reason=format_exc(exc_value),
details=json_dumps(
{
"error": exc_type.__name__,
"error_type": format_exc_type(exc_type),
"message": str(exc_value),
"traceback": tb,
},
default=repr,
),
)
future.set_exception(task_failed)
logger.exception("rescuing exception: {}".format(exc_value))
if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(
func, "raises_on_failure", None
):
raise task_failed
state = "failed"
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(
func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={"args": args, "kwargs": kwargs},
result=future.result,
)
return future