def process(self, poller, token, task): """ :param poller: :type poller: ActivityPoller :param token: :type token: str :param task: :type task: swf.models.ActivityTask """ logger.debug('ActivityWorker.process() pid={}'.format(os.getpid())) try: activity = self.dispatch(task) input = format.decode(task.input) args = input.get('args', ()) kwargs = input.get('kwargs', {}) context = sanitize_activity_context(task.context) context['domain_name'] = poller.domain.name if input.get('meta', {}).get('binaries'): download_binaries(input['meta']['binaries']) result = ActivityTask(activity, *args, context=context, **kwargs).execute() except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() logger.exception("process error: {}".format(str(exc_value))) if isinstance(exc_value, ExecutionError) and len(exc_value.args): details = exc_value.args[0] reason = format_exc(exc_value) # FIXME json.loads and rebuild? else: tb = traceback.format_tb(exc_traceback) reason = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr ) return poller.fail_with_retry( token, task, reason=reason, details=details ) try: logger.info('completing activity') poller.complete_with_retry(token, result) except Exception as err: logger.exception("complete error") reason = 'cannot complete task {}: {} {}'.format( task.activity_id, err.__class__.__name__, err, ) poller.fail_with_retry(token, task, reason)
def process(self, response): """ Process a swf.actors.ActivityWorker poll response.. :param response: :type response: swf.responses.Response """ token = response.task_token task = response.activity_task if self.process_mode == "kubernetes": try: spawn_kubernetes_job(self, response.raw_response) except Exception as err: logger.exception("spawn_kubernetes_job error") reason = "cannot spawn kubernetes job for task {}: {} {}".format( task.activity_id, err.__class__.__name__, err, ) self.fail_with_retry(token, task, reason) else: spawn(self, token, task, self._heartbeat)
def process(self, response): """ Process a swf.actors.ActivityWorker poll response.. :param response: :type response: swf.responses.Response """ token = response.task_token task = response.activity_task if self.process_mode == "kubernetes": try: spawn_kubernetes_job(self, response.raw_response) except Exception as err: logger.exception("spawn_kubernetes_job error") reason = 'cannot spawn kubernetes job for task {}: {} {}'.format( task.activity_id, err.__class__.__name__, err, ) self.fail_with_retry(token, task, reason) else: spawn(self, token, task, self._heartbeat)
def complete_with_retry(self, token, response): """ Complete with retry. :param token: :type token: str :param response: response: decision list, JSON result, ... :type response: Any :return: :rtype: """ try: complete = utils.retry.with_delay( nb_times=self.nb_retries, delay=utils.retry.exponential, log_with=logger.exception, except_on=swf.exceptions.DoesNotExistError, )(self.complete) # Exponential backoff on errors. complete(token, response) except Exception as err: # This is embarrassing because the decider cannot notify SWF of the # task completion. As it will not try again, the task will # timeout (start_to_complete). logger.exception("cannot complete task: %s", str(err))
def decide(self, decision_response, task_list): """ Delegate the decision to the executor, loading it if needed. :param decision_response: an object wrapping the PollForDecisionTask response. :type decision_response: swf.responses.Response :param task_list: :type task_list: Optional[str] :returns: the decisions. :rtype: list[swf.models.decision.base.Decision] """ history = decision_response.history workflow_name = history[0].workflow_type["name"] workflow_executor = self._workflow_executors.get(workflow_name) if not workflow_executor: # Child workflow from another module from . import helpers workflow_executor = helpers.load_workflow_executor( self._domain, workflow_name, task_list=task_list, ) self._workflow_executors[workflow_name] = workflow_executor try: decisions = workflow_executor.replay(decision_response) except Exception as err: import traceback details = traceback.format_exc() message = "workflow decision failed: {}".format(err) logger.exception(message) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail(reason=message, details=details) decisions = [decision] return decisions
def decide(self, decision_response, task_list): """ Delegate the decision to the executor, loading it if needed. :param decision_response: an object wrapping the PollForDecisionTask response. :type decision_response: swf.responses.Response :param task_list: :type task_list: Optional[str] :returns: the decisions. :rtype: list[swf.models.decision.base.Decision] """ history = decision_response.history workflow_name = history[0].workflow_type['name'] workflow_executor = self._workflow_executors.get(workflow_name) if not workflow_executor: # Child workflow from another module from . import helpers workflow_executor = helpers.load_workflow_executor( self._domain, workflow_name, task_list=task_list, ) self._workflow_executors[workflow_name] = workflow_executor try: decisions = workflow_executor.replay(decision_response) except Exception as err: import traceback details = traceback.format_exc() message = "workflow decision failed: {}".format(err) logger.exception(message) decision = swf.models.decision.WorkflowExecutionDecision() decision.fail(reason=message, details=details) decisions = [decision] return decisions
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor' .format(signal_name)) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, 'post_execute'): task.post_execute() state = 'completed' except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() future._exception = exc_value logger.exception('rescuing exception: {}'.format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, 'raises_on_failure', None): tb = traceback.format_tb(exc_traceback) message = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr) raise exceptions.TaskFailed( func.name, message, details, ) state = 'failed' finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task(func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={ 'args': args, 'kwargs': kwargs }, result=future.result) return future
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, 'post_execute'): task.post_execute() state = 'completed' except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() future._exception = exc_value logger.exception('rescuing exception: {}'.format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(func, 'raises_on_failure', None): tb = traceback.format_tb(exc_traceback) message = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr ) raise exceptions.TaskFailed( func.name, message, details, ) state = 'failed' finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={'args': args, 'kwargs': kwargs}, result=future.result) return future
def submit(self, func, *args, **kwargs): logger.info("executing task {}(args={}, kwargs={})".format(func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( "wait_signal({}) before signal was sent: unsupported by the local executor".format( signal_name ) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append( Marker(func.name, func.details) ) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, "activity", None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError("invalid type {} for {}".format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, "post_execute"): task.post_execute() state = "completed" except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) task_failed = exceptions.TaskFailed( name=getattr(task, "name", "unknown"), reason=format_exc(exc_value), details=json_dumps( { "error": exc_type.__name__, "error_type": format_exc_type(exc_type), "message": str(exc_value), "traceback": tb, }, default=repr, ), ) future.set_exception(task_failed) logger.exception("rescuing exception: {}".format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, "raises_on_failure", None ): raise task_failed state = "failed" finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={"args": args, "kwargs": kwargs}, result=future.result, ) return future