def handle(self): if not self.func: raise exceptions.NonRecoverableError('func not found: {0}'. format(self.cloudify_context)) ctx = self.ctx kwargs = self.kwargs if ctx.task_target: # # this operation requires an AMQP client amqp_client_utils.init_amqp_client() else: # task is local (not through celery) so we need clone kwarg # and an amqp client is not required kwargs = copy.deepcopy(kwargs) if self.cloudify_context.get('has_intrinsic_functions'): with state.current_ctx.push(ctx, kwargs): kwargs = ctx._endpoint.evaluate_functions(payload=kwargs) if not self.cloudify_context.get('no_ctx_kwarg'): kwargs['ctx'] = ctx with state.current_ctx.push(ctx, kwargs): try: result = self.func(*self.args, **kwargs) finally: amqp_client_utils.close_amqp_client() if ctx.type == context.NODE_INSTANCE: ctx.instance.update() elif ctx.type == context.RELATIONSHIP_INSTANCE: ctx.source.instance.update() ctx.target.instance.update() if ctx.operation._operation_retry: raise ctx.operation._operation_retry return result
def handle(self): if not self.func: raise exceptions.NonRecoverableError('func not found: {0}'.format( self.cloudify_context)) ctx = self.ctx kwargs = self.kwargs if ctx.task_target: # # this operation requires an AMQP client amqp_client_utils.init_amqp_client() else: # task is local (not through celery) so we need clone kwarg # and an amqp client is not required kwargs = copy.deepcopy(kwargs) if self.cloudify_context.get('has_intrinsic_functions'): with state.current_ctx.push(ctx, kwargs): kwargs = ctx._endpoint.evaluate_functions(payload=kwargs) if not self.cloudify_context.get('no_ctx_kwarg'): kwargs['ctx'] = ctx with state.current_ctx.push(ctx, kwargs): try: result = self.func(*self.args, **kwargs) finally: amqp_client_utils.close_amqp_client() if ctx.type == constants.NODE_INSTANCE: ctx.instance.update() elif ctx.type == constants.RELATIONSHIP_INSTANCE: ctx.source.instance.update() ctx.target.instance.update() if ctx.operation._operation_retry: raise ctx.operation._operation_retry return result
def _amqp_client(self): # initialize an amqp client only when needed, ie. if the task is # not local with_amqp = bool(self.ctx.task_target) if with_amqp: amqp_client_utils.init_events_publisher() try: yield finally: if with_amqp: amqp_client_utils.close_amqp_client()
def _amqp_client(self): # initialize an amqp client only when needed, ie. if the task is # not local with_amqp = bool(self.ctx.task_target) if with_amqp: try: amqp_client_utils.init_events_publisher() except Exception: _, ex, tb = sys.exc_info() # This one should never (!) raise an exception. amqp_client_utils.close_amqp_client() raise exceptions.RecoverableError( 'Failed initializing AMQP connection', causes=[utils.exception_to_error_cause(ex, tb)]) try: yield finally: if with_amqp: amqp_client_utils.close_amqp_client()
def _handle_remote_workflow(self): tenant = self.ctx._context['tenant'].get('original_name', self.ctx.tenant_name) rest = get_rest_client(tenant=tenant) execution = rest.executions.get(self.ctx.execution_id, _include=['status']) if execution.status == Execution.STARTED: self.ctx.resume = True try: amqp_client_utils.init_events_publisher() try: self._workflow_started() except InvalidExecutionUpdateStatus: self._workflow_cancelled() return api.EXECUTION_CANCELLED_RESULT queue = Queue.Queue() t = AMQPWrappedThread(target=self._remote_workflow_child_thread, args=(queue, ), name='Workflow-Child') t.start() # while the child thread is executing the workflow, the parent # thread is polling for 'cancel' requests while also waiting for # messages from the child thread result = None while True: # check if child thread sent a message try: data = queue.get(timeout=5) if 'result' in data: # child thread has terminated result = data['result'] break else: # error occurred in child thread error = data['error'] raise exceptions.ProcessExecutionError( error['message'], error['type'], error['traceback']) except Queue.Empty: pass # A very hacky way to solve an edge case when trying to poll # for the execution status while the DB is downgraded during # a snapshot restore if self.cloudify_context['workflow_id'] == 'restore_snapshot': continue # check for 'cancel' requests execution = rest.executions.get(self.ctx.execution_id, _include=['status']) if execution.status in [ Execution.CANCELLING, Execution.FORCE_CANCELLING, Execution.KILL_CANCELLING ]: # send a 'cancel' message to the child thread. It is up to # the workflow implementation to check for this message # and act accordingly (by stopping and raising an # api.ExecutionCancelled error, or by returning the # deprecated api.EXECUTION_CANCELLED_RESULT as result). # parent thread then goes back to polling for messages from # child thread or possibly 'force-cancelling' requests api.cancel_request = True if execution.status == Execution.KILL_CANCELLING: # if a custom workflow function must attempt some cleanup, # it might attempt to catch SIGTERM, and confirm using this # flag that it is being kill-cancelled api.kill_request = True if execution.status in [ Execution.FORCE_CANCELLING, Execution.KILL_CANCELLING ]: # force-cancel additionally stops this loop immediately result = api.EXECUTION_CANCELLED_RESULT break if result == api.EXECUTION_CANCELLED_RESULT: self._workflow_cancelled() else: self._workflow_succeeded() return result except exceptions.ProcessExecutionError as e: self._workflow_failed(e, e.traceback) raise except BaseException as e: self._workflow_failed(e, traceback.format_exc()) raise finally: amqp_client_utils.close_amqp_client()
def _handle_remote_workflow(self): rest = get_rest_client() amqp_client_utils.init_amqp_client() try: try: self._workflow_started() except InvalidExecutionUpdateStatus: self._workflow_cancelled() return api.EXECUTION_CANCELLED_RESULT queue = Queue.Queue() t = AMQPWrappedThread(target=self._remote_workflow_child_thread, args=(queue,), name='Workflow-Child') t.start() # while the child thread is executing the workflow, the parent # thread is polling for 'cancel' requests while also waiting for # messages from the child thread result = None while True: # check if child thread sent a message try: data = queue.get(timeout=5) if 'result' in data: # child thread has terminated result = data['result'] break else: # error occurred in child thread error = data['error'] raise exceptions.ProcessExecutionError( error['message'], error['type'], error['traceback']) except Queue.Empty: pass # check for 'cancel' requests execution = rest.executions.get(self.ctx.execution_id, _include=['status']) if execution.status == Execution.FORCE_CANCELLING: result = api.EXECUTION_CANCELLED_RESULT break elif execution.status == Execution.CANCELLING: # send a 'cancel' message to the child thread. It is up to # the workflow implementation to check for this message # and act accordingly (by stopping and raising an # api.ExecutionCancelled error, or by returning the # deprecated api.EXECUTION_CANCELLED_RESULT as result). # parent thread then goes back to polling for messages from # child thread or possibly 'force-cancelling' requests api.cancel_request = True if result == api.EXECUTION_CANCELLED_RESULT: self._workflow_cancelled() else: self._workflow_succeeded() return result except exceptions.ProcessExecutionError as e: self._workflow_failed(e, e.traceback) raise except BaseException as e: error = StringIO.StringIO() traceback.print_exc(file=error) self._workflow_failed(e, error.getvalue()) raise finally: amqp_client_utils.close_amqp_client()
def _handle_remote_workflow(self): rest = get_rest_client() amqp_client_utils.init_amqp_client() try: execution = rest.executions.get(self.ctx.execution_id, _include=['status']) if execution.status in (Execution.CANCELLING, Execution.FORCE_CANCELLING): # execution has been requested to be cancelled before it was # even started self._workflow_cancelled() return api.EXECUTION_CANCELLED_RESULT self._workflow_started() queue = Queue.Queue() t = AMQPWrappedThread(target=self._remote_workflow_child_thread, args=(queue,)) t.start() # while the child thread is executing the workflow, the parent # thread is polling for 'cancel' requests while also waiting for # messages from the child thread result = None while True: # check if child thread sent a message try: data = queue.get(timeout=5) if 'result' in data: # child thread has terminated result = data['result'] break else: # error occurred in child thread error = data['error'] raise exceptions.ProcessExecutionError( error['message'], error['type'], error['traceback']) except Queue.Empty: pass # check for 'cancel' requests execution = rest.executions.get(self.ctx.execution_id, _include=['status']) if execution.status == Execution.FORCE_CANCELLING: result = api.EXECUTION_CANCELLED_RESULT break elif execution.status == Execution.CANCELLING: # send a 'cancel' message to the child thread. It is up to # the workflow implementation to check for this message # and act accordingly (by stopping and raising an # api.ExecutionCancelled error, or by returning the # deprecated api.EXECUTION_CANCELLED_RESULT as result). # parent thread then goes back to polling for messages from # child thread or possibly 'force-cancelling' requests api.cancel_request = True if result == api.EXECUTION_CANCELLED_RESULT: self._workflow_cancelled() else: self._workflow_succeeded() return result except exceptions.ProcessExecutionError as e: self._workflow_failed(e, e.traceback) raise except BaseException as e: error = StringIO.StringIO() traceback.print_exc(file=error) self._workflow_failed(e, error.getvalue()) raise finally: amqp_client_utils.close_amqp_client()
def _remote_workflow(ctx, func, args, kwargs): def update_execution_cancelled(): update_execution_status(ctx.execution_id, Execution.CANCELLED) _send_workflow_cancelled_event(ctx) rest = get_rest_client() parent_queue, child_queue = (Queue.Queue(), Queue.Queue()) try: amqp_client_utils.init_amqp_client() if rest.executions.get(ctx.execution_id).status in \ (Execution.CANCELLING, Execution.FORCE_CANCELLING): # execution has been requested to be cancelled before it # was even started update_execution_cancelled() return api.EXECUTION_CANCELLED_RESULT update_execution_status(ctx.execution_id, Execution.STARTED) _send_workflow_started_event(ctx) # the actual execution of the workflow will run in another # thread - this wrapper is the entry point for that # thread, and takes care of forwarding the result or error # back to the parent thread def child_wrapper(): try: ctx.internal.start_event_monitor() workflow_result = _execute_workflow_function( ctx, func, args, kwargs) child_queue.put({'result': workflow_result}) except api.ExecutionCancelled: child_queue.put({ 'result': api.EXECUTION_CANCELLED_RESULT}) except BaseException as workflow_ex: tb = StringIO() traceback.print_exc(file=tb) err = { 'type': type(workflow_ex).__name__, 'message': str(workflow_ex), 'traceback': tb.getvalue() } child_queue.put({'error': err}) finally: ctx.internal.stop_event_monitor() api.queue = parent_queue # starting workflow execution on child thread # AMQPWrappedThread is used to provide it with its own amqp client t = AMQPWrappedThread(target=child_wrapper) t.start() # while the child thread is executing the workflow, # the parent thread is polling for 'cancel' requests while # also waiting for messages from the child thread has_sent_cancelling_action = False result = None execution = None while True: # check if child thread sent a message try: data = child_queue.get(timeout=5) if 'result' in data: # child thread has terminated result = data['result'] break else: # error occurred in child thread error = data['error'] raise exceptions.ProcessExecutionError(error['message'], error['type'], error['traceback']) except Queue.Empty: pass # check for 'cancel' requests execution = rest.executions.get(ctx.execution_id) if execution.status == Execution.FORCE_CANCELLING: result = api.EXECUTION_CANCELLED_RESULT break elif not has_sent_cancelling_action and \ execution.status == Execution.CANCELLING: # send a 'cancel' message to the child thread. It # is up to the workflow implementation to check for # this message and act accordingly (by stopping and # raising an api.ExecutionCancelled error, or by returning # the deprecated api.EXECUTION_CANCELLED_RESULT as result). # parent thread then goes back to polling for # messages from child process or possibly # 'force-cancelling' requests parent_queue.put({'action': 'cancel'}) has_sent_cancelling_action = True # updating execution status and sending events according to # how the execution ended if result == api.EXECUTION_CANCELLED_RESULT: update_execution_cancelled() if execution and execution.status == Execution.FORCE_CANCELLING: # TODO: kill worker externally raise RequestSystemExit() else: update_execution_status(ctx.execution_id, Execution.TERMINATED) _send_workflow_succeeded_event(ctx) return result except RequestSystemExit: raise except BaseException as e: if isinstance(e, exceptions.ProcessExecutionError): error_traceback = e.traceback else: error = StringIO() traceback.print_exc(file=error) error_traceback = error.getvalue() update_execution_status(ctx.execution_id, Execution.FAILED, error_traceback) _send_workflow_failed_event(ctx, e, error_traceback) raise finally: amqp_client_utils.close_amqp_client()
def wrapper(*args, **kwargs): ctx = _find_context_arg(args, kwargs, _is_cloudify_context) if ctx is None: ctx = {} if not _is_cloudify_context(ctx): ctx = context.CloudifyContext(ctx) # remove __cloudify_context raw_context = kwargs.pop(CLOUDIFY_CONTEXT_PROPERTY_KEY, {}) if ctx.task_target: # this operation requires an AMQP client amqp_client_utils.init_amqp_client() else: # task is local (not through celery) so we need to # clone kwarg, and an amqp client is not required kwargs = copy.deepcopy(kwargs) if raw_context.get('has_intrinsic_functions') is True: kwargs = ctx._endpoint.evaluate_functions(payload=kwargs) kwargs['ctx'] = ctx try: current_ctx.set(ctx, kwargs) result = func(*args, **kwargs) except BaseException as e: ctx.logger.error( 'Exception raised on operation [%s] invocation', ctx.task_name, exc_info=True) if ctx.task_target is None: # local task execution # no serialization issues raise # extract exception details # type, value, traceback tpe, value, tb = sys.exc_info() # we re-create the exception here # since it will be sent # over the wire. And the original exception # may cause de-serialization issues # on the other side. # preserve original type in the message message = '{0}: {1}'.format(tpe.__name__, str(e)) # if the exception type is directly one of our exception # than there is no need for conversion and we can just # raise the original exception if type(e) in [exceptions.OperationRetry, exceptions.RecoverableError, exceptions.NonRecoverableError, exceptions.HttpException]: raise # if the exception inherits from our base exceptions, there # still might be a de-serialization problem caused by one of # the types in the inheritance tree. if isinstance(e, exceptions.NonRecoverableError): value = exceptions.NonRecoverableError(message) elif isinstance(e, exceptions.OperationRetry): value = exceptions.OperationRetry(message, e.retry_after) elif isinstance(e, exceptions.RecoverableError): value = exceptions.RecoverableError(message, e.retry_after) else: # convert pure user exceptions # to a RecoverableError value = exceptions.RecoverableError(message) raise type(value), value, tb finally: amqp_client_utils.close_amqp_client() current_ctx.clear() if ctx.type == context.NODE_INSTANCE: ctx.instance.update() elif ctx.type == context.RELATIONSHIP_INSTANCE: ctx.source.instance.update() ctx.target.instance.update() if ctx.operation._operation_retry: raise ctx.operation._operation_retry return result