def start(self, executor): from pipeline.parser import pipeline_parser from pipeline.engine import api from pipeline.utils.context import get_pipeline_context from pipeline.engine.models import FunctionSwitch if FunctionSwitch.objects.is_frozen(): return False, 'engine has been freeze, try later please' with transaction.atomic(): instance = self.__class__.objects.select_for_update().get(id=self.id) if instance.is_started: return False, 'pipeline instance already started.' instance.start_time = timezone.now() instance.is_started = True instance.executor = executor # calculate tree info instance.calculate_tree_info() pipeline_data = instance.execution_data parser = pipeline_parser.WebPipelineAdapter(pipeline_data) pipeline = parser.parser(get_pipeline_context(instance, 'instance')) instance.save() api.start_pipeline(pipeline) return True, {}
def get_act_data(self, act_id, component_code=None, subprocess_stack=None): act_started = True result = True try: inputs = pipeline_api.get_inputs(act_id) outputs = pipeline_api.get_outputs(act_id) except Data.DoesNotExist: act_started = False if component_code: if not act_started: try: instance_data = self.pipeline_instance.execution_data inputs = pipeline_parser.WebPipelineAdapter( instance_data).get_act_inputs( act_id=act_id, subprocess_stack=subprocess_stack, root_pipeline_data=get_pipeline_context( self.pipeline_instance, 'instance')) outputs = {} except Exception as e: inputs = {} result = False message = 'parser pipeline tree error: %s' % e logger.exception(message) outputs = {'ex_data': message} outputs_table = [] try: component = library.ComponentLibrary.get_component_class( component_code) outputs_format = component.outputs_format() except Exception as e: result = False message = 'get component[component_code=%s] format error: %s' % ( component_code, e) logger.exception(message) outputs = {'ex_data': message} else: for outputs_item in outputs_format: value = outputs.get('outputs', {}).get(outputs_item['key'], '') outputs_table.append({ 'name': outputs_item['name'], 'value': value }) else: outputs_table = [] data = { 'inputs': inputs, 'outputs': outputs_table, 'ex_data': outputs.pop('ex_data', '') } return {'result': result, 'data': data, 'message': data['ex_data']}
def get_instance_context(request): instance_id = request.GET.get('instance_id') if instance_id: try: instance = PipelineInstance.objects.get(instance_id=instance_id) except PipelineInstance.DoesNotExist: return action_result(False, 'PipelineInstance object of instance_id:%s does not exist' % instance_id) else: instance = None context = get_pipeline_context(instance, 'instance') return JsonResponse(context)
def get_template_context(request): template_id = request.GET.get('template_id') if template_id: try: template = PipelineTemplate.objects.get(template_id=template_id) except PipelineTemplate.DoesNotExist: return action_result(False, 'PipelineTemplate object of template_id:%s does not exist' % template_id) else: template = None context = get_pipeline_context(template, 'template') return JsonResponse(context)
def start(self, executor, check_workers=True): """ 启动当前流程 @param executor: 执行者 @param check_workers: 是否检测 worker 的状态 @return: 执行结果 """ from pipeline.engine import api from pipeline.utils.context import get_pipeline_context from pipeline.engine.models import FunctionSwitch from pipeline.engine.core.api import workers if FunctionSwitch.objects.is_frozen(): return ActionResult(result=False, message='engine has been freeze, try later please') if check_workers: try: if not workers(): return ActionResult(result=False, message='can not find celery workers, please check worker status') except RabbitMQConnectionError as e: return ActionResult(result=False, message='celery worker status check failed with message: %s, ' 'check rabbitmq status please' % e.message) except RedisConnectionError: return ActionResult(result=False, message='redis connection error, check redis status please') with transaction.atomic(): instance = self.__class__.objects.select_for_update().get(id=self.id) if instance.is_started: return ActionResult(result=False, message='pipeline instance already started.') instance.start_time = timezone.now() instance.is_started = True instance.executor = executor # calculate tree info instance.calculate_tree_info() pipeline_data = instance.execution_data try: parser_cls = import_string(settings.PIPELINE_PARSER_CLASS) except ImportError: return ActionResult(result=False, message='invalid parser class: %s' % settings.PIPELINE_PARSER_CLASS) parser = parser_cls(pipeline_data) pipeline = parser.parse(get_pipeline_context(instance, 'instance')) instance.save() return api.start_pipeline(pipeline, check_workers=check_workers)
def start(self, executor, check_workers=True): """ 启动当前流程 @param executor: 执行者 @param check_workers: 是否检测 worker 的状态 @return: 执行结果 """ with transaction.atomic(): instance = self.__class__.objects.select_for_update().get( id=self.id) if instance.is_started: return ActionResult( result=False, message='pipeline instance already started.') pipeline_data = instance.execution_data try: parser_cls = import_string(settings.PIPELINE_PARSER_CLASS) except ImportError: return ActionResult(result=False, message='invalid parser class: %s' % settings.PIPELINE_PARSER_CLASS) instance.start_time = timezone.now() instance.is_started = True instance.executor = executor parser = parser_cls(pipeline_data) pipeline = parser.parse(get_pipeline_context(instance, 'instance')) # calculate tree info instance.calculate_tree_info() instance.save() act_result = task_service.run_pipeline(pipeline) if not act_result.result: with transaction.atomic(): instance = self.__class__.objects.select_for_update().get( id=self.id) instance.start_time = None instance.is_started = False instance.executor = '' instance.save() return act_result
def start(self, executor): from pipeline.parser import pipeline_parser from pipeline.engine import api from pipeline.utils.context import get_pipeline_context with transaction.atomic(): instance = self.__class__.objects.select_for_update().get( id=self.id) if instance.is_started: return False, 'pipeline instance already started.' instance.start_time = timezone.now() instance.is_started = True instance.executor = executor pipeline_data = instance.execution_data parser = pipeline_parser.WebPipelineAdapter(pipeline_data) pipeline = parser.parser(get_pipeline_context( instance, 'instance')) instance.save() api.start_pipeline(pipeline) return True, {}
def get_node_data(self, node_id, component_code=None, subprocess_stack=None): if not self.has_node(node_id): message = 'node[node_id={node_id}] not found in task[task_id={task_id}]'.format( node_id=node_id, task_id=self.id) return {'result': False, 'message': message, 'data': {}} act_started = True result = True inputs = {} outputs = {} try: inputs = pipeline_api.get_inputs(node_id) outputs = pipeline_api.get_outputs(node_id) except Data.DoesNotExist: act_started = False instance_data = self.pipeline_instance.execution_data if not act_started: try: inputs = WebPipelineAdapter(instance_data).get_act_inputs( act_id=node_id, subprocess_stack=subprocess_stack, root_pipeline_data=get_pipeline_context( self.pipeline_instance, 'instance')) outputs = {} except Exception as e: inputs = {} result = False message = 'parser pipeline tree error: %s' % e logger.exception(message) outputs = {'ex_data': message} if component_code: outputs_table = [] try: component = library.ComponentLibrary.get_component_class( component_code) outputs_format = component.outputs_format() except Exception as e: result = False message = 'get component[component_code=%s] format error: %s' % ( component_code, e) logger.exception(message) outputs = {'ex_data': message} else: outputs_data = outputs.get('outputs', {}) # 在原子定义中的预设输出参数 archived_keys = [] for outputs_item in outputs_format: value = outputs_data.get(outputs_item['key'], '') outputs_table.append({ 'name': outputs_item['name'], 'key': outputs_item['key'], 'value': value, 'preset': True, }) archived_keys.append(outputs_item['key']) # 其他输出参数 for out_key, out_value in outputs_data.items(): if out_key not in archived_keys: outputs_table.append({ 'name': out_key, 'key': out_key, 'value': out_value, 'preset': False, }) else: outputs_table = [{ 'key': key, 'value': val, 'preset': False } for key, val in outputs.get('outputs', {}).items()] if not isinstance(inputs, dict): inputs = {} if not isinstance(outputs, dict): outputs = {} data = { 'inputs': inputs, 'outputs': outputs_table, 'ex_data': outputs.pop('ex_data', '') } return { 'result': result, 'data': data, 'message': '' if result else data['ex_data'] }