def start(self, executor):
from pipeline.parser import pipeline_parser
from pipeline.engine import api
from pipeline.utils.context import get_pipeline_context
from pipeline.engine.models import FunctionSwitch
if FunctionSwitch.objects.is_frozen():
return False, 'engine has been freeze, try later please'
with transaction.atomic():
instance = self.__class__.objects.select_for_update().get(id=self.id)
if instance.is_started:
return False, 'pipeline instance already started.'
instance.start_time = timezone.now()
instance.is_started = True
instance.executor = executor
# calculate tree info
instance.calculate_tree_info()
pipeline_data = instance.execution_data
parser = pipeline_parser.WebPipelineAdapter(pipeline_data)
pipeline = parser.parser(get_pipeline_context(instance, 'instance'))
instance.save()
api.start_pipeline(pipeline)
return True, {}
def get_act_data(self, act_id, component_code=None, subprocess_stack=None):
act_started = True
result = True
try:
inputs = pipeline_api.get_inputs(act_id)
outputs = pipeline_api.get_outputs(act_id)
except Data.DoesNotExist:
act_started = False
if component_code:
if not act_started:
try:
instance_data = self.pipeline_instance.execution_data
inputs = pipeline_parser.WebPipelineAdapter(
instance_data).get_act_inputs(
act_id=act_id,
subprocess_stack=subprocess_stack,
root_pipeline_data=get_pipeline_context(
self.pipeline_instance, 'instance'))
outputs = {}
except Exception as e:
inputs = {}
result = False
message = 'parser pipeline tree error: %s' % e
logger.exception(message)
outputs = {'ex_data': message}
outputs_table = []
try:
component = library.ComponentLibrary.get_component_class(
component_code)
outputs_format = component.outputs_format()
except Exception as e:
result = False
message = 'get component[component_code=%s] format error: %s' % (
component_code, e)
logger.exception(message)
outputs = {'ex_data': message}
else:
for outputs_item in outputs_format:
value = outputs.get('outputs',
{}).get(outputs_item['key'], '')
outputs_table.append({
'name': outputs_item['name'],
'value': value
})
else:
outputs_table = []
data = {
'inputs': inputs,
'outputs': outputs_table,
'ex_data': outputs.pop('ex_data', '')
}
return {'result': result, 'data': data, 'message': data['ex_data']}
def get_instance_context(request):
instance_id = request.GET.get('instance_id')
if instance_id:
try:
instance = PipelineInstance.objects.get(instance_id=instance_id)
except PipelineInstance.DoesNotExist:
return action_result(False, 'PipelineInstance object of instance_id:%s does not exist' % instance_id)
else:
instance = None
context = get_pipeline_context(instance, 'instance')
return JsonResponse(context)
def get_template_context(request):
template_id = request.GET.get('template_id')
if template_id:
try:
template = PipelineTemplate.objects.get(template_id=template_id)
except PipelineTemplate.DoesNotExist:
return action_result(False, 'PipelineTemplate object of template_id:%s does not exist' % template_id)
else:
template = None
context = get_pipeline_context(template, 'template')
return JsonResponse(context)
def start(self, executor, check_workers=True):
"""
启动当前流程
@param executor: 执行者
@param check_workers: 是否检测 worker 的状态
@return: 执行结果
"""
from pipeline.engine import api
from pipeline.utils.context import get_pipeline_context
from pipeline.engine.models import FunctionSwitch
from pipeline.engine.core.api import workers
if FunctionSwitch.objects.is_frozen():
return ActionResult(result=False, message='engine has been freeze, try later please')
if check_workers:
try:
if not workers():
return ActionResult(result=False, message='can not find celery workers, please check worker status')
except RabbitMQConnectionError as e:
return ActionResult(result=False, message='celery worker status check failed with message: %s, '
'check rabbitmq status please' % e.message)
except RedisConnectionError:
return ActionResult(result=False, message='redis connection error, check redis status please')
with transaction.atomic():
instance = self.__class__.objects.select_for_update().get(id=self.id)
if instance.is_started:
return ActionResult(result=False, message='pipeline instance already started.')
instance.start_time = timezone.now()
instance.is_started = True
instance.executor = executor
# calculate tree info
instance.calculate_tree_info()
pipeline_data = instance.execution_data
try:
parser_cls = import_string(settings.PIPELINE_PARSER_CLASS)
except ImportError:
return ActionResult(result=False, message='invalid parser class: %s' % settings.PIPELINE_PARSER_CLASS)
parser = parser_cls(pipeline_data)
pipeline = parser.parse(get_pipeline_context(instance, 'instance'))
instance.save()
return api.start_pipeline(pipeline, check_workers=check_workers)
def start(self, executor, check_workers=True):
"""
启动当前流程
@param executor: 执行者
@param check_workers: 是否检测 worker 的状态
@return: 执行结果
"""
with transaction.atomic():
instance = self.__class__.objects.select_for_update().get(
id=self.id)
if instance.is_started:
return ActionResult(
result=False, message='pipeline instance already started.')
pipeline_data = instance.execution_data
try:
parser_cls = import_string(settings.PIPELINE_PARSER_CLASS)
except ImportError:
return ActionResult(result=False,
message='invalid parser class: %s' %
settings.PIPELINE_PARSER_CLASS)
instance.start_time = timezone.now()
instance.is_started = True
instance.executor = executor
parser = parser_cls(pipeline_data)
pipeline = parser.parse(get_pipeline_context(instance, 'instance'))
# calculate tree info
instance.calculate_tree_info()
instance.save()
act_result = task_service.run_pipeline(pipeline)
if not act_result.result:
with transaction.atomic():
instance = self.__class__.objects.select_for_update().get(
id=self.id)
instance.start_time = None
instance.is_started = False
instance.executor = ''
instance.save()
return act_result
def start(self, executor):
from pipeline.parser import pipeline_parser
from pipeline.engine import api
from pipeline.utils.context import get_pipeline_context
with transaction.atomic():
instance = self.__class__.objects.select_for_update().get(
id=self.id)
if instance.is_started:
return False, 'pipeline instance already started.'
instance.start_time = timezone.now()
instance.is_started = True
instance.executor = executor
pipeline_data = instance.execution_data
parser = pipeline_parser.WebPipelineAdapter(pipeline_data)
pipeline = parser.parser(get_pipeline_context(
instance, 'instance'))
instance.save()
api.start_pipeline(pipeline)
return True, {}
def get_node_data(self,
node_id,
component_code=None,
subprocess_stack=None):
if not self.has_node(node_id):
message = 'node[node_id={node_id}] not found in task[task_id={task_id}]'.format(
node_id=node_id, task_id=self.id)
return {'result': False, 'message': message, 'data': {}}
act_started = True
result = True
inputs = {}
outputs = {}
try:
inputs = pipeline_api.get_inputs(node_id)
outputs = pipeline_api.get_outputs(node_id)
except Data.DoesNotExist:
act_started = False
instance_data = self.pipeline_instance.execution_data
if not act_started:
try:
inputs = WebPipelineAdapter(instance_data).get_act_inputs(
act_id=node_id,
subprocess_stack=subprocess_stack,
root_pipeline_data=get_pipeline_context(
self.pipeline_instance, 'instance'))
outputs = {}
except Exception as e:
inputs = {}
result = False
message = 'parser pipeline tree error: %s' % e
logger.exception(message)
outputs = {'ex_data': message}
if component_code:
outputs_table = []
try:
component = library.ComponentLibrary.get_component_class(
component_code)
outputs_format = component.outputs_format()
except Exception as e:
result = False
message = 'get component[component_code=%s] format error: %s' % (
component_code, e)
logger.exception(message)
outputs = {'ex_data': message}
else:
outputs_data = outputs.get('outputs', {})
# 在原子定义中的预设输出参数
archived_keys = []
for outputs_item in outputs_format:
value = outputs_data.get(outputs_item['key'], '')
outputs_table.append({
'name': outputs_item['name'],
'key': outputs_item['key'],
'value': value,
'preset': True,
})
archived_keys.append(outputs_item['key'])
# 其他输出参数
for out_key, out_value in outputs_data.items():
if out_key not in archived_keys:
outputs_table.append({
'name': out_key,
'key': out_key,
'value': out_value,
'preset': False,
})
else:
outputs_table = [{
'key': key,
'value': val,
'preset': False
} for key, val in outputs.get('outputs', {}).items()]
if not isinstance(inputs, dict):
inputs = {}
if not isinstance(outputs, dict):
outputs = {}
data = {
'inputs': inputs,
'outputs': outputs_table,
'ex_data': outputs.pop('ex_data', '')
}
return {
'result': result,
'data': data,
'message': '' if result else data['ex_data']
}