def export_templates(self, template_id_list):
templates = list(self.filter(id__in=template_id_list).select_related("pipeline_template").values())
pipeline_template_id_list = []
template = {}
for tmpl in templates:
pipeline_template_id_list.append(tmpl["pipeline_template_id"])
tmpl["pipeline_template_str_id"] = tmpl["pipeline_template_id"]
template[tmpl["id"]] = tmpl
try:
pipeline_temp_data = PipelineTemplateWebWrapper.export_templates(pipeline_template_id_list)
except SubprocessExpiredError as e:
raise FlowExportError(str(e))
all_template_ids = set(pipeline_temp_data["template"].keys())
additional_template_id = all_template_ids - set(pipeline_template_id_list)
subprocess_temp_list = list(
self.filter(pipeline_template_id__in=additional_template_id).select_related("pipeline_template").values()
)
for sub_temp in subprocess_temp_list:
sub_temp["pipeline_template_str_id"] = sub_temp["pipeline_template_id"]
template[sub_temp["id"]] = sub_temp
result = {
"template": template,
"pipeline_template_data": pipeline_temp_data,
"exporter_version": TEMPLATE_EXPORTER_VERSION,
}
return result
def export_templates(self, template_id_list):
templates = self.filter(id__in=template_id_list).select_related(
'pipeline_template').values()
pipeline_template_id_list = []
template = {}
for tmpl in templates:
pipeline_template_id_list.append(tmpl['pipeline_template_id'])
tmpl['pipeline_template_str_id'] = tmpl['pipeline_template_id']
template[tmpl['id']] = tmpl
try:
pipeline_temp_data = PipelineTemplateWebWrapper.export_templates(
pipeline_template_id_list)
except SubprocessExpiredError as e:
raise FlowExportError(e.message)
all_template_ids = set(pipeline_temp_data['template'].keys())
additional_template_id = all_template_ids - set(
pipeline_template_id_list)
subprocess_temp_list = self.filter(pipeline_template_id__in=additional_template_id) \
.select_related('pipeline_template') \
.values()
for sub_temp in subprocess_temp_list:
sub_temp['pipeline_template_str_id'] = sub_temp[
'pipeline_template_id']
template[sub_temp['id']] = sub_temp
result = {
'template': template,
'pipeline_template_data': pipeline_temp_data
}
return result
def find_deprecated_plugins_in_unfold_tree(tree, template_model, phases=None):
"""查找子流程未展开的树中已经下线的插件
:param tree: 子流程未展开的树
:type tree: dict
:param template_model: 子流程引用的模板模型
:type template_model: Model
:return: {
"found": True or False,
"plugins": {
"activities": [
{
"id": "act_id",
"name": "act_name",
"component": "component_code",
"version": "component_version",
"subprocess": "subprocess_name"
},
...
],
"variables": [
{
"key": "var_key",
"name": "var_name",
"custom_type": "var_code",
"version": "var version",
"subprocess": "subprocess_name"
},
...
]
}
}
:rtype: dict
"""
check_tree = copy.deepcopy(tree)
# replace template id to pipeline id
replace_template_id(template_model, check_tree)
# unfold subprocess reference
PipelineTemplateWebWrapper.unfold_subprocess(check_tree, template_model)
phases = phases or [DeprecatedPlugin.PLUGIN_PHASE_DEPRECATED]
return find_deprecated_plugins_in_spread_tree(tree=check_tree,
phases=phases)
def _perform_import(self, template_data, check_info, override,
defaults_getter, resource):
template = template_data['template']
tid_to_reuse = {}
# find old template_id for override using
# import_id -> reuse_id
for template_to_be_replaced in check_info['override_template']:
task_template_id = template_to_be_replaced['id']
template_id = template_data['template'][str(
task_template_id)]['pipeline_template_str_id']
tid_to_reuse[template_id] = template_to_be_replaced['template_id']
# import pipeline template first
id_map = PipelineTemplateWebWrapper.import_templates(
template_data['pipeline_template_data'],
override=override,
tid_to_reuse=tid_to_reuse)
old_id_to_new_id = id_map[PipelineTemplateWebWrapper.ID_MAP_KEY]
new_objects = []
new_objects_template_ids = set()
# find templates which had been deleted
if override:
new_objects_template_ids = set(
self.model.objects.filter(id__in=template.keys(),
is_deleted=True).values_list(
'pipeline_template_id',
flat=True))
for tid, template_dict in template.items():
template_dict['pipeline_template_id'] = old_id_to_new_id[
template_dict['pipeline_template_str_id']]
defaults = defaults_getter(template_dict)
# use update or create to avoid id conflict
if override:
obj, created = self.update_or_create(id=tid, defaults=defaults)
if created:
new_objects_template_ids.add(
template_dict['pipeline_template_id'])
else:
new_objects.append(self.model(**defaults))
new_objects_template_ids.add(
template_dict['pipeline_template_id'])
self.model.objects.bulk_create(new_objects)
create_templates = list(
self.model.objects.filter(
pipeline_template_id__in=new_objects_template_ids))
if create_templates:
resource.batch_register_instance(create_templates)
return {
'result': True,
'data': len(template),
'message': 'Successfully imported %s flows' % len(template)
}
def create_pipeline_instance(template, **kwargs):
pipeline_tree = kwargs['pipeline_tree']
replace_template_id(template.__class__, pipeline_tree)
pipeline_template_data = {
'name': kwargs['name'],
'creator': kwargs['creator'],
'description': kwargs.get('description', ''),
}
PipelineTemplateWebWrapper.unfold_subprocess(pipeline_tree)
pipeline_instance = PipelineInstance.objects.create_instance(
template.pipeline_template,
pipeline_tree,
spread=True,
**pipeline_template_data)
return pipeline_instance
def test_always_use_latest(self):
layer_1_t1_tree = {
"activities": {},
"constants": {
"${param}": {
"value": ""
},
"${c1}": {
"value": "constant_value"
}
},
}
# prepare pipeline data
pipeline_data = {
"activities": {
"subproc_1": {
"type": "SubProcess",
"template_id": "layer_1_t1",
"version": "v1",
"always_use_latest": True,
"constants": {
"${param}": {
"value": "${parent_param}"
}
},
}
},
"constants": {
"${parent_param}": "${another_constants}"
},
}
# prepare template model mock
template_model = MagicMock()
get_return = MagicMock()
get_return.get_pipeline_tree_by_version = MagicMock(
return_value=layer_1_t1_tree)
template_model.objects.get = MagicMock(return_value=get_return)
PipelineTemplateWebWrapper.unfold_subprocess(pipeline_data,
template_model)
template_model.objects.get.assert_called_once_with(
pipeline_template__template_id="layer_1_t1")
get_return.get_pipeline_tree_by_version.assert_called_once_with(None)
def import_templates(self, template_data, override):
template = template_data['template']
check_info = self.import_operation_check(template_data)
tid_to_reuse = {}
# operation validation check
if override and (not check_info['can_override']):
return {
'result': False,
'message':
'Unable to override common flows or keep ID when importing business flows data',
'data': 0
}
# find old template_id for override using
# import_id -> reuse_id
for template_to_be_replaced in check_info['override_template']:
task_template_id = template_to_be_replaced['id']
template_id = template_data['template'][str(
task_template_id)]['pipeline_template_str_id']
tid_to_reuse[template_id] = template_to_be_replaced['template_id']
# import pipeline template first
id_map = PipelineTemplateWebWrapper.import_templates(
template_data['pipeline_template_data'],
override=override,
tid_to_reuse=tid_to_reuse)
old_id_to_new_id = id_map[PipelineTemplateWebWrapper.ID_MAP_KEY]
new_objects = []
for tid, tmpl_dict in template.items():
tmpl_dict['pipeline_template_id'] = old_id_to_new_id[
tmpl_dict['pipeline_template_str_id']]
defaults = {
'category': tmpl_dict['category'],
'notify_type': tmpl_dict['notify_type'],
'notify_receivers': tmpl_dict['notify_receivers'],
'time_out': tmpl_dict['time_out'],
'pipeline_template_id': tmpl_dict['pipeline_template_id'],
'is_deleted': False
}
# use update or create to avoid id conflict
if override:
self.update_or_create(id=tid, defaults=defaults)
else:
new_objects.append(self.model(**defaults))
self.model.objects.bulk_create(new_objects)
return {
'result': True,
'data': {
'count': len(template)
},
'message': 'Successfully imported %s common flows' % len(template)
}
def create_pipeline_task(self,
project,
template,
name,
cron,
pipeline_tree,
creator,
template_source=PROJECT):
if template_source == PROJECT and template.project.id != project.id:
raise InvalidOperationException(
"template %s do not belong to project[%s]" %
(template.id, project.name))
PipelineTemplateWebWrapper.unfold_subprocess(pipeline_tree,
template.__class__)
PipelineTemplate.objects.replace_id(pipeline_tree)
extra_info = {
"project_id": project.id,
"category": template.category,
"template_id": template.pipeline_template.template_id,
"template_source": template_source,
"template_num_id": template.id,
"pipeline_formator":
"pipeline_web.parser.format.format_web_data_to_pipeline",
"engine_ver": EngineConfig.ENGINE_VER_V2,
}
queue = settings.PERIODIC_TASK_QUEUE_NAME_V2
trigger_task = BAMBOO_ENGINE_TRIGGER_TASK
return PipelinePeriodicTask.objects.create_task(
name=name,
template=template.pipeline_template,
cron=cron,
data=pipeline_tree,
creator=creator,
timezone=project.time_zone,
extra_info=extra_info,
spread=True,
queue=queue,
trigger_task=trigger_task,
)
def create_pipeline_task(self, business, template, name, cron,
pipeline_tree, creator):
if template.business.id != business.id:
raise InvalidOperationException(
'template %s do not belong to business[%s]' %
(template.id, business.cc_name))
extra_info = {
'business_id': business.id,
'category': template.category,
'template_id': template.pipeline_template.template_id,
'template_num_id': template.id
}
PipelineTemplateWebWrapper.unfold_subprocess(pipeline_tree)
return PipelinePeriodicTask.objects.create_task(
name=name,
template=template.pipeline_template,
cron=cron,
data=pipeline_tree,
creator=creator,
timezone=business.time_zone,
extra_info=extra_info,
spread=True)
def template_wrapper(self):
return PipelineTemplateWebWrapper(self.pipeline_template)
def _perform_import(self, template_data, check_info, override, defaults_getter, operator):
template = template_data["template"]
tid_to_reuse = {}
# find old template_id for override using
# import_id -> reuse_id
for template_to_be_replaced in check_info["override_template"]:
task_template_id = template_to_be_replaced["id"]
template_id = template_data["template"][str(task_template_id)]["pipeline_template_str_id"]
tid_to_reuse[template_id] = template_to_be_replaced["template_id"]
# import pipeline template first
id_map = PipelineTemplateWebWrapper.import_templates(
template_data["pipeline_template_data"], override=override, tid_to_reuse=tid_to_reuse
)
old_id_to_new_id = id_map[PipelineTemplateWebWrapper.ID_MAP_KEY]
new_objects = []
new_objects_template_ids = set()
# find templates which had been deleted
if override:
new_objects_template_ids = set(
self.model.objects.filter(id__in=list(template.keys()), is_deleted=True).values_list(
"pipeline_template_id", flat=True
)
)
for tid, template_dict in list(template.items()):
template_dict["pipeline_template_id"] = old_id_to_new_id[template_dict["pipeline_template_str_id"]]
defaults = defaults_getter(template_dict)
# use update or create to avoid id conflict
if override:
obj, created = self.update_or_create(id=tid, defaults=defaults)
if created:
new_objects_template_ids.add(template_dict["pipeline_template_id"])
else:
new_objects.append(self.model(**defaults))
new_objects_template_ids.add(template_dict["pipeline_template_id"])
# update creator when templates are created
PipelineTemplate.objects.filter(template_id__in=new_objects_template_ids).update(creator=operator)
# update flows map
flows = {info["id"]: info["name"] for info in check_info["new_template"]}
if not override:
self.model.objects.bulk_create(new_objects)
create_templates = list(self.model.objects.filter(pipeline_template_id__in=new_objects_template_ids))
# create flows map
flows = {tmp.id: tmp.name for tmp in create_templates}
# send_signal
if create_templates:
batch_create.send(self.model, instance=create_templates, creator=operator)
return {
"result": True,
"data": {"count": len(template), "flows": flows},
"message": "Successfully imported %s flows" % len(template),
"code": err_code.SUCCESS.code,
}
def test_unfold_subprocess_with_schemes(self):
layer_1_t1_tree = {
"activities": {
"t1_tree_node_1": {
"type": "ServiceActivity",
"optional": True
},
"t1_tree_node_2": {
"type": "ServiceActivity",
"optional": True
},
"t1_tree_node_3": {
"type": "ServiceActivity",
"optional": True
},
"t1_tree_node_4": {
"type": "ServiceActivity",
"optional": True
},
},
"constants": {
"${parent_param2}": {
"value": ""
},
"${c1}": {
"value": "constant_value_1"
}
},
}
# prepare pipeline data
pipeline_data = {
"activities": {
"subproc_1": {
"type": "SubProcess",
"template_id": "layer_1_t1",
"version": "v1",
"scheme_id_list": [1, 2, 3],
"constants": {
"${parent_param2}": {
"value": "${parent_param1}"
}
},
}
},
"constants": {
"${parent_param1}": "${another_constants}"
},
}
def get_pipeline_tree_by_version(v):
return {"v1": layer_1_t1_tree}[v]
# prepare template model mock
template_model = MagicMock()
get_return = MagicMock()
get_return.get_pipeline_tree_by_version = MagicMock(
side_effect=get_pipeline_tree_by_version)
template_model.objects.get = MagicMock(return_value=get_return)
PipelineTemplateWebWrapper.unfold_subprocess(pipeline_data,
template_model)
template_model.objects.get.assert_called_once_with(
pipeline_template__template_id="layer_1_t1")
get_return.get_pipeline_tree_by_version.assert_called_once_with("v1")
MockPipelineTemplateWebPreviewer.get_template_exclude_task_nodes_with_schemes.assert_called(
)
MockPipelineTemplateWebPreviewer.preview_pipeline_tree_exclude_task_nodes.assert_called_once_with(
{
"activities": {
"t1_tree_node_2": {
"type": "ServiceActivity",
"optional": True
},
"t1_tree_node_4": {
"type": "ServiceActivity",
"optional": True
},
},
"constants": {
"${parent_param2}": {
"value": "${parent_param1}"
},
"${c1}": {
"value": "constant_value_1"
},
},
"id": "subproc_1",
},
["t1_tree_node_1", "t1_tree_node_3"],
False,
)
self.assertDictEqual(
pipeline_data,
{
"activities": {
"subproc_1": {
"type": "SubProcess",
"template_id": "layer_1_t1",
"version": "v1",
"scheme_id_list": [1, 2, 3],
"pipeline": {
"activities": {
"t1_tree_node_2": {
"type": "ServiceActivity",
"optional": True
},
"t1_tree_node_4": {
"type": "ServiceActivity",
"optional": True
},
},
"constants": {
"${parent_param2}": {
"value": "${parent_param1}"
},
"${c1}": {
"value": "constant_value_1"
},
},
"id": "subproc_1",
},
}
},
"constants": {
"${parent_param1}": "${another_constants}"
},
},
)
def test_unfold_3_layer_subprocess(self):
layer_3_t1_tree = {
"activities": {
"subproc_2": {
"type": "SubProcess",
"template_id": "layer_3_t2",
"version": "v2",
"constants": {
"${parent_param3}": {
"value": "${parent_param2}"
}
},
}
},
"constants": {
"${parent_param2}": {
"value": ""
},
"${c1}": {
"value": "constant_value_1"
}
},
}
layer_3_t2_tree = {
"activities": {
"subproc_3": {
"type": "SubProcess",
"template_id": "layer_3_t3",
"version": "v3",
"constants": {
"${param}": {
"value": "${parent_param3}"
}
},
}
},
"constants": {
"${parent_param3}": {
"value": ""
},
"${c2}": {
"value": "constant_value_2"
}
},
}
layer_3_t3_tree = {
"activities": {},
"constants": {
"${param}": {
"value": ""
},
"${c3}": {
"value": "constant_value_3"
}
},
}
# prepare pipeline data
pipeline_data = {
"activities": {
"subproc_1": {
"type": "SubProcess",
"template_id": "layer_3_t1",
"version": "v1",
"constants": {
"${parent_param2}": {
"value": "${parent_param1}"
}
},
}
},
"constants": {
"${parent_param1}": "${another_constants}"
},
}
def get_pipeline_tree_by_version(v):
return {
"v1": layer_3_t1_tree,
"v2": layer_3_t2_tree,
"v3": layer_3_t3_tree
}[v]
# prepare template model mock
template_model = MagicMock()
get_return = MagicMock()
get_return.get_pipeline_tree_by_version = MagicMock(
side_effect=get_pipeline_tree_by_version)
template_model.objects.get = MagicMock(return_value=get_return)
PipelineTemplateWebWrapper.unfold_subprocess(pipeline_data,
template_model)
template_model.objects.get.assert_has_calls([
call(pipeline_template__template_id="layer_3_t1"),
call(pipeline_template__template_id="layer_3_t2"),
call(pipeline_template__template_id="layer_3_t3"),
])
get_return.get_pipeline_tree_by_version.assert_has_calls(
[call("v1"), call("v2"), call("v3")])
self.assertDictEqual(
pipeline_data,
{
"activities": {
"subproc_1": {
"type": "SubProcess",
"template_id": "layer_3_t1",
"version": "v1",
"pipeline": {
"id": "subproc_1",
"activities": {
"subproc_2": {
"type": "SubProcess",
"template_id": "layer_3_t2",
"version": "v2",
"pipeline": {
"id": "subproc_2",
"activities": {
"subproc_3": {
"type": "SubProcess",
"template_id": "layer_3_t3",
"version": "v3",
"pipeline": {
"id": "subproc_3",
"activities": {},
"constants": {
"${param}": {
"value":
"${parent_param3}"
},
"${c3}": {
"value":
"constant_value_3"
},
},
},
}
},
"constants": {
"${parent_param3}": {
"value": "${parent_param2}"
},
"${c2}": {
"value": "constant_value_2"
},
},
},
}
},
"constants": {
"${parent_param2}": {
"value": "${parent_param1}"
},
"${c1}": {
"value": "constant_value_1"
},
},
},
}
},
"constants": {
"${parent_param1}": "${another_constants}"
},
},
)
