def setUp(self):
self.test_snapshot = Snapshot.objects.create_snapshot({})
self.test_snapshot.save()
self.test_project = Project.objects.create(
name="proj",
creator="creator",
)
self.test_project.save()
# prepare test data
instance_id = uniqid()
template_id = uniqid()
self.instance_in_pipeline = InstanceInPipeline.objects.create(
instance_id=instance_id,
atom_total=0,
subprocess_total=0,
gateways_total=0)
self.pipeline_template = PipelineTemplate.objects.create(
template_id=template_id,
creator="creator",
snapshot=self.test_snapshot)
self.pipeline_instance = PipelineInstance.objects.create(
instance_id=instance_id,
creator="creator",
snapshot=self.test_snapshot,
template=self.pipeline_template)
self.task_template = TaskTemplate.objects.create(
project=self.test_project,
pipeline_template=self.pipeline_template)
self.taskflow_instance = TaskFlowInstance.objects.create(
project=self.test_project,
pipeline_instance=self.pipeline_instance,
template_id=template_id)
def setUp(self):
self.name = "test"
self.task_template_name = "task_template_name"
self.creator = "tester"
self.extra_info = {"extra_info": "val"}
self.pipeline_tree = {
"constants": {
"key_1": {"value": "val_1", "show_type": "show"},
"key_2": {"value": "val_2", "show_type": "hide"},
},
"activities": {},
"start_event": {
"id": "id1",
"name": "",
"type": "EmptyStartEvent",
"incoming": None,
"outgoing": "flow_id1",
},
"end_event": {"id": "id2", "name": "", "type": "EmptyEndEvent", "incoming": "flow_id1", "outgoing": None},
"gateways": {},
"flows": {"flow_id1": {"id": "flow_id1", "source": "id1", "target": "id2"}},
}
self.project = Project.objects.create(name="test_project", time_zone="Asia/Shanghai", creator="test", desc="")
self.invalid_project = Project.objects.create(
name="invalid_project", time_zone="Asia/Shanghai", creator="test", desc=""
)
self.snapshot = Snapshot.objects.create_snapshot({})
self.pipeline_template = PipelineTemplate.objects.create(
template_id=uniqid(), name=self.task_template_name, creator=self.creator, snapshot=self.snapshot
)
task_template = TaskTemplate(project=self.project, pipeline_template=self.pipeline_template,)
task_template.save()
self.template = task_template
self.task = self.create_a_task()
def build_tree(start_elem, id=None, data=None, replace_id=False):
tree = copy.deepcopy(__skeleton)
elem_queue = Queue.Queue()
processed_elem = set()
tree[__incoming] = {}
elem_queue.put(start_elem)
while not elem_queue.empty():
# get elem
elem = elem_queue.get()
# update node when we meet again
if elem.id in processed_elem:
__update(tree, elem)
continue
# add to queue
for e in elem.outgoing:
elem_queue.put(e)
# mark as processed
processed_elem.add(elem.id)
# tree grow
__grow(tree, elem)
del tree[__incoming]
tree[PE.id] = id or uniqid()
user_data = data.to_dict() if isinstance(data, Data) else data
tree[PE.data] = user_data or tree[PE.data]
if replace_id:
replace_all_id(tree)
return tree
def __init__(self, **kwargs):
self.pipeline = kwargs.get('pipeline', PipelineObject(data=kwargs.get('pipeline_data'),
context=kwargs.get('pipeline_context')))
self.next = mock.MagicMock(return_value=kwargs.get('next', uniqid()))
self.data = kwargs.get('data', MockData())
self.prepare_rerun_data = mock.MagicMock()
super(MockSubprocessActivity, self).__init__(kwargs.get('id'))
def build_tree(start_elem, id=None, data=None):
tree = copy.deepcopy(__skeleton)
elem_queue = Queue.Queue()
processed_elem = set()
tree[__incoming] = {}
elem_queue.put(start_elem)
while not elem_queue.empty():
# get elem
elem = elem_queue.get()
# update node when we meet again
if elem.id in processed_elem:
__update(tree, elem)
continue
# add to queue
for e in elem.outgoing:
elem_queue.put(e)
# mark as processed
processed_elem.add(elem.id)
# tree grow
__grow(tree, elem)
del tree[__incoming]
tree[PE.id] = id or uniqid()
tree[PE.data] = data or tree[PE.data]
return tree
def forced_fail(node_id):
"""
forced fail a node
:param node_id:
:return:
"""
try:
process = PipelineProcess.objects.get(current_node_id=node_id)
except PipelineProcess.DoesNotExist:
return False
node = process.top_pipeline.node(node_id)
if not isinstance(node, ServiceActivity):
return False
result = Status.objects.transit(node_id, to_state=states.FAILED)
if not result:
return False
try:
node.failure_handler(process.root_pipeline.data)
except Exception as e:
pass
with transaction.atomic():
s = Status.objects.get(id=node.id)
ScheduleService.objects.delete_schedule(s.id, s.version)
Data.objects.forced_failed(node_id)
ProcessCeleryTask.objects.revoke(process.id)
process.sleep(adjust_status=True)
s.version = uniqid.uniqid()
s.save()
return True
def retry(self, process, node, inputs):
if RERUN_MAX_LIMIT != 0 and self.get(id=node.id).loop > RERUN_MAX_LIMIT:
return ActionResult(result=False, message='rerun times exceed max limit: {limit}, can not retry'.format(
limit=RERUN_MAX_LIMIT
))
action_res = self.transit(id=node.id, to_state=states.READY, appoint=True)
if not action_res.result:
return action_res
# add retry times
s = Status.objects.get(id=node.id)
s.version = uniqid()
history = History.objects.record(s)
LogEntry.objects.link_history(node_id=node.id, history_id=history.id)
s.retry += 1
s.save()
# update inputs
if inputs:
new_data = DataObject(inputs=inputs, outputs={})
node.data = new_data
Data.objects.write_node_data(node)
# mark
node.next_exec_is_retry()
self.recover_from_block(process.root_pipeline.id, process.subprocess_stack)
signals.node_retry_ready.send(sender=Status, process=process, node=node)
# because node may be updated
process.save()
return action_res
def retry(self, process, node, inputs):
action_res = self.transit(id=node.id,
to_state=states.READY,
appoint=True)
if not action_res.result:
return action_res
# add retry times
s = Status.objects.get(id=node.id)
s.version = uniqid()
history = History.objects.record(s)
LogEntry.objects.link_history(node_id=node.id, history_id=history.id)
s.retry += 1
s.save()
# update inputs
if inputs:
new_data = DataObject(inputs=inputs, outputs={})
node.data = new_data
Data.objects.write_node_data(node)
self.recover_from_block(process.root_pipeline.id,
process.subprocess_stack)
signals.node_retry_ready.send(sender=Status,
process=process,
node=node)
# because node may be updated
process.save()
return action_res
def delete_model(self, template_ids):
if not isinstance(template_ids, list):
template_ids = [template_ids]
qs = self.filter(template_id__in=template_ids)
for template in qs:
template.is_deleted = True
template.name = uniqid()
template.save()
def delete_model(self, instance_ids):
if not isinstance(instance_ids, list):
instance_ids = [instance_ids]
qs = self.filter(instance_id__in=instance_ids)
for instance in qs:
instance.is_deleted = True
instance.name = uniqid()
instance.save()
def setUp(self):
self.test_snapshot = Snapshot.objects.create_snapshot({})
self.test_snapshot.save()
self.test_project = Project.objects.create(
name="proj",
creator="creator",
)
self.test_project.save()
# prepare test data
instance_id = uniqid()
template_id = uniqid()
component_code = uniqid()
node_id = uniqid()
self.component_execute_data = ComponentExecuteData.objects.create(
component_code=component_code,
instance_id=instance_id,
node_id=node_id,
started_time=datetime(2021,
9,
18,
14,
57,
18,
609564,
tzinfo=timezone.utc),
)
self.pipeline_template = PipelineTemplate.objects.create(
template_id=template_id,
creator="creator",
snapshot=self.test_snapshot)
self.pipeline_instance = PipelineInstance.objects.create(
instance_id=instance_id,
creator="creator",
snapshot=self.test_snapshot,
template=self.pipeline_template)
self.taskflow_instance = TaskFlowInstance.objects.create(
project=self.test_project,
pipeline_instance=self.pipeline_instance,
template_id=template_id)
self.task_template = TaskTemplate.objects.create(
project=self.test_project,
pipeline_template=self.pipeline_template)
def __init__(self, id=None, **kwargs):
self.id = id or ('%s%s' % (uniqid(), uniqid()))
self.activity_id = self.id[:32]
self.version = self.id[32:]
self.destroy = mock.MagicMock()
self.service_act = ServiceActObject(interval=None,
id=self.activity_id,
schedule_return=kwargs.get('schedule_return'),
schedule_exception=kwargs.get('schedule_exception'),
timeout=kwargs.get('service_timeout'),
error_ignorable=kwargs.get('service_err_ignore', False),
is_schedule_done=kwargs.get('schedule_done', False),
result_bit=kwargs.get('result_bit', True))
self.callback_data = kwargs.get('callback_data', 'callback_data')
self.wait_callback = kwargs.get('wait_callback', False)
self.process_id = kwargs.get('process_id', uniqid())
self.is_finished = kwargs.get('is_finished', False)
self.schedule_times = 0
self.finish = mock.MagicMock()
self.set_next_schedule = mock.MagicMock()
self.callback = mock.MagicMock()
def delete_model(self, template_ids):
"""
删除模板对象
@param template_ids: 模板对象 ID 列表或 ID
@return:
"""
if not isinstance(template_ids, list):
template_ids = [template_ids]
qs = self.filter(template_id__in=template_ids)
for template in qs:
template.is_deleted = True
template.name = uniqid()
template.save()
def delete_model(self, instance_ids):
"""
删除流程实例对象
@param instance_ids: 实例 ID 或 ID 列表
@return:
"""
if not isinstance(instance_ids, list):
instance_ids = [instance_ids]
qs = self.filter(instance_id__in=instance_ids)
for instance in qs:
instance.is_deleted = True
instance.name = uniqid()
instance.save()
def setUp(self):
self.test_snapshot = Snapshot.objects.create_snapshot({})
self.test_snapshot.save()
self.test_project = Project.objects.create(
name="proj",
creator="creator",
)
self.test_project.save()
# prepare test data
template_id = uniqid()
component_code = uniqid()
node_id = uniqid()
self.component_in_template = ComponentInTemplate.objects.create(
component_code=component_code,
template_id=template_id,
node_id=node_id)
self.pipeline_template = PipelineTemplate.objects.create(
template_id=template_id,
creator="creator",
snapshot=self.test_snapshot)
self.task_template = TaskTemplate.objects.create(
project=self.test_project,
pipeline_template=self.pipeline_template)
def setUp(self):
self.name = 'test'
self.creator = 'tester'
self.extra_info = {'extra_info': 'val'}
self.pipeline_tree = {
'constants': {
'key_1': {
'value': 'val_1',
'show_type': 'show',
},
'key_2': {
'value': 'val_2',
'show_type': 'hide',
}
}
}
self.business = Business.objects.create(
cc_id=1,
cc_name='mock business',
cc_owner='tester',
cc_company='',
life_cycle='2',
executor='',
)
self.invalid_business = Business.objects.create(
cc_id=2,
cc_name='mock business',
cc_owner='tester',
cc_company='',
life_cycle='2',
executor='',
)
self.snapshot, _ = Snapshot.objects.create_or_get_snapshot({})
self.pipeline_template = PipelineTemplate.objects.create(
template_id=uniqid(),
name=self.name,
creator=self.creator,
snapshot=self.snapshot)
task_template = TaskTemplate(
business=self.business,
pipeline_template=self.pipeline_template,
)
task_template.save()
self.template = task_template
self.task = self.create_a_task()
def __init__(self,
loop=0,
id=None,
state=None,
started_time=None,
archived_time=None,
retry=False,
skip=False):
super(MockStatus, self).__init__(id=id)
self.version = uniqid()
self.loop = loop
self.state = state
self.started_time = started_time
self.archived_time = archived_time
self.retry = retry
self.skip = skip
self.save = MagicMock()
def __init__(self,
inputs,
parent_data,
execute_assertion,
schedule_assertion,
name='',
patchers=None,
execute_call_assertion=None,
schedule_call_assertion=None,
service_id=None):
self.inputs = inputs
self.parent_data = parent_data
self.execute_assertion = execute_assertion
self.execute_call_assertion = execute_call_assertion or []
self.schedule_call_assertion = schedule_call_assertion or []
self.schedule_assertion = schedule_assertion
self.name = name
self.patchers = patchers or []
self.service_id = service_id or uniqid()
def forced_fail(node_id, kill=False, ex_data=''):
"""
forced fail a node
:param node_id:
:param kill:
:param ex_data:
:return:
"""
try:
process = PipelineProcess.objects.get(current_node_id=node_id)
except PipelineProcess.DoesNotExist:
return ActionResult(
result=False,
message=
'invalid operation, this node is finished or pipeline have been revoked'
)
node = process.top_pipeline.node(node_id)
if not isinstance(node, ServiceActivity):
return ActionResult(result=False,
message='can\'t not forced fail this type of node')
action_result = Status.objects.transit(node_id, to_state=states.FAILED)
if not action_result.result:
return action_result
try:
node.failure_handler(process.root_pipeline.data)
except Exception:
pass
with transaction.atomic():
s = Status.objects.get(id=node.id)
ScheduleService.objects.delete_schedule(s.id, s.version)
Data.objects.forced_fail(node_id, ex_data)
ProcessCeleryTask.objects.revoke(process.id, kill)
process.sleep(adjust_status=True)
s.version = uniqid.uniqid()
s.save()
return ActionResult(result=True, message='success')
def setUp(self):
self.name = 'test'
self.task_template_name = 'task_template_name'
self.creator = 'tester'
self.extra_info = {'extra_info': 'val'}
self.pipeline_tree = {
'constants': {
'key_1': {
'value': 'val_1',
'show_type': 'show',
},
'key_2': {
'value': 'val_2',
'show_type': 'hide',
}
}
}
self.project = Project.objects.create(name='test_project',
time_zone='Asia/Shanghai',
creator='test',
desc='')
self.invalid_project = Project.objects.create(
name='invalid_project',
time_zone='Asia/Shanghai',
creator='test',
desc='')
self.snapshot, _ = Snapshot.objects.create_or_get_snapshot({})
self.pipeline_template = PipelineTemplate.objects.create(
template_id=uniqid(),
name=self.task_template_name,
creator=self.creator,
snapshot=self.snapshot)
task_template = TaskTemplate(
project=self.project,
pipeline_template=self.pipeline_template,
)
task_template.save()
self.template = task_template
self.task = self.create_a_task()
def convert_atom_from_v2_step_to_v3_act(step, constants):
act_id = uniqid()
v3_act = {
'id': act_id,
'incoming': '',
'outgoing': '',
'name': step['step_name'],
'error_ignorable': bool(step['is_ignore']),
'optional': bool(step['is_adjust']),
'type': 'ServiceActivity',
'loop': 1,
'component': {
'code': '',
'data': {}
}
}
tag_code = step['tag_code']
component_code = component_code_v2_to_v3.get(tag_code)
if not component_code:
raise Exception("unknown tag code: %s" % tag_code)
data = step['tag_data']['data']
tag_data = {}
if tag_code == 'requests':
mount_constant(act_id, tag_code, data, constants)
for key, val in data.items():
hook = True if val['hook'] == 'on' else False
tag_data[tag_v2_to_v3[tag_code][key]] = {
'hook': hook,
'value': val['constant'] if hook else val['value']
}
# TODO another tag
v3_act['component']['code'] = component_code
v3_act['component']['data'] = tag_data
return v3_act
def retry(self, process, node, inputs):
result = self.transit(id=node.id, to_state=states.READY, appoint=True)
if not result:
return result
# add retry times
s = Status.objects.get(id=node.id)
s.version = uniqid()
history = History.objects.record(s)
LogEntry.objects.link_history(node_id=node.id, history_id=history.id)
s.retry += 1
s.save()
# update inputs
if inputs:
new_data = DataObject(inputs=inputs, outputs={})
node.data = new_data
Data.objects.write_node_data(node)
process.save()
self.recover_from_block(process.root_pipeline.id,
process.subprocess_stack)
return result
def create_task(self,
name,
template,
cron,
data,
creator,
timezone=None,
extra_info=None,
spread=False):
snapshot, _ = Snapshot.objects.create_or_get_snapshot(data)
schedule, _ = CrontabSchedule.objects.get_or_create(
minute=cron.get('minute', '*'),
hour=cron.get('hour', '*'),
day_of_week=cron.get('day_of_week', '*'),
day_of_month=cron.get('day_of_month', '*'),
month_of_year=cron.get('month_of_year', '*'),
timezone=timezone or 'UTC')
_ = schedule.schedule # noqa
task = self.create(name=name,
template=template,
snapshot=snapshot,
cron=schedule.__str__(),
creator=creator,
extra_info=extra_info)
kwargs = {'period_task_id': task.id, 'spread': spread}
celery_task = DjCeleryPeriodicTask.objects.create(
crontab=schedule,
name=uniqid(),
task='pipeline.contrib.periodic_task.tasks.periodic_task_start',
enabled=False,
kwargs=json.dumps(kwargs))
task.celery_task = celery_task
task.save()
return task
def main_test():
pipe1 = {
"id": uniqid(),
"activities": {
"d4b43834dbea0981f07db76d3c2d12ce": {
"outgoing": "65085d5739b76b8c3c0152ca1e845a6f",
"incoming": "30722fa74d29e61ce9289c5029a48a49",
"name": "\u8282\u70b9_1",
"error_ignorable": False,
"component": {
"code": "job_fast_execute_script",
"data": {
"job_account": {
"hook": False,
"value": "root"
},
"job_script_timeout": {
"hook": False,
"value": "600"
},
"job_ip_list": {
"hook": False,
"value": "${ip}"
},
"job_content": {
"hook": False,
"value": "echo 1"
},
"job_script_type": {
"hook": False,
"value": "1"
},
"job_script_param": {
"hook": False,
"value": ""
}
}
},
"optional": False,
"type": "ServiceActivity",
"id": "d4b43834dbea0981f07db76d3c2d12ce",
"loop": None
}
},
"end_event": {
"type": "EmptyEndEvent",
"outgoing": "",
"incoming": "65085d5739b76b8c3c0152ca1e845a6f",
"id": "ad3d6364cbf1f3ff4b0a9d0c98a99d99",
"name": ""
},
"outputs": [],
"flows": {
"65085d5739b76b8c3c0152ca1e845a6f": {
"is_default": False,
"source": "d4b43834dbea0981f07db76d3c2d12ce",
"id": "65085d5739b76b8c3c0152ca1e845a6f",
"target": "ad3d6364cbf1f3ff4b0a9d0c98a99d99"
},
"30722fa74d29e61ce9289c5029a48a49": {
"is_default": False,
"source": "3e8f33f335b6695201cc259fb5f365e0",
"id": "30722fa74d29e61ce9289c5029a48a49",
"target": "d4b43834dbea0981f07db76d3c2d12ce"
}
},
"start_event": {
"type": "EmptyStartEvent",
"outgoing": "30722fa74d29e61ce9289c5029a48a49",
"incoming": "",
"id": "3e8f33f335b6695201cc259fb5f365e0",
"name": ""
},
"constants": {
"${ip}": {
"source_tag": "var_ip_picker",
"source_info": {},
"name": "ip",
"index": 0,
"custom_type": "ip",
"value": {
"var_ip_picker": {
"var_ip_custom_value": "",
"var_ip_select_module": ["111"],
"var_ip_input_set": "",
"var_ip_value_type": "ip",
"var_ip_select_set": ["2"],
"var_ip_input_module": "",
"var_ip_method": "select"
}
},
"show_type": "hide",
"source_type": "custom",
"key": "${ip}",
"desc": ""
}
},
"gateways": {}
}
parser_obj = WebPipelineAdapter(pipe1)
run_pipeline(parser_obj.parser())
def transit(self,
id,
to_state,
is_pipeline=False,
appoint=False,
start=False,
name='',
version=None):
"""
尝试改变某个节点的状态
:param id: 节点 ID
:param to_state: 目标状态
:param is_pipeline: 该节点是否是 pipeline
:param appoint: 该动作是否由用户发起（非引擎内部操作）
:param start: 是否刷新其开始事件
:param name: 节点名称
:param version: 节点版本
:return:
"""
defaults = {'name': name, 'state': to_state, 'version': uniqid()}
if start:
defaults['started_time'] = timezone.now()
_, created = self.get_or_create(id=id, defaults=defaults)
# reservation or first creation
if created:
return True
with transaction.atomic():
kwargs = {'id': id}
if version:
kwargs['version'] = version
try:
status = self.select_for_update().get(**kwargs)
except Status.DoesNotExist:
return False
if states.can_transit(from_state=status.state,
to_state=to_state,
is_pipeline=is_pipeline,
appoint=appoint):
# 在冻结状态下不能改变 pipeline 的状态
if is_pipeline:
subprocess_rel = SubProcessRelationship.objects.filter(
subprocess_id=id)
if subprocess_rel:
process = PipelineProcess.objects.get(
id=subprocess_rel[0].process_id)
if process.is_frozen:
return False
processes = PipelineProcess.objects.filter(
root_pipeline_id=id)
if processes and processes[0].is_frozen:
return False
status.state = to_state
if name:
status.name = name
if start:
status.started_time = timezone.now()
if to_state in states.ARCHIVED_STATES:
status.archived_time = timezone.now()
status.save()
return True
else:
return False
def __init__(self, id=None, node=None):
self.next = mock.MagicMock(return_value=node or uniqid())
super(StartEventObject, self).__init__(id=id)
def __init__(self, **kwargs):
self.outgoing = kwargs.get('outgoing', MockSequenceCollection())
self.converge_gateway_id = kwargs.get('converge_gateway_id', uniqid())
def transit(self,
id,
to_state,
is_pipeline=False,
appoint=False,
start=False,
name='',
version=None,
unchanged_pass=False):
"""
尝试改变某个节点的状态
:param id: 节点 ID
:param to_state: 目标状态
:param is_pipeline: 该节点是否是 pipeline
:param appoint: 该动作是否由用户发起（非引擎内部操作）
:param start: 是否刷新其开始时间
:param name: 节点名称
:param version: 节点版本
:param unchanged_pass: 当 to_state 与当前节点状态相同时则视为操作成功
:return:
"""
defaults = {'name': name, 'state': to_state, 'version': uniqid()}
if start:
defaults['started_time'] = timezone.now()
status, created = self.get_or_create(id=id, defaults=defaults)
# reservation or first creation
if created:
return ActionResult(result=True, message='success', extra=status)
with transaction.atomic():
kwargs = {'id': id}
if version:
kwargs['version'] = version
try:
status = self.select_for_update().get(**kwargs)
except Status.DoesNotExist:
return ActionResult(
result=False,
message='node not exists or not be executed yet')
if unchanged_pass and status.state == to_state:
return ActionResult(result=True,
message='success',
extra=status)
if states.can_transit(from_state=status.state,
to_state=to_state,
is_pipeline=is_pipeline,
appoint=appoint):
# 在冻结状态下不能改变 pipeline 的状态
if is_pipeline:
subprocess_rel = SubProcessRelationship.objects.filter(
subprocess_id=id)
if subprocess_rel:
process = PipelineProcess.objects.get(
id=subprocess_rel[0].process_id)
if process.is_frozen:
return ActionResult(
result=False,
message=
'engine is frozen, can not perform operation')
processes = PipelineProcess.objects.filter(
root_pipeline_id=id)
if processes and processes[0].is_frozen:
return ActionResult(
result=False,
message=
'engine is frozen, can not perform operation')
if name:
status.name = name
if start:
status.started_time = timezone.now()
if to_state in states.ARCHIVED_STATES:
status.archived_time = timezone.now()
# from FINISHED to RUNNING
if states.is_rerunning(from_state=status.state,
to_state=to_state):
history = History.objects.record(status, is_rerunning=True)
if history:
LogEntry.objects.link_history(node_id=status.id,
history_id=history.id)
status.loop += 1
status.skip = False
status.version = uniqid()
status.state = to_state
status.save()
return ActionResult(result=True,
message='success',
extra=status)
else:
return ActionResult(
result=False,
message='can\'t transit state(%s) from %s to %s' %
(id, status.state, to_state),
extra=status)
def setUp(self):
self.subprocess_id = uniqid()
self.process_id = uniqid()
def __init__(self, id=None, name=''):
self.id = id or uniqid()
self.name = name
