def test_simple_schedule_pipeline():
start_event_a_id = node_uniqid()
act_b_id = node_uniqid()
end_event_c_id = node_uniqid()
start_event = EmptyStartEvent(start_event_a_id)
act = ServiceActivity(act_b_id,
service=ScheduleService(),
data=DataObject({}))
end_event = EmptyEndEvent(end_event_c_id)
flow_ab = SequenceFlow('ab', start_event, act)
flow_bc = SequenceFlow('bc', act, end_event)
start_event.outgoing.add_flow(flow_ab)
act.incoming.add_flow(flow_ab)
act.outgoing.add_flow(flow_bc)
end_event.incoming.add_flow(flow_bc)
spec = PipelineSpec(start_event,
end_event, [flow_ab, flow_bc], [act], [],
data=DataObject({}),
context=context.Context({}))
pipeline = Pipeline(node_uniqid(), spec)
run_pipeline(pipeline)
def subprocess():
start_event_a_id = node_uniqid()
act_b_id = node_uniqid()
end_event_c_id = node_uniqid()
start_event = EmptyStartEvent(start_event_a_id)
act = ServiceActivity(act_b_id,
service=RetryTestService(),
data=DataObject({'data': '0'}))
end_event = EmptyEndEvent(end_event_c_id)
flow_ab = SequenceFlow('ab', start_event, act)
flow_bc = SequenceFlow('bc', act, end_event)
start_event.outgoing.add_flow(flow_ab)
act.incoming.add_flow(flow_ab)
act.outgoing.add_flow(flow_bc)
end_event.incoming.add_flow(flow_bc)
spec = PipelineSpec(start_event,
end_event, [flow_ab, flow_bc], [act], [],
data=DataObject({}),
context=context.Context({}))
pipeline = Pipeline(node_uniqid(), spec)
return pipeline
def get_simple_pipeline():
start_event_a_id = node_uniqid()
act_b_id = node_uniqid()
end_event_c_id = node_uniqid()
start_event = EmptyStartEvent(start_event_a_id)
act = ServiceActivity(id=act_b_id,
service=RetryTestService(),
data=DataObject({
'data': '0',
'value2': '2',
'value3': '3',
'timing': 1000
}))
end_event = EmptyEndEvent(end_event_c_id)
flow_ab = SequenceFlow('ab', start_event, act)
flow_bc = SequenceFlow('bc', act, end_event)
start_event.outgoing.add_flow(flow_ab)
act.incoming.add_flow(flow_ab)
act.outgoing.add_flow(flow_bc)
end_event.incoming.add_flow(flow_bc)
spec = PipelineSpec(start_event,
end_event, [flow_ab, flow_bc], [act], [],
data=DataObject({}),
context=context.Context({}))
return Pipeline(node_uniqid(), spec)
def test_inputs_copy(self):
inputs = {'args': '1', 'kwargs': {'1': 1, '2': 2}}
data_object = DataObject(inputs=inputs)
inputs_copy = data_object.inputs_copy()
self.assertIsInstance(inputs_copy, FancyDict)
self.assertEqual(inputs_copy, inputs)
self.assertFalse(inputs is inputs_copy)
def test_outputs_copy(self):
outputs = {'args': '1', 'kwargs': {'1': 1, '2': 2}}
data_object = DataObject(inputs={}, outputs=outputs)
outputs_copy = data_object.outputs_copy()
self.assertIsInstance(outputs_copy, FancyDict)
self.assertEqual(outputs_copy, outputs)
self.assertFalse(outputs_copy is outputs)
def test_write_node_data(self):
node = IdentifyObject()
data_obj = DataObject({'input_key': 'value'},
outputs={'output_key': 'value'})
node.data = data_obj
Data.objects.write_node_data(node)
data = Data.objects.get(id=node.id)
self.assertEqual(data.inputs, data_obj.inputs)
self.assertEqual(data.outputs, data_obj.outputs)
self.assertIsNone(data.ex_data)
data_obj.inputs = {'new_inputs': 'new_value'}
Data.objects.write_node_data(node, ex_data='ex_data')
data = Data.objects.get(id=node.id)
self.assertEqual(data.inputs, data_obj.inputs)
self.assertEqual(data.outputs, data_obj.outputs)
self.assertEqual(data.ex_data, 'ex_data')
data_obj.outputs.ex_data = 'new_ex_data'
Data.objects.write_node_data(node, ex_data='ex_data')
data = Data.objects.get(id=node.id)
self.assertEqual(data.inputs, data_obj.inputs)
self.assertEqual(data.outputs, data_obj.outputs)
self.assertEqual(data.ex_data, 'new_ex_data')
def test_sequence_flow(self):
flow_id = '1'
source = ServiceActivity(id='1', service=None, data=DataObject({}))
target = ServiceActivity(id='2', service=None, data=DataObject({}))
flow = SequenceFlow(flow_id, source, target)
self.assertTrue(isinstance(flow, FlowElement))
self.assertEqual(flow_id, flow.id)
self.assertEqual(source, flow.source)
self.assertEqual(target, flow.target)
self.assertEqual(False, flow.is_default)
def test_service_activity(self):
act_id = '1'
service = FabricService()
inputs = {'args': [1, 2, 3], 'kwargs': {'1': 1, '2': 2}}
service_act = ServiceActivity(id=act_id,
service=service,
data=DataObject(inputs))
service_act.execute(DataObject(inputs))
print(service_act.get_result_bit())
self.assertTrue(isinstance(service_act, Activity))
self.assertEqual(service, service_act.service)
def retry(self, process, node, inputs):
action_res = self.transit(id=node.id,
to_state=states.READY,
appoint=True)
if not action_res.result:
return action_res
# add retry times
s = Status.objects.get(id=node.id)
s.version = uniqid()
history = History.objects.record(s)
LogEntry.objects.link_history(node_id=node.id, history_id=history.id)
s.retry += 1
s.save()
# update inputs
if inputs:
new_data = DataObject(inputs=inputs, outputs={})
node.data = new_data
Data.objects.write_node_data(node)
self.recover_from_block(process.root_pipeline.id,
process.subprocess_stack)
signals.node_retry_ready.send(sender=Status,
process=process,
node=node)
# because node may be updated
process.save()
return action_res
def retry(self, process, node, inputs):
if RERUN_MAX_LIMIT != 0 and self.get(id=node.id).loop > RERUN_MAX_LIMIT:
return ActionResult(result=False, message='rerun times exceed max limit: {limit}, can not retry'.format(
limit=RERUN_MAX_LIMIT
))
action_res = self.transit(id=node.id, to_state=states.READY, appoint=True)
if not action_res.result:
return action_res
# add retry times
s = Status.objects.get(id=node.id)
s.version = uniqid()
history = History.objects.record(s)
LogEntry.objects.link_history(node_id=node.id, history_id=history.id)
s.retry += 1
s.save()
# update inputs
if inputs:
new_data = DataObject(inputs=inputs, outputs={})
node.data = new_data
Data.objects.write_node_data(node)
# mark
node.next_exec_is_retry()
self.recover_from_block(process.root_pipeline.id, process.subprocess_stack)
signals.node_retry_ready.send(sender=Status, process=process, node=node)
# because node may be updated
process.save()
return action_res
def test_service_activity(self):
act_id = '1'
service = 'a_service'
inputs = {'args': [1, 2, 3], 'kwargs': {'1': 1, '2': 2}}
service_act = ServiceActivity(id=act_id,
service=service,
data=DataObject(inputs))
self.assertTrue(isinstance(service_act, Activity))
self.assertEqual(service, service_act.service)
def data_for_execution(self, context, pipeline_data):
data_dict = self.clean_execute_data(context)
inputs = {}
for key, tag_info in data_dict.items():
if tag_info is None:
raise ComponentDataLackException('Lack of inputs: %s' % key)
inputs[key] = get_variable(key, tag_info, context, pipeline_data)
return DataObject(inputs)
def test_end_event(self):
start_event = EmptyStartEvent(id='a')
act = ServiceActivity(id='b', service=None, data=DataObject({}))
end_event = EmptyEndEvent(id='c')
flow_ab = SequenceFlow('ab', start_event, act)
flow_bc = SequenceFlow('bc', act, end_event)
start_event.outgoing.add_flow(flow_ab)
act.incoming.add_flow(flow_ab)
act.outgoing.add_flow(flow_bc)
end_event.incoming.add_flow(flow_bc)
spec = PipelineSpec(start_event, end_event, [flow_ab, flow_bc], [act],
[], None, None)
pipeline = Pipeline('pipeline', spec)
self.assertEqual(end_event, pipeline.end_event)
def test_data_object(self):
inputs = {'args': '1', 'kwargs': {'1': 1, '2': 2}}
self.assertRaises(exceptions.DataTypeErrorException, DataObject, None)
data_object = DataObject(inputs)
self.assertIsInstance(data_object, DataObject)
self.assertEqual(data_object.get_inputs(), inputs)
self.assertEqual(data_object.get_outputs(), {})
self.assertEqual(data_object.get_one_of_inputs('args'), '1')
self.assertIsNone(data_object.get_one_of_outputs('args'))
self.assertRaises(exceptions.DataTypeErrorException,
data_object.reset_outputs, None)
self.assertTrue(data_object.reset_outputs({'a': str}))
def get_empty_parallel_gateway_test_pipeline():
start_event_a_id = node_uniqid()
gateway_b_id = node_uniqid()
gateway_h_id = node_uniqid()
end_event_i_id = node_uniqid()
start_event_a = EmptyStartEvent(start_event_a_id)
gateway_b = ParallelGateway(gateway_b_id, gateway_h_id)
gateway_h = ConvergeGateway(gateway_h_id)
end_event_i = EmptyEndEvent(end_event_i_id)
flow_ab = SequenceFlow('ab', start_event_a, gateway_b)
flow_bc = SequenceFlow('bc', gateway_b, gateway_h)
flow_bd = SequenceFlow('bd', gateway_b, gateway_h)
flow_be = SequenceFlow('be', gateway_b, gateway_h)
flow_hi = SequenceFlow('hi', gateway_h, end_event_i)
start_event_a.outgoing.add_flow(flow_ab)
gateway_b.incoming.add_flow(flow_ab)
gateway_b.outgoing.add_flow(flow_bc)
gateway_b.outgoing.add_flow(flow_bd)
gateway_b.outgoing.add_flow(flow_be)
gateway_h.incoming.add_flow(flow_bc)
gateway_h.incoming.add_flow(flow_bd)
gateway_h.incoming.add_flow(flow_be)
gateway_h.outgoing.add_flow(flow_hi)
end_event_i.incoming.add_flow(flow_hi)
spec = PipelineSpec(start_event_a,
end_event_i, [flow_ab, flow_bc, flow_bd, flow_be], [],
[gateway_b, gateway_h],
data=DataObject({}),
context=context.Context(act_outputs={}, output_key=[]))
return Pipeline(node_uniqid(), spec)
def retry(self, process, node, inputs):
result = self.transit(id=node.id, to_state=states.READY, appoint=True)
if not result:
return result
# add retry times
s = Status.objects.get(id=node.id)
s.version = uniqid()
history = History.objects.record(s)
LogEntry.objects.link_history(node_id=node.id, history_id=history.id)
s.retry += 1
s.save()
# update inputs
if inputs:
new_data = DataObject(inputs=inputs, outputs={})
node.data = new_data
Data.objects.write_node_data(node)
process.save()
self.recover_from_block(process.root_pipeline.id,
process.subprocess_stack)
return result
def retry_activity(act_id, inputs):
if not inputs:
api.retry_activity(act_id)
origin_inputs = api.get_inputs(act_id)
origin_inputs[0][0].data = DataObject(inputs)
api.retry_activity(act_id, args=origin_inputs[0])
def test_data_object(self):
inputs = {'args': '1', 'kwargs': {'1': 1, '2': 2}}
self.assertRaises(exceptions.DataTypeErrorException, DataObject, None)
data_object = DataObject(inputs)
self.assertIsInstance(data_object, DataObject)
self.assertIsInstance(data_object.inputs, FancyDict)
self.assertIsInstance(data_object.outputs, FancyDict)
self.assertEqual(data_object.get_inputs(), inputs)
self.assertEqual(data_object.get_outputs(), {})
self.assertEqual(data_object.get_one_of_inputs('args'), '1')
self.assertEqual(data_object.inputs.args, '1')
self.assertIsNone(data_object.get_one_of_outputs('args'))
self.assertRaises(exceptions.DataTypeErrorException,
data_object.reset_outputs, None)
self.assertTrue(data_object.reset_outputs({'a': str}))
self.assertEqual(data_object.outputs.a, str)
data_object.update_outputs({'args': '1', 'kwargs': {'1': 1, '2': 2}})
self.assertEqual(data_object.get_outputs(), {
'a': str,
'args': '1',
'kwargs': {
'1': 1,
'2': 2
}
})
self.assertEqual(
jsonschema.validate(json.loads(data_object.serializer()),
BASE_PARAM), None)
def _parse(self,
root_pipeline_data=None,
params=None,
is_subprocess=False,
parent_context=None):
if root_pipeline_data is None:
root_pipeline_data = {}
if params is None:
params = {}
pipeline_data = deepcopy(
root_pipeline_data) if is_subprocess else root_pipeline_data
pipeline_inputs = self.pipeline_tree[PE.data][PE.inputs]
act_outputs = {}
scope_info = {}
process_params = {}
for key, info in pipeline_inputs.items():
if info.get(PE.source_act):
act_outputs.setdefault(info[PE.source_act],
{}).update({info[PE.source_key]: key})
continue
if info.get(PE.is_param, False):
info = params.get(key, info)
if is_subprocess:
process_params.update({key: info})
continue
scope_info.update({key: info})
output_keys = self.pipeline_tree[PE.data][PE.outputs].keys()
context = Context(act_outputs, output_keys)
for key, info in scope_info.items():
var = get_variable(key, info, context, pipeline_data)
context.set_global_var(key, var)
if is_subprocess:
if parent_context is None:
raise exceptions.DataTypeErrorException(
'parent context of subprocess cannot be none')
for key, info in process_params.items():
var = get_variable(key, info, parent_context, pipeline_data)
pipeline_data.update({key: var})
start = self.pipeline_tree[PE.start_event]
start_cls = getattr(event, start[PE.type])
start_event = start_cls(id=start[PE.id], name=start[PE.name])
end = self.pipeline_tree[PE.end_event]
end_cls = getattr(event, end[PE.type])
end_event = end_cls(id=end[PE.id], name=end[PE.name])
acts = self.pipeline_tree[PE.activities]
act_objs = []
for act in acts.values():
act_cls = getattr(activity, act[PE.type])
if act[PE.type] == PE.ServiceActivity:
component = ComponentLibrary.get_component(
act[PE.component][PE.code], act[PE.component][PE.inputs])
service = component.service()
data = component.data_for_execution(context, pipeline_data)
act_objs.append(
act_cls(id=act[PE.id],
service=service,
name=act[PE.name],
data=data,
error_ignorable=act.get(PE.error_ignorable, False),
skippable=act.get(PE.skippable, True),
can_retry=act.get(PE.can_retry, True),
timeout=act.get(PE.timeout)))
elif act[PE.type] == PE.SubProcess:
sub_tree = act[PE.pipeline]
params = act[PE.params]
sub_parser = PipelineParser(pipeline_tree=sub_tree)
act_objs.append(
act_cls(id=act[PE.id],
pipeline=sub_parser._parse(
root_pipeline_data=root_pipeline_data,
params=params,
is_subprocess=True,
parent_context=context),
name=act[PE.name]))
elif act[PE.type] == PE.LoopServiceActivity:
act_cls = getattr(activity, act[PE.type])
component = ComponentLibrary.get_component(
act[PE.component][PE.code], act[PE.component][PE.inputs])
service = component.service()
data = component.data_for_execution(context, pipeline_data)
act_objs.append(
act_cls(id=act[PE.id],
service=service,
name=act[PE.name],
data=data,
error_ignorable=act.get(PE.error_ignorable, False),
loop_times=act[PE.loop_times]))
else:
raise exceptions.FlowTypeError(u"Unknown Activity type: %s" %
act[PE.type])
gateways = self.pipeline_tree[PE.gateways]
flows = self.pipeline_tree[PE.flows]
gateway_objs = []
for gw in gateways.values():
gw_cls = getattr(gateway, gw[PE.type])
if gw[PE.type] in [PE.ParallelGateway]:
gateway_objs.append(
gw_cls(id=gw[PE.id],
converge_gateway_id=gw[PE.converge_gateway_id],
name=gw[PE.name]))
elif gw[PE.type] in [PE.ExclusiveGateway, PE.ConvergeGateway]:
gateway_objs.append(gw_cls(id=gw[PE.id], name=gw[PE.name]))
else:
raise exceptions.FlowTypeError(u"Unknown Gateway type: %s" %
gw[PE.type])
flow_objs_dict = {}
for fl in flows.values():
flow_nodes = act_objs + gateway_objs
if fl[PE.source] == start[PE.id]:
source = start_event
else:
source = filter(lambda x: x.id == fl[PE.source], flow_nodes)[0]
if fl[PE.target] == end[PE.id]:
target = end_event
else:
target = filter(lambda x: x.id == fl[PE.target], flow_nodes)[0]
flow_objs_dict[fl[PE.id]] = base.SequenceFlow(
fl[PE.id], source, target)
flow_objs = flow_objs_dict.values()
# add incoming and outgoing flow to acts
if not isinstance(start[PE.outgoing], list):
start[PE.outgoing] = [start[PE.outgoing]]
for outgoing_id in start[PE.outgoing]:
start_event.outgoing.add_flow(flow_objs_dict[outgoing_id])
if not isinstance(end[PE.incoming], list):
end[PE.incoming] = [end[PE.incoming]]
for incoming_id in end[PE.incoming]:
end_event.incoming.add_flow(flow_objs_dict[incoming_id])
for act in act_objs:
incoming = acts[act.id][PE.incoming]
if isinstance(incoming, list):
for s in incoming:
act.incoming.add_flow(flow_objs_dict[s])
else:
act.incoming.add_flow(flow_objs_dict[incoming])
act.outgoing.add_flow(flow_objs_dict[acts[act.id][PE.outgoing]])
for gw in gateway_objs:
if isinstance(gw, gateway.ExclusiveGateway):
for flow_id, con in gateways[gw.id][PE.conditions].items():
con_obj = gateway.Condition(
con[PE.evaluate],
flow_objs_dict[flow_id],
)
gw.add_condition(con_obj)
gw.incoming.add_flow(
flow_objs_dict[gateways[gw.id][PE.incoming]])
for outgoing_id in gateways[gw.id][PE.outgoing]:
gw.outgoing.add_flow(flow_objs_dict[outgoing_id])
elif isinstance(gw, gateway.ParallelGateway):
gw.incoming.add_flow(
flow_objs_dict[gateways[gw.id][PE.incoming]])
for outgoing_id in gateways[gw.id][PE.outgoing]:
gw.outgoing.add_flow(flow_objs_dict[outgoing_id])
elif isinstance(gw, gateway.ConvergeGateway):
for incoming_id in gateways[gw.id][PE.incoming]:
gw.incoming.add_flow(flow_objs_dict[incoming_id])
gw.outgoing.add_flow(
flow_objs_dict[gateways[gw.id][PE.outgoing]])
else:
raise exceptions.FlowTypeError(u"Unknown Gateway type: %s" %
type(gw))
pipeline_data = DataObject(pipeline_data)
pipeline_spec = PipelineSpec(start_event, end_event, flow_objs,
act_objs, gateway_objs, pipeline_data,
context)
return Pipeline(self.pipeline_tree[PE.id], pipeline_spec)
def get_exclusive_gateway_pipeline(result):
start_event_a_id = node_uniqid()
act_b_id = node_uniqid()
gateway_c_id = node_uniqid()
act_d_id = node_uniqid()
act_e_id = node_uniqid()
act_f_id = node_uniqid()
gateway_g_id = node_uniqid()
end_event_h_id = node_uniqid()
start_event_a = EmptyStartEvent(start_event_a_id)
b_act = ServiceActivity(act_b_id,
service=EchoService(),
data=DataObject({'a': 3}))
c_gateway = ExclusiveGateway(gateway_c_id, gateway_g_id)
d_act = ServiceActivity(act_d_id,
service=LogService(),
data=DataObject({'node': 'd'}))
e_act = ServiceActivity(act_e_id,
service=EchoService(),
data=DataObject({'node': 'e'}))
f_act = ServiceActivity(act_f_id,
service=TestService(),
data=DataObject({'node': 'f'}))
g_gateway = ConvergeGateway(gateway_g_id)
end_event_h = EmptyEndEvent(end_event_h_id)
ab_flow = SequenceFlow('ab', start_event_a, b_act)
bc_flow = SequenceFlow('bc', b_act, c_gateway)
cd_flow = SequenceFlow('cd', c_gateway, d_act)
ce_flow = SequenceFlow('ce', c_gateway, e_act)
cf_flow = SequenceFlow('cf', c_gateway, f_act)
dg_flow = SequenceFlow('dg', d_act, g_gateway)
eg_flow = SequenceFlow('eg', e_act, g_gateway)
fg_flow = SequenceFlow('fg', f_act, g_gateway)
gh_flow = SequenceFlow('gh', g_gateway, end_event_h)
start_event_a.outgoing.add_flow(ab_flow)
b_act.incoming.add_flow(ab_flow)
b_act.outgoing.add_flow(bc_flow)
c_gateway.incoming.add_flow(bc_flow)
c_gateway.outgoing.add_flow(cd_flow)
c_gateway.outgoing.add_flow(ce_flow)
c_gateway.outgoing.add_flow(cf_flow)
d_act.incoming.add_flow(cd_flow)
d_act.outgoing.add_flow(dg_flow)
e_act.incoming.add_flow(ce_flow)
e_act.outgoing.add_flow(eg_flow)
f_act.incoming.add_flow(cf_flow)
f_act.outgoing.add_flow(fg_flow)
g_gateway.incoming.add_flow(dg_flow)
g_gateway.incoming.add_flow(eg_flow)
g_gateway.incoming.add_flow(fg_flow)
g_gateway.outgoing.add_flow(gh_flow)
end_event_h.incoming.add_flow(gh_flow)
c_gateway.add_condition(Condition('result == 1', cd_flow))
c_gateway.add_condition(Condition('result == 2', ce_flow))
c_gateway.add_condition(Condition('result == 3', cf_flow))
spec = PipelineSpec(start_event_a,
end_event_h, [
ab_flow, bc_flow, cd_flow, ce_flow, cf_flow,
dg_flow, eg_flow, fg_flow, gh_flow
], [b_act, d_act, e_act, f_act],
[c_gateway, g_gateway],
data=DataObject({}),
context=context.Context(act_outputs={},
scope={'result': result}))
return Pipeline(node_uniqid(), spec)
def get_parallel_gateway_test_pipeline():
start_event_a_id = node_uniqid()
gateway_b_id = node_uniqid()
act_c_id = node_uniqid()
act_d_id = node_uniqid()
act_e_id = node_uniqid()
act_f_id = node_uniqid()
act_g_id = node_uniqid()
gateway_h_id = node_uniqid()
end_event_i_id = node_uniqid()
start_event_a = EmptyStartEvent(start_event_a_id)
gateway_b = ParallelGateway(gateway_b_id, gateway_h_id)
act_c = ServiceActivity(act_c_id,
service=SleepService(),
data=DataObject({'data': '1'}))
act_d = ServiceActivity(act_d_id,
service=SleepService(),
data=DataObject({'data': '1'}))
act_e = ServiceActivity(act_e_id,
service=SleepService(),
data=DataObject({'data': '1'}))
act_f = ServiceActivity(act_f_id,
service=SleepService(),
data=DataObject({'node_1': 'd'}))
act_g = ServiceActivity(act_g_id,
service=SleepService(),
data=DataObject({'node_2': 'd'}))
gateway_h = ConvergeGateway(gateway_h_id)
end_event_i = EmptyEndEvent(end_event_i_id)
flow_ab = SequenceFlow('ab', start_event_a, gateway_b)
flow_bc = SequenceFlow('bc', gateway_b, act_c)
flow_bd = SequenceFlow('bd', gateway_b, act_d)
flow_be = SequenceFlow('be', gateway_b, act_e)
flow_cf = SequenceFlow('cf', act_c, act_f)
flow_dg = SequenceFlow('dg', act_d, act_g)
flow_fh = SequenceFlow('fh', act_f, gateway_h)
flow_gh = SequenceFlow('gh', act_g, gateway_h)
flow_eh = SequenceFlow('eh', act_e, gateway_h)
flow_hi = SequenceFlow('hi', gateway_h, end_event_i)
start_event_a.outgoing.add_flow(flow_ab)
gateway_b.incoming.add_flow(flow_ab)
gateway_b.outgoing.add_flow(flow_bc)
gateway_b.outgoing.add_flow(flow_bd)
gateway_b.outgoing.add_flow(flow_be)
act_c.incoming.add_flow(flow_bc)
act_d.incoming.add_flow(flow_bd)
act_e.incoming.add_flow(flow_be)
act_c.outgoing.add_flow(flow_cf)
act_d.outgoing.add_flow(flow_dg)
act_e.outgoing.add_flow(flow_eh)
act_f.incoming.add_flow(flow_cf)
act_g.incoming.add_flow(flow_dg)
act_f.outgoing.add_flow(flow_fh)
act_g.outgoing.add_flow(flow_gh)
gateway_h.incoming.add_flow(flow_fh)
gateway_h.incoming.add_flow(flow_gh)
gateway_h.incoming.add_flow(flow_eh)
gateway_h.outgoing.add_flow(flow_hi)
end_event_i.incoming.add_flow(flow_hi)
spec = PipelineSpec(
start_event_a,
end_event_i, [
flow_ab, flow_bc, flow_bd, flow_be, flow_cf, flow_dg, flow_fh,
flow_gh, flow_eh
], [act_c, act_d, act_e, act_f, act_g], [gateway_b, gateway_h],
data=DataObject({}),
context=context.Context(act_outputs={
act_f_id: {
'node_1': 'node_1_heihei'
},
act_d_id: {
'data': 'data_haha'
}
},
output_key=['node_1_heihei', 'data_haha']))
return Pipeline(node_uniqid(), spec)
def _parse(self,
root_pipeline_data=None,
root_pipeline_params=None,
params=None,
is_subprocess=False,
parent_context=None):
"""
@summary: parse pipeline and subprocess recursively
@param root_pipeline_data: root data from root pipeline parsing, witch will be passed to subprocess recursively
@param root_pipeline_params: params from root pipeline for all subprocess
@param params: params from parent for son subprocess
@param is_subprocess: whither is subprocess
@param parent_context: parent context for activity of subprocess to resolving inputs
@return: Pipeline object
"""
if root_pipeline_data is None:
root_pipeline_data = {}
if root_pipeline_params is None:
root_pipeline_params = {}
if params is None:
params = {}
pipeline_inputs = self.pipeline_tree[PE.data][PE.inputs]
classification = classify_inputs(pipeline_inputs, params,
is_subprocess, root_pipeline_params)
output_keys = self.pipeline_tree[PE.data][PE.outputs]
context = Context(classification['act_outputs'], output_keys)
for key, info in classification['scope_info'].items():
var = get_variable(key, info, context, root_pipeline_data)
context.set_global_var(key, var)
pipeline_data = deepcopy(root_pipeline_data)
if is_subprocess:
if parent_context is None:
raise exceptions.DataTypeErrorException(
'parent context of subprocess cannot be none')
for key, info in classification['subprocess_params'].items():
var = get_variable(key, info, parent_context, pipeline_data)
pipeline_data.update({key: var})
start = self.pipeline_tree[PE.start_event]
start_cls = FlowNodeClsFactory.get_node_cls(start[PE.type])
start_event = start_cls(id=start[PE.id], name=start[PE.name])
end = self.pipeline_tree[PE.end_event]
end_cls = FlowNodeClsFactory.get_node_cls(end[PE.type])
end_event = end_cls(id=end[PE.id],
name=end[PE.name],
data=DataObject({}))
acts = self.pipeline_tree[PE.activities]
act_objs = []
for act in acts.values():
act_cls = FlowNodeClsFactory.get_node_cls(act[PE.type])
if act[PE.type] == PE.ServiceActivity:
component = ComponentLibrary.get_component(
act[PE.component][PE.code], act[PE.component][PE.inputs])
service = component.service()
data = component.data_for_execution(context, pipeline_data)
handler_path = act.get('failure_handler')
failure_handler = import_string(
handler_path) if handler_path else None
act_objs.append(
act_cls(id=act[PE.id],
service=service,
name=act[PE.name],
data=data,
error_ignorable=act.get(PE.error_ignorable, False),
skippable=act.get(PE.skippable)
or act.get(PE.skippable_old, True),
retryable=act.get(PE.retryable)
or act.get(PE.retryable_old, True),
timeout=act.get(PE.timeout),
failure_handler=failure_handler))
elif act[PE.type] == PE.SubProcess:
sub_tree = act[PE.pipeline]
params = act[PE.params]
sub_parser = PipelineParser(pipeline_tree=sub_tree)
act_objs.append(
act_cls(id=act[PE.id],
pipeline=sub_parser._parse(
root_pipeline_data=root_pipeline_data,
root_pipeline_params=root_pipeline_params,
params=params,
is_subprocess=True,
parent_context=context),
name=act[PE.name]))
else:
raise exceptions.FlowTypeError(u"Unknown Activity type: %s" %
act[PE.type])
gateways = self.pipeline_tree[PE.gateways]
flows = self.pipeline_tree[PE.flows]
gateway_objs = []
for gw in gateways.values():
gw_cls = FlowNodeClsFactory.get_node_cls(gw[PE.type])
if gw[PE.type] in {
PE.ParallelGateway, PE.ConditionalParallelGateway
}:
gateway_objs.append(
gw_cls(id=gw[PE.id],
converge_gateway_id=gw[PE.converge_gateway_id],
name=gw[PE.name]))
elif gw[PE.type] in {PE.ExclusiveGateway, PE.ConvergeGateway}:
gateway_objs.append(gw_cls(id=gw[PE.id], name=gw[PE.name]))
else:
raise exceptions.FlowTypeError(u"Unknown Gateway type: %s" %
gw[PE.type])
flow_objs_dict = {}
for fl in flows.values():
flow_nodes = act_objs + gateway_objs
if fl[PE.source] == start[PE.id]:
source = start_event
else:
source = filter(lambda x: x.id == fl[PE.source], flow_nodes)[0]
if fl[PE.target] == end[PE.id]:
target = end_event
else:
target = filter(lambda x: x.id == fl[PE.target], flow_nodes)[0]
flow_objs_dict[fl[PE.id]] = SequenceFlow(fl[PE.id], source, target)
flow_objs = flow_objs_dict.values()
# add incoming and outgoing flow to acts
if not isinstance(start[PE.outgoing], list):
start[PE.outgoing] = [start[PE.outgoing]]
for outgoing_id in start[PE.outgoing]:
start_event.outgoing.add_flow(flow_objs_dict[outgoing_id])
if not isinstance(end[PE.incoming], list):
end[PE.incoming] = [end[PE.incoming]]
for incoming_id in end[PE.incoming]:
end_event.incoming.add_flow(flow_objs_dict[incoming_id])
for act in act_objs:
incoming = acts[act.id][PE.incoming]
if isinstance(incoming, list):
for s in incoming:
act.incoming.add_flow(flow_objs_dict[s])
else:
act.incoming.add_flow(flow_objs_dict[incoming])
act.outgoing.add_flow(flow_objs_dict[acts[act.id][PE.outgoing]])
for gw in gateway_objs:
if isinstance(gw, ExclusiveGateway) or isinstance(
gw, ConditionalParallelGateway):
for flow_id, con in gateways[gw.id][PE.conditions].items():
con_obj = Condition(con[PE.evaluate],
flow_objs_dict[flow_id])
gw.add_condition(con_obj)
if isinstance(gateways[gw.id][PE.incoming], list):
for incoming_id in gateways[gw.id][PE.incoming]:
gw.incoming.add_flow(flow_objs_dict[incoming_id])
else:
gw.incoming.add_flow(
flow_objs_dict[gateways[gw.id][PE.incoming]])
for outgoing_id in gateways[gw.id][PE.outgoing]:
gw.outgoing.add_flow(flow_objs_dict[outgoing_id])
elif isinstance(gw, ParallelGateway):
if isinstance(gateways[gw.id][PE.incoming], list):
for incoming_id in gateways[gw.id][PE.incoming]:
gw.incoming.add_flow(flow_objs_dict[incoming_id])
else:
gw.incoming.add_flow(
flow_objs_dict[gateways[gw.id][PE.incoming]])
for outgoing_id in gateways[gw.id][PE.outgoing]:
gw.outgoing.add_flow(flow_objs_dict[outgoing_id])
elif isinstance(gw, ConvergeGateway):
for incoming_id in gateways[gw.id][PE.incoming]:
gw.incoming.add_flow(flow_objs_dict[incoming_id])
gw.outgoing.add_flow(
flow_objs_dict[gateways[gw.id][PE.outgoing]])
else:
raise exceptions.FlowTypeError(u"Unknown Gateway type: %s" %
type(gw))
context.duplicate_variables()
pipeline_data = DataObject(pipeline_data)
pipeline_spec = PipelineSpec(start_event, end_event, flow_objs,
act_objs, gateway_objs, pipeline_data,
context)
return Pipeline(self.pipeline_tree[PE.id], pipeline_spec)
def parser(self, root_pipeline_data=None):
if root_pipeline_data is None:
root_pipeline_data = {}
pipeline_inputs = self.pipeline_tree['data']['inputs']
act_outputs = {}
scope_info = {}
for key, info in pipeline_inputs.iteritems():
if info.get('source_act'):
act_outputs.setdefault(info['source_act'],
{}).update({info['source_key']: key})
else:
scope_info.update({key: info})
output_keys = self.pipeline_tree['data']['outputs'].keys()
context = Context(act_outputs, output_keys)
for key, info in scope_info.iteritems():
value = get_variable(key, info, context, root_pipeline_data)
context.set_global_var(key, value)
start = self.pipeline_tree['start_event']
start_cls = getattr(event, start['type'])
start_event = start_cls(id=start['id'],
name=start['name'])
end = self.pipeline_tree['end_event']
end_cls = getattr(event, end['type'])
end_event = end_cls(id=end['id'],
name=end['name'])
acts = self.pipeline_tree['activities']
act_objs = []
for act in acts.values():
act_cls = getattr(activity, act['type'])
if act['type'] == 'ServiceActivity':
component = ComponentLibrary.get_component(
act['component']['code'], act['component']['inputs']
)
service = component.service()
data = component.data_for_execution(context, root_pipeline_data)
act_objs.append(act_cls(id=act['id'],
service=service,
name=act['name'],
data=data,
error_ignorable=act.get('error_ignorable', False)))
elif act['type'] == 'SubProcess':
pipeline_info = act['pipeline']
sub_parser = PipelineParser(pipeline_info)
act_objs.append(act_cls(id=act['id'],
pipeline=sub_parser.parser(root_pipeline_data),
name=act['name']))
else:
raise exceptions.FlowTypeError(u"Unknown Activity type: %s" %
act['type'])
gateways = self.pipeline_tree['gateways']
flows = self.pipeline_tree['flows']
gateway_objs = []
for gw in gateways.values():
gw_cls = getattr(gateway, gw['type'])
if gw['type'] in ['ParallelGateway']:
gateway_objs.append(
gw_cls(id=gw['id'],
converge_gateway_id=gw['converge_gateway_id'],
name=gw['name']))
elif gw['type'] in ['ExclusiveGateway', 'ConvergeGateway']:
gateway_objs.append(gw_cls(id=gw['id'],
name=gw['name']))
else:
raise exceptions.FlowTypeError(u"Unknown Gateway type: %s" %
gw['type'])
flow_objs_dict = {}
for fl in flows.values():
flow_nodes = act_objs + gateway_objs
if fl['source'] == start['id']:
source = start_event
else:
source = filter(lambda x: x.id == fl['source'], flow_nodes)[0]
if fl['target'] == end['id']:
target = end_event
else:
target = filter(lambda x: x.id == fl['target'], flow_nodes)[0]
flow_objs_dict[fl['id']] = base.SequenceFlow(fl['id'],
source,
target)
flow_objs = flow_objs_dict.values()
# add incoming and outgoing flow to acts
if not isinstance(start['outgoing'], list):
start['outgoing'] = [start['outgoing']]
for outgoing_id in start['outgoing']:
start_event.outgoing.add_flow(flow_objs_dict[outgoing_id])
if not isinstance(end['incoming'], list):
end['incoming'] = [end['incoming']]
for incoming_id in end['incoming']:
end_event.incoming.add_flow(flow_objs_dict[incoming_id])
for act in act_objs:
act.incoming.add_flow(flow_objs_dict[acts[act.id]['incoming']])
act.outgoing.add_flow(flow_objs_dict[acts[act.id]['outgoing']])
for gw in gateway_objs:
if isinstance(gw, gateway.ExclusiveGateway):
for flow_id, con in gateways[gw.id]['conditions'].iteritems():
con_obj = gateway.Condition(
con['evaluate'],
flow_objs_dict[flow_id],
)
gw.add_condition(con_obj)
gw.incoming.add_flow(
flow_objs_dict[gateways[gw.id]['incoming']]
)
for outgoing_id in gateways[gw.id]['outgoing']:
gw.outgoing.add_flow(flow_objs_dict[outgoing_id])
elif isinstance(gw, gateway.ParallelGateway):
gw.incoming.add_flow(
flow_objs_dict[gateways[gw.id]['incoming']]
)
for outgoing_id in gateways[gw.id]['outgoing']:
gw.outgoing.add_flow(flow_objs_dict[outgoing_id])
elif isinstance(gw, gateway.ConvergeGateway):
for incoming_id in gateways[gw.id]['incoming']:
gw.incoming.add_flow(flow_objs_dict[incoming_id])
gw.outgoing.add_flow(
flow_objs_dict[gateways[gw.id]['outgoing']]
)
else:
raise exceptions.FlowTypeError(u"Unknown Gateway type: %s" %
type(gw))
root_pipeline_data = DataObject(root_pipeline_data)
pipeline_spec = PipelineSpec(start_event, end_event, flow_objs,
act_objs, gateway_objs, root_pipeline_data,
context)
return Pipeline(self.pipeline_tree['id'], pipeline_spec)
import logging
import mock
from django.test import TestCase
from mock import MagicMock
from pipeline.core.data.base import DataObject
from pipeline_plugins.components.collections.sites.open.job.base import GetJobHistoryResultMixin
TEST_INPUTS = {
"job_success_id": 12345,
"biz_cc_id": 11111,
"executor": "executor"
}
TEST_DATA = DataObject(TEST_INPUTS)
TEST_PARENT_DATA = DataObject(TEST_INPUTS)
logger = logging.getLogger("component")
GET_CLIENT_BY_USER = "******"
GET_JOB_INSTANCE_URL = "pipeline_plugins.components.collections.sites.open.job.base.get_job_instance_status"
GET_JOB_STATUS_RETURN = {
"result": True,
"code": 0,
"message": "",
"data": {
"finished":
True,
"job_instance": {
"job_instance_id": 100,
def test_component(self):
component = self._component_cls({})
for no, case in enumerate(self._cases):
try:
patchers = [patcher.mock_patcher() for patcher in case.patchers]
with patch_context(patchers):
bound_service = component.service()
setattr(bound_service, 'id', case.service_id)
data = DataObject(inputs=case.inputs)
parent_data = DataObject(inputs=case.parent_data)
# execute result check
do_continue = self._do_case_assert(service=bound_service,
method='execute',
args=(data, parent_data),
assertion=case.execute_assertion,
no=no,
name=case.name)
for call_assertion in case.execute_call_assertion:
self._do_call_assertion(name=case.name,
no=no,
assertion=call_assertion)
if do_continue:
self.case_pass(case)
continue
if bound_service.need_schedule():
if bound_service.interval is None:
# callback case
self._do_case_assert(service=bound_service,
method='schedule',
args=(data, parent_data, case.schedule_assertion.callback_data),
assertion=case.schedule_assertion,
no=no,
name=case.name)
else:
# schedule case
assertions = case.schedule_assertion
assertions = assertions if isinstance(assertions, list) else [assertions]
for assertion in assertions:
do_continue = self._do_case_assert(service=bound_service,
method='schedule',
args=(data, parent_data),
assertion=assertion,
no=no,
name=case.name)
self.assertEqual(assertion.schedule_finished,
bound_service.is_schedule_finished(),
msg=self._format_failure_message(
no=no,
name=case.name,
msg='schedule_finished assertion failed:'
'\nexpected: {expected}\nactual: {actual}'.format(
expected=assertion.schedule_finished, # noqa
actual=bound_service.is_schedule_finished()))) # noqa
if do_continue:
break
for call_assertion in case.schedule_call_assertion:
self._do_call_assertion(name=case.name,
no=no,
assertion=call_assertion)
self.case_pass(case)
except Exception:
self.case_fail(case)
logger.error(traceback.format_exc())
