def test_connected(self):
graph = Graph()
graph.add_node(1)
graph.add_node(2)
graph.add_node(3)
graph.add_node(4)
self.assertFalse(graph.connected())
graph.add_edge(1, 2)
graph.add_edge(3, 4)
self.assertFalse(graph.connected())
graph.add_edge(2, 3)
graph.add_edge(4, 1)
self.assertTrue(graph.connected())
class Workflow(Process):
def __init__(self,
process_id,
inputs,
outputs,
requirements,
hints,
label,
description,
steps,
context,
data_links=None):
super(Workflow, self).__init__(process_id, inputs, outputs,
requirements, hints, label, description)
self.graph = Graph()
self.executor = context.executor
self.steps = steps
self.data_links = data_links or []
self.context = context
self.port_step_index = {}
for step in steps:
node = AppNode(step.app, {})
self.add_node(step.id, node)
for inp in step.inputs:
self.port_step_index[inp.id] = step.id
self.move_connect_to_datalink(inp)
if inp.value:
node.inputs[inp.id] = inp.value
for out in step.outputs:
self.port_step_index[out.id] = step.id
for inp in self.inputs:
self.add_node(inp.id, inp)
for out in self.outputs:
self.move_connect_to_datalink(out)
self.add_node(out.id, out)
# dedupe links
s = {tuple(dl.items()) for dl in self.data_links}
self.data_links = [dict(dl) for dl in s]
for dl in self.data_links:
dst = dl['destination'].lstrip('#')
src = dl['source'].lstrip('#')
if src in self.port_step_index and dst in self.port_step_index:
rel = Relation(src, dst, dl.get('position', 0))
src = self.port_step_index[src]
dst = self.port_step_index[dst]
elif src in self._inputs:
rel = InputRelation(dst, dl.get('position', 0))
dst = self.port_step_index[dst]
elif dst in self._outputs:
rel = OutputRelation(src, dl.get('position', 0))
src = self.port_step_index[src]
else:
raise RabixError("invalid data link %s" % dl)
self.graph.add_edge(src, dst, rel)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
def move_connect_to_datalink(self, port):
for src in port.source:
self.data_links.append({
'source': src,
'destination': '#' + port.id
})
del port.source[:]
# Graph.add_node silently fails if node already exists
def add_node(self, node_id, node):
if node_id in self.graph.nodes:
raise ValidationError('Duplicate node ID: %s' % node_id)
self.graph.add_node(node_id, node)
def run(self, job):
eg = ExecutionGraph(self, job)
while eg.has_next():
next_id, next = eg.next_job()
self.executor.execute(next, eg.job_done, next_id)
return eg.outputs
def to_dict(self, context):
d = super(Workflow, self).to_dict(context)
d.update({
"class": "Workflow",
'steps': [step.to_dict(context) for step in self.steps]
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {}
for k, v in six.iteritems(d):
if k == 'steps':
converted[k] = [Step.from_dict(context, s) for s in v]
else:
converted[k] = context.from_dict(v)
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'steps':
converted['steps'],
'data_links':
converted.get('dataLinks'),
'context':
context,
'inputs': [
InputParameter.from_dict(context, i)
for i in converted['inputs']
],
'outputs': [
WorkflowOutput.from_dict(context, o)
for o in converted['outputs']
]
})
return cls(**kwargs)
class Workflow(Process):
def __init__(self, process_id, inputs, outputs, requirements, hints, label,
description, steps, context, data_links=None):
super(Workflow, self).__init__(
process_id, inputs, outputs, requirements,
hints, label, description
)
self.graph = Graph()
self.executor = context.executor
self.steps = steps
self.data_links = data_links or []
self.context = context
self.port_step_index = {}
for step in steps:
node = AppNode(step.app, {})
self.add_node(step.id, node)
for inp in step.inputs:
self.port_step_index[inp.id] = step.id
self.move_connect_to_datalink(inp)
if inp.value:
node.inputs[inp.id] = inp.value
for out in step.outputs:
self.port_step_index[out.id] = step.id
for inp in self.inputs:
self.add_node(inp.id, inp)
for out in self.outputs:
self.move_connect_to_datalink(out)
self.add_node(out.id, out)
for dl in self.data_links:
dst = dl['destination'].lstrip('#')
src = dl['source'].lstrip('#')
if src in self.port_step_index and dst in self.port_step_index:
rel = Relation(src, dst, dl.get('position', 0))
src = self.port_step_index[src]
dst = self.port_step_index[dst]
elif src in self._inputs:
rel = InputRelation(dst, dl.get('position', 0))
dst = self.port_step_index[dst]
elif dst in self._outputs:
rel = OutputRelation(src, dl.get('position', 0))
src = self.port_step_index[src]
else:
raise RabixError("invalid data link %s" % dl)
self.graph.add_edge(src, dst, rel)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
def move_connect_to_datalink(self, port):
for dl in port.connect:
dl['destination'] = '#'+port.id
self.data_links.append(dl)
del port.connect[:]
# Graph.add_node silently fails if node already exists
def add_node(self, node_id, node):
if node_id in self.graph.nodes:
raise ValidationError('Duplicate node ID: %s' % node_id)
self.graph.add_node(node_id, node)
def hide_nodes(self, type):
for node_id in self.graph.node_list():
node = self.graph.node_data(node_id)
if isinstance(node, type):
self.graph.hide_node(node_id)
def run(self, job):
eg = ExecutionGraph(self, job)
while eg.has_next():
next_id, next = eg.next_job()
self.executor.execute(next, eg.job_done, next_id)
return eg.outputs
def to_dict(self, context):
d = super(Workflow, self).to_dict(context)
d.update({
"class": "Workflow",
'steps': [step.to_dict(context) for step in self.steps]
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {}
for k, v in six.iteritems(d):
if k == 'steps':
converted[k] = [Step.from_dict(context, s) for s in v]
else:
converted[k] = context.from_dict(v)
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'steps': converted['steps'],
'data_links': converted.get('dataLinks'),
'context': context,
'inputs': [InputParameter.from_dict(context, i)
for i in converted['inputs']],
'outputs': [WorkflowOutput.from_dict(context, o)
for o in converted['outputs']]
})
return cls(**kwargs)
class WorkflowApp(App):
def __init__(self, app_id, steps, context,
inputs=None, outputs=None, to=None,
app_description=None,
annotations=None,
platform_features=None):
self.graph = Graph()
self.inputs = inputs or []
self.outputs = outputs or []
self.executor = context.executor
self.steps = steps
self.to = to or {}
self.context = context
for step in steps:
self.add_node(step.id, AppNode(step.app, {}))
for step in steps:
# inputs
for input_port, input_val in six.iteritems(step.inputs):
inp = wrap_in_list(input_val)
for item in inp:
self.add_edge_or_input(step, input_port, item)
# outputs
if step.outputs:
for output_port, output_val in six.iteritems(step.outputs):
self.to[output_val['$to']] = output_port
if isinstance(step.app, WorkflowApp):
output_node = step.app.get_output(step.app.to.get(output_port))
else:
output_node = step.app.get_output(output_port)
output_id = output_val['$to']
self.add_node(output_id, output_node)
self.graph.add_edge(
step.id, output_id, OutputRelation(output_port)
)
# output_node.id = output_val['$to']
self.outputs.append(output_node)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
schema = {
"@type": "JsonSchema",
"type": "object",
"properties": {},
"required": []
}
for inp in self.inputs:
schema['properties'][inp.id] = inp.validator.schema
if inp.required:
schema['required'].append(inp.id)
super(WorkflowApp, self).__init__(
app_id, JsonSchema(context, schema), self.outputs,
app_description=app_description,
annotations=annotations,
platform_features=platform_features
)
def add_edge_or_input(self, step, input_name, input_val):
node_id = step.id
if isinstance(input_val, dict) and '$from' in input_val:
frm = wrap_in_list(input_val['$from'])
for inp in frm:
if '.' in inp:
node, outp = inp.split('.')
self.graph.add_edge(node, node_id, Relation(outp, input_name))
else:
# TODO: merge input schemas if one input goes to different apps
input = step.app.get_input(input_name)
if inp not in self.graph.nodes:
self.add_node(inp, input)
self.graph.add_edge(
inp, node_id, InputRelation(input_name)
)
wf_input = copy.deepcopy(input)
wf_input.id = inp
self.inputs.append(wf_input)
else:
self.graph.node_data(node_id).inputs[input_name] = input_val
# Graph.add_node silently fails if node already exists
def add_node(self, node_id, node):
if node_id in self.graph.nodes:
raise ValidationError('Duplicate node ID: %s' % node_id)
self.graph.add_node(node_id, node)
def hide_nodes(self, type):
for node_id in self.graph.node_list():
node = self.graph.node_data(node_id)
if isinstance(node, type):
self.graph.hide_node(node_id)
def run(self, job):
eg = ExecutionGraph(self, job)
while eg.has_next():
next_id, next = eg.next_job()
self.executor.execute(next, eg.job_done, next_id)
return eg.outputs
def to_dict(self, context):
d = super(WorkflowApp, self).to_dict(context)
d.update({
"@type": "Workflow",
'steps': [step.to_dict(context) for step in self.steps]
})
return d
@classmethod
def from_dict(cls, context, d):
steps = [Step(
step['id'], context.from_dict(step['app']),
step['inputs'], step.get('outputs')
)
for step in d['steps']]
return cls(
d.get('@id', six.text_type(uuid4())),
steps,
context
)
