def testGroupings():
prod = NumberProducer(10)
cons1 = TestOneInOneOut()
cons1.inputconnections["input"]["type"] = [0]
cons2 = TestOneInOneOut()
cons1.inputconnections["input"]["type"] = "all"
cons3 = TestOneInOneOut()
cons1.inputconnections["input"]["type"] = "global"
cons4 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, "output", cons1, "input")
graph.connect(cons1, "output", cons2, "input")
graph.connect(cons2, "output", cons3, "input")
graph.connect(cons3, "output", cons4, "input")
args = argparse.Namespace
args.num = 5
args.simple = False
args.results = True
result_queue = process(graph, inputs={prod: 1}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_(cons4.id, name)
tools.eq_("output", output)
results.extend(data)
item = result_queue.get()
tools.eq_(Counter(range(10)), Counter(results))
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
prev = prod
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
return graph
def testPipeline():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
results = simple_process.process_and_return(graph, inputs={ prod : [ {}, {}, {}, {}, {} ] } )
tools.eq_({ cons2.id : { 'output' : [1, 2, 3, 4, 5] } }, results)
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
prev = prod
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
args.num = 3
process(graph, inputs={prod: [{}, {}, {}, {}, {}]}, args=args)
def testPipelineWithInputId():
graph = WorkflowGraph()
first = TestOneInOneOut()
prev = first
for i in range(5):
cons = TestOneInOneOut()
graph.connect(prev, 'output', cons, 'input')
prev = cons
results = simple_process.process(graph, {first.id: [{'input': 1}]})
tools.eq_({(prev.id, 'output'): [1]}, results)
def testPipeline():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
args = argparse.Namespace
args.num = 5
args.simple = False
process(graph, inputs={prod: [{}, {}, {}]}, args=args)
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
last = TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
results = simple_process.process_and_return(graph, {prod: [{}]})
tools.eq_({last.id: {'output': ['1', '1']}}, results)
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
last = TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
args.num = 4
process(graph, inputs={prod: [{}]}, args=args)
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
results = simple_process.process(graph, {prod: [{}, {}, {}, {}, {}]})
tools.eq_(
{
(cons1.id, 'output'): [1, 2, 3, 4, 5],
(cons2.id, 'output'): [1, 2, 3, 4, 5]
}, results)
def testNotEnoughProcesses():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
args = argparse.Namespace
args.num = 1
args.simple = False
args.results = True
message = process(graph, inputs={prod: 5}, args=args)
tools.ok_('Not enough processes' in message)
def testGroupings():
prod = NumberProducer(10)
cons1 = TestOneInOneOut()
cons1.inputconnections['input']['type'] = [0]
cons2 = TestOneInOneOut()
cons1.inputconnections['input']['type'] = 'all'
cons3 = TestOneInOneOut()
cons1.inputconnections['input']['type'] = 'global'
cons4 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
graph.connect(cons2, 'output', cons3, 'input')
graph.connect(cons3, 'output', cons4, 'input')
args = argparse.Namespace
args.num = 5
args.simple = False
args.results = True
result_queue = process(graph, inputs={prod: 1}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_(cons4.id, name)
tools.eq_('output', output)
results.extend(data)
item = result_queue.get()
tools.eq_(Counter(range(10)), Counter(results))
def testComposite():
comp = CompositePE()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
comp.connect(cons1, 'output', cons2, 'input')
comp._map_input('comp_input', cons1, 'input')
comp._map_output('comp_output', cons2, 'output')
prod = TestProducer()
cons = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', comp, 'comp_input')
graph.connect(comp, 'comp_output', cons, 'input')
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({cons.id: {'output': list(range(1, 11))}}, results)
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
results = simple_process.process_and_return(graph, {prod: 5})
tools.eq_(
{
cons1.id: {
'output': list(range(1, 6))
},
cons2.id: {
'output': list(range(1, 6))
}
}, results)
def test_types():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestOneInOneOut()
graph.connect(prod, 'output', cons, 'input')
graph.propagate_types()
tools.eq_(prod.outputconnections['output']['type'],
cons.inputconnections['input']['type'])
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
prev = prod
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
args.num = 3
process(graph, inputs={prod: [{}, {}, {}, {}, {}]}, args=args)
#print '='*20 + 'PIPELINE' + '='*20
#testPipeline()
#print '='*20 + 'SQUARE ' + '='*20
#testSquare()
#print '='*20 + 'TEE ' + '='*20
#testTee()
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
args.num = 3
args.results = True
result_queue = process(graph, inputs={prod: 5}, args=args)
results = defaultdict(list)
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_('output', output)
results[name].append(data)
item = result_queue.get()
tools.eq_(list(range(1, 6)), results[cons1.id])
tools.eq_(list(range(1, 6)), results[cons2.id])
def testPipeline():
graph = WorkflowGraph()
prod = TestProducer()
prev = prod
for i in range(5):
cons = TestOneInOneOut()
graph.connect(prev, 'output', cons, 'input')
prev = cons
results = simple_process.process(graph, {prod: [{}, {}, {}, {}, {}]})
tools.eq_({(prev.id, 'output'): [1, 2, 3, 4, 5]}, results)
def testPipelineSimple():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
args = argparse.Namespace
args.num = 4
args.simple = True
args.results = True
result_queue = process(graph, inputs={prod: 5}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_((cons2.id, 'output'), output)
results.extend(data)
item = result_queue.get()
tools.eq_(Counter(range(1, 6)), Counter(results))
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOutWriter()
last = TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
return graph
def testPipeline(graph):
prod = TestProducer()
prev = prod
part1 = [prod]
part2 = []
for i in range(5):
cons = TestOneInOneOut()
part2.append(cons)
graph.connect(prev, 'output', cons, 'input')
prev = cons
graph.partitions = [part1, part2]
return graph
def testCompositeWithCreateParams():
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
def create_graph(graph, connections):
for i in range(connections):
graph.connect(cons1, 'output', cons2, 'input')
comp = CompositePE(create_graph, {'connections': 2})
comp._map_input('comp_input', cons1, 'input')
comp._map_output('comp_output', cons2, 'output')
prod = TestProducer()
cons = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', comp, 'comp_input')
graph.connect(comp, 'comp_output', cons, 'input')
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 10})
expected = []
for i in range(1, 11):
expected += [i, i]
tools.eq_({cons.id: {'output': expected}}, results)
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
last = TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
args.num = 4
args.results = True
result_queue = process(graph, inputs={prod: 10}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_(last.id, name)
tools.eq_('output', output)
results.append(data)
item = result_queue.get()
expected = {str(i): 2 for i in range(1, 11)}
tools.eq_(expected, Counter(results))
def test_dot_composite():
def inc(a):
return a+1
def dec(a):
return a-1
graph = WorkflowGraph()
prod = TestProducer()
comp = create_iterative_chain([inc, dec])
cons = TestOneInOneOut()
graph.connect(prod, 'output', comp, 'input')
graph.connect(comp, 'output', cons, 'input')
graph.inputmappings = {'input': (prod, 'input')}
root_prod = TestProducer()
root_graph = WorkflowGraph()
root_graph.connect(root_prod, 'output', graph, 'input')
dot = draw(root_graph)
tools.ok_('subgraph cluster_' in dot)
def testInputsAndOutputs():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestOneInOneOut()
cons._add_output("output", tuple_type=["integer"])
cons._add_input("input", grouping=[0], tuple_type=["integer"])
cons.setInputTypes({"input": ["number"]})
tools.eq_({"output": ["number"]}, cons.getOutputTypes())
cons._add_output("output2")
try:
cons.getOutputTypes()
except Exception:
pass
graph.connect(prod, "output", cons, "input")
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({cons.id: {"output": list(range(1, 11))}}, results)
def testInputsAndOutputs():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestOneInOneOut()
cons._add_output('output', tuple_type=['integer'])
cons._add_input('input', grouping=[0], tuple_type=['integer'])
cons.setInputTypes({'input': ['number']})
tools.eq_({'output': ['number']}, cons.getOutputTypes())
cons._add_output('output2')
try:
cons.getOutputTypes()
except Exception:
pass
graph.connect(prod, 'output', cons, 'input')
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({cons.id: {'output': list(range(1, 11))}}, results)
def create_graph(graph):
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(cons1, 'output', cons2, 'input')
graph._map_input('comp_input', cons1, 'input')
graph._map_output('comp_output', cons2, 'output')
import processor
from dispel4py.workflow_graph import WorkflowGraph
from dispel4py.examples.graph_testing.testing_PEs import TestProducer, TestOneInOneOut
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
graph.partitions = [[prod], [cons1, cons2]]
ubergraph = processor.create_partitioned(graph)
processes, inputmappings, outputmappings = processor.assign_and_connect(
ubergraph, 2)
print processes
print inputmappings
print outputmappings
import multi_process
inputs = {prod: [{}]}
mapped_inputs = processor.map_inputs_to_partitions(ubergraph, inputs)
print 'MAPPED INPUTS: %s' % mapped_inputs
multi_process.process(ubergraph, 2, inputs=mapped_inputs)
from dispel4py.examples.graph_testing.testing_PEs import TestProducer, TestOneInOneOut from dispel4py.workflow_graph import WorkflowGraph prod = TestProducer() cons1a = TestOneInOneOut() cons1b = TestOneInOneOut() cons2 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', cons1a, 'input') graph.connect(prod, 'output', cons1b, 'input') graph.connect(cons1a, 'output', cons2, 'input')
def test_dot_pipeline():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestOneInOneOut()
graph.connect(prod, 'output', cons, 'input')
draw(graph)
