def runYesProv():
elapsed_time=0
for x in range(0,num):
graph = createGraphWithProv()
elapsed_time=0
start_time = time.time()
process(graph, inputs=input_data, args=args)
elapsed_time+= time.time() - start_time
print ("PROV ELAPSED TIME: "+str(elapsed_time/num))
def runYesProv():
elapsed_time = 0
for x in range(0, num):
graph = createGraphWithProv()
elapsed_time = 0
start_time = time.time()
process(graph, inputs=input_data, args=args)
elapsed_time += time.time() - start_time
print("PROV ELAPSED TIME: " + str(elapsed_time / num))
def testTee():
graph = WorkflowGraph()
prod = t.TestProducer()
prev = prod
cons1 = t.TestOneInOneOut()
cons2 = t.TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
args.num = 3
process(graph, inputs={prod: [{}, {}, {}, {}, {}]}, args=args)
def runNoProv():
elapsed_time=0
for x in range(0,num):
time.sleep(2)
graph = createWf()
elapsed_time=0
start_time = time.time()
process(graph, inputs=input_data, args=args)
elapsed_time=elapsed_time+( time.time() - start_time)
print ("NO PROV ELAPSED TIME: "+str(elapsed_time/num))
def runNoProv():
elapsed_time = 0
for x in range(0, num):
time.sleep(2)
graph = createWf()
elapsed_time = 0
start_time = time.time()
process(graph, inputs=input_data, args=args)
elapsed_time = elapsed_time + (time.time() - start_time)
print("NO PROV ELAPSED TIME: " + str(elapsed_time / num))
def testPipeline():
prod = t.TestProducer()
cons1 = t.TestOneInOneOut()
cons2 = t.TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
args = argparse.Namespace
args.num = 5
args.simple = False
process(graph, inputs={prod: [{}, {}, {}]}, args=args)
def testSquare():
graph = WorkflowGraph()
prod = t.TestProducer(2)
cons1 = t.TestOneInOneOut()
cons2 = t.TestOneInOneOut()
last = t.TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
args.num = 4
process(graph, inputs={prod: [{}]}, args=args)
def testGroupings():
prod = NumberProducer(10)
cons1 = TestOneInOneOut()
cons1.inputconnections['input']['type'] = [0]
cons2 = TestOneInOneOut()
cons1.inputconnections['input']['type'] = 'all'
cons3 = TestOneInOneOut()
cons1.inputconnections['input']['type'] = 'global'
cons4 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
graph.connect(cons2, 'output', cons3, 'input')
graph.connect(cons3, 'output', cons4, 'input')
args = argparse.Namespace
args.num = 5
args.simple = False
args.results = True
result_queue = process(graph, inputs={prod: 1}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_(cons4.id, name)
tools.eq_('output', output)
results.extend(data)
item = result_queue.get()
tools.eq_(Counter(range(10)), Counter(results))
def testGroupings():
prod = NumberProducer(10)
cons1 = TestOneInOneOut()
cons1.inputconnections["input"]["type"] = [0]
cons2 = TestOneInOneOut()
cons1.inputconnections["input"]["type"] = "all"
cons3 = TestOneInOneOut()
cons1.inputconnections["input"]["type"] = "global"
cons4 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, "output", cons1, "input")
graph.connect(cons1, "output", cons2, "input")
graph.connect(cons2, "output", cons3, "input")
graph.connect(cons3, "output", cons4, "input")
args = argparse.Namespace
args.num = 5
args.simple = False
args.results = True
result_queue = process(graph, inputs={prod: 1}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_(cons4.id, name)
tools.eq_("output", output)
results.extend(data)
item = result_queue.get()
tools.eq_(Counter(range(10)), Counter(results))
def testNotEnoughProcesses():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, "output", cons1, "input")
graph.connect(cons1, "output", cons2, "input")
args = argparse.Namespace
args.num = 1
args.simple = False
args.results = True
message = process(graph, inputs={prod: 5}, args=args)
tools.ok_("Not enough processes" in message)
def testNotEnoughProcesses():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
args = argparse.Namespace
args.num = 1
args.simple = False
args.results = True
message = process(graph, inputs={prod: 5}, args=args)
tools.ok_('Not enough processes' in message)
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, "output", cons1, "input")
graph.connect(prod, "output", cons2, "input")
args.num = 3
args.results = True
result_queue = process(graph, inputs={prod: 5}, args=args)
results = defaultdict(list)
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_("output", output)
results[name].append(data)
item = result_queue.get()
tools.eq_(list(range(1, 6)), results[cons1.id])
tools.eq_(list(range(1, 6)), results[cons2.id])
def testTee():
graph = WorkflowGraph()
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(prod, 'output', cons2, 'input')
args.num = 3
args.results = True
result_queue = process(graph, inputs={prod: 5}, args=args)
results = defaultdict(list)
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_('output', output)
results[name].append(data)
item = result_queue.get()
tools.eq_(list(range(1, 6)), results[cons1.id])
tools.eq_(list(range(1, 6)), results[cons2.id])
def testPipelineSimple():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, "output", cons1, "input")
graph.connect(cons1, "output", cons2, "input")
args = argparse.Namespace
args.num = 4
args.simple = True
args.results = True
result_queue = process(graph, inputs={prod: 5}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_((cons2.id, "output"), output)
results.extend(data)
item = result_queue.get()
tools.eq_(Counter(range(1, 6)), Counter(results))
def testPipelineSimple():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
args = argparse.Namespace
args.num = 4
args.simple = True
args.results = True
result_queue = process(graph, inputs={prod: 5}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_((cons2.id, 'output'), output)
results.extend(data)
item = result_queue.get()
tools.eq_(Counter(range(1, 6)), Counter(results))
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
last = TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
args.num = 4
args.results = True
result_queue = process(graph, inputs={prod: 10}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_(last.id, name)
tools.eq_('output', output)
results.append(data)
item = result_queue.get()
expected = {str(i): 2 for i in range(1, 11)}
tools.eq_(expected, Counter(results))
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
last = TestTwoInOneOut()
graph.connect(prod, "output0", cons1, "input")
graph.connect(prod, "output1", cons2, "input")
graph.connect(cons1, "output", last, "input0")
graph.connect(cons2, "output", last, "input1")
args.num = 4
args.results = True
result_queue = process(graph, inputs={prod: 10}, args=args)
results = []
item = result_queue.get()
while item != STATUS_TERMINATED:
name, output, data = item
tools.eq_(last.id, name)
tools.eq_("output", output)
results.append(data)
item = result_queue.get()
expected = {str(i): 2 for i in range(1, 11)}
tools.eq_(expected, Counter(results))
