def runYesProv(): elapsed_time=0 for x in range(0,num): graph = createGraphWithProv() elapsed_time=0 start_time = time.time() process(graph, inputs=input_data, args=args) elapsed_time+= time.time() - start_time print ("PROV ELAPSED TIME: "+str(elapsed_time/num))
def runYesProv(): elapsed_time = 0 for x in range(0, num): graph = createGraphWithProv() elapsed_time = 0 start_time = time.time() process(graph, inputs=input_data, args=args) elapsed_time += time.time() - start_time print("PROV ELAPSED TIME: " + str(elapsed_time / num))
def testTee(): graph = WorkflowGraph() prod = t.TestProducer() prev = prod cons1 = t.TestOneInOneOut() cons2 = t.TestOneInOneOut() graph.connect(prod, 'output', cons1, 'input') graph.connect(prod, 'output', cons2, 'input') args.num = 3 process(graph, inputs={prod: [{}, {}, {}, {}, {}]}, args=args)
def runNoProv(): elapsed_time=0 for x in range(0,num): time.sleep(2) graph = createWf() elapsed_time=0 start_time = time.time() process(graph, inputs=input_data, args=args) elapsed_time=elapsed_time+( time.time() - start_time) print ("NO PROV ELAPSED TIME: "+str(elapsed_time/num))
def runNoProv(): elapsed_time = 0 for x in range(0, num): time.sleep(2) graph = createWf() elapsed_time = 0 start_time = time.time() process(graph, inputs=input_data, args=args) elapsed_time = elapsed_time + (time.time() - start_time) print("NO PROV ELAPSED TIME: " + str(elapsed_time / num))
def testPipeline(): prod = t.TestProducer() cons1 = t.TestOneInOneOut() cons2 = t.TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', cons1, 'input') graph.connect(cons1, 'output', cons2, 'input') args = argparse.Namespace args.num = 5 args.simple = False process(graph, inputs={prod: [{}, {}, {}]}, args=args)
def testSquare(): graph = WorkflowGraph() prod = t.TestProducer(2) cons1 = t.TestOneInOneOut() cons2 = t.TestOneInOneOut() last = t.TestTwoInOneOut() graph.connect(prod, 'output0', cons1, 'input') graph.connect(prod, 'output1', cons2, 'input') graph.connect(cons1, 'output', last, 'input0') graph.connect(cons2, 'output', last, 'input1') args.num = 4 process(graph, inputs={prod: [{}]}, args=args)
def testGroupings(): prod = NumberProducer(10) cons1 = TestOneInOneOut() cons1.inputconnections['input']['type'] = [0] cons2 = TestOneInOneOut() cons1.inputconnections['input']['type'] = 'all' cons3 = TestOneInOneOut() cons1.inputconnections['input']['type'] = 'global' cons4 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', cons1, 'input') graph.connect(cons1, 'output', cons2, 'input') graph.connect(cons2, 'output', cons3, 'input') graph.connect(cons3, 'output', cons4, 'input') args = argparse.Namespace args.num = 5 args.simple = False args.results = True result_queue = process(graph, inputs={prod: 1}, args=args) results = [] item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_(cons4.id, name) tools.eq_('output', output) results.extend(data) item = result_queue.get() tools.eq_(Counter(range(10)), Counter(results))
def testGroupings(): prod = NumberProducer(10) cons1 = TestOneInOneOut() cons1.inputconnections["input"]["type"] = [0] cons2 = TestOneInOneOut() cons1.inputconnections["input"]["type"] = "all" cons3 = TestOneInOneOut() cons1.inputconnections["input"]["type"] = "global" cons4 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, "output", cons1, "input") graph.connect(cons1, "output", cons2, "input") graph.connect(cons2, "output", cons3, "input") graph.connect(cons3, "output", cons4, "input") args = argparse.Namespace args.num = 5 args.simple = False args.results = True result_queue = process(graph, inputs={prod: 1}, args=args) results = [] item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_(cons4.id, name) tools.eq_("output", output) results.extend(data) item = result_queue.get() tools.eq_(Counter(range(10)), Counter(results))
def testNotEnoughProcesses(): prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, "output", cons1, "input") graph.connect(cons1, "output", cons2, "input") args = argparse.Namespace args.num = 1 args.simple = False args.results = True message = process(graph, inputs={prod: 5}, args=args) tools.ok_("Not enough processes" in message)
def testNotEnoughProcesses(): prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', cons1, 'input') graph.connect(cons1, 'output', cons2, 'input') args = argparse.Namespace args.num = 1 args.simple = False args.results = True message = process(graph, inputs={prod: 5}, args=args) tools.ok_('Not enough processes' in message)
def testTee(): graph = WorkflowGraph() prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph.connect(prod, "output", cons1, "input") graph.connect(prod, "output", cons2, "input") args.num = 3 args.results = True result_queue = process(graph, inputs={prod: 5}, args=args) results = defaultdict(list) item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_("output", output) results[name].append(data) item = result_queue.get() tools.eq_(list(range(1, 6)), results[cons1.id]) tools.eq_(list(range(1, 6)), results[cons2.id])
def testTee(): graph = WorkflowGraph() prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph.connect(prod, 'output', cons1, 'input') graph.connect(prod, 'output', cons2, 'input') args.num = 3 args.results = True result_queue = process(graph, inputs={prod: 5}, args=args) results = defaultdict(list) item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_('output', output) results[name].append(data) item = result_queue.get() tools.eq_(list(range(1, 6)), results[cons1.id]) tools.eq_(list(range(1, 6)), results[cons2.id])
def testPipelineSimple(): prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, "output", cons1, "input") graph.connect(cons1, "output", cons2, "input") args = argparse.Namespace args.num = 4 args.simple = True args.results = True result_queue = process(graph, inputs={prod: 5}, args=args) results = [] item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_((cons2.id, "output"), output) results.extend(data) item = result_queue.get() tools.eq_(Counter(range(1, 6)), Counter(results))
def testPipelineSimple(): prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', cons1, 'input') graph.connect(cons1, 'output', cons2, 'input') args = argparse.Namespace args.num = 4 args.simple = True args.results = True result_queue = process(graph, inputs={prod: 5}, args=args) results = [] item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_((cons2.id, 'output'), output) results.extend(data) item = result_queue.get() tools.eq_(Counter(range(1, 6)), Counter(results))
def testSquare(): graph = WorkflowGraph() prod = TestProducer(2) cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() last = TestTwoInOneOut() graph.connect(prod, 'output0', cons1, 'input') graph.connect(prod, 'output1', cons2, 'input') graph.connect(cons1, 'output', last, 'input0') graph.connect(cons2, 'output', last, 'input1') args.num = 4 args.results = True result_queue = process(graph, inputs={prod: 10}, args=args) results = [] item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_(last.id, name) tools.eq_('output', output) results.append(data) item = result_queue.get() expected = {str(i): 2 for i in range(1, 11)} tools.eq_(expected, Counter(results))
def testSquare(): graph = WorkflowGraph() prod = TestProducer(2) cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() last = TestTwoInOneOut() graph.connect(prod, "output0", cons1, "input") graph.connect(prod, "output1", cons2, "input") graph.connect(cons1, "output", last, "input0") graph.connect(cons2, "output", last, "input1") args.num = 4 args.results = True result_queue = process(graph, inputs={prod: 10}, args=args) results = [] item = result_queue.get() while item != STATUS_TERMINATED: name, output, data = item tools.eq_(last.id, name) tools.eq_("output", output) results.append(data) item = result_queue.get() expected = {str(i): 2 for i in range(1, 11)} tools.eq_(expected, Counter(results))