def testIterative(): graph = WorkflowGraph() prod = TestProducer() cons = TestIterative() graph.connect(prod, 'output', cons, 'input') results = simple_process.process_and_return(graph, {prod: 25}) tools.eq_({cons.id: {'output': list(range(1, 26))}}, results)
def testAvg(): avg_wf = graph_avg() avg_wf.flatten() results = simple_process.process_and_return(avg_wf, inputs={ 'NumberProducer' : [ {} ] } ) tools.eq_(1, len(results)) for key in results: tools.eq_({ 'output' :[[499.5, 1000, 499500]]} , results[key])
def testConsumer(): graph = WorkflowGraph() prod = TestProducer() cons = PrintDataConsumer() graph.connect(prod, "output", cons, "input") results = simple_process.process_and_return(graph, {prod: 10}) tools.eq_({}, results)
def testConsumer(): graph = WorkflowGraph() prod = TestProducer() cons = PrintDataConsumer() graph.connect(prod, 'output', cons, 'input') results = simple_process.process_and_return(graph, {prod: 10}) tools.eq_({}, results)
def testWriter(): graph = WorkflowGraph() prod = TestProducer() cons1 = TestOneInOneOutWriter() graph.connect(prod, "output", cons1, "input") results = simple_process.process_and_return(graph, {prod: 5}) tools.eq_({cons1.id: {"output": list(range(1, 6))}}, results)
def testContinuousReduce(): prod = NumberProducer() test = TestPE() graph = WorkflowGraph() graph.connect(prod, 'output', test, 'input') results = simple_process.process_and_return(graph, {prod: 5}) tools.eq_({test.id: {'output': [[0] for i in range(5)]}}, results)
def testIterative(): graph = WorkflowGraph() prod = TestProducer() cons = TestIterative() graph.connect(prod, "output", cons, "input") results = simple_process.process_and_return(graph, {prod: 25}) tools.eq_({cons.id: {"output": list(range(1, 26))}}, results)
def esgf_workflow(source, worker, monitor=None, headers=None): graph = WorkflowGraph() # TODO: configure limit esgsearch = EsgSearch( url=wps_url(), search_url=source.get('url', 'https://esgf-data.dkrz.de/esg-search'), constraints=source.get( 'constraints', source.get('facets')), # facets for backward compatibility query=source.get('query'), limit=source.get('limit', 100), search_type='File', distrib=source.get('distrib'), replica=source.get('replica'), latest=source.get('latest'), temporal=source.get('temporal'), start=source.get('start'), end=source.get('end')) esgsearch.set_monitor(monitor, 0, 10) download = Download(url=wps_url(), headers=headers) download.set_monitor(monitor, 10, 50) doit = GenericWPS(headers=headers, **worker) doit.set_monitor(monitor, 50, 100) graph.connect(esgsearch, esgsearch.OUTPUT_NAME, download, download.INPUT_NAME) graph.connect(download, download.OUTPUT_NAME, doit, doit.INPUT_NAME) result = simple_process.process_and_return(graph, inputs={esgsearch: [{}]}) status_location = result[doit.id][doit.STATUS_LOCATION_NAME][0] status = result[doit.id][doit.STATUS_NAME][0] return dict(worker=dict(status_location=status_location, status=status))
def testWriter(): graph = WorkflowGraph() prod = TestProducer() cons1 = TestOneInOneOutWriter() graph.connect(prod, 'output', cons1, 'input') results = simple_process.process_and_return(graph, {prod: 5}) tools.eq_({cons1.id: {'output': list(range(1, 6))}}, results)
def testCount(): count_wf = graph_count() count_wf.flatten() results = simple_process.process_and_return(count_wf, inputs={ 'NumberProducer' : [ {} ] } ) print results tools.eq_(1, len(results)) for key in results: tools.eq_({ 'output' :[[1000]]} , results[key])
def testAvg(): avg_wf = graph_avg() avg_wf.flatten() results = simple_process.process_and_return( avg_wf, inputs={'NumberProducer': [{}]}) tools.eq_(1, len(results)) for key in results: tools.eq_({'output': [[499.5, 1000, 499500]]}, results[key])
def testSum(): sum_wf = graph_sum() sum_wf.flatten() results = simple_process.process_and_return(sum_wf, inputs={ 'NumberProducer' : [ {} ] } ) # there should be only one result tools.eq_(1, len(results)) for key in results: tools.eq_({ 'output' :[[499500]]} , results[key])
def testCount(): count_wf = graph_count() count_wf.flatten() results = simple_process.process_and_return( count_wf, inputs={'NumberProducer': [{}]}) tools.eq_(1, len(results)) for key in results: tools.eq_({'output': [[1000]]}, results[key])
def testPipeline(): prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, "output", cons1, "input") graph.connect(cons1, "output", cons2, "input") results = simple_process.process_and_return(graph, inputs={prod: 5}) tools.eq_({cons2.id: {"output": list(range(1, 6))}}, results)
def testWriter(): graph = WorkflowGraph() prod = t.TestProducer() prev = prod cons1 = t.TestOneInOneOutWriter() graph.connect(prod, 'output', cons1, 'input') results = simple_process.process_and_return(graph, {prod: [{}, {}, {}, {}, {}]}) assert {cons1.id: {'output': [1, 2, 3, 4, 5]}} == results
def testSum(): sum_wf = graph_sum() sum_wf.flatten() results = simple_process.process_and_return( sum_wf, inputs={'NumberProducer': [{}]}) # there should be only one result tools.eq_(1, len(results)) for key in results: tools.eq_({'output': [[499500]]}, results[key])
def testPipeline(): prod = TestProducer() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', cons1, 'input') graph.connect(cons1, 'output', cons2, 'input') results = simple_process.process_and_return(graph, inputs={prod: 5}) tools.eq_({cons2.id: {'output': list(range(1, 6))}}, results)
def testPipeline(): prod = t.TestProducer() cons1 = t.TestOneInOneOut() cons2 = t.TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', cons1, 'input') graph.connect(cons1, 'output', cons2, 'input') results = simple_process.process_and_return( graph, inputs={prod: [{}, {}, {}, {}, {}]}) assert {cons2.id: {'output': [1, 2, 3, 4, 5]}} == results
def testStdDev(): stddev_wf = graph_stddev() stddev_wf.flatten() results = simple_process.process_and_return( stddev_wf, inputs={'NumberProducer': [{}]}) tools.eq_(1, len(results)) for key in results: # (41.51433802981722, 288.8182819698227, 1000, 499500) tools.eq_(1000, results[key]['output'][0][2]) tools.eq_(499500, results[key]['output'][0][3])
def testMinMax(): min_max_wf = graph_min_max() min_max_wf.flatten() results = simple_process.process_and_return( min_max_wf, inputs={'NumberProducer': [{}]}) tools.eq_(2, len(results)) for key in results: if key.startswith('MinPE'): tools.eq_({'output': [[0]]}, results[key]) else: tools.eq_({'output': [[999]]}, results[key])
def testMinMax(): min_max_wf = graph_min_max() min_max_wf.flatten() results = simple_process.process_and_return(min_max_wf, inputs={ 'NumberProducer' : [ {} ] } ) print results tools.eq_(2, len(results)) for key in results: if key.startswith('MinPE'): tools.eq_({ 'output' :[[0]]} , results[key]) else: tools.eq_({ 'output' :[[999]]} , results[key])
def testSquare(): graph = WorkflowGraph() prod = TestProducer(2) cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() last = TestTwoInOneOut() graph.connect(prod, "output0", cons1, "input") graph.connect(prod, "output1", cons2, "input") graph.connect(cons1, "output", last, "input0") graph.connect(cons2, "output", last, "input1") results = simple_process.process_and_return(graph, {prod: 1}) tools.eq_({last.id: {"output": ["1", "1"]}}, results)
def testSquare(): graph = WorkflowGraph() prod = TestProducer(2) cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() last = TestTwoInOneOut() graph.connect(prod, 'output0', cons1, 'input') graph.connect(prod, 'output1', cons2, 'input') graph.connect(cons1, 'output', last, 'input0') graph.connect(cons2, 'output', last, 'input1') results = simple_process.process_and_return(graph, {prod: 1}) tools.eq_({last.id: {'output': ['1', '1']}}, results)
def testSquare(): graph = WorkflowGraph() prod = t.TestProducer(2) cons1 = t.TestOneInOneOut() cons2 = t.TestOneInOneOut() last = t.TestTwoInOneOut() graph.connect(prod, 'output0', cons1, 'input') graph.connect(prod, 'output1', cons2, 'input') graph.connect(cons1, 'output', last, 'input0') graph.connect(cons2, 'output', last, 'input1') results = simple_process.process_and_return(graph, {prod: [{}]}) assert {last.id: {'output': ['1', '1']}} == results
def testComposite(): comp = CompositePE() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() comp.connect(cons1, 'output', cons2, 'input') comp._map_input('comp_input', cons1, 'input') comp._map_output('comp_output', cons2, 'output') prod = TestProducer() cons = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', comp, 'comp_input') graph.connect(comp, 'comp_output', cons, 'input') graph.flatten() results = simple_process.process_and_return(graph, {prod: 10}) tools.eq_({cons.id: {'output': list(range(1, 11))}}, results)
def thredds_workflow(source, worker, monitor=None, headers=None): graph = WorkflowGraph() download = ThreddsDownload(url=wps_url(), headers=headers, **source) download.set_monitor(monitor, 10, 50) doit = GenericWPS(headers=headers, **worker) doit.set_monitor(monitor, 50, 100) graph.connect(download, download.OUTPUT_NAME, doit, doit.INPUT_NAME) result = simple_process.process_and_return(graph, inputs={download: [{}]}) status_location = result[doit.id][doit.STATUS_LOCATION_NAME][0] status = result[doit.id][doit.STATUS_NAME][0] return dict(worker=dict(status_location=status_location, status=status))
def testComposite(): comp = CompositePE() cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() comp.connect(cons1, "output", cons2, "input") comp._map_input("comp_input", cons1, "input") comp._map_output("comp_output", cons2, "output") prod = TestProducer() cons = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, "output", comp, "comp_input") graph.connect(comp, "comp_output", cons, "input") graph.flatten() results = simple_process.process_and_return(graph, {prod: 10}) tools.eq_({cons.id: {"output": list(range(1, 11))}}, results)
def testInputsAndOutputs(): graph = WorkflowGraph() prod = TestProducer() cons = TestOneInOneOut() cons._add_output("output", tuple_type=["integer"]) cons._add_input("input", grouping=[0], tuple_type=["integer"]) cons.setInputTypes({"input": ["number"]}) tools.eq_({"output": ["number"]}, cons.getOutputTypes()) cons._add_output("output2") try: cons.getOutputTypes() except Exception: pass graph.connect(prod, "output", cons, "input") results = simple_process.process_and_return(graph, {prod: 10}) tools.eq_({cons.id: {"output": list(range(1, 11))}}, results)
def testInputsAndOutputs(): graph = WorkflowGraph() prod = TestProducer() cons = TestOneInOneOut() cons._add_output('output', tuple_type=['integer']) cons._add_input('input', grouping=[0], tuple_type=['integer']) cons.setInputTypes({'input': ['number']}) tools.eq_({'output': ['number']}, cons.getOutputTypes()) cons._add_output('output2') try: cons.getOutputTypes() except Exception: pass graph.connect(prod, 'output', cons, 'input') results = simple_process.process_and_return(graph, {prod: 10}) tools.eq_({cons.id: {'output': list(range(1, 11))}}, results)
def testCreateChain(): def add(a, b): return a + b def mult(a, b): return a * b def is_odd(a): return a % 2 == 1 c = [(add, {'b': 1}), (mult, {'b': 3}), is_odd] chain = create_iterative_chain(c) prod = TestProducer() graph = WorkflowGraph() graph.connect(prod, 'output', chain, 'input') graph.flatten() results = simple_process.process_and_return(graph, {prod: 2}) for key, value in results.items(): tools.eq_({'output': [False, True]}, value)
def testCreateChain(): def add(a, b): return a + b def mult(a, b): return a * b def is_odd(a): return a % 2 == 1 c = [(add, {"b": 1}), (mult, {"b": 3}), is_odd] chain = create_iterative_chain(c) prod = TestProducer() graph = WorkflowGraph() graph.connect(prod, "output", chain, "input") graph.flatten() results = simple_process.process_and_return(graph, {prod: 2}) for key, value in results.items(): tools.eq_({"output": [False, True]}, value)
def testCompositeWithCreateParams(): cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() def create_graph(graph, connections): for i in range(connections): graph.connect(cons1, 'output', cons2, 'input') comp = CompositePE(create_graph, {'connections': 2}) comp._map_input('comp_input', cons1, 'input') comp._map_output('comp_output', cons2, 'output') prod = TestProducer() cons = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, 'output', comp, 'comp_input') graph.connect(comp, 'comp_output', cons, 'input') graph.flatten() results = simple_process.process_and_return(graph, {prod: 10}) expected = [] for i in range(1, 11): expected += [i, i] tools.eq_({cons.id: {'output': expected}}, results)
def solr_workflow(source, worker, monitor=None, headers=None): graph = WorkflowGraph() solrsearch = SolrSearch(url=source.get('url'), query=source.get('query'), filter_query=source.get('filter_query')) solrsearch.set_monitor(monitor, 0, 10) download = Download(url=wps_url(), headers=headers) download.set_monitor(monitor, 10, 50) doit = GenericWPS(headers=headers, **worker) doit.set_monitor(monitor, 50, 100) graph.connect(solrsearch, solrsearch.OUTPUT_NAME, download, download.INPUT_NAME) graph.connect(download, download.OUTPUT_NAME, doit, doit.INPUT_NAME) result = simple_process.process_and_return(graph, inputs={solrsearch: [{}]}) status_location = result[doit.id][doit.STATUS_LOCATION_NAME][0] status = result[doit.id][doit.STATUS_NAME][0] return dict(worker=dict(status_location=status_location, status=status))
def testCompositeWithCreateParams(): cons1 = TestOneInOneOut() cons2 = TestOneInOneOut() def create_graph(graph, connections): for i in range(connections): graph.connect(cons1, "output", cons2, "input") comp = CompositePE(create_graph, {"connections": 2}) comp._map_input("comp_input", cons1, "input") comp._map_output("comp_output", cons2, "output") prod = TestProducer() cons = TestOneInOneOut() graph = WorkflowGraph() graph.connect(prod, "output", comp, "comp_input") graph.connect(comp, "comp_output", cons, "input") graph.flatten() results = simple_process.process_and_return(graph, {prod: 10}) expected = [] for i in range(1, 11): expected += [i, i] tools.eq_({cons.id: {"output": expected}}, results)