def testIterative():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestIterative()
graph.connect(prod, 'output', cons, 'input')
results = simple_process.process_and_return(graph, {prod: 25})
tools.eq_({cons.id: {'output': list(range(1, 26))}}, results)
def testAvg():
avg_wf = graph_avg()
avg_wf.flatten()
results = simple_process.process_and_return(avg_wf, inputs={ 'NumberProducer' : [ {} ] } )
tools.eq_(1, len(results))
for key in results:
tools.eq_({ 'output' :[[499.5, 1000, 499500]]} , results[key])
def testConsumer():
graph = WorkflowGraph()
prod = TestProducer()
cons = PrintDataConsumer()
graph.connect(prod, "output", cons, "input")
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({}, results)
def testConsumer():
graph = WorkflowGraph()
prod = TestProducer()
cons = PrintDataConsumer()
graph.connect(prod, 'output', cons, 'input')
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({}, results)
def testWriter():
graph = WorkflowGraph()
prod = TestProducer()
cons1 = TestOneInOneOutWriter()
graph.connect(prod, "output", cons1, "input")
results = simple_process.process_and_return(graph, {prod: 5})
tools.eq_({cons1.id: {"output": list(range(1, 6))}}, results)
def testContinuousReduce():
prod = NumberProducer()
test = TestPE()
graph = WorkflowGraph()
graph.connect(prod, 'output', test, 'input')
results = simple_process.process_and_return(graph, {prod: 5})
tools.eq_({test.id: {'output': [[0] for i in range(5)]}}, results)
def testIterative():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestIterative()
graph.connect(prod, "output", cons, "input")
results = simple_process.process_and_return(graph, {prod: 25})
tools.eq_({cons.id: {"output": list(range(1, 26))}}, results)
def esgf_workflow(source, worker, monitor=None, headers=None):
graph = WorkflowGraph()
# TODO: configure limit
esgsearch = EsgSearch(
url=wps_url(),
search_url=source.get('url', 'https://esgf-data.dkrz.de/esg-search'),
constraints=source.get(
'constraints',
source.get('facets')), # facets for backward compatibility
query=source.get('query'),
limit=source.get('limit', 100),
search_type='File',
distrib=source.get('distrib'),
replica=source.get('replica'),
latest=source.get('latest'),
temporal=source.get('temporal'),
start=source.get('start'),
end=source.get('end'))
esgsearch.set_monitor(monitor, 0, 10)
download = Download(url=wps_url(), headers=headers)
download.set_monitor(monitor, 10, 50)
doit = GenericWPS(headers=headers, **worker)
doit.set_monitor(monitor, 50, 100)
graph.connect(esgsearch, esgsearch.OUTPUT_NAME, download,
download.INPUT_NAME)
graph.connect(download, download.OUTPUT_NAME, doit, doit.INPUT_NAME)
result = simple_process.process_and_return(graph, inputs={esgsearch: [{}]})
status_location = result[doit.id][doit.STATUS_LOCATION_NAME][0]
status = result[doit.id][doit.STATUS_NAME][0]
return dict(worker=dict(status_location=status_location, status=status))
def testContinuousReduce():
prod = NumberProducer()
test = TestPE()
graph = WorkflowGraph()
graph.connect(prod, 'output', test, 'input')
results = simple_process.process_and_return(graph, {prod: 5})
tools.eq_({test.id: {'output': [[0] for i in range(5)]}}, results)
def testWriter():
graph = WorkflowGraph()
prod = TestProducer()
cons1 = TestOneInOneOutWriter()
graph.connect(prod, 'output', cons1, 'input')
results = simple_process.process_and_return(graph, {prod: 5})
tools.eq_({cons1.id: {'output': list(range(1, 6))}}, results)
def testCount():
count_wf = graph_count()
count_wf.flatten()
results = simple_process.process_and_return(count_wf, inputs={ 'NumberProducer' : [ {} ] } )
print results
tools.eq_(1, len(results))
for key in results:
tools.eq_({ 'output' :[[1000]]} , results[key])
def testAvg():
avg_wf = graph_avg()
avg_wf.flatten()
results = simple_process.process_and_return(
avg_wf, inputs={'NumberProducer': [{}]})
tools.eq_(1, len(results))
for key in results:
tools.eq_({'output': [[499.5, 1000, 499500]]}, results[key])
def testSum():
sum_wf = graph_sum()
sum_wf.flatten()
results = simple_process.process_and_return(sum_wf, inputs={ 'NumberProducer' : [ {} ] } )
# there should be only one result
tools.eq_(1, len(results))
for key in results:
tools.eq_({ 'output' :[[499500]]} , results[key])
def testCount():
count_wf = graph_count()
count_wf.flatten()
results = simple_process.process_and_return(
count_wf, inputs={'NumberProducer': [{}]})
tools.eq_(1, len(results))
for key in results:
tools.eq_({'output': [[1000]]}, results[key])
def testPipeline():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, "output", cons1, "input")
graph.connect(cons1, "output", cons2, "input")
results = simple_process.process_and_return(graph, inputs={prod: 5})
tools.eq_({cons2.id: {"output": list(range(1, 6))}}, results)
def testWriter():
graph = WorkflowGraph()
prod = t.TestProducer()
prev = prod
cons1 = t.TestOneInOneOutWriter()
graph.connect(prod, 'output', cons1, 'input')
results = simple_process.process_and_return(graph,
{prod: [{}, {}, {}, {}, {}]})
assert {cons1.id: {'output': [1, 2, 3, 4, 5]}} == results
def testSum():
sum_wf = graph_sum()
sum_wf.flatten()
results = simple_process.process_and_return(
sum_wf, inputs={'NumberProducer': [{}]})
# there should be only one result
tools.eq_(1, len(results))
for key in results:
tools.eq_({'output': [[499500]]}, results[key])
def testPipeline():
prod = TestProducer()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
results = simple_process.process_and_return(graph, inputs={prod: 5})
tools.eq_({cons2.id: {'output': list(range(1, 6))}}, results)
def testPipeline():
prod = t.TestProducer()
cons1 = t.TestOneInOneOut()
cons2 = t.TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', cons1, 'input')
graph.connect(cons1, 'output', cons2, 'input')
results = simple_process.process_and_return(
graph, inputs={prod: [{}, {}, {}, {}, {}]})
assert {cons2.id: {'output': [1, 2, 3, 4, 5]}} == results
def testStdDev():
stddev_wf = graph_stddev()
stddev_wf.flatten()
results = simple_process.process_and_return(
stddev_wf, inputs={'NumberProducer': [{}]})
tools.eq_(1, len(results))
for key in results:
# (41.51433802981722, 288.8182819698227, 1000, 499500)
tools.eq_(1000, results[key]['output'][0][2])
tools.eq_(499500, results[key]['output'][0][3])
def testMinMax():
min_max_wf = graph_min_max()
min_max_wf.flatten()
results = simple_process.process_and_return(
min_max_wf, inputs={'NumberProducer': [{}]})
tools.eq_(2, len(results))
for key in results:
if key.startswith('MinPE'):
tools.eq_({'output': [[0]]}, results[key])
else:
tools.eq_({'output': [[999]]}, results[key])
def testMinMax():
min_max_wf = graph_min_max()
min_max_wf.flatten()
results = simple_process.process_and_return(min_max_wf, inputs={ 'NumberProducer' : [ {} ] } )
print results
tools.eq_(2, len(results))
for key in results:
if key.startswith('MinPE'):
tools.eq_({ 'output' :[[0]]} , results[key])
else:
tools.eq_({ 'output' :[[999]]} , results[key])
def testStdDev():
stddev_wf = graph_stddev()
stddev_wf.flatten()
results = simple_process.process_and_return(
stddev_wf,
inputs={'NumberProducer': [{}]})
tools.eq_(1, len(results))
for key in results:
# (41.51433802981722, 288.8182819698227, 1000, 499500)
tools.eq_(1000, results[key]['output'][0][2])
tools.eq_(499500, results[key]['output'][0][3])
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
last = TestTwoInOneOut()
graph.connect(prod, "output0", cons1, "input")
graph.connect(prod, "output1", cons2, "input")
graph.connect(cons1, "output", last, "input0")
graph.connect(cons2, "output", last, "input1")
results = simple_process.process_and_return(graph, {prod: 1})
tools.eq_({last.id: {"output": ["1", "1"]}}, results)
def testSquare():
graph = WorkflowGraph()
prod = TestProducer(2)
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
last = TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
results = simple_process.process_and_return(graph, {prod: 1})
tools.eq_({last.id: {'output': ['1', '1']}}, results)
def testSquare():
graph = WorkflowGraph()
prod = t.TestProducer(2)
cons1 = t.TestOneInOneOut()
cons2 = t.TestOneInOneOut()
last = t.TestTwoInOneOut()
graph.connect(prod, 'output0', cons1, 'input')
graph.connect(prod, 'output1', cons2, 'input')
graph.connect(cons1, 'output', last, 'input0')
graph.connect(cons2, 'output', last, 'input1')
results = simple_process.process_and_return(graph, {prod: [{}]})
assert {last.id: {'output': ['1', '1']}} == results
def testComposite():
comp = CompositePE()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
comp.connect(cons1, 'output', cons2, 'input')
comp._map_input('comp_input', cons1, 'input')
comp._map_output('comp_output', cons2, 'output')
prod = TestProducer()
cons = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', comp, 'comp_input')
graph.connect(comp, 'comp_output', cons, 'input')
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({cons.id: {'output': list(range(1, 11))}}, results)
def thredds_workflow(source, worker, monitor=None, headers=None):
graph = WorkflowGraph()
download = ThreddsDownload(url=wps_url(), headers=headers, **source)
download.set_monitor(monitor, 10, 50)
doit = GenericWPS(headers=headers, **worker)
doit.set_monitor(monitor, 50, 100)
graph.connect(download, download.OUTPUT_NAME, doit, doit.INPUT_NAME)
result = simple_process.process_and_return(graph, inputs={download: [{}]})
status_location = result[doit.id][doit.STATUS_LOCATION_NAME][0]
status = result[doit.id][doit.STATUS_NAME][0]
return dict(worker=dict(status_location=status_location, status=status))
def testComposite():
comp = CompositePE()
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
comp.connect(cons1, "output", cons2, "input")
comp._map_input("comp_input", cons1, "input")
comp._map_output("comp_output", cons2, "output")
prod = TestProducer()
cons = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, "output", comp, "comp_input")
graph.connect(comp, "comp_output", cons, "input")
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({cons.id: {"output": list(range(1, 11))}}, results)
def testInputsAndOutputs():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestOneInOneOut()
cons._add_output("output", tuple_type=["integer"])
cons._add_input("input", grouping=[0], tuple_type=["integer"])
cons.setInputTypes({"input": ["number"]})
tools.eq_({"output": ["number"]}, cons.getOutputTypes())
cons._add_output("output2")
try:
cons.getOutputTypes()
except Exception:
pass
graph.connect(prod, "output", cons, "input")
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({cons.id: {"output": list(range(1, 11))}}, results)
def testInputsAndOutputs():
graph = WorkflowGraph()
prod = TestProducer()
cons = TestOneInOneOut()
cons._add_output('output', tuple_type=['integer'])
cons._add_input('input', grouping=[0], tuple_type=['integer'])
cons.setInputTypes({'input': ['number']})
tools.eq_({'output': ['number']}, cons.getOutputTypes())
cons._add_output('output2')
try:
cons.getOutputTypes()
except Exception:
pass
graph.connect(prod, 'output', cons, 'input')
results = simple_process.process_and_return(graph, {prod: 10})
tools.eq_({cons.id: {'output': list(range(1, 11))}}, results)
def testCreateChain():
def add(a, b):
return a + b
def mult(a, b):
return a * b
def is_odd(a):
return a % 2 == 1
c = [(add, {'b': 1}), (mult, {'b': 3}), is_odd]
chain = create_iterative_chain(c)
prod = TestProducer()
graph = WorkflowGraph()
graph.connect(prod, 'output', chain, 'input')
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 2})
for key, value in results.items():
tools.eq_({'output': [False, True]}, value)
def testCreateChain():
def add(a, b):
return a + b
def mult(a, b):
return a * b
def is_odd(a):
return a % 2 == 1
c = [(add, {"b": 1}), (mult, {"b": 3}), is_odd]
chain = create_iterative_chain(c)
prod = TestProducer()
graph = WorkflowGraph()
graph.connect(prod, "output", chain, "input")
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 2})
for key, value in results.items():
tools.eq_({"output": [False, True]}, value)
def testCompositeWithCreateParams():
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
def create_graph(graph, connections):
for i in range(connections):
graph.connect(cons1, 'output', cons2, 'input')
comp = CompositePE(create_graph, {'connections': 2})
comp._map_input('comp_input', cons1, 'input')
comp._map_output('comp_output', cons2, 'output')
prod = TestProducer()
cons = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, 'output', comp, 'comp_input')
graph.connect(comp, 'comp_output', cons, 'input')
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 10})
expected = []
for i in range(1, 11):
expected += [i, i]
tools.eq_({cons.id: {'output': expected}}, results)
def solr_workflow(source, worker, monitor=None, headers=None):
graph = WorkflowGraph()
solrsearch = SolrSearch(url=source.get('url'),
query=source.get('query'),
filter_query=source.get('filter_query'))
solrsearch.set_monitor(monitor, 0, 10)
download = Download(url=wps_url(), headers=headers)
download.set_monitor(monitor, 10, 50)
doit = GenericWPS(headers=headers, **worker)
doit.set_monitor(monitor, 50, 100)
graph.connect(solrsearch, solrsearch.OUTPUT_NAME, download,
download.INPUT_NAME)
graph.connect(download, download.OUTPUT_NAME, doit, doit.INPUT_NAME)
result = simple_process.process_and_return(graph,
inputs={solrsearch: [{}]})
status_location = result[doit.id][doit.STATUS_LOCATION_NAME][0]
status = result[doit.id][doit.STATUS_NAME][0]
return dict(worker=dict(status_location=status_location, status=status))
def testCompositeWithCreateParams():
cons1 = TestOneInOneOut()
cons2 = TestOneInOneOut()
def create_graph(graph, connections):
for i in range(connections):
graph.connect(cons1, "output", cons2, "input")
comp = CompositePE(create_graph, {"connections": 2})
comp._map_input("comp_input", cons1, "input")
comp._map_output("comp_output", cons2, "output")
prod = TestProducer()
cons = TestOneInOneOut()
graph = WorkflowGraph()
graph.connect(prod, "output", comp, "comp_input")
graph.connect(comp, "comp_output", cons, "input")
graph.flatten()
results = simple_process.process_and_return(graph, {prod: 10})
expected = []
for i in range(1, 11):
expected += [i, i]
tools.eq_({cons.id: {"output": expected}}, results)
