def test_foreach_commandlinetask(self):
"""run CommandLineTask for a list of input data"""
inputs = ["black", "white", 2.5, 17]
worklist = [{"source": {"type": "data", "value": s}} for s in inputs]
spec = {"work list": worklist}
task = {
"_fw_name": "CommandLineTask",
"command_spec": {
"command": ["echo"],
"file set": {"source": {"type": "stdout"}, "target": {"type": "path", "value": "."}},
"work list": "work list",
},
"inputs": ["work list"],
"outputs": ["file set"],
}
params = {"task": task, "split": "work list", "number of chunks": 2}
action = ForeachTask(**params).run_task(spec)
outputs = []
for detour in action.detours:
action = detour.tasks[0].run_task(detour.spec)
ofile = action.mod_spec[0]["_push"]["file set"]["value"]
self.assertTrue(os.path.exists(ofile))
with open(ofile) as fptr:
outputs.append(fptr.read().strip())
os.remove(ofile)
ref_str = " ".join(str(inp) for inp in inputs)
out_str = " ".join(str(out) for out in outputs)
self.assertEqual(out_str, ref_str)
def test_foreach_pytask(self):
"""run PyTask for a list of numbers"""
numbers = [0, 1, 2, 3, 4]
power = 2
spec = {"numbers": numbers, "power": power}
task = {
"_fw_name": "PyTask",
"func": afunc.__module__ + "." + afunc.__name__,
"inputs": ["numbers", "power"],
"outputs": ["results"],
}
params = {"task": task, "split": "numbers", "number of chunks": 3}
action = ForeachTask(**params).run_task(spec)
results = []
for detour in action.detours:
action = detour.tasks[0].run_task(detour.spec)
for mod in action.mod_spec:
results.append(mod["_push"]["results"])
for (number, result) in zip(numbers, results):
self.assertEqual(result, pow(number, power))
def test_foreach_pytask(self):
""" run PyTask for a list of numbers """
numbers = [0, 1, 2, 3, 4]
power = 2
spec = {'numbers': numbers, 'power': power}
task = {
'_fw_name': 'PyTask',
'func': afunc.__module__ + '.' + afunc.__name__,
'inputs': ['numbers', 'power'],
'outputs': ['results']
}
params = {'task': task, 'split': 'numbers', 'number of chunks': 3}
action = ForeachTask(**params).run_task(spec)
results = []
for detour in action.detours:
action = detour.tasks[0].run_task(detour.spec)
for mod in action.mod_spec:
results.append(mod['_push']['results'])
for (number, result) in zip(numbers, results):
self.assertEqual(result, pow(number, power))
def test_foreachtask_workflow():
"""
Workflow to test fireworks ForeachTask with a dummy workflow
"""
simple_task = SimpleTestTask(spec={'dft_params':[]}, inputs=['dft_params'])
simple_dict = simple_task.to_dict()
#resimple_task = SimpleTestTask.from_dict(simple_dict)
#print(simple_task)
print(simple_dict)
#print(resimple_task)
firetask1 = ForeachTask(task=simple_dict, split='dft_params', spec={'dft_params':[1,2,3]})
fw1 = Firework([simple_task], spec={'dft_params':[-1,-2,-3], 'dft_outputs':{} }, fw_id=1)
fw2 = Firework([firetask1], fw_id=2)
fw3 = Firework([firetask1], fw_id=3)
fw4 = Firework([simple_task], fw_id=4)
#wf = Firework([simple_task, firetask1, simple_task], spec={'dft_params':[1,2,3], 'ranking': [-3,-1] })
workflow = Workflow([fw1, fw2, fw4], {1: [2], 2: [4],})
return workflow
def test_foreach_commandlinetask(self):
""" run CommandLineTask for a list of input data """
inputs = ['black', 'white', 2.5, 17]
worklist = [{'source': {'type': 'data', 'value': s}} for s in inputs]
spec = {'work list': worklist}
task = {
'_fw_name': 'CommandLineTask',
'command_spec': {
'command': ['echo'],
'file set': {
'source': {
'type': 'stdout'
},
'target': {
'type': 'path',
'value': '.'
}
},
'work list': 'work list'
},
'inputs': ['work list'],
'outputs': ['file set']
}
params = {'task': task, 'split': 'work list', 'number of chunks': 2}
action = ForeachTask(**params).run_task(spec)
outputs = []
for detour in action.detours:
action = detour.tasks[0].run_task(detour.spec)
ofile = action.mod_spec[0]['_push']['file set']['value']
self.assertTrue(os.path.exists(ofile))
with open(ofile, 'r') as fptr:
outputs.append(fptr.read().strip())
os.remove(ofile)
ref_str = ' '.join(str(inp) for inp in inputs)
out_str = ' '.join(str(out) for out in outputs)
self.assertEqual(out_str, ref_str)