def test_task_auto_kwargs(self):
p = PyTask(func="json.dumps",
obj={"hello": "world"},
stored_data_varname="json",
auto_kwargs=True)
a = p.run_task({})
self.assertEqual(a.stored_data["json"], '{"hello": "world"}')
p = PyTask(func="pow", args=[3, 2], stored_data_varname="data")
a = p.run_task({})
self.assertEqual(a.stored_data["data"], 9)
p = PyTask(func="print", args=[3])
p.run_task({})
def test_task_auto_kwargs(self):
p = PyTask(func='json.dumps',
obj={'hello': 'world'},
stored_data_varname="json",
auto_kwargs=True)
a = p.run_task({})
self.assertEqual(a.stored_data['json'], '{"hello": "world"}')
p = PyTask(func='pow', args=[3, 2], stored_data_varname='data')
a = p.run_task({})
self.assertEqual(a.stored_data['data'], 9)
p = PyTask(func='print', args=[3])
p.run_task({})
def test_fizzle(self):
p = PyTask(func="fireworks.tests.mongo_tests.throw_error", args=["Testing; this error is normal."])
fw = Firework(p)
self.lp.add_wf(fw)
self.assertTrue(launch_rocket(self.lp, self.fworker))
self.assertEqual(self.lp.get_fw_by_id(1).state, 'FIZZLED')
self.assertFalse(launch_rocket(self.lp, self.fworker))
def test_copy(self):
"""Test that we can produce a copy of a Workflow but that the copy
has unique fw_ids.
"""
fws = []
for i in range(5):
fw = Firework([PyTask(func="print", args=[i])], fw_id=i, name=i)
fws.append(fw)
wf = Workflow(fws, links_dict={0: [1, 2, 3], 1: [4], 2: [4]})
wf_copy = Workflow.from_wflow(wf)
# now we compare to the original to make sure dependencies are same.
# have to do gymnastics because ids will NOT be the same
# but names are retained
for fw in wf_copy.fws:
children = wf_copy.links.get(fw.fw_id, list())
orig_id = fw.name
orig_children = wf.links.get(orig_id, list())
for child_id, orig_child_id in zip(children, orig_children):
self.assertEqual(orig_child_id, wf_copy.id_fw[child_id].name)
def test_init(self):
fws = []
for i in range(5):
fw = Firework([PyTask(func="print", args=[i])], fw_id=i)
fws.append(fw)
wf = Workflow(fws, links_dict={0: [1, 2, 3], 1: [4], 2: [4]})
self.assertIsInstance(wf, Workflow)
self.assertRaises(ValueError,
Workflow,
fws,
links_dict={
0: [1, 2, 3],
1: [4],
100: [4]
})
self.assertRaises(ValueError,
Workflow,
fws,
links_dict={
0: [1, 2, 3],
1: [4],
2: [100]
})
def test_archive(self):
p = PyTask(func="fireworks.tests.mongo_tests.throw_error",
args=["This should not happen"])
fw = Firework(p)
self.lp.add_wf(fw)
self.lp.archive_wf(fw.fw_id)
self.assertFalse(launch_rocket(self.lp, self.fworker))
def test_get_time_report_for_wf(self):
task = PyTask(func="time.sleep", args=[0.5])
fw1 = Firework([task],
spec={
'wf_task_index': "test1_1",
"nproc": 16
},
fw_id=1)
fw2 = Firework([task],
spec={
'wf_task_index': "test2_1",
"nproc": 16
},
fw_id=2)
wf = Workflow([fw1, fw2])
self.lp.add_wf(wf)
rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR)
wf = self.lp.get_wf_by_fw_id(1)
assert wf.state == "COMPLETED"
tr = get_time_report_for_wf(wf)
assert tr.n_fws == 2
assert tr.total_run_time > 1
def setUp(self):
# Define a timed fireWork
fw_timer = Firework(PyTask(func='time.sleep', args=[5]), name="timer")
self.lp.add_wf(fw_timer)
# Get assigned fwid for timer firework
self.fw_id = self.lp.get_fw_ids({'name': 'timer'}, limit=1)[0]
self.old_wd = os.getcwd()
def test_task_auto_kwargs(self):
p = PyTask(func="json.dumps", obj={"hello": "world"},
stored_data_varname="json", auto_kwargs=True)
a = p.run_task({})
self.assertEqual(a.stored_data["json"], '{"hello": "world"}')
p = PyTask(func="pow", args=[3, 2], stored_data_varname="data")
a = p.run_task({})
self.assertEqual(a.stored_data["data"], 9)
p = PyTask(func="print", args=[3])
p.run_task({})
def test_task_data_flow(self):
""" test dataflow parameters: inputs, outputs and chunk_number """
params = {
'func': 'pow',
'inputs': ['arg', 'power', 'modulo'],
'stored_data_varname': 'data'
}
spec = {'arg': 2, 'power': 3, 'modulo': None}
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data['data'], 8)
params['outputs'] = ['result']
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data['data'], 8)
self.assertEqual(action.update_spec['result'], 8)
params['chunk_number'] = 0
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data['data'], 8)
self.assertEqual(action.mod_spec[0]['_push']['result'], 8)
params['args'] = [2, 3]
params['inputs'] = ['modulo']
spec = {'modulo': 3}
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data['data'], 2)
self.assertEqual(action.mod_spec[0]['_push']['result'], 2)
params['func'] = afunc.__module__ + '.' + afunc.__name__
params['args'] = [3, 3]
params['inputs'] = ['array']
spec = {'array': [1, 2]}
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data['data'], [1, 2])
self.assertEqual(action.mod_spec[0]['_push']['result'], 1)
self.assertEqual(action.mod_spec[1]['_push']['result'], 2)
del params['chunk_number']
action = PyTask(**params).run_task(spec)
self.assertEqual(action.update_spec['result'][0], 1)
self.assertEqual(action.update_spec['result'][1], 2)
params['outputs'] = ['first', 'second']
action = PyTask(**params).run_task(spec)
self.assertEqual(action.update_spec['first'], 1)
self.assertEqual(action.update_spec['second'], 2)
def test_task_data_flow(self):
"""test dataflow parameters: inputs, outputs and chunk_number"""
params = {
"func": "pow",
"inputs": ["arg", "power", "modulo"],
"stored_data_varname": "data"
}
spec = {"arg": 2, "power": 3, "modulo": None}
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data["data"], 8)
params["outputs"] = ["result"]
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data["data"], 8)
self.assertEqual(action.update_spec["result"], 8)
params["chunk_number"] = 0
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data["data"], 8)
self.assertEqual(action.mod_spec[0]["_push"]["result"], 8)
params["args"] = [2, 3]
params["inputs"] = ["modulo"]
spec = {"modulo": 3}
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data["data"], 2)
self.assertEqual(action.mod_spec[0]["_push"]["result"], 2)
params["func"] = afunc.__module__ + "." + afunc.__name__
params["args"] = [3, 3]
params["inputs"] = ["array"]
spec = {"array": [1, 2]}
action = PyTask(**params).run_task(spec)
self.assertEqual(action.stored_data["data"], [1, 2])
self.assertEqual(action.mod_spec[0]["_push"]["result"], 1)
self.assertEqual(action.mod_spec[1]["_push"]["result"], 2)
del params["chunk_number"]
action = PyTask(**params).run_task(spec)
self.assertEqual(action.update_spec["result"][0], 1)
self.assertEqual(action.update_spec["result"][1], 2)
params["outputs"] = ["first", "second"]
action = PyTask(**params).run_task(spec)
self.assertEqual(action.update_spec["first"], 1)
self.assertEqual(action.update_spec["second"], 2)
def test_directory(self):
doc_class = self.get_document_class_from_mixin(DirectoryMixin)
doc = doc_class()
# create and run a fireworks workflow
task = PyTask(func="time.sleep", args=[0.5])
wf = Workflow([
Firework(task, spec={'wf_task_index': "1"}, fw_id=1),
Firework(task, spec={'wf_task_index': "2"}, fw_id=2)
])
self.lp.add_wf(wf)
rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR)
wf = self.lp.get_wf_by_fw_id(1)
doc.set_dir_names_from_fws_wf(wf)
assert len(doc.dir_names) == 2
if has_mongodb():
doc.save()
query_result = doc_class.objects()
assert len(query_result) == 1
assert len(query_result[0].dir_names) == 2
def setUp(self):
# define the individual FireWorks used in the Workflow
# Parent Firework
fw_p = Firework(ScriptTask.from_str(
'echo "Cronus is the ruler of titans"', {'store_stdout': True}),
name="parent",
fw_id=1)
# Sibling fireworks
#fw_s1 = Firework(ScriptTask.from_str(
# 'echo "Zeus is son of Cronus"',
# {'store_stdout':True}), name="sib1", fw_id=2, parents=fw_p)
# Timed firework
fw_s1 = Firework(PyTask(func='time.sleep', args=[5]),
name="sib1",
fw_id=2,
parents=fw_p)
fw_s2 = Firework(ScriptTask.from_str(
'echo "Poisedon is brother of Zeus"', {'store_stdout': True}),
name="sib2",
fw_id=3,
parents=fw_p)
fw_s3 = Firework(ScriptTask.from_str('echo "Hades is brother of Zeus"',
{'store_stdout': True}),
name="sib3",
fw_id=4,
parents=fw_p)
fw_s4 = Firework(ScriptTask.from_str(
'echo "Demeter is sister & wife of Zeus"', {'store_stdout': True}),
name="sib4",
fw_id=5,
parents=fw_p)
fw_s5 = Firework(ScriptTask.from_str(
'echo "Lapetus is son of Oceanus"', {'store_stdout': True}),
name="cousin1",
fw_id=6)
# Children fireworks
fw_c1 = Firework(ScriptTask.from_str('echo "Ares is son of Zeus"',
{'store_stdout': True}),
name="c1",
fw_id=7,
parents=fw_s1)
fw_c2 = Firework(ScriptTask.from_str(
'echo "Persephone is daughter of Zeus & Demeter and wife of Hades"',
{'store_stdout': True}),
name="c2",
fw_id=8,
parents=[fw_s1, fw_s4])
fw_c3 = Firework(ScriptTask.from_str(
'echo "Makaria is daughter of Hades & Persephone"',
{'store_stdout': True}),
name="c3",
fw_id=9,
parents=[fw_s3, fw_c2])
fw_c4 = Firework(ScriptTask.from_str(
'echo "Dione is descendant of Lapetus"', {'store_stdout': True}),
name="c4",
fw_id=10,
parents=fw_s5)
fw_c5 = Firework(ScriptTask.from_str(
'echo "Aphrodite is son of of Zeus and Dione"',
{'store_stdout': True}),
name="c5",
fw_id=11,
parents=[fw_s1, fw_c4])
fw_c6 = Firework(ScriptTask.from_str(
'echo "Atlas is son of of Lapetus"', {'store_stdout': True}),
name="c6",
fw_id=12,
parents=fw_s5)
fw_c7 = Firework(ScriptTask.from_str(
'echo "Maia is daughter of Atlas"', {'store_stdout': True}),
name="c7",
fw_id=13,
parents=fw_c6)
fw_c8 = Firework(ScriptTask.from_str(
'echo "Hermes is daughter of Maia and Zeus"',
{'store_stdout': True}),
name="c8",
fw_id=14,
parents=[fw_s1, fw_c7])
# assemble Workflow from FireWorks and their connections by id
workflow = Workflow([
fw_p, fw_s1, fw_s2, fw_s3, fw_s4, fw_s5, fw_c1, fw_c2, fw_c3,
fw_c4, fw_c5, fw_c6, fw_c7, fw_c8
])
self.lp.add_wf(workflow)
# Give names to fw_ids
self.zeus_fw_id = 2
self.zeus_child_fw_ids = set([7, 8, 9, 11, 14])
self.lapetus_desc_fw_ids = set([6, 10, 12, 13])
self.zeus_sib_fw_ids = set([3, 4, 5])
self.par_fw_id = 1
self.all_ids = self.zeus_child_fw_ids | self.lapetus_desc_fw_ids | \
self.zeus_sib_fw_ids | set([self.zeus_fw_id]) | \
set([self.par_fw_id])
self.old_wd = os.getcwd()