def test_files_in_out(self):
# create the Workflow that passes files_in and files_out
fw1 = Firework(
[ScriptTask.from_str('echo "This is the first FireWork" > test1')],
spec={"_files_out": {"fwtest1": "test1"}}, fw_id=1)
fw2 = Firework([ScriptTask.from_str('gzip hello')], fw_id=2,
parents=[fw1],
spec={"_files_in": {"fwtest1": "hello"},
"_files_out": {"fw2": "hello.gz"}})
fw3 = Firework([ScriptTask.from_str('cat fwtest.2')], fw_id=3,
parents=[fw2],
spec={"_files_in": {"fw2": "fwtest.2"}})
wf = Workflow([fw1, fw2, fw3],
{fw1: [fw2], fw2: [fw3]})
# store workflow and launch it locally
self.lp.add_wf(wf)
launch_rocket(self.lp, self.fworker)
self.assertTrue(os.path.exists("test1"))
launch_rocket(self.lp, self.fworker)
self.assertTrue(os.path.exists("hello.gz"))
launch_rocket(self.lp, self.fworker)
self.assertTrue(os.path.exists("fwtest.2"))
for f in ["test1", "hello.gz", "fwtest.2"]:
os.remove(f)
def test_tracker_failed_fw(self):
"""
Add a bad firetask to workflow and test the tracking
"""
self._teardown([self.dest1])
try:
fts = []
for i in range(5,50):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> '+ self.dest1,
{'store_stdout':True})
fts.append(ft)
fts.append(ScriptTask.from_str('cat 4 >> ' + self.dest1))
for i in range(51,100):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> ' + self.dest1,
{'store_stdout':True})
fts.append(ft)
fw = Firework(fts, spec={'_trackers':[self.tracker1]}, fw_id=21, name='test_fw')
self.lp.add_wf(fw)
try:
print("===========================================")
print("Bad rocket launched. The failure below is OK")
print("===========================================")
launch_rocket(self.lp, self.fworker)
except:
pass
self.assertEqual('48\n49',self.tracker1.track_file())
finally:
self._teardown([self.dest1])
def test_preserve_fworker(self):
fw1 = Firework(
[ScriptTask.from_str('echo "Testing preserve FWorker"')],
spec={"_preserve_fworker": True},
fw_id=1)
fw2 = Firework(
[ScriptTask.from_str('echo "Testing preserve FWorker pt 2"')],
spec={"target": 1},
parents=[fw1],
fw_id=2)
self.lp.add_wf(Workflow([fw1, fw2]))
launch_rocket(self.lp, self.fworker)
target_fw_id = self.lp.get_fw_ids({"spec.target": 1})[0]
modified_spec = self.lp.get_fw_by_id(target_fw_id).spec
"""
cnt = 0
while '_fworker' not in modified_spec and cnt < 5:
modified_spec = self.lp.get_fw_by_id(target_fw_id).spec
print(modified_spec)
time.sleep(5)
cnt += 1
"""
self.assertIsNotNone(modified_spec['_fworker'])
def basic_wf_ex():
print '--- BASIC WORKFLOW EXAMPLE ---'
# setup
launchpad = setup()
# add FireWorks
task1 = ScriptTask.from_str('echo "Ingrid is the CEO."')
task2 = ScriptTask.from_str('echo "Jill is a manager."')
task3 = ScriptTask.from_str('echo "Jack is a manager."')
task4 = ScriptTask.from_str('echo "Kip is an intern."')
fw1 = FireWork(task1, fw_id=-1)
fw2 = FireWork(task2, fw_id=-2)
fw3 = FireWork(task3, fw_id=-3)
fw4 = FireWork(task4, fw_id=-4)
# make workflow
workflow = Workflow([fw1, fw2, fw3, fw4], {
-1: [-2, -3],
-2: [-4],
-3: [-4]
})
launchpad.add_wf(workflow)
# launch Rocket
rapidfire(launchpad)
def test_parentconnector(self):
fw1 = Firework(ScriptTask.from_str('echo "1"'))
fw2 = Firework(ScriptTask.from_str('echo "1"'), parents=fw1)
fw3 = Firework(ScriptTask.from_str('echo "1"'), parents=[fw1, fw2])
self.assertEqual(Workflow([fw1, fw2, fw3]).links, {fw1.fw_id: [fw2.fw_id, fw3.fw_id], fw2.fw_id: [fw3.fw_id], fw3.fw_id: []})
self.assertRaises(ValueError, Workflow, [fw1, fw3]) # can't make this
def test_early_exit(self):
os.chdir(MODULE_DIR)
script_text = "echo hello from process $PPID; sleep 2"
fw1 = Firework(ScriptTask.from_str(shell_cmd=script_text,
parameters={"stdout_file": "task.out"}),
fw_id=1)
fw2 = Firework(ScriptTask.from_str(shell_cmd=script_text,
parameters={"stdout_file": "task.out"}),
fw_id=2)
fw3 = Firework(ScriptTask.from_str(shell_cmd=script_text,
parameters={"stdout_file": "task.out"}),
fw_id=3)
fw4 = Firework(ScriptTask.from_str(shell_cmd=script_text,
parameters={"stdout_file": "task.out"}),
fw_id=4)
wf = Workflow([fw1, fw2, fw3, fw4], {1: [2, 3], 2: [4], 3: [4]})
self.lp.add_wf(wf)
launch_multiprocess(self.lp, FWorker(), 'DEBUG', 0, 2, sleep_time=0.5)
fw2 = self.lp.get_fw_by_id(2)
fw3 = self.lp.get_fw_by_id(3)
with open(os.path.join(fw2.launches[0].launch_dir, "task.out")) as f:
fw2_text = f.read()
with open(os.path.join(fw3.launches[0].launch_dir, "task.out")) as f:
fw3_text = f.read()
self.assertNotEqual(fw2_text, fw3_text)
def test_tracker_failed_fw(self):
"""
Add a bad firetask to workflow and test the tracking
"""
self._teardown([self.dest1])
try:
fts = []
for i in range(5,50):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> '+ self.dest1,
{'store_stdout':True})
fts.append(ft)
fts.append(ScriptTask.from_str('cat 4 >> ' + self.dest1))
for i in range(51,100):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> ' + self.dest1,
{'store_stdout':True})
fts.append(ft)
fw = Firework(fts, spec={'_trackers':[self.tracker1]}, fw_id=21, name='test_fw')
self.lp.add_wf(fw)
try:
print("===========================================")
print("Bad rocket launched. The failure below is OK")
print("===========================================")
launch_rocket(self.lp, self.fworker)
except:
pass
self.assertEqual('48\n49',self.tracker1.track_file())
finally:
self._teardown([self.dest1])
def test_files_in_out(self):
# create the Workflow that passes files_in and files_out
fw1 = Firework(
[ScriptTask.from_str('echo "This is the first FireWork" > test1')],
spec={"_files_out": {"fwtest1": "test1"}}, fw_id=1)
fw2 = Firework([ScriptTask.from_str('gzip hello')], fw_id=2,
parents=[fw1],
spec={"_files_in": {"fwtest1": "hello"},
"_files_out": {"fw2": "hello.gz"}})
fw3 = Firework([ScriptTask.from_str('cat fwtest.2')], fw_id=3,
parents=[fw2],
spec={"_files_in": {"fw2": "fwtest.2"}})
wf = Workflow([fw1, fw2, fw3],
{fw1: [fw2], fw2: [fw3]})
# store workflow and launch it locally
self.lp.add_wf(wf)
launch_rocket(self.lp, self.fworker)
self.assertTrue(os.path.exists("test1"))
launch_rocket(self.lp, self.fworker)
self.assertTrue(os.path.exists("hello.gz"))
launch_rocket(self.lp, self.fworker)
self.assertTrue(os.path.exists("fwtest.2"))
for f in ["test1", "hello.gz", "fwtest.2"]:
os.remove(f)
def test_multi_fw(self):
test1 = ScriptTask.from_str("python -c 'print \"test1\"'", {'store_stdout': True})
test2 = ScriptTask.from_str("python -c 'print \"test2\"'", {'store_stdout': True})
fw = FireWork([test1, test2])
self.lp.add_wf(fw)
launch_rocket(self.lp)
time.sleep(3)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test2\n')
def test_parentconnector(self):
fw1 = Firework(ScriptTask.from_str('echo "1"'))
fw2 = Firework(ScriptTask.from_str('echo "1"'), parents=fw1)
fw3 = Firework(ScriptTask.from_str('echo "1"'), parents=[fw1, fw2])
self.assertEqual(
Workflow([fw1, fw2, fw3]).links, {fw1.fw_id: [fw2.fw_id, fw3.fw_id], fw2.fw_id: [fw3.fw_id], fw3.fw_id: []}
)
self.assertRaises(ValueError, Workflow, [fw1, fw3]) # can't make this
def run_task(self, fw_spec):
print('Running the Multiple Detour Task')
dt1 = Firework(
ScriptTask.from_str('echo "this is intermediate job 1"'))
dt2 = Firework(
ScriptTask.from_str('echo "this is intermediate job 2"'))
dt3 = Firework(
ScriptTask.from_str('echo "this is intermediate job 3"'))
return FWAction(detours=[dt1, dt2, dt3])
def test_scripttask(self):
if os.path.exists('hello.txt'):
os.remove('hello.txt')
s = ScriptTask({'script':"echo 'hello world'",'stdout_file':'hello.txt'})
s.run_task({})
self.assertTrue(os.path.exists('hello.txt'))
with open('hello.txt') as fp:
line = fp.readlines()[0]
self.assertTrue('hello world' in line)
os.remove('hello.txt')
def test_multi_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
test2 = ScriptTask.from_str("python -c 'print(\"test2\")'",
{'store_stdout': True})
fw = Firework([test1, test2])
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
self.assertEqual(
self.lp.get_launch_by_id(1).action.stored_data['stdout'],
"test2\n")
def test_multi_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
test2 = ScriptTask.from_str("python -c 'print(\"test2\")'",
{'store_stdout': True})
fw = Firework([test1, test2])
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
self.assertEqual(
self.lp.get_launch_by_id(1).action.stored_data['stdout'],
"test2\n")
def test_org_wf(self):
test1 = ScriptTask.from_str("python -c 'print \"test1\"'", {'store_stdout': True})
test2 = ScriptTask.from_str("python -c 'print \"test2\"'", {'store_stdout': True})
fw1 = FireWork(test1, fw_id=-1)
fw2 = FireWork(test2, fw_id=-2)
wf = Workflow([fw1, fw2], {-1: -2})
self.lp.add_wf(wf)
launch_rocket(self.lp)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
launch_rocket(self.lp)
self.assertEqual(self.lp.get_launch_by_id(2).action.stored_data['stdout'], 'test2\n')
def test_multi_fw(self):
test1 = ScriptTask.from_str("python -c 'print \"test1\"'",
{'store_stdout': True})
test2 = ScriptTask.from_str("python -c 'print \"test2\"'",
{'store_stdout': True})
fw = FireWork([test1, test2])
self.lp.add_wf(fw)
launch_rocket(self.lp)
time.sleep(3)
self.assertEqual(
self.lp.get_launch_by_id(1).action.stored_data['stdout'],
'test2\n')
def test_fwconnector(self):
fw1 = Firework(ScriptTask.from_str('echo "1"'))
fw2 = Firework(ScriptTask.from_str('echo "1"'))
wf1 = Workflow([fw1, fw2], {fw1.fw_id: fw2.fw_id})
self.assertEqual(wf1.links, {fw1.fw_id: [fw2.fw_id], fw2.fw_id: []})
wf2 = Workflow([fw1, fw2], {fw1: fw2})
self.assertEqual(wf2.links, {fw1.fw_id: [fw2.fw_id], fw2.fw_id: []})
wf3 = Workflow([fw1, fw2])
self.assertEqual(wf3.links, {fw1.fw_id: [], fw2.fw_id: []})
def test_fwconnector(self):
fw1 = Firework(ScriptTask.from_str('echo "1"'))
fw2 = Firework(ScriptTask.from_str('echo "1"'))
wf1 = Workflow([fw1, fw2], {fw1.fw_id: fw2.fw_id})
self.assertEqual(wf1.links, {fw1.fw_id: [fw2.fw_id], fw2.fw_id: []})
wf2 = Workflow([fw1, fw2], {fw1: fw2})
self.assertEqual(wf2.links, {fw1.fw_id: [fw2.fw_id], fw2.fw_id: []})
wf3 = Workflow([fw1, fw2])
self.assertEqual(wf3.links, {fw1.fw_id: [], fw2.fw_id: []})
def test_spec_copy(self):
task1 = ScriptTask.from_str('echo "Task 1"')
task2 = ScriptTask.from_str('echo "Task 2"')
spec = {'_category': 'dummy_category'}
fw1 = Firework(task1, fw_id=1, name='Task 1', spec=spec)
fw2 = Firework(task2, fw_id=2, name='Task 2', spec=spec)
self.lp.add_wf(Workflow([fw1, fw2]))
self.assertEqual(self.lp.get_fw_by_id(1).tasks[0]['script'][0], 'echo "Task 1"')
self.assertEqual(self.lp.get_fw_by_id(2).tasks[0]['script'][0], 'echo "Task 2"')
def test_scripttask(self):
if os.path.exists("hello.txt"):
os.remove("hello.txt")
s = ScriptTask({
"script": 'echo "hello world"',
"stdout_file": "hello.txt"
})
s.run_task({})
self.assertTrue(os.path.exists("hello.txt"))
with open("hello.txt") as fp:
line = fp.readlines()[0]
self.assertTrue("hello world" in line)
os.remove("hello.txt")
def test_spec_copy(self):
task1 = ScriptTask.from_str('echo "Task 1"')
task2 = ScriptTask.from_str('echo "Task 2"')
spec = {'_category': 'dummy_category'}
fw1 = Firework(task1, fw_id=1, name='Task 1', spec=spec)
fw2 = Firework(task2, fw_id=2, name='Task 2', spec=spec)
self.lp.add_wf(Workflow([fw1, fw2]))
self.assertEqual(self.lp.get_fw_by_id(1).tasks[0]['script'][0], 'echo "Task 1"')
self.assertEqual(self.lp.get_fw_by_id(2).tasks[0]['script'][0], 'echo "Task 2"')
def test_scripttask(self):
if os.path.exists('hello.txt'):
os.remove('hello.txt')
s = ScriptTask({
'script': 'echo "hello world"',
'stdout_file': 'hello.txt'
})
s.run_task({})
self.assertTrue(os.path.exists('hello.txt'))
with open('hello.txt') as fp:
line = fp.readlines()[0]
self.assertTrue('hello world' in line)
os.remove('hello.txt')
def test_category_pt2(self):
task1 = ScriptTask.from_str('echo "Task 1"')
task2 = ScriptTask.from_str('echo "Task 2"')
fw1 = Firework(task1, fw_id=1, name='Task 1')
fw2 = Firework(task2, fw_id=2, name='Task 2')
self.lp.add_wf(Workflow([fw1, fw2]))
self.assertFalse(self.lp.run_exists(FWorker(category="dummy_category")))
self.assertTrue(self.lp.run_exists(FWorker(category="__none__")))
self.assertTrue(self.lp.run_exists(FWorker())) # can run any category
self.assertFalse(self.lp.run_exists(FWorker(category=["dummy_category",
"other category"])))
def test_category_pt2(self):
task1 = ScriptTask.from_str('echo "Task 1"')
task2 = ScriptTask.from_str('echo "Task 2"')
fw1 = Firework(task1, fw_id=1, name='Task 1')
fw2 = Firework(task2, fw_id=2, name='Task 2')
self.lp.add_wf(Workflow([fw1, fw2]))
self.assertFalse(self.lp.run_exists(FWorker(category="dummy_category")))
self.assertTrue(self.lp.run_exists(FWorker(category="__none__")))
self.assertTrue(self.lp.run_exists(FWorker())) # can run any category
self.assertFalse(self.lp.run_exists(FWorker(category=["dummy_category",
"other category"])))
def multiple_tasks_ex():
print('--- MULTIPLE FIRETASKS EXAMPLE ---')
# setup
launchpad = setup()
# add FireWorks
firetask1 = ScriptTask.from_str('echo "This is TASK #1"')
firetask2 = ScriptTask.from_str('echo "This is TASK #2"')
firetask3 = ScriptTask.from_str('echo "This is TASK #3"')
fw = FireWork([firetask1, firetask2, firetask3])
launchpad.add_wf(fw)
# launch Rocket
rapidfire(launchpad, FWorker())
def test_org_wf(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
test2 = ScriptTask.from_str("python -c 'print(\"test2\")'",
{'store_stdout': True})
fw1 = Firework(test1, fw_id=-1)
fw2 = Firework(test2, fw_id=-2)
wf = Workflow([fw1, fw2], {-1: -2})
self.lp.add_wf(wf)
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'],
'test1\n')
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.get_launch_by_id(2).action.stored_data['stdout'],
'test2\n')
def multiple_tasks_ex():
print("--- MULTIPLE FIRETASKS EXAMPLE ---")
# setup
launchpad = setup()
# add FireWorks
firetask1 = ScriptTask.from_str('echo "This is TASK #1"')
firetask2 = ScriptTask.from_str('echo "This is TASK #2"')
firetask3 = ScriptTask.from_str('echo "This is TASK #3"')
fw = Firework([firetask1, firetask2, firetask3])
launchpad.add_wf(fw)
# launch Rocket
rapidfire(launchpad, FWorker())
def test_tracker(self):
"""
Launch a workflow and track the files
"""
self._teardown([self.dest1])
try:
fts = []
for i in range(5, 100):
ft = ScriptTask.from_str(
'echo "' + str(i) + '" >> ' + self.dest1,
{"store_stdout": True})
fts.append(ft)
fw = Firework(fts,
spec={"_trackers": [self.tracker1]},
fw_id=20,
name="test_fw")
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
# print (self.tracker1.track_file())
self.assertEqual("98\n99", self.tracker1.track_file())
finally:
self._teardown([self.dest1])
def test_backgroundtask(self):
dest1 = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'hello.txt')
self._teardown([dest1])
try:
test1 = ScriptTask.from_str(
"python -c 'print(\"testing background...\")'",
{'store_stdout': True})
bg_task1 = BackgroundTask(FileWriteTask(
{'files_to_write': [{
'filename': dest1,
'contents': 'hello'
}]}),
num_launches=1,
run_on_finish=True)
fw = Firework(test1, spec={'_background_tasks': [bg_task1]})
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
with open(dest1) as f:
self.assertEqual(f.read(), 'hello')
finally:
self._teardown([dest1])
def test_add_wf(self):
fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello")
self.lp.add_wf(fw)
wf_id = self.lp.get_wf_ids()
self.assertEqual(len(wf_id), 1)
for fw_id in self.lp.get_wf_ids():
wf = self.lp.get_wf_by_fw_id_lzyfw(fw_id)
self.assertEqual(len(wf.id_fw.keys()), 1)
fw2 = Firework(ScriptTask.from_str('echo "goodbye"'), name="goodbye")
wf = Workflow([fw, fw2], name='test_workflow')
self.lp.add_wf(wf)
#fw = self.lp.get_fw_ids()
#self.assertEqual(len(wf.id_fw.keys()), 2)
fw_ids = self.lp.get_fw_ids()
self.assertEqual(len(fw_ids), 3)
self.lp.reset('',require_password=False)
def test_basic_fw_offline(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
fw = Firework(test1)
self.lp.add_wf(fw)
fw, launch_id = self.lp.reserve_fw(FWorker(), os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
with open(os.path.join(os.getcwd(), "FW_offline.json")) as f:
fwo = json.load(f)
self.assertEquals(fwo["state"], "COMPLETED")
self.assertEquals(fwo["launch_id"], 1)
self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False})
with open(os.path.join(os.getcwd(), "FW_ping.json")) as f:
fwp = json.load(f)
self.assertIsNotNone(fwp["ping_time"])
l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1})
self.lp.recover_offline(l['launch_id'])
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test_basic_fw_offline(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
fw = Firework(test1)
self.lp.add_wf(fw)
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
with open(os.path.join(os.getcwd(), "FW_offline.json")) as f:
fwo = json.load(f)
self.assertEquals(fwo["state"], "COMPLETED")
self.assertEquals(fwo["launch_id"], 1)
self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False, 'defuse_workflow': False})
with open(os.path.join(os.getcwd(), "FW_ping.json")) as f:
fwp = json.load(f)
self.assertIsNotNone(fwp["ping_time"])
l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1})
self.lp.recover_offline(l['launch_id'])
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test_add_wf(self):
fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello")
self.lp.add_wf(fw)
wf_id = self.lp.get_wf_ids()
self.assertEqual(len(wf_id), 1)
for fw_id in self.lp.get_wf_ids():
wf = self.lp.get_wf_by_fw_id_lzyfw(fw_id)
self.assertEqual(len(wf.id_fw.keys()), 1)
fw2 = Firework(ScriptTask.from_str('echo "goodbye"'), name="goodbye")
wf = Workflow([fw, fw2], name='test_workflow')
self.lp.add_wf(wf)
#fw = self.lp.get_fw_ids()
#self.assertEqual(len(wf.id_fw.keys()), 2)
fw_ids = self.lp.get_fw_ids()
self.assertEqual(len(fw_ids), 3)
self.lp.reset('', require_password=False)
def test_force_lock_removal(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True})
fw = Firework(test1, {"_dupefinder": DupeFinderExact()}, fw_id=1)
self.lp.add_wf(fw)
# add a manual lock
with WFLock(self.lp, 1):
with WFLock(self.lp, 1, kill=True, expire_secs=1):
self.assertTrue(True) # dummy to make sure we got here
def test_basic_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
fw = FireWork(test1)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[
'stdout'], 'test1\n')
def test_basic_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
fw = FireWork(test1)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[
'stdout'], str(six.b('test1\n')))
def test_reset(self):
# Store some test fireworks
# Atempt couple of ways to reset the lp and check
fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello")
wf = Workflow([fw], name='test_workflow')
self.lp.add_wf(wf)
self.assertRaises(ValueError, self.lp.reset, '', False, 0)
self.assertEqual(self.lp.workflows.count(), 1)
self.lp.reset('',require_password=False)
self.assertFalse(self.lp.get_fw_ids())
self.assertFalse(self.lp.get_wf_ids())
# test failsafe in a strict way
for x in range(30):
self.lp.add_wf(Workflow([Firework(ScriptTask.from_str('echo "hello"'))]))
self.assertRaises(ValueError, self.lp.reset, '')
self.lp.reset('', False, 100) # reset back
def test_reset(self):
# Store some test fireworks
# Atempt couple of ways to reset the lp and check
fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello")
wf = Workflow([fw], name='test_workflow')
self.lp.add_wf(wf)
self.assertRaises(ValueError, self.lp.reset, '', False, 0)
self.assertEqual(self.lp.workflows.count(), 1)
self.lp.reset('',require_password=False)
self.assertFalse(self.lp.get_fw_ids())
self.assertFalse(self.lp.get_wf_ids())
# test failsafe in a strict way
for x in range(30):
self.lp.add_wf(Workflow([Firework(ScriptTask.from_str('echo "hello"'))]))
self.assertRaises(ValueError, self.lp.reset, '')
self.lp.reset('', False, 100) # reset back
def test_dupefinder(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True})
fw = Firework(test1, {"_dupefinder": DupeFinderExact()})
self.lp.add_wf(fw)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.launches.count(), 1)
def add_wf(j, dest, tracker, name):
fts = []
for i in range(j,j+25):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> '+ dest,
{'store_stdout':True})
fts.append(ft)
fw1 = Firework(fts, spec={'_trackers':[tracker]},
fw_id=j+1, name=name+'1')
fts = []
for i in range(j+25,j+50):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> ' + dest,
{'store_stdout':True})
fts.append(ft)
fw2 = Firework(fts, spec={'_trackers':[tracker]},
fw_id=j+2, name=name+'2')
wf = Workflow([fw1, fw2], links_dict={fw1:[fw2]})
self.lp.add_wf(wf)
def add_wf(j, dest, tracker, name):
fts = []
for i in range(j,j+25):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> '+ dest,
{'store_stdout':True})
fts.append(ft)
fw1 = Firework(fts, spec={'_trackers':[tracker]},
fw_id=j+1, name=name+'1')
fts = []
for i in range(j+25,j+50):
ft = ScriptTask.from_str('echo "' + str(i) + '" >> ' + dest,
{'store_stdout':True})
fts.append(ft)
fw2 = Firework(fts, spec={'_trackers':[tracker]},
fw_id=j+2, name=name+'2')
wf = Workflow([fw1, fw2], links_dict={fw1:[fw2]})
self.lp.add_wf(wf)
def test_reset(self):
# Store some test fireworks
# Atempt couple of ways to reset the lp and check
fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello")
wf = Workflow([fw], name="test_workflow")
self.lp.add_wf(wf)
self.lp.reset("", require_password=False)
self.assertFalse(self.lp.get_fw_ids())
self.assertFalse(self.lp.get_wf_ids())
def test_dupefinder(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True})
fw = Firework(test1, {"_dupefinder": DupeFinderExact()})
self.lp.add_wf(fw)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.launches.count(), 1)
def test_checkout_fw(self):
os.chdir(MODULE_DIR)
self.lp.add_wf(Firework(ScriptTask.from_str(
shell_cmd='echo "hello 1"',
parameters={"stdout_file": "task.out"}), fw_id=1))
self.lp.add_wf(Firework(ScriptTask.from_str(
shell_cmd='echo "hello 2"',
parameters={"stdout_file": "task.out"}), fw_id=2))
launch_multiprocess(self.lp, FWorker(), 'DEBUG', 0, 2, 10)
fw1 = self.lp.get_fw_by_id(1)
fw2 = self.lp.get_fw_by_id(2)
self.assertEqual(fw1.launches[0].state_history[-1]["state"],
"COMPLETED")
self.assertEqual(fw2.launches[0].state_history[-1]["state"],
"COMPLETED")
with open(os.path.join(fw1.launches[0].launch_dir, "task.out")) as f:
self.assertEqual(f.readlines(), ['hello 1\n'])
with open(os.path.join(fw2.launches[0].launch_dir, "task.out")) as f:
self.assertEqual(f.readlines(), ['hello 2\n'])
def test_delete_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True})
fw = Firework(test1)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[
'stdout'], 'test1\n')
self.lp.delete_wf(fw.fw_id)
self.assertRaises(ValueError, self.lp.get_fw_by_id, fw.fw_id)
self.assertRaises(ValueError, self.lp.get_launch_by_id, 1)
def test_delete_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True})
fw = Firework(test1)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[
'stdout'], 'test1\n')
self.lp.delete_wf(fw.fw_id)
self.assertRaises(ValueError, self.lp.get_fw_by_id, fw.fw_id)
self.assertRaises(ValueError, self.lp.get_launch_by_id, 1)
def test_multi_detour(self):
fw1 = Firework([MultipleDetourTask()], fw_id=1)
fw2 = Firework([ScriptTask.from_str('echo "DONE"')], parents=[fw1], fw_id=2)
self.lp.add_wf(Workflow([fw1, fw2]))
rapidfire(self.lp)
links = self.lp.get_wf_by_fw_id(1).links
self.assertEqual(set(links[1]), set([2, 3, 4, 5]))
self.assertEqual(set(links[2]), set([]))
self.assertEqual(set(links[3]), set([2]))
self.assertEqual(set(links[4]), set([2]))
self.assertEqual(set(links[5]), set([2]))
def test_multi_detour(self):
fw1 = Firework([MultipleDetourTask()], fw_id=1)
fw2 = Firework([ScriptTask.from_str('echo "DONE"')], parents=[fw1], fw_id=2)
self.lp.add_wf(Workflow([fw1, fw2]))
rapidfire(self.lp)
links = self.lp.get_wf_by_fw_id(1).links
self.assertEqual(set(links[1]), set([2, 3, 4, 5]))
self.assertEqual(set(links[2]), set([]))
self.assertEqual(set(links[3]), set([2]))
self.assertEqual(set(links[4]), set([2]))
self.assertEqual(set(links[5]), set([2]))
def test_preserve_fworker(self):
fw1 = Firework([ScriptTask.from_str('echo "Testing preserve FWorker"')], spec={"_preserve_fworker": True}, fw_id=1)
fw2 = Firework([ScriptTask.from_str('echo "Testing preserve FWorker pt 2"')], spec={"target": 1}, parents=[fw1], fw_id=2)
self.lp.add_wf(Workflow([fw1, fw2]))
launch_rocket(self.lp, self.fworker)
target_fw_id = self.lp.get_fw_ids({"spec.target": 1})[0]
modified_spec = self.lp.get_fw_by_id(target_fw_id).spec
"""
cnt = 0
while '_fworker' not in modified_spec and cnt < 5:
modified_spec = self.lp.get_fw_by_id(target_fw_id).spec
print(modified_spec)
time.sleep(5)
cnt += 1
"""
self.assertIsNotNone(modified_spec['_fworker'])
def test_duplicate_delete_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True})
fw = Firework(test1, {"_dupefinder": DupeFinderExact()})
self.lp.add_wf(fw)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
launch_rocket(self.lp, self.fworker)
self.lp.delete_wf(2)
self.assertRaises(ValueError, self.lp.get_fw_by_id, 2)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[
'stdout'], 'test1\n')
def setUp(self):
fireworks.core.firework.EXCEPT_DETAILS_ON_RERUN = True
self.error_test_dict = {'error': 'description', 'error_code': 1}
fw = Firework([ExecutionCounterTask(),
ScriptTask.from_str('date +"%s %N"', parameters={'stdout_file': 'date_file'}),
ExceptionTestTask(exc_details=self.error_test_dict)])
self.lp.add_wf(fw)
ExecutionCounterTask.exec_counter = 0
ExceptionTestTask.exec_counter = 0
self.old_wd = os.getcwd()
def test_parallel_fibadder(self):
# this is really testing to see if a Workflow can handle multiple FWs updating it at once
parent = FireWork(ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}), fw_id=1)
fib1 = FireWork(FibonacciAdderTask(), {'smaller': 0, 'larger': 1, 'stop_point': 30}, fw_id=2)
fib2 = FireWork(FibonacciAdderTask(), {'smaller': 0, 'larger': 1, 'stop_point': 30}, fw_id=3)
wf = Workflow([parent, fib1, fib2], {1: [2, 3]})
self.lp.add_wf(wf)
p = Pool(NCORES_PARALLEL_TEST)
creds_array = [self.lp.to_dict()] * NCORES_PARALLEL_TEST
p.map(random_launch, creds_array)
def setUp(self):
# set the defaults in the init of wflock to break the lock quickly
fireworks.core.launchpad.WFLock(3, False).__init__.__func__.__defaults__= (3, False)
self.error_test_dict = {'error': 'description', 'error_code': 1}
fw_slow = Firework(SlowAdditionTask(), spec={'seconds': 10}, fw_id=1)
fw_fast = Firework(WaitWFLockTask(), fw_id=2, spec={'_add_launchpad_and_fw_id': True})
fw_child = Firework(ScriptTask.from_str('echo "child"'), fw_id=3)
wf = Workflow([fw_slow, fw_fast, fw_child], {fw_slow: fw_child, fw_fast: fw_child})
self.lp.add_wf(wf)
self.old_wd = os.getcwd()
def add_scripts(args):
lp = get_lp(args)
args.names = args.names if args.names else [None] * len(args.scripts)
args.wf_name = args.wf_name if args.wf_name else args.names[0]
fws = []
links = {}
for idx, s in enumerate(args.scripts):
fws.append(Firework(ScriptTask({'script': s, 'use_shell': True}), name=args.names[idx], fw_id=idx))
if idx != 0:
links[idx-1] = idx
lp.add_wf(Workflow(fws, links, args.wf_name))
def test_duplicate_delete_fw(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True})
fw = Firework(test1, {"_dupefinder": DupeFinderExact()})
self.lp.add_wf(fw)
self.lp.add_wf(fw)
launch_rocket(self.lp, self.fworker)
launch_rocket(self.lp, self.fworker)
self.lp.delete_wf(2)
self.assertRaises(ValueError, self.lp.get_fw_by_id, 2)
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[
'stdout'], 'test1\n')
def basic_fw_ex():
print('--- BASIC FIREWORK EXAMPLE ---')
# setup
launchpad = setup()
# add FireWork
firetask = ScriptTask.from_str('echo "howdy, your job launched successfully!"')
firework = FireWork(firetask)
launchpad.add_wf(firework)
# launch Rocket
launch_rocket(launchpad, FWorker())
def setUp(self):
fireworks.core.firework.EXCEPT_DETAILS_ON_RERUN = True
self.error_test_dict = {'error': 'description', 'error_code': 1}
fw = Firework(ScriptTask.from_str(
'echo "test offline"',
{'store_stdout':True}), name="offline_fw", fw_id=1)
self.lp.add_wf(fw)
self.launch_dir = os.path.join(MODULE_DIR, "launcher_offline")
os.makedirs(self.launch_dir)
self.old_wd = os.getcwd()
