def test_recover_errors(self):
fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir)
fw = self.lp.get_fw_by_id(1)
with cd(self.launch_dir):
setup_offline_job(self.lp, fw, launch_id)
# remove the directory to cause an exception
shutil.rmtree(self.launch_dir)
# recover ignoring errors
self.assertIsNotNone(
self.lp.recover_offline(launch_id,
ignore_errors=True,
print_errors=True))
fw = self.lp.get_fw_by_id(launch_id)
self.assertEqual(fw.state, 'RESERVED')
#fizzle
self.assertIsNotNone(
self.lp.recover_offline(launch_id, ignore_errors=False))
fw = self.lp.get_fw_by_id(launch_id)
self.assertEqual(fw.state, 'FIZZLED')
def test_basic_fw_offline(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
fw = Firework(test1)
self.lp.add_wf(fw)
fw, launch_id = self.lp.reserve_fw(FWorker(), os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
with open(os.path.join(os.getcwd(), "FW_offline.json")) as f:
fwo = json.load(f)
self.assertEquals(fwo["state"], "COMPLETED")
self.assertEquals(fwo["launch_id"], 1)
self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False})
with open(os.path.join(os.getcwd(), "FW_ping.json")) as f:
fwp = json.load(f)
self.assertIsNotNone(fwp["ping_time"])
l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1})
self.lp.recover_offline(l['launch_id'])
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test_basic_fw_offline(self):
test1 = ScriptTask.from_str("python -c 'print(\"test1\")'",
{'store_stdout': True})
fw = Firework(test1)
self.lp.add_wf(fw)
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
with open(os.path.join(os.getcwd(), "FW_offline.json")) as f:
fwo = json.load(f)
self.assertEquals(fwo["state"], "COMPLETED")
self.assertEquals(fwo["launch_id"], 1)
self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False, 'defuse_workflow': False})
with open(os.path.join(os.getcwd(), "FW_ping.json")) as f:
fwp = json.load(f)
self.assertIsNotNone(fwp["ping_time"])
l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1})
self.lp.recover_offline(l['launch_id'])
self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test__recover_completed(self):
fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir)
fw = self.lp.get_fw_by_id(1)
with cd(self.launch_dir):
setup_offline_job(self.lp, fw, launch_id)
# launch rocket without launchpad to trigger offline mode
launch_rocket(launchpad=None, fworker=self.fworker, fw_id=1)
self.assertIsNone(self.lp.recover_offline(launch_id))
fw = self.lp.get_fw_by_id(launch_id)
self.assertEqual(fw.state, 'COMPLETED')
def test__recover_completed(self):
fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir)
fw = self.lp.get_fw_by_id(1)
with cd(self.launch_dir):
setup_offline_job(self.lp, fw, launch_id)
# launch rocket without launchpad to trigger offline mode
launch_rocket(launchpad=None, fworker=self.fworker, fw_id=1)
self.assertIsNone(self.lp.recover_offline(launch_id))
fw = self.lp.get_fw_by_id(launch_id)
self.assertEqual(fw.state, 'COMPLETED')
def test_recover_errors(self):
fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir)
fw = self.lp.get_fw_by_id(1)
with cd(self.launch_dir):
setup_offline_job(self.lp, fw, launch_id)
# remove the directory to cause an exception
shutil.rmtree(self.launch_dir)
# recover ignoring errors
self.assertIsNotNone(self.lp.recover_offline(launch_id, ignore_errors=True, print_errors=True))
fw = self.lp.get_fw_by_id(launch_id)
self.assertEqual(fw.state, 'RESERVED')
#fizzle
self.assertIsNotNone(self.lp.recover_offline(launch_id, ignore_errors=False))
fw = self.lp.get_fw_by_id(launch_id)
self.assertEqual(fw.state, 'FIZZLED')
def test_offline_fw_passinfo(self):
fw1 = Firework([AdditionTask()], {"input_array": [1, 1]}, name="1")
fw2 = Firework([AdditionTask()], {"input_array": [2, 2]}, name="2")
fw3 = Firework([AdditionTask()], {"input_array": [3]},
parents=[fw1, fw2],
name="3")
wf = Workflow([fw1, fw2, fw3])
self.lp.add_wf(wf)
# make dirs for launching jobs
cur_dir = os.path.dirname(os.path.abspath(__file__))
os.mkdir(os.path.join(cur_dir, "launcher_1"))
os.mkdir(os.path.join(cur_dir, "launcher_2"))
os.mkdir(os.path.join(cur_dir, "launcher_3"))
# launch two parent jobs
os.chdir(os.path.join(cur_dir, "launcher_1"))
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
os.chdir(os.path.join(cur_dir, "launcher_2"))
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
# recover jobs
for l in self.lp.offline_runs.find(
{
"completed": False,
"deprecated": False
}, {"launch_id": 1}):
fw = self.lp.recover_offline(l['launch_id'])
# launch child job
os.chdir(os.path.join(cur_dir, "launcher_3"))
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
last_fw_id = fw.fw_id
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
# recover jobs
for l in self.lp.offline_runs.find(
{
"completed": False,
"deprecated": False
}, {"launch_id": 1}):
fw = self.lp.recover_offline(l['launch_id'])
# confirm the sum in the child job
child_fw = self.lp.get_fw_by_id(last_fw_id)
self.assertEqual(set(child_fw.spec['input_array']), set([2, 3, 4]))
self.assertEqual(child_fw.launches[0].action.stored_data["sum"], 9)
def test_offline_fw_passinfo(self):
fw1 = Firework([AdditionTask()], {"input_array": [1,1]}, name="1")
fw2 = Firework([AdditionTask()], {"input_array": [2,2]}, name="2")
fw3 = Firework([AdditionTask()], {"input_array": [3]}, parents=[fw1, fw2], name="3")
wf = Workflow([fw1, fw2, fw3])
self.lp.add_wf(wf)
# make dirs for launching jobs
cur_dir = os.path.dirname(os.path.abspath(__file__))
os.mkdir(os.path.join(cur_dir, "launcher_1"))
os.mkdir(os.path.join(cur_dir, "launcher_2"))
os.mkdir(os.path.join(cur_dir, "launcher_3"))
# launch two parent jobs
os.chdir(os.path.join(cur_dir, "launcher_1"))
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
os.chdir(os.path.join(cur_dir, "launcher_2"))
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
# recover jobs
for l in self.lp.offline_runs.find({"completed": False, "deprecated": False}, {"launch_id": 1}):
fw = self.lp.recover_offline(l['launch_id'])
# launch child job
os.chdir(os.path.join(cur_dir, "launcher_3"))
fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd())
last_fw_id = fw.fw_id
setup_offline_job(self.lp, fw, launch_id)
launch_rocket(None, self.fworker)
# recover jobs
for l in self.lp.offline_runs.find({"completed": False, "deprecated": False}, {"launch_id": 1}):
fw = self.lp.recover_offline(l['launch_id'])
# confirm the sum in the child job
child_fw = self.lp.get_fw_by_id(last_fw_id)
self.assertEqual(set(child_fw.spec['input_array']), set([2, 3, 4]))
self.assertEqual(child_fw.launches[0].action.stored_data["sum"], 9)