def test_recover_errors(self): fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir) fw = self.lp.get_fw_by_id(1) with cd(self.launch_dir): setup_offline_job(self.lp, fw, launch_id) # remove the directory to cause an exception shutil.rmtree(self.launch_dir) # recover ignoring errors self.assertIsNotNone( self.lp.recover_offline(launch_id, ignore_errors=True, print_errors=True)) fw = self.lp.get_fw_by_id(launch_id) self.assertEqual(fw.state, 'RESERVED') #fizzle self.assertIsNotNone( self.lp.recover_offline(launch_id, ignore_errors=False)) fw = self.lp.get_fw_by_id(launch_id) self.assertEqual(fw.state, 'FIZZLED')
def test_basic_fw_offline(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1) self.lp.add_wf(fw) fw, launch_id = self.lp.reserve_fw(FWorker(), os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) with open(os.path.join(os.getcwd(), "FW_offline.json")) as f: fwo = json.load(f) self.assertEquals(fwo["state"], "COMPLETED") self.assertEquals(fwo["launch_id"], 1) self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False}) with open(os.path.join(os.getcwd(), "FW_ping.json")) as f: fwp = json.load(f) self.assertIsNotNone(fwp["ping_time"]) l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1}) self.lp.recover_offline(l['launch_id']) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test_basic_fw_offline(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1) self.lp.add_wf(fw) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) with open(os.path.join(os.getcwd(), "FW_offline.json")) as f: fwo = json.load(f) self.assertEquals(fwo["state"], "COMPLETED") self.assertEquals(fwo["launch_id"], 1) self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False, 'defuse_workflow': False}) with open(os.path.join(os.getcwd(), "FW_ping.json")) as f: fwp = json.load(f) self.assertIsNotNone(fwp["ping_time"]) l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1}) self.lp.recover_offline(l['launch_id']) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test__recover_completed(self): fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir) fw = self.lp.get_fw_by_id(1) with cd(self.launch_dir): setup_offline_job(self.lp, fw, launch_id) # launch rocket without launchpad to trigger offline mode launch_rocket(launchpad=None, fworker=self.fworker, fw_id=1) self.assertIsNone(self.lp.recover_offline(launch_id)) fw = self.lp.get_fw_by_id(launch_id) self.assertEqual(fw.state, 'COMPLETED')
def test__recover_completed(self): fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir) fw = self.lp.get_fw_by_id(1) with cd(self.launch_dir): setup_offline_job(self.lp, fw, launch_id) # launch rocket without launchpad to trigger offline mode launch_rocket(launchpad=None, fworker=self.fworker, fw_id=1) self.assertIsNone(self.lp.recover_offline(launch_id)) fw = self.lp.get_fw_by_id(launch_id) self.assertEqual(fw.state, 'COMPLETED')
def test_recover_errors(self): fw, launch_id = self.lp.reserve_fw(self.fworker, self.launch_dir) fw = self.lp.get_fw_by_id(1) with cd(self.launch_dir): setup_offline_job(self.lp, fw, launch_id) # remove the directory to cause an exception shutil.rmtree(self.launch_dir) # recover ignoring errors self.assertIsNotNone(self.lp.recover_offline(launch_id, ignore_errors=True, print_errors=True)) fw = self.lp.get_fw_by_id(launch_id) self.assertEqual(fw.state, 'RESERVED') #fizzle self.assertIsNotNone(self.lp.recover_offline(launch_id, ignore_errors=False)) fw = self.lp.get_fw_by_id(launch_id) self.assertEqual(fw.state, 'FIZZLED')
def test_offline_fw_passinfo(self): fw1 = Firework([AdditionTask()], {"input_array": [1, 1]}, name="1") fw2 = Firework([AdditionTask()], {"input_array": [2, 2]}, name="2") fw3 = Firework([AdditionTask()], {"input_array": [3]}, parents=[fw1, fw2], name="3") wf = Workflow([fw1, fw2, fw3]) self.lp.add_wf(wf) # make dirs for launching jobs cur_dir = os.path.dirname(os.path.abspath(__file__)) os.mkdir(os.path.join(cur_dir, "launcher_1")) os.mkdir(os.path.join(cur_dir, "launcher_2")) os.mkdir(os.path.join(cur_dir, "launcher_3")) # launch two parent jobs os.chdir(os.path.join(cur_dir, "launcher_1")) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) os.chdir(os.path.join(cur_dir, "launcher_2")) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) # recover jobs for l in self.lp.offline_runs.find( { "completed": False, "deprecated": False }, {"launch_id": 1}): fw = self.lp.recover_offline(l['launch_id']) # launch child job os.chdir(os.path.join(cur_dir, "launcher_3")) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) last_fw_id = fw.fw_id setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) # recover jobs for l in self.lp.offline_runs.find( { "completed": False, "deprecated": False }, {"launch_id": 1}): fw = self.lp.recover_offline(l['launch_id']) # confirm the sum in the child job child_fw = self.lp.get_fw_by_id(last_fw_id) self.assertEqual(set(child_fw.spec['input_array']), set([2, 3, 4])) self.assertEqual(child_fw.launches[0].action.stored_data["sum"], 9)
def test_offline_fw_passinfo(self): fw1 = Firework([AdditionTask()], {"input_array": [1,1]}, name="1") fw2 = Firework([AdditionTask()], {"input_array": [2,2]}, name="2") fw3 = Firework([AdditionTask()], {"input_array": [3]}, parents=[fw1, fw2], name="3") wf = Workflow([fw1, fw2, fw3]) self.lp.add_wf(wf) # make dirs for launching jobs cur_dir = os.path.dirname(os.path.abspath(__file__)) os.mkdir(os.path.join(cur_dir, "launcher_1")) os.mkdir(os.path.join(cur_dir, "launcher_2")) os.mkdir(os.path.join(cur_dir, "launcher_3")) # launch two parent jobs os.chdir(os.path.join(cur_dir, "launcher_1")) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) os.chdir(os.path.join(cur_dir, "launcher_2")) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) # recover jobs for l in self.lp.offline_runs.find({"completed": False, "deprecated": False}, {"launch_id": 1}): fw = self.lp.recover_offline(l['launch_id']) # launch child job os.chdir(os.path.join(cur_dir, "launcher_3")) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) last_fw_id = fw.fw_id setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) # recover jobs for l in self.lp.offline_runs.find({"completed": False, "deprecated": False}, {"launch_id": 1}): fw = self.lp.recover_offline(l['launch_id']) # confirm the sum in the child job child_fw = self.lp.get_fw_by_id(last_fw_id) self.assertEqual(set(child_fw.spec['input_array']), set([2, 3, 4])) self.assertEqual(child_fw.launches[0].action.stored_data["sum"], 9)