def test_files_in_out(self): # create the Workflow that passes files_in and files_out fw1 = Firework( [ScriptTask.from_str('echo "This is the first FireWork" > test1')], spec={"_files_out": {"fwtest1": "test1"}}, fw_id=1) fw2 = Firework([ScriptTask.from_str('gzip hello')], fw_id=2, parents=[fw1], spec={"_files_in": {"fwtest1": "hello"}, "_files_out": {"fw2": "hello.gz"}}) fw3 = Firework([ScriptTask.from_str('cat fwtest.2')], fw_id=3, parents=[fw2], spec={"_files_in": {"fw2": "fwtest.2"}}) wf = Workflow([fw1, fw2, fw3], {fw1: [fw2], fw2: [fw3]}) # store workflow and launch it locally self.lp.add_wf(wf) launch_rocket(self.lp, self.fworker) self.assertTrue(os.path.exists("test1")) launch_rocket(self.lp, self.fworker) self.assertTrue(os.path.exists("hello.gz")) launch_rocket(self.lp, self.fworker) self.assertTrue(os.path.exists("fwtest.2")) for f in ["test1", "hello.gz", "fwtest.2"]: os.remove(f)
def test_parentconnector(self): fw1 = Firework(ScriptTask.from_str('echo "1"')) fw2 = Firework(ScriptTask.from_str('echo "1"'), parents=fw1) fw3 = Firework(ScriptTask.from_str('echo "1"'), parents=[fw1, fw2]) self.assertEqual(Workflow([fw1, fw2, fw3]).links, {fw1.fw_id: [fw2.fw_id, fw3.fw_id], fw2.fw_id: [fw3.fw_id], fw3.fw_id: []}) self.assertRaises(ValueError, Workflow, [fw1, fw3]) # can't make this
def test_preserve_fworker(self): fw1 = Firework( [ScriptTask.from_str('echo "Testing preserve FWorker"')], spec={"_preserve_fworker": True}, fw_id=1) fw2 = Firework( [ScriptTask.from_str('echo "Testing preserve FWorker pt 2"')], spec={"target": 1}, parents=[fw1], fw_id=2) self.lp.add_wf(Workflow([fw1, fw2])) launch_rocket(self.lp, self.fworker) target_fw_id = self.lp.get_fw_ids({"spec.target": 1})[0] modified_spec = self.lp.get_fw_by_id(target_fw_id).spec """ cnt = 0 while '_fworker' not in modified_spec and cnt < 5: modified_spec = self.lp.get_fw_by_id(target_fw_id).spec print(modified_spec) time.sleep(5) cnt += 1 """ self.assertIsNotNone(modified_spec['_fworker'])
def test_tracker_failed_fw(self): """ Add a bad firetask to workflow and test the tracking """ self._teardown([self.dest1]) try: fts = [] for i in range(5,50): ft = ScriptTask.from_str('echo "' + str(i) + '" >> '+ self.dest1, {'store_stdout':True}) fts.append(ft) fts.append(ScriptTask.from_str('cat 4 >> ' + self.dest1)) for i in range(51,100): ft = ScriptTask.from_str('echo "' + str(i) + '" >> ' + self.dest1, {'store_stdout':True}) fts.append(ft) fw = Firework(fts, spec={'_trackers':[self.tracker1]}, fw_id=21, name='test_fw') self.lp.add_wf(fw) try: print("===========================================") print("Bad rocket launched. The failure below is OK") print("===========================================") launch_rocket(self.lp, self.fworker) except: pass self.assertEqual('48\n49',self.tracker1.track_file()) finally: self._teardown([self.dest1])
def test_early_exit(self): os.chdir(MODULE_DIR) script_text = "echo hello from process $PPID; sleep 2" fw1 = Firework(ScriptTask.from_str(shell_cmd=script_text, parameters={"stdout_file": "task.out"}), fw_id=1) fw2 = Firework(ScriptTask.from_str(shell_cmd=script_text, parameters={"stdout_file": "task.out"}), fw_id=2) fw3 = Firework(ScriptTask.from_str(shell_cmd=script_text, parameters={"stdout_file": "task.out"}), fw_id=3) fw4 = Firework(ScriptTask.from_str(shell_cmd=script_text, parameters={"stdout_file": "task.out"}), fw_id=4) wf = Workflow([fw1, fw2, fw3, fw4], {1: [2, 3], 2: [4], 3: [4]}) self.lp.add_wf(wf) launch_multiprocess(self.lp, FWorker(), 'DEBUG', 0, 2, sleep_time=0.5) fw2 = self.lp.get_fw_by_id(2) fw3 = self.lp.get_fw_by_id(3) with open(os.path.join(fw2.launches[0].launch_dir, "task.out")) as f: fw2_text = f.read() with open(os.path.join(fw3.launches[0].launch_dir, "task.out")) as f: fw3_text = f.read() self.assertNotEqual(fw2_text, fw3_text)
def basic_wf_ex(): print '--- BASIC WORKFLOW EXAMPLE ---' # setup launchpad = setup() # add FireWorks task1 = ScriptTask.from_str('echo "Ingrid is the CEO."') task2 = ScriptTask.from_str('echo "Jill is a manager."') task3 = ScriptTask.from_str('echo "Jack is a manager."') task4 = ScriptTask.from_str('echo "Kip is an intern."') fw1 = FireWork(task1, fw_id=-1) fw2 = FireWork(task2, fw_id=-2) fw3 = FireWork(task3, fw_id=-3) fw4 = FireWork(task4, fw_id=-4) # make workflow workflow = Workflow([fw1, fw2, fw3, fw4], { -1: [-2, -3], -2: [-4], -3: [-4] }) launchpad.add_wf(workflow) # launch Rocket rapidfire(launchpad)
def test_multi_fw(self): test1 = ScriptTask.from_str("python -c 'print \"test1\"'", {'store_stdout': True}) test2 = ScriptTask.from_str("python -c 'print \"test2\"'", {'store_stdout': True}) fw = FireWork([test1, test2]) self.lp.add_wf(fw) launch_rocket(self.lp) time.sleep(3) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test2\n')
def test_parentconnector(self): fw1 = Firework(ScriptTask.from_str('echo "1"')) fw2 = Firework(ScriptTask.from_str('echo "1"'), parents=fw1) fw3 = Firework(ScriptTask.from_str('echo "1"'), parents=[fw1, fw2]) self.assertEqual( Workflow([fw1, fw2, fw3]).links, {fw1.fw_id: [fw2.fw_id, fw3.fw_id], fw2.fw_id: [fw3.fw_id], fw3.fw_id: []} ) self.assertRaises(ValueError, Workflow, [fw1, fw3]) # can't make this
def run_task(self, fw_spec): print('Running the Multiple Detour Task') dt1 = Firework( ScriptTask.from_str('echo "this is intermediate job 1"')) dt2 = Firework( ScriptTask.from_str('echo "this is intermediate job 2"')) dt3 = Firework( ScriptTask.from_str('echo "this is intermediate job 3"')) return FWAction(detours=[dt1, dt2, dt3])
def test_multi_fw(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) test2 = ScriptTask.from_str("python -c 'print(\"test2\")'", {'store_stdout': True}) fw = Firework([test1, test2]) self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) self.assertEqual( self.lp.get_launch_by_id(1).action.stored_data['stdout'], "test2\n")
def test_org_wf(self): test1 = ScriptTask.from_str("python -c 'print \"test1\"'", {'store_stdout': True}) test2 = ScriptTask.from_str("python -c 'print \"test2\"'", {'store_stdout': True}) fw1 = FireWork(test1, fw_id=-1) fw2 = FireWork(test2, fw_id=-2) wf = Workflow([fw1, fw2], {-1: -2}) self.lp.add_wf(wf) launch_rocket(self.lp) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n') launch_rocket(self.lp) self.assertEqual(self.lp.get_launch_by_id(2).action.stored_data['stdout'], 'test2\n')
def test_fwconnector(self): fw1 = Firework(ScriptTask.from_str('echo "1"')) fw2 = Firework(ScriptTask.from_str('echo "1"')) wf1 = Workflow([fw1, fw2], {fw1.fw_id: fw2.fw_id}) self.assertEqual(wf1.links, {fw1.fw_id: [fw2.fw_id], fw2.fw_id: []}) wf2 = Workflow([fw1, fw2], {fw1: fw2}) self.assertEqual(wf2.links, {fw1.fw_id: [fw2.fw_id], fw2.fw_id: []}) wf3 = Workflow([fw1, fw2]) self.assertEqual(wf3.links, {fw1.fw_id: [], fw2.fw_id: []})
def test_multi_fw(self): test1 = ScriptTask.from_str("python -c 'print \"test1\"'", {'store_stdout': True}) test2 = ScriptTask.from_str("python -c 'print \"test2\"'", {'store_stdout': True}) fw = FireWork([test1, test2]) self.lp.add_wf(fw) launch_rocket(self.lp) time.sleep(3) self.assertEqual( self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test2\n')
def test_spec_copy(self): task1 = ScriptTask.from_str('echo "Task 1"') task2 = ScriptTask.from_str('echo "Task 2"') spec = {'_category': 'dummy_category'} fw1 = Firework(task1, fw_id=1, name='Task 1', spec=spec) fw2 = Firework(task2, fw_id=2, name='Task 2', spec=spec) self.lp.add_wf(Workflow([fw1, fw2])) self.assertEqual(self.lp.get_fw_by_id(1).tasks[0]['script'][0], 'echo "Task 1"') self.assertEqual(self.lp.get_fw_by_id(2).tasks[0]['script'][0], 'echo "Task 2"')
def test_category_pt2(self): task1 = ScriptTask.from_str('echo "Task 1"') task2 = ScriptTask.from_str('echo "Task 2"') fw1 = Firework(task1, fw_id=1, name='Task 1') fw2 = Firework(task2, fw_id=2, name='Task 2') self.lp.add_wf(Workflow([fw1, fw2])) self.assertFalse(self.lp.run_exists(FWorker(category="dummy_category"))) self.assertTrue(self.lp.run_exists(FWorker(category="__none__"))) self.assertTrue(self.lp.run_exists(FWorker())) # can run any category self.assertFalse(self.lp.run_exists(FWorker(category=["dummy_category", "other category"])))
def multiple_tasks_ex(): print("--- MULTIPLE FIRETASKS EXAMPLE ---") # setup launchpad = setup() # add FireWorks firetask1 = ScriptTask.from_str('echo "This is TASK #1"') firetask2 = ScriptTask.from_str('echo "This is TASK #2"') firetask3 = ScriptTask.from_str('echo "This is TASK #3"') fw = Firework([firetask1, firetask2, firetask3]) launchpad.add_wf(fw) # launch Rocket rapidfire(launchpad, FWorker())
def multiple_tasks_ex(): print('--- MULTIPLE FIRETASKS EXAMPLE ---') # setup launchpad = setup() # add FireWorks firetask1 = ScriptTask.from_str('echo "This is TASK #1"') firetask2 = ScriptTask.from_str('echo "This is TASK #2"') firetask3 = ScriptTask.from_str('echo "This is TASK #3"') fw = FireWork([firetask1, firetask2, firetask3]) launchpad.add_wf(fw) # launch Rocket rapidfire(launchpad, FWorker())
def test_org_wf(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) test2 = ScriptTask.from_str("python -c 'print(\"test2\")'", {'store_stdout': True}) fw1 = Firework(test1, fw_id=-1) fw2 = Firework(test2, fw_id=-2) wf = Workflow([fw1, fw2], {-1: -2}) self.lp.add_wf(wf) launch_rocket(self.lp, self.fworker) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n') launch_rocket(self.lp, self.fworker) self.assertEqual(self.lp.get_launch_by_id(2).action.stored_data['stdout'], 'test2\n')
def test_tracker(self): """ Launch a workflow and track the files """ self._teardown([self.dest1]) try: fts = [] for i in range(5, 100): ft = ScriptTask.from_str( 'echo "' + str(i) + '" >> ' + self.dest1, {"store_stdout": True}) fts.append(ft) fw = Firework(fts, spec={"_trackers": [self.tracker1]}, fw_id=20, name="test_fw") self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) # print (self.tracker1.track_file()) self.assertEqual("98\n99", self.tracker1.track_file()) finally: self._teardown([self.dest1])
def test_add_wf(self): fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello") self.lp.add_wf(fw) wf_id = self.lp.get_wf_ids() self.assertEqual(len(wf_id), 1) for fw_id in self.lp.get_wf_ids(): wf = self.lp.get_wf_by_fw_id_lzyfw(fw_id) self.assertEqual(len(wf.id_fw.keys()), 1) fw2 = Firework(ScriptTask.from_str('echo "goodbye"'), name="goodbye") wf = Workflow([fw, fw2], name='test_workflow') self.lp.add_wf(wf) #fw = self.lp.get_fw_ids() #self.assertEqual(len(wf.id_fw.keys()), 2) fw_ids = self.lp.get_fw_ids() self.assertEqual(len(fw_ids), 3) self.lp.reset('',require_password=False)
def test_backgroundtask(self): dest1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'hello.txt') self._teardown([dest1]) try: test1 = ScriptTask.from_str( "python -c 'print(\"testing background...\")'", {'store_stdout': True}) bg_task1 = BackgroundTask(FileWriteTask( {'files_to_write': [{ 'filename': dest1, 'contents': 'hello' }]}), num_launches=1, run_on_finish=True) fw = Firework(test1, spec={'_background_tasks': [bg_task1]}) self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) with open(dest1) as f: self.assertEqual(f.read(), 'hello') finally: self._teardown([dest1])
def test_basic_fw_offline(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1) self.lp.add_wf(fw) fw, launch_id = self.lp.reserve_fw(FWorker(), os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) with open(os.path.join(os.getcwd(), "FW_offline.json")) as f: fwo = json.load(f) self.assertEquals(fwo["state"], "COMPLETED") self.assertEquals(fwo["launch_id"], 1) self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False}) with open(os.path.join(os.getcwd(), "FW_ping.json")) as f: fwp = json.load(f) self.assertIsNotNone(fwp["ping_time"]) l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1}) self.lp.recover_offline(l['launch_id']) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test_basic_fw_offline(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1) self.lp.add_wf(fw) fw, launch_id = self.lp.reserve_fw(self.fworker, os.getcwd()) setup_offline_job(self.lp, fw, launch_id) launch_rocket(None, self.fworker) with open(os.path.join(os.getcwd(), "FW_offline.json")) as f: fwo = json.load(f) self.assertEquals(fwo["state"], "COMPLETED") self.assertEquals(fwo["launch_id"], 1) self.assertEquals(fwo["fwaction"], {'update_spec': {}, 'mod_spec': [], 'stored_data': {'returncode': 0, 'stdout': u'test1\n', 'all_returncodes': [0]}, 'exit': False, 'detours': [], 'additions': [], 'defuse_children': False, 'defuse_workflow': False}) with open(os.path.join(os.getcwd(), "FW_ping.json")) as f: fwp = json.load(f) self.assertIsNotNone(fwp["ping_time"]) l = self.lp.offline_runs.find_one({"completed": False, "deprecated": False}, {"launch_id": 1}) self.lp.recover_offline(l['launch_id']) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data['stdout'], 'test1\n')
def test_add_wf(self): fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello") self.lp.add_wf(fw) wf_id = self.lp.get_wf_ids() self.assertEqual(len(wf_id), 1) for fw_id in self.lp.get_wf_ids(): wf = self.lp.get_wf_by_fw_id_lzyfw(fw_id) self.assertEqual(len(wf.id_fw.keys()), 1) fw2 = Firework(ScriptTask.from_str('echo "goodbye"'), name="goodbye") wf = Workflow([fw, fw2], name='test_workflow') self.lp.add_wf(wf) #fw = self.lp.get_fw_ids() #self.assertEqual(len(wf.id_fw.keys()), 2) fw_ids = self.lp.get_fw_ids() self.assertEqual(len(fw_ids), 3) self.lp.reset('', require_password=False)
def test_force_lock_removal(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1, {"_dupefinder": DupeFinderExact()}, fw_id=1) self.lp.add_wf(fw) # add a manual lock with WFLock(self.lp, 1): with WFLock(self.lp, 1, kill=True, expire_secs=1): self.assertTrue(True) # dummy to make sure we got here
def test_basic_fw(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = FireWork(test1) self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[ 'stdout'], 'test1\n')
def test_basic_fw(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = FireWork(test1) self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[ 'stdout'], str(six.b('test1\n')))
def test_dupefinder(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1, {"_dupefinder": DupeFinderExact()}) self.lp.add_wf(fw) self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) launch_rocket(self.lp, self.fworker) self.assertEqual(self.lp.launches.count(), 1)
def test_reset(self): # Store some test fireworks # Atempt couple of ways to reset the lp and check fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello") wf = Workflow([fw], name="test_workflow") self.lp.add_wf(wf) self.lp.reset("", require_password=False) self.assertFalse(self.lp.get_fw_ids()) self.assertFalse(self.lp.get_wf_ids())
def test_reset(self): # Store some test fireworks # Atempt couple of ways to reset the lp and check fw = Firework(ScriptTask.from_str('echo "hello"'), name="hello") wf = Workflow([fw], name='test_workflow') self.lp.add_wf(wf) self.assertRaises(ValueError, self.lp.reset, '', False, 0) self.assertEqual(self.lp.workflows.count(), 1) self.lp.reset('',require_password=False) self.assertFalse(self.lp.get_fw_ids()) self.assertFalse(self.lp.get_wf_ids()) # test failsafe in a strict way for x in range(30): self.lp.add_wf(Workflow([Firework(ScriptTask.from_str('echo "hello"'))])) self.assertRaises(ValueError, self.lp.reset, '') self.lp.reset('', False, 100) # reset back
def add_wf(j, dest, tracker, name): fts = [] for i in range(j,j+25): ft = ScriptTask.from_str('echo "' + str(i) + '" >> '+ dest, {'store_stdout':True}) fts.append(ft) fw1 = Firework(fts, spec={'_trackers':[tracker]}, fw_id=j+1, name=name+'1') fts = [] for i in range(j+25,j+50): ft = ScriptTask.from_str('echo "' + str(i) + '" >> ' + dest, {'store_stdout':True}) fts.append(ft) fw2 = Firework(fts, spec={'_trackers':[tracker]}, fw_id=j+2, name=name+'2') wf = Workflow([fw1, fw2], links_dict={fw1:[fw2]}) self.lp.add_wf(wf)
def test_delete_fw(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1) self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[ 'stdout'], 'test1\n') self.lp.delete_wf(fw.fw_id) self.assertRaises(ValueError, self.lp.get_fw_by_id, fw.fw_id) self.assertRaises(ValueError, self.lp.get_launch_by_id, 1)
def test_checkout_fw(self): os.chdir(MODULE_DIR) self.lp.add_wf(Firework(ScriptTask.from_str( shell_cmd='echo "hello 1"', parameters={"stdout_file": "task.out"}), fw_id=1)) self.lp.add_wf(Firework(ScriptTask.from_str( shell_cmd='echo "hello 2"', parameters={"stdout_file": "task.out"}), fw_id=2)) launch_multiprocess(self.lp, FWorker(), 'DEBUG', 0, 2, 10) fw1 = self.lp.get_fw_by_id(1) fw2 = self.lp.get_fw_by_id(2) self.assertEqual(fw1.launches[0].state_history[-1]["state"], "COMPLETED") self.assertEqual(fw2.launches[0].state_history[-1]["state"], "COMPLETED") with open(os.path.join(fw1.launches[0].launch_dir, "task.out")) as f: self.assertEqual(f.readlines(), ['hello 1\n']) with open(os.path.join(fw2.launches[0].launch_dir, "task.out")) as f: self.assertEqual(f.readlines(), ['hello 2\n'])
def test_multi_detour(self): fw1 = Firework([MultipleDetourTask()], fw_id=1) fw2 = Firework([ScriptTask.from_str('echo "DONE"')], parents=[fw1], fw_id=2) self.lp.add_wf(Workflow([fw1, fw2])) rapidfire(self.lp) links = self.lp.get_wf_by_fw_id(1).links self.assertEqual(set(links[1]), set([2, 3, 4, 5])) self.assertEqual(set(links[2]), set([])) self.assertEqual(set(links[3]), set([2])) self.assertEqual(set(links[4]), set([2])) self.assertEqual(set(links[5]), set([2]))
def test_preserve_fworker(self): fw1 = Firework([ScriptTask.from_str('echo "Testing preserve FWorker"')], spec={"_preserve_fworker": True}, fw_id=1) fw2 = Firework([ScriptTask.from_str('echo "Testing preserve FWorker pt 2"')], spec={"target": 1}, parents=[fw1], fw_id=2) self.lp.add_wf(Workflow([fw1, fw2])) launch_rocket(self.lp, self.fworker) target_fw_id = self.lp.get_fw_ids({"spec.target": 1})[0] modified_spec = self.lp.get_fw_by_id(target_fw_id).spec """ cnt = 0 while '_fworker' not in modified_spec and cnt < 5: modified_spec = self.lp.get_fw_by_id(target_fw_id).spec print(modified_spec) time.sleep(5) cnt += 1 """ self.assertIsNotNone(modified_spec['_fworker'])
def setUp(self): fireworks.core.firework.EXCEPT_DETAILS_ON_RERUN = True self.error_test_dict = {'error': 'description', 'error_code': 1} fw = Firework([ExecutionCounterTask(), ScriptTask.from_str('date +"%s %N"', parameters={'stdout_file': 'date_file'}), ExceptionTestTask(exc_details=self.error_test_dict)]) self.lp.add_wf(fw) ExecutionCounterTask.exec_counter = 0 ExceptionTestTask.exec_counter = 0 self.old_wd = os.getcwd()
def test_parallel_fibadder(self): # this is really testing to see if a Workflow can handle multiple FWs updating it at once parent = FireWork(ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}), fw_id=1) fib1 = FireWork(FibonacciAdderTask(), {'smaller': 0, 'larger': 1, 'stop_point': 30}, fw_id=2) fib2 = FireWork(FibonacciAdderTask(), {'smaller': 0, 'larger': 1, 'stop_point': 30}, fw_id=3) wf = Workflow([parent, fib1, fib2], {1: [2, 3]}) self.lp.add_wf(wf) p = Pool(NCORES_PARALLEL_TEST) creds_array = [self.lp.to_dict()] * NCORES_PARALLEL_TEST p.map(random_launch, creds_array)
def test_duplicate_delete_fw(self): test1 = ScriptTask.from_str("python -c 'print(\"test1\")'", {'store_stdout': True}) fw = Firework(test1, {"_dupefinder": DupeFinderExact()}) self.lp.add_wf(fw) self.lp.add_wf(fw) launch_rocket(self.lp, self.fworker) launch_rocket(self.lp, self.fworker) self.lp.delete_wf(2) self.assertRaises(ValueError, self.lp.get_fw_by_id, 2) self.assertEqual(self.lp.get_launch_by_id(1).action.stored_data[ 'stdout'], 'test1\n')
def setUp(self): # set the defaults in the init of wflock to break the lock quickly fireworks.core.launchpad.WFLock(3, False).__init__.__func__.__defaults__= (3, False) self.error_test_dict = {'error': 'description', 'error_code': 1} fw_slow = Firework(SlowAdditionTask(), spec={'seconds': 10}, fw_id=1) fw_fast = Firework(WaitWFLockTask(), fw_id=2, spec={'_add_launchpad_and_fw_id': True}) fw_child = Firework(ScriptTask.from_str('echo "child"'), fw_id=3) wf = Workflow([fw_slow, fw_fast, fw_child], {fw_slow: fw_child, fw_fast: fw_child}) self.lp.add_wf(wf) self.old_wd = os.getcwd()
def basic_fw_ex(): print('--- BASIC FIREWORK EXAMPLE ---') # setup launchpad = setup() # add FireWork firetask = ScriptTask.from_str('echo "howdy, your job launched successfully!"') firework = FireWork(firetask) launchpad.add_wf(firework) # launch Rocket launch_rocket(launchpad, FWorker())
def basic_fw_ex(): print("--- BASIC FIREWORK EXAMPLE ---") # setup launchpad = setup() # add Firework firetask = ScriptTask.from_str('echo "howdy, your job launched successfully!"') firework = Firework(firetask) launchpad.add_wf(firework) # launch Rocket launch_rocket(launchpad, FWorker())
def setUp(self): fireworks.core.firework.EXCEPT_DETAILS_ON_RERUN = True self.error_test_dict = {'error': 'description', 'error_code': 1} fw = Firework(ScriptTask.from_str( 'echo "test offline"', {'store_stdout':True}), name="offline_fw", fw_id=1) self.lp.add_wf(fw) self.launch_dir = os.path.join(MODULE_DIR, "launcher_offline") os.makedirs(self.launch_dir) self.old_wd = os.getcwd()
def setUp(self): fireworks.core.firework.EXCEPT_DETAILS_ON_RERUN = True self.error_test_dict = {'error': 'description', 'error_code': 1} fw = Firework(ScriptTask.from_str('echo "test offline"', {'store_stdout': True}), name="offline_fw", fw_id=1) self.lp.add_wf(fw) self.launch_dir = os.path.join(MODULE_DIR, "launcher_offline") os.makedirs(self.launch_dir) self.old_wd = os.getcwd()
def basic_wf_ex(): print('--- BASIC WORKFLOW EXAMPLE ---') # setup launchpad = setup() # add FireWorks task1 = ScriptTask.from_str('echo "Ingrid is the CEO."') task2 = ScriptTask.from_str('echo "Jill is a manager."') task3 = ScriptTask.from_str('echo "Jack is a manager."') task4 = ScriptTask.from_str('echo "Kip is an intern."') fw1 = FireWork(task1, fw_id=1) fw2 = FireWork(task2, fw_id=2) fw3 = FireWork(task3, fw_id=3) fw4 = FireWork(task4, fw_id=4) # make workflow workflow = Workflow([fw1, fw2, fw3, fw4], {1: [2, 3], 2: [4], 3: [4]}) launchpad.add_wf(workflow) # launch Rocket rapidfire(launchpad, FWorker())