def test_task_auto_kwargs(self): p = PyTask(func="json.dumps", obj={"hello": "world"}, stored_data_varname="json", auto_kwargs=True) a = p.run_task({}) self.assertEqual(a.stored_data["json"], '{"hello": "world"}') p = PyTask(func="pow", args=[3, 2], stored_data_varname="data") a = p.run_task({}) self.assertEqual(a.stored_data["data"], 9) p = PyTask(func="print", args=[3]) p.run_task({})
def test_task_auto_kwargs(self): p = PyTask(func='json.dumps', obj={'hello': 'world'}, stored_data_varname="json", auto_kwargs=True) a = p.run_task({}) self.assertEqual(a.stored_data['json'], '{"hello": "world"}') p = PyTask(func='pow', args=[3, 2], stored_data_varname='data') a = p.run_task({}) self.assertEqual(a.stored_data['data'], 9) p = PyTask(func='print', args=[3]) p.run_task({})
def test_fizzle(self): p = PyTask(func="fireworks.tests.mongo_tests.throw_error", args=["Testing; this error is normal."]) fw = Firework(p) self.lp.add_wf(fw) self.assertTrue(launch_rocket(self.lp, self.fworker)) self.assertEqual(self.lp.get_fw_by_id(1).state, 'FIZZLED') self.assertFalse(launch_rocket(self.lp, self.fworker))
def test_copy(self): """Test that we can produce a copy of a Workflow but that the copy has unique fw_ids. """ fws = [] for i in range(5): fw = Firework([PyTask(func="print", args=[i])], fw_id=i, name=i) fws.append(fw) wf = Workflow(fws, links_dict={0: [1, 2, 3], 1: [4], 2: [4]}) wf_copy = Workflow.from_wflow(wf) # now we compare to the original to make sure dependencies are same. # have to do gymnastics because ids will NOT be the same # but names are retained for fw in wf_copy.fws: children = wf_copy.links.get(fw.fw_id, list()) orig_id = fw.name orig_children = wf.links.get(orig_id, list()) for child_id, orig_child_id in zip(children, orig_children): self.assertEqual(orig_child_id, wf_copy.id_fw[child_id].name)
def test_init(self): fws = [] for i in range(5): fw = Firework([PyTask(func="print", args=[i])], fw_id=i) fws.append(fw) wf = Workflow(fws, links_dict={0: [1, 2, 3], 1: [4], 2: [4]}) self.assertIsInstance(wf, Workflow) self.assertRaises(ValueError, Workflow, fws, links_dict={ 0: [1, 2, 3], 1: [4], 100: [4] }) self.assertRaises(ValueError, Workflow, fws, links_dict={ 0: [1, 2, 3], 1: [4], 2: [100] })
def test_archive(self): p = PyTask(func="fireworks.tests.mongo_tests.throw_error", args=["This should not happen"]) fw = Firework(p) self.lp.add_wf(fw) self.lp.archive_wf(fw.fw_id) self.assertFalse(launch_rocket(self.lp, self.fworker))
def test_get_time_report_for_wf(self): task = PyTask(func="time.sleep", args=[0.5]) fw1 = Firework([task], spec={ 'wf_task_index': "test1_1", "nproc": 16 }, fw_id=1) fw2 = Firework([task], spec={ 'wf_task_index': "test2_1", "nproc": 16 }, fw_id=2) wf = Workflow([fw1, fw2]) self.lp.add_wf(wf) rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR) wf = self.lp.get_wf_by_fw_id(1) assert wf.state == "COMPLETED" tr = get_time_report_for_wf(wf) assert tr.n_fws == 2 assert tr.total_run_time > 1
def setUp(self): # Define a timed fireWork fw_timer = Firework(PyTask(func='time.sleep', args=[5]), name="timer") self.lp.add_wf(fw_timer) # Get assigned fwid for timer firework self.fw_id = self.lp.get_fw_ids({'name': 'timer'}, limit=1)[0] self.old_wd = os.getcwd()
def test_task_auto_kwargs(self): p = PyTask(func="json.dumps", obj={"hello": "world"}, stored_data_varname="json", auto_kwargs=True) a = p.run_task({}) self.assertEqual(a.stored_data["json"], '{"hello": "world"}') p = PyTask(func="pow", args=[3, 2], stored_data_varname="data") a = p.run_task({}) self.assertEqual(a.stored_data["data"], 9) p = PyTask(func="print", args=[3]) p.run_task({})
def test_task_data_flow(self): """ test dataflow parameters: inputs, outputs and chunk_number """ params = { 'func': 'pow', 'inputs': ['arg', 'power', 'modulo'], 'stored_data_varname': 'data' } spec = {'arg': 2, 'power': 3, 'modulo': None} action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data['data'], 8) params['outputs'] = ['result'] action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data['data'], 8) self.assertEqual(action.update_spec['result'], 8) params['chunk_number'] = 0 action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data['data'], 8) self.assertEqual(action.mod_spec[0]['_push']['result'], 8) params['args'] = [2, 3] params['inputs'] = ['modulo'] spec = {'modulo': 3} action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data['data'], 2) self.assertEqual(action.mod_spec[0]['_push']['result'], 2) params['func'] = afunc.__module__ + '.' + afunc.__name__ params['args'] = [3, 3] params['inputs'] = ['array'] spec = {'array': [1, 2]} action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data['data'], [1, 2]) self.assertEqual(action.mod_spec[0]['_push']['result'], 1) self.assertEqual(action.mod_spec[1]['_push']['result'], 2) del params['chunk_number'] action = PyTask(**params).run_task(spec) self.assertEqual(action.update_spec['result'][0], 1) self.assertEqual(action.update_spec['result'][1], 2) params['outputs'] = ['first', 'second'] action = PyTask(**params).run_task(spec) self.assertEqual(action.update_spec['first'], 1) self.assertEqual(action.update_spec['second'], 2)
def test_task_data_flow(self): """test dataflow parameters: inputs, outputs and chunk_number""" params = { "func": "pow", "inputs": ["arg", "power", "modulo"], "stored_data_varname": "data" } spec = {"arg": 2, "power": 3, "modulo": None} action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data["data"], 8) params["outputs"] = ["result"] action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data["data"], 8) self.assertEqual(action.update_spec["result"], 8) params["chunk_number"] = 0 action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data["data"], 8) self.assertEqual(action.mod_spec[0]["_push"]["result"], 8) params["args"] = [2, 3] params["inputs"] = ["modulo"] spec = {"modulo": 3} action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data["data"], 2) self.assertEqual(action.mod_spec[0]["_push"]["result"], 2) params["func"] = afunc.__module__ + "." + afunc.__name__ params["args"] = [3, 3] params["inputs"] = ["array"] spec = {"array": [1, 2]} action = PyTask(**params).run_task(spec) self.assertEqual(action.stored_data["data"], [1, 2]) self.assertEqual(action.mod_spec[0]["_push"]["result"], 1) self.assertEqual(action.mod_spec[1]["_push"]["result"], 2) del params["chunk_number"] action = PyTask(**params).run_task(spec) self.assertEqual(action.update_spec["result"][0], 1) self.assertEqual(action.update_spec["result"][1], 2) params["outputs"] = ["first", "second"] action = PyTask(**params).run_task(spec) self.assertEqual(action.update_spec["first"], 1) self.assertEqual(action.update_spec["second"], 2)
def test_directory(self): doc_class = self.get_document_class_from_mixin(DirectoryMixin) doc = doc_class() # create and run a fireworks workflow task = PyTask(func="time.sleep", args=[0.5]) wf = Workflow([ Firework(task, spec={'wf_task_index': "1"}, fw_id=1), Firework(task, spec={'wf_task_index': "2"}, fw_id=2) ]) self.lp.add_wf(wf) rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR) wf = self.lp.get_wf_by_fw_id(1) doc.set_dir_names_from_fws_wf(wf) assert len(doc.dir_names) == 2 if has_mongodb(): doc.save() query_result = doc_class.objects() assert len(query_result) == 1 assert len(query_result[0].dir_names) == 2
def setUp(self): # define the individual FireWorks used in the Workflow # Parent Firework fw_p = Firework(ScriptTask.from_str( 'echo "Cronus is the ruler of titans"', {'store_stdout': True}), name="parent", fw_id=1) # Sibling fireworks #fw_s1 = Firework(ScriptTask.from_str( # 'echo "Zeus is son of Cronus"', # {'store_stdout':True}), name="sib1", fw_id=2, parents=fw_p) # Timed firework fw_s1 = Firework(PyTask(func='time.sleep', args=[5]), name="sib1", fw_id=2, parents=fw_p) fw_s2 = Firework(ScriptTask.from_str( 'echo "Poisedon is brother of Zeus"', {'store_stdout': True}), name="sib2", fw_id=3, parents=fw_p) fw_s3 = Firework(ScriptTask.from_str('echo "Hades is brother of Zeus"', {'store_stdout': True}), name="sib3", fw_id=4, parents=fw_p) fw_s4 = Firework(ScriptTask.from_str( 'echo "Demeter is sister & wife of Zeus"', {'store_stdout': True}), name="sib4", fw_id=5, parents=fw_p) fw_s5 = Firework(ScriptTask.from_str( 'echo "Lapetus is son of Oceanus"', {'store_stdout': True}), name="cousin1", fw_id=6) # Children fireworks fw_c1 = Firework(ScriptTask.from_str('echo "Ares is son of Zeus"', {'store_stdout': True}), name="c1", fw_id=7, parents=fw_s1) fw_c2 = Firework(ScriptTask.from_str( 'echo "Persephone is daughter of Zeus & Demeter and wife of Hades"', {'store_stdout': True}), name="c2", fw_id=8, parents=[fw_s1, fw_s4]) fw_c3 = Firework(ScriptTask.from_str( 'echo "Makaria is daughter of Hades & Persephone"', {'store_stdout': True}), name="c3", fw_id=9, parents=[fw_s3, fw_c2]) fw_c4 = Firework(ScriptTask.from_str( 'echo "Dione is descendant of Lapetus"', {'store_stdout': True}), name="c4", fw_id=10, parents=fw_s5) fw_c5 = Firework(ScriptTask.from_str( 'echo "Aphrodite is son of of Zeus and Dione"', {'store_stdout': True}), name="c5", fw_id=11, parents=[fw_s1, fw_c4]) fw_c6 = Firework(ScriptTask.from_str( 'echo "Atlas is son of of Lapetus"', {'store_stdout': True}), name="c6", fw_id=12, parents=fw_s5) fw_c7 = Firework(ScriptTask.from_str( 'echo "Maia is daughter of Atlas"', {'store_stdout': True}), name="c7", fw_id=13, parents=fw_c6) fw_c8 = Firework(ScriptTask.from_str( 'echo "Hermes is daughter of Maia and Zeus"', {'store_stdout': True}), name="c8", fw_id=14, parents=[fw_s1, fw_c7]) # assemble Workflow from FireWorks and their connections by id workflow = Workflow([ fw_p, fw_s1, fw_s2, fw_s3, fw_s4, fw_s5, fw_c1, fw_c2, fw_c3, fw_c4, fw_c5, fw_c6, fw_c7, fw_c8 ]) self.lp.add_wf(workflow) # Give names to fw_ids self.zeus_fw_id = 2 self.zeus_child_fw_ids = set([7, 8, 9, 11, 14]) self.lapetus_desc_fw_ids = set([6, 10, 12, 13]) self.zeus_sib_fw_ids = set([3, 4, 5]) self.par_fw_id = 1 self.all_ids = self.zeus_child_fw_ids | self.lapetus_desc_fw_ids | \ self.zeus_sib_fw_ids | set([self.zeus_fw_id]) | \ set([self.par_fw_id]) self.old_wd = os.getcwd()