コード例 #1
0
 def test_task_auto_kwargs(self):
     p = PyTask(func="json.dumps",
                obj={"hello": "world"},
                stored_data_varname="json",
                auto_kwargs=True)
     a = p.run_task({})
     self.assertEqual(a.stored_data["json"], '{"hello": "world"}')
     p = PyTask(func="pow", args=[3, 2], stored_data_varname="data")
     a = p.run_task({})
     self.assertEqual(a.stored_data["data"], 9)
     p = PyTask(func="print", args=[3])
     p.run_task({})
コード例 #2
0
ファイル: test_script_task.py プロジェクト: FilipchukB/P1
 def test_task_auto_kwargs(self):
     p = PyTask(func='json.dumps',
                obj={'hello': 'world'},
                stored_data_varname="json",
                auto_kwargs=True)
     a = p.run_task({})
     self.assertEqual(a.stored_data['json'], '{"hello": "world"}')
     p = PyTask(func='pow', args=[3, 2], stored_data_varname='data')
     a = p.run_task({})
     self.assertEqual(a.stored_data['data'], 9)
     p = PyTask(func='print', args=[3])
     p.run_task({})
コード例 #3
0
ファイル: mongo_tests.py プロジェクト: sseyler/fireworks
 def test_fizzle(self):
     p = PyTask(func="fireworks.tests.mongo_tests.throw_error", args=["Testing; this error is normal."])
     fw = Firework(p)
     self.lp.add_wf(fw)
     self.assertTrue(launch_rocket(self.lp, self.fworker))
     self.assertEqual(self.lp.get_fw_by_id(1).state, 'FIZZLED')
     self.assertFalse(launch_rocket(self.lp, self.fworker))
コード例 #4
0
    def test_copy(self):
        """Test that we can produce a copy of a Workflow but that the copy
        has unique fw_ids.

        """
        fws = []
        for i in range(5):
            fw = Firework([PyTask(func="print", args=[i])], fw_id=i, name=i)
            fws.append(fw)

        wf = Workflow(fws, links_dict={0: [1, 2, 3], 1: [4], 2: [4]})

        wf_copy = Workflow.from_wflow(wf)

        # now we compare to the original to make sure dependencies are same.
        # have to do gymnastics because ids will NOT be the same
        # but names are retained
        for fw in wf_copy.fws:
            children = wf_copy.links.get(fw.fw_id, list())
            orig_id = fw.name

            orig_children = wf.links.get(orig_id, list())

            for child_id, orig_child_id in zip(children, orig_children):
                self.assertEqual(orig_child_id, wf_copy.id_fw[child_id].name)
コード例 #5
0
    def test_init(self):

        fws = []
        for i in range(5):
            fw = Firework([PyTask(func="print", args=[i])], fw_id=i)

            fws.append(fw)
        wf = Workflow(fws, links_dict={0: [1, 2, 3], 1: [4], 2: [4]})
        self.assertIsInstance(wf, Workflow)
        self.assertRaises(ValueError,
                          Workflow,
                          fws,
                          links_dict={
                              0: [1, 2, 3],
                              1: [4],
                              100: [4]
                          })
        self.assertRaises(ValueError,
                          Workflow,
                          fws,
                          links_dict={
                              0: [1, 2, 3],
                              1: [4],
                              2: [100]
                          })
コード例 #6
0
ファイル: mongo_tests.py プロジェクト: FilipchukB/P1
 def test_archive(self):
     p = PyTask(func="fireworks.tests.mongo_tests.throw_error",
                args=["This should not happen"])
     fw = Firework(p)
     self.lp.add_wf(fw)
     self.lp.archive_wf(fw.fw_id)
     self.assertFalse(launch_rocket(self.lp, self.fworker))
コード例 #7
0
ファイル: test_fw_utils.py プロジェクト: fagan2888/abiflows
    def test_get_time_report_for_wf(self):
        task = PyTask(func="time.sleep", args=[0.5])
        fw1 = Firework([task],
                       spec={
                           'wf_task_index': "test1_1",
                           "nproc": 16
                       },
                       fw_id=1)
        fw2 = Firework([task],
                       spec={
                           'wf_task_index': "test2_1",
                           "nproc": 16
                       },
                       fw_id=2)
        wf = Workflow([fw1, fw2])
        self.lp.add_wf(wf)

        rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR)

        wf = self.lp.get_wf_by_fw_id(1)

        assert wf.state == "COMPLETED"

        tr = get_time_report_for_wf(wf)

        assert tr.n_fws == 2
        assert tr.total_run_time > 1
コード例 #8
0
ファイル: test_launchpad.py プロジェクト: zbwang/fireworks
    def setUp(self):
        # Define a timed fireWork
        fw_timer = Firework(PyTask(func='time.sleep', args=[5]), name="timer")
        self.lp.add_wf(fw_timer)

        # Get assigned fwid for timer firework
        self.fw_id = self.lp.get_fw_ids({'name': 'timer'}, limit=1)[0]

        self.old_wd = os.getcwd()
コード例 #9
0
 def test_task_auto_kwargs(self):
     p = PyTask(func="json.dumps", obj={"hello": "world"},
                    stored_data_varname="json", auto_kwargs=True)
     a = p.run_task({})
     self.assertEqual(a.stored_data["json"], '{"hello": "world"}')
     p = PyTask(func="pow", args=[3, 2], stored_data_varname="data")
     a = p.run_task({})
     self.assertEqual(a.stored_data["data"], 9)
     p = PyTask(func="print", args=[3])
     p.run_task({})
コード例 #10
0
ファイル: test_script_task.py プロジェクト: FilipchukB/P1
    def test_task_data_flow(self):
        """ test dataflow parameters: inputs, outputs and chunk_number """
        params = {
            'func': 'pow',
            'inputs': ['arg', 'power', 'modulo'],
            'stored_data_varname': 'data'
        }
        spec = {'arg': 2, 'power': 3, 'modulo': None}
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data['data'], 8)

        params['outputs'] = ['result']
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data['data'], 8)
        self.assertEqual(action.update_spec['result'], 8)

        params['chunk_number'] = 0
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data['data'], 8)
        self.assertEqual(action.mod_spec[0]['_push']['result'], 8)

        params['args'] = [2, 3]
        params['inputs'] = ['modulo']
        spec = {'modulo': 3}
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data['data'], 2)
        self.assertEqual(action.mod_spec[0]['_push']['result'], 2)

        params['func'] = afunc.__module__ + '.' + afunc.__name__
        params['args'] = [3, 3]
        params['inputs'] = ['array']
        spec = {'array': [1, 2]}
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data['data'], [1, 2])
        self.assertEqual(action.mod_spec[0]['_push']['result'], 1)
        self.assertEqual(action.mod_spec[1]['_push']['result'], 2)

        del params['chunk_number']
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.update_spec['result'][0], 1)
        self.assertEqual(action.update_spec['result'][1], 2)

        params['outputs'] = ['first', 'second']
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.update_spec['first'], 1)
        self.assertEqual(action.update_spec['second'], 2)
コード例 #11
0
    def test_task_data_flow(self):
        """test dataflow parameters: inputs, outputs and chunk_number"""
        params = {
            "func": "pow",
            "inputs": ["arg", "power", "modulo"],
            "stored_data_varname": "data"
        }
        spec = {"arg": 2, "power": 3, "modulo": None}
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data["data"], 8)

        params["outputs"] = ["result"]
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data["data"], 8)
        self.assertEqual(action.update_spec["result"], 8)

        params["chunk_number"] = 0
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data["data"], 8)
        self.assertEqual(action.mod_spec[0]["_push"]["result"], 8)

        params["args"] = [2, 3]
        params["inputs"] = ["modulo"]
        spec = {"modulo": 3}
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data["data"], 2)
        self.assertEqual(action.mod_spec[0]["_push"]["result"], 2)

        params["func"] = afunc.__module__ + "." + afunc.__name__
        params["args"] = [3, 3]
        params["inputs"] = ["array"]
        spec = {"array": [1, 2]}
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.stored_data["data"], [1, 2])
        self.assertEqual(action.mod_spec[0]["_push"]["result"], 1)
        self.assertEqual(action.mod_spec[1]["_push"]["result"], 2)

        del params["chunk_number"]
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.update_spec["result"][0], 1)
        self.assertEqual(action.update_spec["result"][1], 2)

        params["outputs"] = ["first", "second"]
        action = PyTask(**params).run_task(spec)
        self.assertEqual(action.update_spec["first"], 1)
        self.assertEqual(action.update_spec["second"], 2)
コード例 #12
0
    def test_directory(self):
        doc_class = self.get_document_class_from_mixin(DirectoryMixin)
        doc = doc_class()

        # create and run a fireworks workflow
        task = PyTask(func="time.sleep", args=[0.5])
        wf = Workflow([
            Firework(task, spec={'wf_task_index': "1"}, fw_id=1),
            Firework(task, spec={'wf_task_index': "2"}, fw_id=2)
        ])
        self.lp.add_wf(wf)
        rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR)
        wf = self.lp.get_wf_by_fw_id(1)

        doc.set_dir_names_from_fws_wf(wf)
        assert len(doc.dir_names) == 2

        if has_mongodb():
            doc.save()
            query_result = doc_class.objects()
            assert len(query_result) == 1
            assert len(query_result[0].dir_names) == 2
コード例 #13
0
ファイル: test_launchpad.py プロジェクト: zbwang/fireworks
    def setUp(self):
        # define the individual FireWorks used in the Workflow
        # Parent Firework
        fw_p = Firework(ScriptTask.from_str(
            'echo "Cronus is the ruler of titans"', {'store_stdout': True}),
                        name="parent",
                        fw_id=1)
        # Sibling fireworks
        #fw_s1 = Firework(ScriptTask.from_str(
        #    'echo "Zeus is son of Cronus"',
        #    {'store_stdout':True}), name="sib1", fw_id=2, parents=fw_p)
        # Timed firework
        fw_s1 = Firework(PyTask(func='time.sleep', args=[5]),
                         name="sib1",
                         fw_id=2,
                         parents=fw_p)
        fw_s2 = Firework(ScriptTask.from_str(
            'echo "Poisedon is brother of Zeus"', {'store_stdout': True}),
                         name="sib2",
                         fw_id=3,
                         parents=fw_p)
        fw_s3 = Firework(ScriptTask.from_str('echo "Hades is brother of Zeus"',
                                             {'store_stdout': True}),
                         name="sib3",
                         fw_id=4,
                         parents=fw_p)
        fw_s4 = Firework(ScriptTask.from_str(
            'echo "Demeter is sister & wife of Zeus"', {'store_stdout': True}),
                         name="sib4",
                         fw_id=5,
                         parents=fw_p)
        fw_s5 = Firework(ScriptTask.from_str(
            'echo "Lapetus is son of Oceanus"', {'store_stdout': True}),
                         name="cousin1",
                         fw_id=6)
        # Children fireworks
        fw_c1 = Firework(ScriptTask.from_str('echo "Ares is son of Zeus"',
                                             {'store_stdout': True}),
                         name="c1",
                         fw_id=7,
                         parents=fw_s1)
        fw_c2 = Firework(ScriptTask.from_str(
            'echo "Persephone is daughter of Zeus & Demeter and wife of Hades"',
            {'store_stdout': True}),
                         name="c2",
                         fw_id=8,
                         parents=[fw_s1, fw_s4])
        fw_c3 = Firework(ScriptTask.from_str(
            'echo "Makaria is daughter of Hades & Persephone"',
            {'store_stdout': True}),
                         name="c3",
                         fw_id=9,
                         parents=[fw_s3, fw_c2])
        fw_c4 = Firework(ScriptTask.from_str(
            'echo "Dione is descendant of Lapetus"', {'store_stdout': True}),
                         name="c4",
                         fw_id=10,
                         parents=fw_s5)
        fw_c5 = Firework(ScriptTask.from_str(
            'echo "Aphrodite is son of of Zeus and Dione"',
            {'store_stdout': True}),
                         name="c5",
                         fw_id=11,
                         parents=[fw_s1, fw_c4])
        fw_c6 = Firework(ScriptTask.from_str(
            'echo "Atlas is son of of Lapetus"', {'store_stdout': True}),
                         name="c6",
                         fw_id=12,
                         parents=fw_s5)
        fw_c7 = Firework(ScriptTask.from_str(
            'echo "Maia is daughter of Atlas"', {'store_stdout': True}),
                         name="c7",
                         fw_id=13,
                         parents=fw_c6)
        fw_c8 = Firework(ScriptTask.from_str(
            'echo "Hermes is daughter of Maia and Zeus"',
            {'store_stdout': True}),
                         name="c8",
                         fw_id=14,
                         parents=[fw_s1, fw_c7])

        # assemble Workflow from FireWorks and their connections by id
        workflow = Workflow([
            fw_p, fw_s1, fw_s2, fw_s3, fw_s4, fw_s5, fw_c1, fw_c2, fw_c3,
            fw_c4, fw_c5, fw_c6, fw_c7, fw_c8
        ])
        self.lp.add_wf(workflow)

        # Give names to fw_ids
        self.zeus_fw_id = 2
        self.zeus_child_fw_ids = set([7, 8, 9, 11, 14])
        self.lapetus_desc_fw_ids = set([6, 10, 12, 13])
        self.zeus_sib_fw_ids = set([3, 4, 5])
        self.par_fw_id = 1
        self.all_ids = self.zeus_child_fw_ids | self.lapetus_desc_fw_ids | \
                       self.zeus_sib_fw_ids | set([self.zeus_fw_id]) | \
                       set([self.par_fw_id])

        self.old_wd = os.getcwd()