Esempio n. 1
0
    def test_submit_failed_check(self):
        test_executor = TestCmdExecutor(self.config, "1", False, 0, True,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)

        def expected_error(fail):
            self.assertIsInstance(fail, Failure)
            self.assertTrue(
                fail.value.message.startswith(
                    "Exception at check_status with reason"))

        def unexpected_success(value):
            self.fail(
                "Unexpected success - failure expected. Return value: %s of type %s"
                % (str(value), type(value)))

        d.addCallbacks(unexpected_success, expected_error)
        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec", taskprops['path'])
        self.assertEqual("test_exec", taskprops['name'])
    def test_nested_task(self):
        pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a',), ('b',))
        subgraph =  G(
            C(START_TICK, 'a', 1,'a'),
            C(START_TICK, 'context', 1,'context'),
            T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}),
            C(1, 'b', FINAL_TICK,'b'),
        )
        
        subgraph_task_tick=subgraph.get_all_ticks()[0]
        graph=Graph()
        task=NestedGraphTask(subgraph, 'test_nested')
        tick=START_TICK+1
        graph.add_task(tick, task, {'name':'test_nested','path':'test_nested'})
        source=Endpoint(START_TICK, 'a')
        dest=Endpoint(tick, 'a')
        graph.connect(source,dest)
        source=Endpoint(START_TICK, 'context')
        dest=Endpoint(tick, 'context')
        graph.connect(source,dest)
        source=Endpoint(tick, 'b')
        dest=Endpoint(FINAL_TICK, 'b')
        graph.connect(source,dest)

        task.refine(graph, Tick.parse_tick(1), {'context':{},'a':{}})

        expected_graph_task_tick=subgraph_task_tick<<Tick.parse_tick(1)        
        expected =  G(
            C(START_TICK, 'a', expected_graph_task_tick,'a'),
            C(START_TICK, 'context', expected_graph_task_tick,'context'),
            T(expected_graph_task_tick, exec_task, {'name': 'test_exec', 'path':'test_nested.test_exec'}),
            C(expected_graph_task_tick, 'b', FINAL_TICK,'b'),
        )
        utils.assert_graph_equal(expected, graph)
Esempio n. 3
0
    def test_submit_failed_submit(self):
        test_executor = TestCmdExecutor(self.config, "1", True, 0, False,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)

        def expected_error(failure):
            self.assertIsInstance(failure, Failure)
            self.assertTrue(
                failure.value.message.startswith(
                    "Exception at submit with reason"))
            return "Exception has occurred as expected."

        def unexpected_success(value):
            return Failure(
                ValueError(
                    "Exception did not occur as expected. Value %s found" %
                    str(value)))

        d.addCallback(unexpected_success)
        d.addErrback(expected_error)

        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec", taskprops['path'])
        self.assertEqual("test_exec", taskprops['name'])
Esempio n. 4
0
 def test_task_by_property(self):
     pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
     exec_task1 = ExecTask("exec_name1", pkg, ('a', ), ('b', ))
     exec_task2 = ExecTask("exec_name2", pkg, ('a', ), ('b', ))
     exec_task3 = ExecTask("exec_name3", pkg, ('a', ), ('b', ))
     graph = G(
         C(START_TICK, 'a', 1, 'a'),
         C(START_TICK, 'context', 1, 'context'),
         T(1, exec_task1, {
             'name': 'test_exec1',
             'path': 'test_exec1',
             'prop1': 'a'
         }),
         T(2, exec_task2, {
             'name': 'test_exec2',
             'path': 'test_exec2',
             'prop1': 'a'
         }),
         T(3, exec_task3, {
             'name': 'test_exec3',
             'path': 'test_exec3',
             'prop1': 'b'
         }),
         C(1, 'b', FINAL_TICK, 'b'),
     )
     self.assertEquals([Tick.parse_tick(1)],
                       get_ticks_by_property(graph, 'name', 'test_exec1'))
     self.assertEquals(
         [Tick.parse_tick(1), Tick.parse_tick(2)],
         get_ticks_by_property(graph, 'prop1', 'a'))
     self.assertEquals([], get_ticks_by_property(graph, 'prop2', 'a'))
Esempio n. 5
0
    def test_submit_error(self):
        test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
                                        JOB_ERROR)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)

        def expected_error(fail):
            if isinstance(fail, Failure):
                self.assertTrue(True)
            else:
                self.fail("Type of return value not as expected: %s" %
                          type(fail))

        def unexpected_success(value):
            self.fail("Return value not as expected: %s of type %s" %
                      (str(value), type(value)))

        d.addCallbacks(unexpected_success, expected_error)

        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        summary = taskprops['summary']
        self.assertEqual("test_exec", summary.dfpath)
        self.assertEqual("test_exec", summary.outdir)
        self.assertEqual("1", summary.pid)
        self.assertEqual(JOB_ERROR, summary.status)
 def test_submit_completed(self):
     test_executor=TestCmdExecutor(self.config, "1", False, 0, False, JOB_COMPLETED)
     self.node_callbacks._cmd_executor=test_executor
     tick=Tick.parse_tick(1)
     d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
     expected={'c':'test_exec/c.xml'}
     self.assertEqual(expected, d.result.result)
     
     tick=Tick.parse_tick(1)
     taskprops=self.g.get_task_properties(tick)
     summary=taskprops['summary']
     self.assertEqual("test_exec",summary.dfpath)
     self.assertEqual("test_exec",summary.outdir)
     self.assertEqual("1",summary.pid)
     self.assertEqual(JOB_COMPLETED,summary.status)
 def test_task_by_property(self):
     pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
     exec_task1 = ExecTask("exec_name1", pkg, ('a',), ('b',))
     exec_task2 = ExecTask("exec_name2", pkg, ('a',), ('b',))
     exec_task3 = ExecTask("exec_name3", pkg, ('a',), ('b',))
     graph =  G(
         C(START_TICK, 'a', 1,'a'),
         C(START_TICK, 'context', 1,'context'),
         T(1, exec_task1, {'name': 'test_exec1', 'path':'test_exec1', 'prop1':'a'}),
         T(2, exec_task2, {'name': 'test_exec2', 'path':'test_exec2', 'prop1':'a'}),
         T(3, exec_task3, {'name': 'test_exec3', 'path':'test_exec3', 'prop1':'b'}),
         C(1, 'b', FINAL_TICK,'b'),
     )
     self.assertEquals([Tick.parse_tick(1)], get_ticks_by_property(graph, 'name', 'test_exec1'))
     self.assertEquals([Tick.parse_tick(1), Tick.parse_tick(2)], get_ticks_by_property(graph, 'prop1', 'a'))
     self.assertEquals([], get_ticks_by_property(graph, 'prop2', 'a'))
 def test_submit_error_after_some_checks(self):
     num_checks=2
     test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_ERROR)
     self.node_callbacks._cmd_executor=test_executor
     tick=Tick.parse_tick(1)
     t1=datetime.datetime.now()
     d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
     
     def expected_error(fail):
         if isinstance(fail, Failure):
             self.assertTrue(True)
             t2=datetime.datetime.now()
             self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
             tick=Tick.parse_tick(1)
             taskprops=self.g.get_task_properties(tick)
             summary=taskprops['summary']
             self.assertEqual("test_exec",summary.dfpath)
             self.assertEqual("test_exec",summary.outdir)
             self.assertEqual("1",summary.pid)
             self.assertEqual(JOB_ERROR,summary.status)
         else:
             self.fail("Type of return value not as expected: %s"%type(fail))
     
     def unexpected_success(value):
         self.fail("Return value not as expected: %s of type %s"%(str(value),type(value)))
     
     d.addCallbacks(unexpected_success, expected_error)
     return d
Esempio n. 9
0
    def test_submit_completed_after_some_checks(self):
        num_checks = 2
        test_executor = TestCmdExecutor(self.config, "1", False, 2, False,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        t1 = datetime.datetime.now()
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)
        expected = {'c': 'test_exec/c.xml'}

        def check_success(_):
            t2 = datetime.datetime.now()
            self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
            self.assertEqual(expected, d.result.result)
            tick = Tick.parse_tick(1)
            taskprops = self.g.get_task_properties(tick)
            summary = taskprops['summary']
            self.assertEqual("test_exec", summary.dfpath)
            self.assertEqual("test_exec", summary.outdir)
            self.assertEqual("1", summary.pid)
            self.assertEqual(JOB_COMPLETED, summary.status)

        d.addCallback(check_success)
        return d
    def test_evaluate_reduce_task(self):
        testdir=tempfile.mkdtemp()
        workdir=os.path.join(testdir,'workdir')
        os.mkdir(workdir)
        local_workdir=os.path.join(testdir,'local_workdir')
        os.makedirs(os.path.join(local_workdir,'ps.reduce'))
        testfile=os.path.join(local_workdir, 'ps.reduce','cd_list')
        with open(testfile,'wb') as f:
            pickle.dump([('x_1','y_1'), ('x_2','y_2'), ('x_3','y_3')],f)
        context={'transporter': LocalFileTransporter(), 'workdir':workdir, 'local_workdir':local_workdir}

        reduce_task=_reducePortsTask(3, ['c','d'], 'cd_list')
        reduce_tick=START_TICK+3<<Tick.parse_tick(1)        
        g =  G(
            C(START_TICK, 'context', reduce_tick,'context'),
            T(reduce_tick, reduce_task, {'name': 'reduce', 'path':'ps.reduce'}),
            C(reduce_tick, 'cd_list', FINAL_TICK,'cd_list'),
        )
        
        inputs={}
        inputs['context']=context
        inputs['c_1']='x_1'
        inputs['d_1']='y_1'
        inputs['c_2']='x_2'
        inputs['d_2']='y_2'
        inputs['c_3']='x_3'
        inputs['d_3']='y_3'                
        result=[('x_1','y_1'),('x_2','y_2'),('x_3','y_3')]
        reduce_task.evaluate(g, reduce_tick, reduce_task, inputs)
        outputfile=os.path.join(workdir,'ps.reduce','cd_list')
        self.assertTrue(os.path.exists(outputfile))
        with open(outputfile,'r') as f:
            output=pickle.load(f)
            self.assertEqual(result, output)
Esempio n. 11
0
    def test_submit_completed(self):
        test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)
        expected = {'c': 'test_exec/c.xml'}
        self.assertEqual(expected, d.result.result)

        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        summary = taskprops['summary']
        self.assertEqual("test_exec", summary.dfpath)
        self.assertEqual("test_exec", summary.outdir)
        self.assertEqual("1", summary.pid)
        self.assertEqual(JOB_COMPLETED, summary.status)
Esempio n. 12
0
    def test_nested_task(self):
        pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a', ), ('b', ))
        subgraph = G(
            C(START_TICK, 'a', 1, 'a'),
            C(START_TICK, 'context', 1, 'context'),
            T(1, exec_task, {
                'name': 'test_exec',
                'path': 'test_exec'
            }),
            C(1, 'b', FINAL_TICK, 'b'),
        )

        subgraph_task_tick = subgraph.get_all_ticks()[0]
        graph = Graph()
        task = NestedGraphTask(subgraph, 'test_nested')
        tick = START_TICK + 1
        graph.add_task(tick, task, {
            'name': 'test_nested',
            'path': 'test_nested'
        })
        source = Endpoint(START_TICK, 'a')
        dest = Endpoint(tick, 'a')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'context')
        dest = Endpoint(tick, 'context')
        graph.connect(source, dest)
        source = Endpoint(tick, 'b')
        dest = Endpoint(FINAL_TICK, 'b')
        graph.connect(source, dest)

        task.refine(graph, Tick.parse_tick(1), {'context': {}, 'a': {}})

        expected_graph_task_tick = subgraph_task_tick << Tick.parse_tick(1)
        expected = G(
            C(START_TICK, 'a', expected_graph_task_tick, 'a'),
            C(START_TICK, 'context', expected_graph_task_tick, 'context'),
            T(expected_graph_task_tick, exec_task, {
                'name': 'test_exec',
                'path': 'test_nested.test_exec'
            }),
            C(expected_graph_task_tick, 'b', FINAL_TICK, 'b'),
        )
        utils.assert_graph_equal(expected, graph)
    def test_submit_failed_check(self):
        test_executor=TestCmdExecutor(self.config, "1", False, 0, True, JOB_COMPLETED)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        
        def expected_error(fail):
            self.assertIsInstance(fail, Failure)
            self.assertTrue(fail.value.message.startswith("Exception at check_status with reason"))
        
        def unexpected_success(value):
            self.fail("Unexpected success - failure expected. Return value: %s of type %s"%(str(value),type(value)))

        d.addCallbacks(unexpected_success, expected_error)
        tick=Tick.parse_tick(1)
        taskprops=self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec",taskprops['path'])
        self.assertEqual("test_exec",taskprops['name'])
 def check_success(_):
     t2=datetime.datetime.now()
     self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
     self.assertEqual(expected, d.result.result)        
     tick=Tick.parse_tick(1)
     taskprops=self.g.get_task_properties(tick)
     summary=taskprops['summary']
     self.assertEqual("test_exec",summary.dfpath)
     self.assertEqual("test_exec",summary.outdir)
     self.assertEqual("1",summary.pid)
     self.assertEqual(JOB_COMPLETED,summary.status)      
Esempio n. 15
0
 def check_success(_):
     t2 = datetime.datetime.now()
     self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
     self.assertEqual(expected, d.result.result)
     tick = Tick.parse_tick(1)
     taskprops = self.g.get_task_properties(tick)
     summary = taskprops['summary']
     self.assertEqual("test_exec", summary.dfpath)
     self.assertEqual("test_exec", summary.outdir)
     self.assertEqual("1", summary.pid)
     self.assertEqual(JOB_COMPLETED, summary.status)
    def test_submit_failed_submit(self):
        test_executor=TestCmdExecutor(self.config, "1", True, 0, False, JOB_COMPLETED)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        
        def expected_error(failure):
            self.assertIsInstance(failure, Failure)
            self.assertTrue(failure.value.message.startswith("Exception at submit with reason"))
            return "Exception has occurred as expected."
        
        def unexpected_success(value):
            return Failure(ValueError("Exception did not occur as expected. Value %s found"%str(value)))

        d.addCallback(unexpected_success)
        d.addErrback(expected_error)

        tick=Tick.parse_tick(1)
        taskprops=self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec",taskprops['path'])
        self.assertEqual("test_exec",taskprops['name'])
Esempio n. 17
0
 def test_eval_fail(self):
     g = G(
         T(1, tasks.ConstTask(None)),
         C(1, "value", FINAL_TICK, "retval")
     )
     d = self.target.execute(g, {})
     self.next_ready()[-1].errback(failure.Failure(MockError()))
     
     
     f = twistit.extract_failure(d)
     self.assertTrue(f.check(traverser.EvaluationError))
     self.assertTrue(f.value.cause.check(MockError))
     self.assertEqual(Tick.parse_tick(1), f.value.tick)
Esempio n. 18
0
 def test_refine_body_twice(self):
     # def f(x):
     #     while x:
     #         pass
     #     return x
         
     with graph_factory():
         g = G(
           C(START_TICK, "x", 1, "$test"),
           C(START_TICK, "x", 1, "x"),
           T(1, tasks.WhileTask(False, False, G("body",
               C(START_TICK, "x", 1 , "$test"),
               C(START_TICK, "x", 1 , "x"),
               T(1, tasks.WhileTask(True, False, G("body"), G("else"))),
           ), G("else"))),
           C(START_TICK, "x", FINAL_TICK, "retval")
         )
         
         
     target = g.get_task(START_TICK + 1)
     target.refine(g, START_TICK + 1, {"$test":True})
     
     target = g.get_task(Tick.parse_tick((1,1,1)))
     target.refine(g, Tick.parse_tick((1,1,1)), {"$test":True})
     
     with graph_factory():
         expected = G(
           C(START_TICK, "x", (1,2,1), "$test"),
           C(START_TICK, "x", (1,2,1), "x"),
           T((1,2,1), tasks.WhileTask(True, False, G("body",
               C(START_TICK, "x", 1 , "$test"),
               C(START_TICK, "x", 1 , "x"),
               T(1, tasks.WhileTask(True, False, G("body"), G("else"))),
           ), G("else"))),
           C(START_TICK, "x", FINAL_TICK, "retval")
         )
         
         
     utils.assert_graph_equal(expected, g)
 def expected_error(fail):
     if isinstance(fail, Failure):
         self.assertTrue(True)
         t2=datetime.datetime.now()
         self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
         tick=Tick.parse_tick(1)
         taskprops=self.g.get_task_properties(tick)
         summary=taskprops['summary']
         self.assertEqual("test_exec",summary.dfpath)
         self.assertEqual("test_exec",summary.outdir)
         self.assertEqual("1",summary.pid)
         self.assertEqual(JOB_ERROR,summary.status)
     else:
         self.fail("Type of return value not as expected: %s"%type(fail))
Esempio n. 20
0
 def test_refine_body_twice(self):
     # def f(lst, x):
     #     for x in lst:
     #         pass
     #     return x
         
     with graph_factory():
         g = G(
           C(START_TICK, "lst", 1, "iterable"),
           T(1, tasks.IterTask()),
           C(1, "value", 2, "$iterator"),
           C(START_TICK, "x", 2, "x"),
           T(2, tasks.ForTask(False, False, G("body",
               C(START_TICK, "$target",1 , "x"),
               C(START_TICK, "$iterator", 1, "$iterator"),
               T(1, tasks.ForTask(True, False, G("body"), G("else"))),
               C(1, "x", FINAL_TICK, "x")
           ), G("else"))),
           C(2, "x", FINAL_TICK, "retval")
         )
         
     it = iter([1,2,3])
     
     target_tick = START_TICK + 2
     target = g.get_task(target_tick)
     target.refine(g, target_tick, {"$iterator":it})
     
     target_tick = Tick.parse_tick((2,1,2,1))
     target = g.get_task(target_tick)
     target.refine(g, target_tick, {"$iterator":it})                
     
     with graph_factory():
         expected = G(
           C(START_TICK, "lst", 1, "iterable"),
           T(1, tasks.IterTask()),
           T((2,1,1), tasks.ConstTask(1)),
           T((2,2,1), tasks.ConstTask(2)),
           C(1, "value", (2,2,2,1), "$iterator"),
           C((2,2,1), "value", (2,2,2,1), "x"),
           T((2,2,2,1), tasks.ForTask(True, False, G("body",
               C(START_TICK, "$target",1 , "x"),
               C(START_TICK, "$iterator", 1, "$iterator"),
               T(1, tasks.ForTask(True, False, G("body"), G("else"))),
               C(1, "x", FINAL_TICK, "x")
           ), G("else"))),
           C((2,2,2,1), "x", FINAL_TICK, "retval")
         )
         
     utils.assert_graph_equal(expected, g)
Esempio n. 21
0
 def expected_error(fail):
     if isinstance(fail, Failure):
         self.assertTrue(True)
         t2 = datetime.datetime.now()
         self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
         tick = Tick.parse_tick(1)
         taskprops = self.g.get_task_properties(tick)
         summary = taskprops['summary']
         self.assertEqual("test_exec", summary.dfpath)
         self.assertEqual("test_exec", summary.outdir)
         self.assertEqual("1", summary.pid)
         self.assertEqual(JOB_ERROR, summary.status)
     else:
         self.fail("Type of return value not as expected: %s" %
                   type(fail))
Esempio n. 22
0
 def test_refine_fail(self):
     task = MockTask("in")
     g = G(
         C(START_TICK, "a", 1, "in"),
         T(1, task),
         C(1, "out", FINAL_TICK, "retval")
     )
     d = self.target.execute(g, {"a": "refinedata"})
     
     self.next_refine()[-1].errback(failure.Failure(MockError()))
     
     f = twistit.extract_failure(d)
     self.assertTrue(f.check(traverser.RefineError))
     self.assertTrue(f.value.cause.check(MockError))
     self.assertEqual(Tick.parse_tick(1), f.value.tick)
Esempio n. 23
0
    def test_evaluate_reduce_task(self):
        testdir = tempfile.mkdtemp()
        workdir = os.path.join(testdir, 'workdir')
        os.mkdir(workdir)
        local_workdir = os.path.join(testdir, 'local_workdir')
        os.makedirs(os.path.join(local_workdir, 'ps.reduce'))
        testfile = os.path.join(local_workdir, 'ps.reduce', 'cd_list')
        with open(testfile, 'wb') as f:
            pickle.dump([('x_1', 'y_1'), ('x_2', 'y_2'), ('x_3', 'y_3')], f)
        context = {
            'transporter': LocalFileTransporter(),
            'workdir': workdir,
            'local_workdir': local_workdir
        }

        reduce_task = _reducePortsTask(3, ['c', 'd'], 'cd_list')
        reduce_tick = START_TICK + 3 << Tick.parse_tick(1)
        g = G(
            C(START_TICK, 'context', reduce_tick, 'context'),
            T(reduce_tick, reduce_task, {
                'name': 'reduce',
                'path': 'ps.reduce'
            }),
            C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'),
        )

        inputs = {}
        inputs['context'] = context
        inputs['c_1'] = 'x_1'
        inputs['d_1'] = 'y_1'
        inputs['c_2'] = 'x_2'
        inputs['d_2'] = 'y_2'
        inputs['c_3'] = 'x_3'
        inputs['d_3'] = 'y_3'
        result = [('x_1', 'y_1'), ('x_2', 'y_2'), ('x_3', 'y_3')]
        reduce_task.evaluate(g, reduce_tick, reduce_task, inputs)
        outputfile = os.path.join(workdir, 'ps.reduce', 'cd_list')
        self.assertTrue(os.path.exists(outputfile))
        with open(outputfile, 'r') as f:
            output = pickle.load(f)
            self.assertEqual(result, output)
 def test_submit_completed_after_some_checks(self):
     num_checks=2
     test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_COMPLETED)
     self.node_callbacks._cmd_executor=test_executor
     tick=Tick.parse_tick(1)
     t1=datetime.datetime.now()
     d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
     expected={'c':'test_exec/c.xml'}
     
     def check_success(_):
         t2=datetime.datetime.now()
         self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
         self.assertEqual(expected, d.result.result)        
         tick=Tick.parse_tick(1)
         taskprops=self.g.get_task_properties(tick)
         summary=taskprops['summary']
         self.assertEqual("test_exec",summary.dfpath)
         self.assertEqual("test_exec",summary.outdir)
         self.assertEqual("1",summary.pid)
         self.assertEqual(JOB_COMPLETED,summary.status)      
     
     d.addCallback(check_success)
     return d
Esempio n. 25
0
    def test_parallel_split(self):
        pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a', 'b'), ('c', 'd'))
        subgraph = G(
            C(START_TICK, 'a', 1, 'a'),
            C(START_TICK, 'b', 1, 'b'),
            C(START_TICK, 'context', 1, 'context'),
            T(1, exec_task, {
                'name': 'test_exec',
                'path': 'test_exec'
            }),
            C(1, 'c', FINAL_TICK, 'c'),
            C(1, 'd', FINAL_TICK, 'd'),
        )

        subgraph_task_tick = subgraph.get_all_ticks()[0]
        graph = Graph()
        task = ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
        tick = START_TICK + 1
        graph.add_task(tick, task, {'name': 'ps', 'path': 'ps'})
        source = Endpoint(START_TICK, 'a')
        dest = Endpoint(tick, 'a')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'b')
        dest = Endpoint(tick, 'b')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'context')
        dest = Endpoint(tick, 'context')
        graph.connect(source, dest)
        source = Endpoint(tick, 'cd_list')
        dest = Endpoint(FINAL_TICK, 'cd_list')
        graph.connect(source, dest)

        testdir = tempfile.mkdtemp()
        workdir = os.path.join(testdir, 'workdir')
        os.mkdir(workdir)
        testfile = os.path.join(workdir, 'testfile.pickle')
        local_workdir = os.path.join(testdir, 'local_workdir')
        os.mkdir(local_workdir)
        with open(testfile, 'wb') as f:
            pickle.dump(['x', 'y', 'z'], f)
        context = {
            'transporter': LocalFileTransporter(),
            'workdir': workdir,
            'local_workdir': local_workdir
        }
        task.refine(graph, Tick.parse_tick(1), {
            'b': 'testfile.pickle',
            'context': context
        })

        mapTask = _mapPortsTask('b', ['x', 'y', 'z'])
        map_task_tick = START_TICK + 1 << tick

        iterations_tick = map_task_tick + 1
        it_1 = START_TICK + 1 << iterations_tick
        it_2 = START_TICK + 2 << iterations_tick
        it_3 = START_TICK + 3 << iterations_tick
        task_tick_1 = subgraph_task_tick << it_1
        task_tick_2 = subgraph_task_tick << it_2
        task_tick_3 = subgraph_task_tick << it_3

        reduceTask = _reducePortsTask(3, ['c', 'd'], 'cd_list')
        reduce_tick = iterations_tick + 1

        expected = G(
            C(START_TICK, 'context', map_task_tick, 'context'),
            C(START_TICK, 'a', task_tick_1, 'a'),
            C(START_TICK, 'a', task_tick_2, 'a'),
            C(START_TICK, 'a', task_tick_3, 'a'),
            C(START_TICK, 'b', map_task_tick, 'inputlist'),
            C(START_TICK, 'context', task_tick_1, 'context'),
            C(START_TICK, 'context', task_tick_2, 'context'),
            C(START_TICK, 'context', task_tick_3, 'context'),
            T(map_task_tick, mapTask, {
                'name': 'map',
                'path': 'ps.map'
            }),
            C(map_task_tick, 'b_1', task_tick_1, 'b'),
            C(map_task_tick, 'b_2', task_tick_2, 'b'),
            C(map_task_tick, 'b_3', task_tick_3, 'b'),
            T(task_tick_1, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.1.test_exec'
            }),
            T(task_tick_2, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.2.test_exec'
            }),
            T(task_tick_3, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.3.test_exec'
            }),
            C(task_tick_1, 'c', reduce_tick, 'c_1'),
            C(task_tick_2, 'c', reduce_tick, 'c_2'),
            C(task_tick_3, 'c', reduce_tick, 'c_3'),
            C(task_tick_1, 'd', reduce_tick, 'd_1'),
            C(task_tick_2, 'd', reduce_tick, 'd_2'),
            C(task_tick_3, 'd', reduce_tick, 'd_3'),
            C(START_TICK, 'context', reduce_tick, 'context'),
            T(reduce_tick, reduceTask, {
                'name': 'reduce',
                'path': 'ps.reduce'
            }),
            C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'),
        )
        utils.assert_graph_equal(expected, graph)
        self.assertTrue(
            os.path.exists(os.path.join(local_workdir, 'testfile.pickle')))
    def test_parallel_split(self):
        pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a','b'), ('c','d'))
        subgraph =  G(
            C(START_TICK, 'a', 1,'a'),
            C(START_TICK, 'b', 1,'b'),
            C(START_TICK, 'context', 1,'context'),
            T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}),
            C(1, 'c', FINAL_TICK,'c'),
            C(1, 'd', FINAL_TICK,'d'),
        )
        
        subgraph_task_tick=subgraph.get_all_ticks()[0]
        graph=Graph()
        task=ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
        tick=START_TICK+1
        graph.add_task(tick, task, {'name':'ps','path':'ps'})
        source=Endpoint(START_TICK, 'a')
        dest=Endpoint(tick, 'a')
        graph.connect(source,dest)
        source=Endpoint(START_TICK, 'b')
        dest=Endpoint(tick, 'b')
        graph.connect(source,dest)
        source=Endpoint(START_TICK, 'context')
        dest=Endpoint(tick, 'context')
        graph.connect(source,dest)
        source=Endpoint(tick, 'cd_list')
        dest=Endpoint(FINAL_TICK, 'cd_list')
        graph.connect(source,dest)

        testdir=tempfile.mkdtemp()
        workdir=os.path.join(testdir,'workdir')
        os.mkdir(workdir)
        testfile=os.path.join(workdir, 'testfile.pickle')
        local_workdir=os.path.join(testdir,'local_workdir')
        os.mkdir(local_workdir)
        with open(testfile,'wb') as f:
            pickle.dump(['x','y','z'],f)
        context={'transporter': LocalFileTransporter(), 'workdir':workdir, 'local_workdir':local_workdir}
        task.refine(graph, Tick.parse_tick(1), {'b':'testfile.pickle', 'context':context})

        mapTask=_mapPortsTask('b', ['x','y','z'])
        map_task_tick=START_TICK+1<<tick

        iterations_tick=map_task_tick+1
        it_1 = START_TICK + 1 << iterations_tick
        it_2 = START_TICK + 2 << iterations_tick
        it_3 = START_TICK + 3 << iterations_tick
        task_tick_1=subgraph_task_tick<<it_1
        task_tick_2=subgraph_task_tick<<it_2
        task_tick_3=subgraph_task_tick<<it_3
                
        reduceTask=_reducePortsTask(3, ['c','d'], 'cd_list')
        reduce_tick=iterations_tick+1
        
        expected =  G(
            C(START_TICK, 'context', map_task_tick,'context'),
            C(START_TICK, 'a', task_tick_1,'a'),
            C(START_TICK, 'a', task_tick_2,'a'),
            C(START_TICK, 'a', task_tick_3,'a'),
            C(START_TICK, 'b', map_task_tick,'inputlist'),
            C(START_TICK, 'context', task_tick_1,'context'),
            C(START_TICK, 'context', task_tick_2,'context'),
            C(START_TICK, 'context', task_tick_3,'context'),
            T(map_task_tick, mapTask, {'name': 'map', 'path':'ps.map'}),
            C(map_task_tick, 'b_1', task_tick_1,'b'),
            C(map_task_tick, 'b_2', task_tick_2,'b'),
            C(map_task_tick, 'b_3', task_tick_3,'b'),
            T(task_tick_1, exec_task, {'name': 'test_exec', 'path':'ps.iterations.1.test_exec'}),
            T(task_tick_2, exec_task, {'name': 'test_exec', 'path':'ps.iterations.2.test_exec'}),
            T(task_tick_3, exec_task, {'name': 'test_exec', 'path':'ps.iterations.3.test_exec'}),
            C(task_tick_1, 'c', reduce_tick,'c_1'),
            C(task_tick_2, 'c', reduce_tick,'c_2'),
            C(task_tick_3, 'c', reduce_tick,'c_3'),
            C(task_tick_1, 'd', reduce_tick,'d_1'),
            C(task_tick_2, 'd', reduce_tick,'d_2'),
            C(task_tick_3, 'd', reduce_tick,'d_3'),
            C(START_TICK, 'context', reduce_tick,'context'),
            T(reduce_tick, reduceTask, {'name': 'reduce', 'path':'ps.reduce'}),
            C(reduce_tick, 'cd_list', FINAL_TICK,'cd_list'),
        )
        utils.assert_graph_equal(expected, graph)        
        self.assertTrue(os.path.exists(os.path.join(local_workdir,'testfile.pickle')))