def setUp(self):
     self.config = _testconfig()
     _prepare_testpkgrepos(self.config.pkgRepository)
     self.node_callbacks=NodeCallbacks(self.config, self.config.credentials, self.config.pkgRepository)
     self.g = _create_graph(self.config.pkgRepository)
     execname='test_exec'
     self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"), ("a","b"), ("c",))
     context={CHECKSTATUS_TIMEOUT : self.config.drmConfig.statusCheckTimeout, CHECKSTATUS_TIME : self.config.drmConfig.statusCheckPollTime}
     context[WORKDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun")
     context[LOGDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun","logs")        
     self.inputs={ "a":"dummypath_a", "b":"dummypath_b", CONTEXT:context}    
Exemple #2
0
 def setUp(self):
     self.config = _testconfig()
     _prepare_testpkgrepos(self.config.pkgRepository)
     self.node_callbacks = NodeCallbacks(self.config,
                                         self.config.credentials,
                                         self.config.pkgRepository)
     self.g = _create_graph(self.config.pkgRepository)
     execname = 'test_exec'
     self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"),
                          ("a", "b"), ("c", ))
     context = {
         CHECKSTATUS_TIMEOUT: self.config.drmConfig.statusCheckTimeout,
         CHECKSTATUS_TIME: self.config.drmConfig.statusCheckPollTime
     }
     context[WORKDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
                                     "testrun")
     context[LOGDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
                                    "testrun", "logs")
     self.inputs = {
         "a": "dummypath_a",
         "b": "dummypath_b",
         CONTEXT: context
     }
Exemple #3
0
    def initialize(self):
        '''
        First, configures PYTHONPATH so that the pipeline specification can be parsed.
        Then, loads pipeline and creates the design time graph.
        Finally, it prepares all for executing the pipeline by creating a runtime context
        and instantiating a graph traverser. 
        '''
        add_to_path([self.pkgRepository, self.pipelineDir])
        self.pipeline = load_pipeline_from_file(self.path_to_script)

        # build the design time dataflow graph
        self.dataflow = build_graph(self.pipeline)

        # initialize the context
        self.data[CONTEXT] = context.create_context(self)

        # instantiate the traverser
        self.callbacks = NodeCallbacks(self.config, self.credentials,
                                       self.pkgRepository)
        self.traverser = Traverser(self.callbacks.schedule_refinement,
                                   self.callbacks.submit_task)
class TestNodeCallbacks(unittest.TestCase):

    def setUp(self):
        self.config = _testconfig()
        _prepare_testpkgrepos(self.config.pkgRepository)
        self.node_callbacks=NodeCallbacks(self.config, self.config.credentials, self.config.pkgRepository)
        self.g = _create_graph(self.config.pkgRepository)
        execname='test_exec'
        self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"), ("a","b"), ("c",))
        context={CHECKSTATUS_TIMEOUT : self.config.drmConfig.statusCheckTimeout, CHECKSTATUS_TIME : self.config.drmConfig.statusCheckPollTime}
        context[WORKDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun")
        context[LOGDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun","logs")        
        self.inputs={ "a":"dummypath_a", "b":"dummypath_b", CONTEXT:context}    
        
        
    @with_reactor
    def test_submit_completed_after_some_checks(self):
        num_checks=2
        test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_COMPLETED)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        t1=datetime.datetime.now()
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        expected={'c':'test_exec/c.xml'}
        
        def check_success(_):
            t2=datetime.datetime.now()
            self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
            self.assertEqual(expected, d.result.result)        
            tick=Tick.parse_tick(1)
            taskprops=self.g.get_task_properties(tick)
            summary=taskprops['summary']
            self.assertEqual("test_exec",summary.dfpath)
            self.assertEqual("test_exec",summary.outdir)
            self.assertEqual("1",summary.pid)
            self.assertEqual(JOB_COMPLETED,summary.status)      
        
        d.addCallback(check_success)
        return d
     

    def test_submit_completed(self):
        test_executor=TestCmdExecutor(self.config, "1", False, 0, False, JOB_COMPLETED)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        expected={'c':'test_exec/c.xml'}
        self.assertEqual(expected, d.result.result)
        
        tick=Tick.parse_tick(1)
        taskprops=self.g.get_task_properties(tick)
        summary=taskprops['summary']
        self.assertEqual("test_exec",summary.dfpath)
        self.assertEqual("test_exec",summary.outdir)
        self.assertEqual("1",summary.pid)
        self.assertEqual(JOB_COMPLETED,summary.status)
        

    def test_submit_error(self):
        test_executor=TestCmdExecutor(self.config, "1", False, 0, False, JOB_ERROR)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        
        def expected_error(fail):
            if isinstance(fail, Failure):
                self.assertTrue(True)
            else:
                self.fail("Type of return value not as expected: %s"%type(fail))
        
        def unexpected_success(value):
            self.fail("Return value not as expected: %s of type %s"%(str(value),type(value)))
        
        d.addCallbacks(unexpected_success, expected_error)
        
        tick=Tick.parse_tick(1)
        taskprops=self.g.get_task_properties(tick)
        summary=taskprops['summary']
        self.assertEqual("test_exec",summary.dfpath)
        self.assertEqual("test_exec",summary.outdir)
        self.assertEqual("1",summary.pid)
        self.assertEqual(JOB_ERROR,summary.status)


    @with_reactor
    def test_submit_error_after_some_checks(self):
        num_checks=2
        test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_ERROR)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        t1=datetime.datetime.now()
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        
        def expected_error(fail):
            if isinstance(fail, Failure):
                self.assertTrue(True)
                t2=datetime.datetime.now()
                self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
                tick=Tick.parse_tick(1)
                taskprops=self.g.get_task_properties(tick)
                summary=taskprops['summary']
                self.assertEqual("test_exec",summary.dfpath)
                self.assertEqual("test_exec",summary.outdir)
                self.assertEqual("1",summary.pid)
                self.assertEqual(JOB_ERROR,summary.status)
            else:
                self.fail("Type of return value not as expected: %s"%type(fail))
        
        def unexpected_success(value):
            self.fail("Return value not as expected: %s of type %s"%(str(value),type(value)))
        
        d.addCallbacks(unexpected_success, expected_error)
        return d
        

    def test_submit_failed_submit(self):
        test_executor=TestCmdExecutor(self.config, "1", True, 0, False, JOB_COMPLETED)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        
        def expected_error(failure):
            self.assertIsInstance(failure, Failure)
            self.assertTrue(failure.value.message.startswith("Exception at submit with reason"))
            return "Exception has occurred as expected."
        
        def unexpected_success(value):
            return Failure(ValueError("Exception did not occur as expected. Value %s found"%str(value)))

        d.addCallback(unexpected_success)
        d.addErrback(expected_error)

        tick=Tick.parse_tick(1)
        taskprops=self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec",taskprops['path'])
        self.assertEqual("test_exec",taskprops['name'])


    def test_submit_failed_check(self):
        test_executor=TestCmdExecutor(self.config, "1", False, 0, True, JOB_COMPLETED)
        self.node_callbacks._cmd_executor=test_executor
        tick=Tick.parse_tick(1)
        d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
        
        def expected_error(fail):
            self.assertIsInstance(fail, Failure)
            self.assertTrue(fail.value.message.startswith("Exception at check_status with reason"))
        
        def unexpected_success(value):
            self.fail("Unexpected success - failure expected. Return value: %s of type %s"%(str(value),type(value)))

        d.addCallbacks(unexpected_success, expected_error)
        tick=Tick.parse_tick(1)
        taskprops=self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec",taskprops['path'])
        self.assertEqual("test_exec",taskprops['name'])
Exemple #5
0
class TestNodeCallbacks(unittest.TestCase):
    def setUp(self):
        self.config = _testconfig()
        _prepare_testpkgrepos(self.config.pkgRepository)
        self.node_callbacks = NodeCallbacks(self.config,
                                            self.config.credentials,
                                            self.config.pkgRepository)
        self.g = _create_graph(self.config.pkgRepository)
        execname = 'test_exec'
        self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"),
                             ("a", "b"), ("c", ))
        context = {
            CHECKSTATUS_TIMEOUT: self.config.drmConfig.statusCheckTimeout,
            CHECKSTATUS_TIME: self.config.drmConfig.statusCheckPollTime
        }
        context[WORKDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
                                        "testrun")
        context[LOGDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
                                       "testrun", "logs")
        self.inputs = {
            "a": "dummypath_a",
            "b": "dummypath_b",
            CONTEXT: context
        }

    @with_reactor
    def test_submit_completed_after_some_checks(self):
        num_checks = 2
        test_executor = TestCmdExecutor(self.config, "1", False, 2, False,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        t1 = datetime.datetime.now()
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)
        expected = {'c': 'test_exec/c.xml'}

        def check_success(_):
            t2 = datetime.datetime.now()
            self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
            self.assertEqual(expected, d.result.result)
            tick = Tick.parse_tick(1)
            taskprops = self.g.get_task_properties(tick)
            summary = taskprops['summary']
            self.assertEqual("test_exec", summary.dfpath)
            self.assertEqual("test_exec", summary.outdir)
            self.assertEqual("1", summary.pid)
            self.assertEqual(JOB_COMPLETED, summary.status)

        d.addCallback(check_success)
        return d

    def test_submit_completed(self):
        test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)
        expected = {'c': 'test_exec/c.xml'}
        self.assertEqual(expected, d.result.result)

        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        summary = taskprops['summary']
        self.assertEqual("test_exec", summary.dfpath)
        self.assertEqual("test_exec", summary.outdir)
        self.assertEqual("1", summary.pid)
        self.assertEqual(JOB_COMPLETED, summary.status)

    def test_submit_error(self):
        test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
                                        JOB_ERROR)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)

        def expected_error(fail):
            if isinstance(fail, Failure):
                self.assertTrue(True)
            else:
                self.fail("Type of return value not as expected: %s" %
                          type(fail))

        def unexpected_success(value):
            self.fail("Return value not as expected: %s of type %s" %
                      (str(value), type(value)))

        d.addCallbacks(unexpected_success, expected_error)

        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        summary = taskprops['summary']
        self.assertEqual("test_exec", summary.dfpath)
        self.assertEqual("test_exec", summary.outdir)
        self.assertEqual("1", summary.pid)
        self.assertEqual(JOB_ERROR, summary.status)

    @with_reactor
    def test_submit_error_after_some_checks(self):
        num_checks = 2
        test_executor = TestCmdExecutor(self.config, "1", False, 2, False,
                                        JOB_ERROR)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        t1 = datetime.datetime.now()
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)

        def expected_error(fail):
            if isinstance(fail, Failure):
                self.assertTrue(True)
                t2 = datetime.datetime.now()
                self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
                tick = Tick.parse_tick(1)
                taskprops = self.g.get_task_properties(tick)
                summary = taskprops['summary']
                self.assertEqual("test_exec", summary.dfpath)
                self.assertEqual("test_exec", summary.outdir)
                self.assertEqual("1", summary.pid)
                self.assertEqual(JOB_ERROR, summary.status)
            else:
                self.fail("Type of return value not as expected: %s" %
                          type(fail))

        def unexpected_success(value):
            self.fail("Return value not as expected: %s of type %s" %
                      (str(value), type(value)))

        d.addCallbacks(unexpected_success, expected_error)
        return d

    def test_submit_failed_submit(self):
        test_executor = TestCmdExecutor(self.config, "1", True, 0, False,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)

        def expected_error(failure):
            self.assertIsInstance(failure, Failure)
            self.assertTrue(
                failure.value.message.startswith(
                    "Exception at submit with reason"))
            return "Exception has occurred as expected."

        def unexpected_success(value):
            return Failure(
                ValueError(
                    "Exception did not occur as expected. Value %s found" %
                    str(value)))

        d.addCallback(unexpected_success)
        d.addErrback(expected_error)

        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec", taskprops['path'])
        self.assertEqual("test_exec", taskprops['name'])

    def test_submit_failed_check(self):
        test_executor = TestCmdExecutor(self.config, "1", False, 0, True,
                                        JOB_COMPLETED)
        self.node_callbacks._cmd_executor = test_executor
        tick = Tick.parse_tick(1)
        d = self.node_callbacks.submit_task(self.g, tick, self.task,
                                            self.inputs)

        def expected_error(fail):
            self.assertIsInstance(fail, Failure)
            self.assertTrue(
                fail.value.message.startswith(
                    "Exception at check_status with reason"))

        def unexpected_success(value):
            self.fail(
                "Unexpected success - failure expected. Return value: %s of type %s"
                % (str(value), type(value)))

        d.addCallbacks(unexpected_success, expected_error)
        tick = Tick.parse_tick(1)
        taskprops = self.g.get_task_properties(tick)
        self.assertFalse("summary" in taskprops.keys())
        self.assertEqual("test_exec", taskprops['path'])
        self.assertEqual("test_exec", taskprops['name'])