def setUp(self):
self.config = _testconfig()
_prepare_testpkgrepos(self.config.pkgRepository)
self.node_callbacks=NodeCallbacks(self.config, self.config.credentials, self.config.pkgRepository)
self.g = _create_graph(self.config.pkgRepository)
execname='test_exec'
self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"), ("a","b"), ("c",))
context={CHECKSTATUS_TIMEOUT : self.config.drmConfig.statusCheckTimeout, CHECKSTATUS_TIME : self.config.drmConfig.statusCheckPollTime}
context[WORKDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun")
context[LOGDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun","logs")
self.inputs={ "a":"dummypath_a", "b":"dummypath_b", CONTEXT:context}
def setUp(self):
self.config = _testconfig()
_prepare_testpkgrepos(self.config.pkgRepository)
self.node_callbacks = NodeCallbacks(self.config,
self.config.credentials,
self.config.pkgRepository)
self.g = _create_graph(self.config.pkgRepository)
execname = 'test_exec'
self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"),
("a", "b"), ("c", ))
context = {
CHECKSTATUS_TIMEOUT: self.config.drmConfig.statusCheckTimeout,
CHECKSTATUS_TIME: self.config.drmConfig.statusCheckPollTime
}
context[WORKDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
"testrun")
context[LOGDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
"testrun", "logs")
self.inputs = {
"a": "dummypath_a",
"b": "dummypath_b",
CONTEXT: context
}
def initialize(self):
'''
First, configures PYTHONPATH so that the pipeline specification can be parsed.
Then, loads pipeline and creates the design time graph.
Finally, it prepares all for executing the pipeline by creating a runtime context
and instantiating a graph traverser.
'''
add_to_path([self.pkgRepository, self.pipelineDir])
self.pipeline = load_pipeline_from_file(self.path_to_script)
# build the design time dataflow graph
self.dataflow = build_graph(self.pipeline)
# initialize the context
self.data[CONTEXT] = context.create_context(self)
# instantiate the traverser
self.callbacks = NodeCallbacks(self.config, self.credentials,
self.pkgRepository)
self.traverser = Traverser(self.callbacks.schedule_refinement,
self.callbacks.submit_task)
class TestNodeCallbacks(unittest.TestCase):
def setUp(self):
self.config = _testconfig()
_prepare_testpkgrepos(self.config.pkgRepository)
self.node_callbacks=NodeCallbacks(self.config, self.config.credentials, self.config.pkgRepository)
self.g = _create_graph(self.config.pkgRepository)
execname='test_exec'
self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"), ("a","b"), ("c",))
context={CHECKSTATUS_TIMEOUT : self.config.drmConfig.statusCheckTimeout, CHECKSTATUS_TIME : self.config.drmConfig.statusCheckPollTime}
context[WORKDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun")
context[LOGDIR]=os.path.join(self.config.wsConfig.workspaceRoot,"testrun","logs")
self.inputs={ "a":"dummypath_a", "b":"dummypath_b", CONTEXT:context}
@with_reactor
def test_submit_completed_after_some_checks(self):
num_checks=2
test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
t1=datetime.datetime.now()
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
expected={'c':'test_exec/c.xml'}
def check_success(_):
t2=datetime.datetime.now()
self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
self.assertEqual(expected, d.result.result)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_COMPLETED,summary.status)
d.addCallback(check_success)
return d
def test_submit_completed(self):
test_executor=TestCmdExecutor(self.config, "1", False, 0, False, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
expected={'c':'test_exec/c.xml'}
self.assertEqual(expected, d.result.result)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_COMPLETED,summary.status)
def test_submit_error(self):
test_executor=TestCmdExecutor(self.config, "1", False, 0, False, JOB_ERROR)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
else:
self.fail("Type of return value not as expected: %s"%type(fail))
def unexpected_success(value):
self.fail("Return value not as expected: %s of type %s"%(str(value),type(value)))
d.addCallbacks(unexpected_success, expected_error)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_ERROR,summary.status)
@with_reactor
def test_submit_error_after_some_checks(self):
num_checks=2
test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_ERROR)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
t1=datetime.datetime.now()
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
t2=datetime.datetime.now()
self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_ERROR,summary.status)
else:
self.fail("Type of return value not as expected: %s"%type(fail))
def unexpected_success(value):
self.fail("Return value not as expected: %s of type %s"%(str(value),type(value)))
d.addCallbacks(unexpected_success, expected_error)
return d
def test_submit_failed_submit(self):
test_executor=TestCmdExecutor(self.config, "1", True, 0, False, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
def expected_error(failure):
self.assertIsInstance(failure, Failure)
self.assertTrue(failure.value.message.startswith("Exception at submit with reason"))
return "Exception has occurred as expected."
def unexpected_success(value):
return Failure(ValueError("Exception did not occur as expected. Value %s found"%str(value)))
d.addCallback(unexpected_success)
d.addErrback(expected_error)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec",taskprops['path'])
self.assertEqual("test_exec",taskprops['name'])
def test_submit_failed_check(self):
test_executor=TestCmdExecutor(self.config, "1", False, 0, True, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
def expected_error(fail):
self.assertIsInstance(fail, Failure)
self.assertTrue(fail.value.message.startswith("Exception at check_status with reason"))
def unexpected_success(value):
self.fail("Unexpected success - failure expected. Return value: %s of type %s"%(str(value),type(value)))
d.addCallbacks(unexpected_success, expected_error)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec",taskprops['path'])
self.assertEqual("test_exec",taskprops['name'])
class TestNodeCallbacks(unittest.TestCase):
def setUp(self):
self.config = _testconfig()
_prepare_testpkgrepos(self.config.pkgRepository)
self.node_callbacks = NodeCallbacks(self.config,
self.config.credentials,
self.config.pkgRepository)
self.g = _create_graph(self.config.pkgRepository)
execname = 'test_exec'
self.task = ExecTask(execname, PackageSource("testpkg", "pkgdefs"),
("a", "b"), ("c", ))
context = {
CHECKSTATUS_TIMEOUT: self.config.drmConfig.statusCheckTimeout,
CHECKSTATUS_TIME: self.config.drmConfig.statusCheckPollTime
}
context[WORKDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
"testrun")
context[LOGDIR] = os.path.join(self.config.wsConfig.workspaceRoot,
"testrun", "logs")
self.inputs = {
"a": "dummypath_a",
"b": "dummypath_b",
CONTEXT: context
}
@with_reactor
def test_submit_completed_after_some_checks(self):
num_checks = 2
test_executor = TestCmdExecutor(self.config, "1", False, 2, False,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
t1 = datetime.datetime.now()
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
expected = {'c': 'test_exec/c.xml'}
def check_success(_):
t2 = datetime.datetime.now()
self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
self.assertEqual(expected, d.result.result)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_COMPLETED, summary.status)
d.addCallback(check_success)
return d
def test_submit_completed(self):
test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
expected = {'c': 'test_exec/c.xml'}
self.assertEqual(expected, d.result.result)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_COMPLETED, summary.status)
def test_submit_error(self):
test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
JOB_ERROR)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
else:
self.fail("Type of return value not as expected: %s" %
type(fail))
def unexpected_success(value):
self.fail("Return value not as expected: %s of type %s" %
(str(value), type(value)))
d.addCallbacks(unexpected_success, expected_error)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_ERROR, summary.status)
@with_reactor
def test_submit_error_after_some_checks(self):
num_checks = 2
test_executor = TestCmdExecutor(self.config, "1", False, 2, False,
JOB_ERROR)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
t1 = datetime.datetime.now()
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
t2 = datetime.datetime.now()
self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_ERROR, summary.status)
else:
self.fail("Type of return value not as expected: %s" %
type(fail))
def unexpected_success(value):
self.fail("Return value not as expected: %s of type %s" %
(str(value), type(value)))
d.addCallbacks(unexpected_success, expected_error)
return d
def test_submit_failed_submit(self):
test_executor = TestCmdExecutor(self.config, "1", True, 0, False,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
def expected_error(failure):
self.assertIsInstance(failure, Failure)
self.assertTrue(
failure.value.message.startswith(
"Exception at submit with reason"))
return "Exception has occurred as expected."
def unexpected_success(value):
return Failure(
ValueError(
"Exception did not occur as expected. Value %s found" %
str(value)))
d.addCallback(unexpected_success)
d.addErrback(expected_error)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec", taskprops['path'])
self.assertEqual("test_exec", taskprops['name'])
def test_submit_failed_check(self):
test_executor = TestCmdExecutor(self.config, "1", False, 0, True,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
def expected_error(fail):
self.assertIsInstance(fail, Failure)
self.assertTrue(
fail.value.message.startswith(
"Exception at check_status with reason"))
def unexpected_success(value):
self.fail(
"Unexpected success - failure expected. Return value: %s of type %s"
% (str(value), type(value)))
d.addCallbacks(unexpected_success, expected_error)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec", taskprops['path'])
self.assertEqual("test_exec", taskprops['name'])
