def _add_external_deps(self): other_workflow = WorkflowDef('some_other_workflow', 'some_schedule', 'some_emails') template = CommandJobTemplate('some_template', 'some_command') parent_external_job = JobDef('parent_external_job', template, other_workflow) other_workflow.add_job(parent_external_job) external_job = JobDef('external_job', template, other_workflow) external_job.add_dep(parent_external_job) other_workflow.add_job(external_job) self.job1.add_dep(external_job)
class WorkflowDefTestCase(unittest.TestCase): def setUp(self): # 1370354400 = 06/04/2012 2:00p UTC self.next_run_time = 1370354400 # 1370289240 = 06/03/2013 7:53p UTC self.previous_run_time = 1370289240 self.workflow_schedule = WorkflowSchedule(next_run_time=self.next_run_time, recurrence_seconds=24 * 60 * 60, workflow='some_workflow', emails=['some_email']) self.workflow = WorkflowDef('some_workflow', self.workflow_schedule, 'some_emails') self.job1 = JobDef('some_job_1', CommandJobTemplate('some_template1', 'some_command1'), self.workflow) self.workflow.add_job(self.job1) self.job2 = JobDef('some_job_2', CommandJobTemplate('some_template2', 'some_command2'), self.workflow) self.workflow.add_job(self.job2) self.job2.add_dep(self.job1) self.job3 = JobDef('some_job_3', CommandJobTemplate('some_template3', 'some_command3'), self.workflow) self.workflow.add_job(self.job3) self.job4 = JobDef('some_job_4', CommandJobTemplate('some_template4', 'some_command4'), self.workflow) self.workflow.add_job(self.job4) self.job4.add_dep(self.job3) def _add_final_job(self): template = CommandJobTemplate('some_template', 'some_command') final_job = JobDef('final_job', template, self.workflow) self.workflow.add_job(final_job) final_job.add_dep(self.job2) final_job.add_dep(self.job4) def _add_external_deps(self): other_workflow = WorkflowDef('some_other_workflow', 'some_schedule', 'some_emails') template = CommandJobTemplate('some_template', 'some_command') parent_external_job = JobDef('parent_external_job', template, other_workflow) other_workflow.add_job(parent_external_job) external_job = JobDef('external_job', template, other_workflow) external_job.add_dep(parent_external_job) other_workflow.add_job(external_job) self.job1.add_dep(external_job) def test_to_string(self): """Test conversion of job to string.""" self.assertEqual( "workflow:some_workflow, schedule:WorkflowSchedule(next_run_time=2013-06-04 " "14:00:00 UTC, recurrence=1 day, 0:00:00, overrun_policy=SKIP, " "parser_params={}, " "workflow=some_workflow, email=['some_email'], max_running_instances=3), " "notify_emails:some_emails, jobs:{'some_job_3': " "job:some_workflow.some_job_3, deps:[], 'some_job_2': " "job:some_workflow.some_job_2, deps:[some_workflow.some_job_1], " "'some_job_1': job:some_workflow.some_job_1, deps:[], " "'some_job_4': job:some_workflow.some_job_4, " "deps:[some_workflow.some_job_3]}", str(self.workflow)) def test_get_leaf_jobs(self): """Test retrieval of leaf jobs.""" leaf_jobs = self.workflow.get_leaf_jobs() leaf_jobs.sort() expected_jobs = [self.job2, self.job4] expected_jobs.sort() self.assertEqual(expected_jobs, leaf_jobs) def test_verify(self): """Test workflow verification.""" # TODO(pawel): figure out why putting this at the top level does not # work. from pinball_ext.workflow.parser import WorkflowVerificationException # Raise an exception if the workflow does not have a final job. self.assertRaises(WorkflowVerificationException, self.workflow.verify) # Verify that the workflow has a final job and is acyclic. self._add_final_job() self.workflow.verify() # Raise an exception if the workflow has a cycle. self.job1.add_dep(self.job2) self.assertRaises(Exception, self.workflow.verify) @mock.patch('time.time') def test_get_schedule_token(self, time_mock): # the value must be a float time_mock.return_value = 1.0 * self.previous_run_time self._add_final_job() token = self.workflow.get_schedule_token() self.assertEqual('/schedule/workflow/some_workflow', token.name) schedule = pickle.loads(token.data) self.assertEqual(self.next_run_time, schedule.next_run_time) self.assertEqual(24 * 60 * 60, schedule.recurrence_seconds) self.assertEqual(OverrunPolicy.SKIP, schedule.overrun_policy) self.assertEqual('some_workflow', schedule.workflow) def test_get_workflow_tokens(self): self._add_final_job() self._add_external_deps() tokens = self.workflow.get_workflow_tokens() # There should be 5 (4 local and 1 external) workflow job tokens, 1 # final job token, and 2 top-level job event tokens. self.assertEqual(5 + 1 + 2, len(tokens))
def _add_final_job(self): template = CommandJobTemplate('some_template', 'some_command') final_job = JobDef('final_job', template, self.workflow) self.workflow.add_job(final_job) final_job.add_dep(self.job2) final_job.add_dep(self.job4)
class WorkflowDefTestCase(unittest.TestCase): def setUp(self): # 1370354400 = 06/04/2012 2:00p UTC self.next_run_time = 1370354400 # 1370289240 = 06/03/2013 7:53p UTC self.previous_run_time = 1370289240 self.workflow_schedule = WorkflowSchedule( next_run_time=self.next_run_time, recurrence_seconds=24 * 60 * 60, workflow='some_workflow', emails=['some_email']) self.workflow = WorkflowDef('some_workflow', self.workflow_schedule, 'some_emails') self.job1 = JobDef( 'some_job_1', CommandJobTemplate('some_template1', 'some_command1'), self.workflow) self.workflow.add_job(self.job1) self.job2 = JobDef( 'some_job_2', CommandJobTemplate('some_template2', 'some_command2'), self.workflow) self.workflow.add_job(self.job2) self.job2.add_dep(self.job1) self.job3 = JobDef( 'some_job_3', CommandJobTemplate('some_template3', 'some_command3'), self.workflow) self.workflow.add_job(self.job3) self.job4 = JobDef( 'some_job_4', CommandJobTemplate('some_template4', 'some_command4'), self.workflow) self.workflow.add_job(self.job4) self.job4.add_dep(self.job3) def _add_final_job(self): template = CommandJobTemplate('some_template', 'some_command') final_job = JobDef('final_job', template, self.workflow) self.workflow.add_job(final_job) final_job.add_dep(self.job2) final_job.add_dep(self.job4) def _add_external_deps(self): other_workflow = WorkflowDef('some_other_workflow', 'some_schedule', 'some_emails') template = CommandJobTemplate('some_template', 'some_command') parent_external_job = JobDef('parent_external_job', template, other_workflow) other_workflow.add_job(parent_external_job) external_job = JobDef('external_job', template, other_workflow) external_job.add_dep(parent_external_job) other_workflow.add_job(external_job) self.job1.add_dep(external_job) def test_to_string(self): """Test conversion of job to string.""" self.assertEqual( "workflow:some_workflow, schedule:WorkflowSchedule(next_run_time=2013-06-04 " "14:00:00 UTC, recurrence=1 day, 0:00:00, overrun_policy=SKIP, " "parser_params={}, " "workflow=some_workflow, email=['some_email'], max_running_instances=3), " "notify_emails:some_emails, jobs:{'some_job_3': " "job:some_workflow.some_job_3, deps:[], 'some_job_2': " "job:some_workflow.some_job_2, deps:[some_workflow.some_job_1], " "'some_job_1': job:some_workflow.some_job_1, deps:[], " "'some_job_4': job:some_workflow.some_job_4, " "deps:[some_workflow.some_job_3]}", str(self.workflow)) def test_get_leaf_jobs(self): """Test retrieval of leaf jobs.""" leaf_jobs = self.workflow.get_leaf_jobs() leaf_jobs.sort() expected_jobs = [self.job2, self.job4] expected_jobs.sort() self.assertEqual(expected_jobs, leaf_jobs) def test_verify(self): """Test workflow verification.""" # TODO(pawel): figure out why putting this at the top level does not # work. from pinball_ext.workflow.parser import WorkflowVerificationException # Raise an exception if the workflow does not have a final job. self.assertRaises(WorkflowVerificationException, self.workflow.verify) # Verify that the workflow has a final job and is acyclic. self._add_final_job() self.workflow.verify() # Raise an exception if the workflow has a cycle. self.job1.add_dep(self.job2) self.assertRaises(Exception, self.workflow.verify) @mock.patch('time.time') def test_get_schedule_token(self, time_mock): # the value must be a float time_mock.return_value = 1.0 * self.previous_run_time self._add_final_job() token = self.workflow.get_schedule_token() self.assertEqual('/schedule/workflow/some_workflow', token.name) schedule = pickle.loads(token.data) self.assertEqual(self.next_run_time, schedule.next_run_time) self.assertEqual(24 * 60 * 60, schedule.recurrence_seconds) self.assertEqual(OverrunPolicy.SKIP, schedule.overrun_policy) self.assertEqual('some_workflow', schedule.workflow) def test_get_workflow_tokens(self): self._add_final_job() self._add_external_deps() tokens = self.workflow.get_workflow_tokens() # There should be 5 (4 local and 1 external) workflow job tokens, 1 # final job token, and 2 top-level job event tokens. self.assertEqual(5 + 1 + 2, len(tokens))