def setUp(self):
# 1370354400 = 06/04/2012 2:00p UTC
self.next_run_time = 1370354400
# 1370289240 = 06/03/2013 7:53p UTC
self.previous_run_time = 1370289240
self.workflow_schedule = WorkflowSchedule(
next_run_time=self.next_run_time,
recurrence_seconds=24 * 60 * 60,
workflow='some_workflow',
emails=['some_email'])
self.workflow = WorkflowDef('some_workflow', self.workflow_schedule,
'some_emails')
self.job1 = JobDef(
'some_job_1', CommandJobTemplate('some_template1',
'some_command1'), self.workflow)
self.workflow.add_job(self.job1)
self.job2 = JobDef(
'some_job_2', CommandJobTemplate('some_template2',
'some_command2'), self.workflow)
self.workflow.add_job(self.job2)
self.job2.add_dep(self.job1)
self.job3 = JobDef(
'some_job_3', CommandJobTemplate('some_template3',
'some_command3'), self.workflow)
self.workflow.add_job(self.job3)
self.job4 = JobDef(
'some_job_4', CommandJobTemplate('some_template4',
'some_command4'), self.workflow)
self.workflow.add_job(self.job4)
self.job4.add_dep(self.job3)
def _add_external_deps(self):
other_workflow = WorkflowDef('some_other_workflow', 'some_schedule',
'some_emails')
template = CommandJobTemplate('some_template', 'some_command')
parent_external_job = JobDef('parent_external_job', template,
other_workflow)
other_workflow.add_job(parent_external_job)
external_job = JobDef('external_job', template, other_workflow)
external_job.add_dep(parent_external_job)
other_workflow.add_job(external_job)
self.job1.add_dep(external_job)
def setUp(self):
# 1370354400 = 06/04/2012 2:00p UTC
self.next_run_time = 1370354400
# 1370289240 = 06/03/2013 7:53p UTC
self.previous_run_time = 1370289240
self.workflow_schedule = WorkflowSchedule(next_run_time=self.next_run_time,
recurrence_seconds=24 * 60 * 60,
workflow='some_workflow',
emails=['some_email'])
self.workflow = WorkflowDef('some_workflow',
self.workflow_schedule,
'some_emails')
self.job1 = JobDef('some_job_1', CommandJobTemplate('some_template1', 'some_command1'),
self.workflow)
self.workflow.add_job(self.job1)
self.job2 = JobDef('some_job_2', CommandJobTemplate('some_template2', 'some_command2'),
self.workflow)
self.workflow.add_job(self.job2)
self.job2.add_dep(self.job1)
self.job3 = JobDef('some_job_3', CommandJobTemplate('some_template3', 'some_command3'),
self.workflow)
self.workflow.add_job(self.job3)
self.job4 = JobDef('some_job_4', CommandJobTemplate('some_template4', 'some_command4'),
self.workflow)
self.workflow.add_job(self.job4)
self.job4.add_dep(self.job3)
def setUp(self):
self.workflow = WorkflowDef('some_workflow', 'some_schedule',
'some_emails')
template = CommandJobTemplate('some_template', 'some_command')
output_template = CommandJobTemplate('output_template',
'output_command',
priority=30.0)
self.job = JobDef('some_job', template, self.workflow)
self.output_job = JobDef('output_job', output_template, self.workflow)
self.job.outputs = [self.output_job]
def _add_external_deps(self):
other_workflow = WorkflowDef('some_other_workflow', 'some_schedule',
'some_emails')
template = CommandJobTemplate('some_template', 'some_command')
parent_external_job = JobDef('parent_external_job', template,
other_workflow)
other_workflow.add_job(parent_external_job)
external_job = JobDef('external_job', template, other_workflow)
external_job.add_dep(parent_external_job)
other_workflow.add_job(external_job)
self.job1.add_dep(external_job)
class WorkflowDefTestCase(unittest.TestCase):
def setUp(self):
# 1370354400 = 06/04/2012 2:00p UTC
self.next_run_time = 1370354400
# 1370289240 = 06/03/2013 7:53p UTC
self.previous_run_time = 1370289240
self.workflow_schedule = WorkflowSchedule(next_run_time=self.next_run_time,
recurrence_seconds=24 * 60 * 60,
workflow='some_workflow',
emails=['some_email'])
self.workflow = WorkflowDef('some_workflow',
self.workflow_schedule,
'some_emails')
self.job1 = JobDef('some_job_1', CommandJobTemplate('some_template1', 'some_command1'),
self.workflow)
self.workflow.add_job(self.job1)
self.job2 = JobDef('some_job_2', CommandJobTemplate('some_template2', 'some_command2'),
self.workflow)
self.workflow.add_job(self.job2)
self.job2.add_dep(self.job1)
self.job3 = JobDef('some_job_3', CommandJobTemplate('some_template3', 'some_command3'),
self.workflow)
self.workflow.add_job(self.job3)
self.job4 = JobDef('some_job_4', CommandJobTemplate('some_template4', 'some_command4'),
self.workflow)
self.workflow.add_job(self.job4)
self.job4.add_dep(self.job3)
def _add_final_job(self):
template = CommandJobTemplate('some_template', 'some_command')
final_job = JobDef('final_job', template, self.workflow)
self.workflow.add_job(final_job)
final_job.add_dep(self.job2)
final_job.add_dep(self.job4)
def _add_external_deps(self):
other_workflow = WorkflowDef('some_other_workflow', 'some_schedule',
'some_emails')
template = CommandJobTemplate('some_template', 'some_command')
parent_external_job = JobDef('parent_external_job', template,
other_workflow)
other_workflow.add_job(parent_external_job)
external_job = JobDef('external_job', template, other_workflow)
external_job.add_dep(parent_external_job)
other_workflow.add_job(external_job)
self.job1.add_dep(external_job)
def test_to_string(self):
"""Test conversion of job to string."""
self.assertEqual(
"workflow:some_workflow, schedule:WorkflowSchedule(next_run_time=2013-06-04 "
"14:00:00 UTC, recurrence=1 day, 0:00:00, overrun_policy=SKIP, "
"parser_params={}, "
"workflow=some_workflow, email=['some_email'], max_running_instances=3), "
"notify_emails:some_emails, jobs:{'some_job_3': "
"job:some_workflow.some_job_3, deps:[], 'some_job_2': "
"job:some_workflow.some_job_2, deps:[some_workflow.some_job_1], "
"'some_job_1': job:some_workflow.some_job_1, deps:[], "
"'some_job_4': job:some_workflow.some_job_4, "
"deps:[some_workflow.some_job_3]}", str(self.workflow))
def test_get_leaf_jobs(self):
"""Test retrieval of leaf jobs."""
leaf_jobs = self.workflow.get_leaf_jobs()
leaf_jobs.sort()
expected_jobs = [self.job2, self.job4]
expected_jobs.sort()
self.assertEqual(expected_jobs, leaf_jobs)
def test_verify(self):
"""Test workflow verification."""
# TODO(pawel): figure out why putting this at the top level does not
# work.
from pinball_ext.workflow.parser import WorkflowVerificationException
# Raise an exception if the workflow does not have a final job.
self.assertRaises(WorkflowVerificationException, self.workflow.verify)
# Verify that the workflow has a final job and is acyclic.
self._add_final_job()
self.workflow.verify()
# Raise an exception if the workflow has a cycle.
self.job1.add_dep(self.job2)
self.assertRaises(Exception, self.workflow.verify)
@mock.patch('time.time')
def test_get_schedule_token(self, time_mock):
# the value must be a float
time_mock.return_value = 1.0 * self.previous_run_time
self._add_final_job()
token = self.workflow.get_schedule_token()
self.assertEqual('/schedule/workflow/some_workflow', token.name)
schedule = pickle.loads(token.data)
self.assertEqual(self.next_run_time, schedule.next_run_time)
self.assertEqual(24 * 60 * 60, schedule.recurrence_seconds)
self.assertEqual(OverrunPolicy.SKIP, schedule.overrun_policy)
self.assertEqual('some_workflow', schedule.workflow)
def test_get_workflow_tokens(self):
self._add_final_job()
self._add_external_deps()
tokens = self.workflow.get_workflow_tokens()
# There should be 5 (4 local and 1 external) workflow job tokens, 1
# final job token, and 2 top-level job event tokens.
self.assertEqual(5 + 1 + 2, len(tokens))
class WorkflowDefTestCase(unittest.TestCase):
def setUp(self):
# 1370354400 = 06/04/2012 2:00p UTC
self.next_run_time = 1370354400
# 1370289240 = 06/03/2013 7:53p UTC
self.previous_run_time = 1370289240
self.workflow_schedule = WorkflowSchedule(
next_run_time=self.next_run_time,
recurrence_seconds=24 * 60 * 60,
workflow='some_workflow',
emails=['some_email'])
self.workflow = WorkflowDef('some_workflow', self.workflow_schedule,
'some_emails')
self.job1 = JobDef(
'some_job_1', CommandJobTemplate('some_template1',
'some_command1'), self.workflow)
self.workflow.add_job(self.job1)
self.job2 = JobDef(
'some_job_2', CommandJobTemplate('some_template2',
'some_command2'), self.workflow)
self.workflow.add_job(self.job2)
self.job2.add_dep(self.job1)
self.job3 = JobDef(
'some_job_3', CommandJobTemplate('some_template3',
'some_command3'), self.workflow)
self.workflow.add_job(self.job3)
self.job4 = JobDef(
'some_job_4', CommandJobTemplate('some_template4',
'some_command4'), self.workflow)
self.workflow.add_job(self.job4)
self.job4.add_dep(self.job3)
def _add_final_job(self):
template = CommandJobTemplate('some_template', 'some_command')
final_job = JobDef('final_job', template, self.workflow)
self.workflow.add_job(final_job)
final_job.add_dep(self.job2)
final_job.add_dep(self.job4)
def _add_external_deps(self):
other_workflow = WorkflowDef('some_other_workflow', 'some_schedule',
'some_emails')
template = CommandJobTemplate('some_template', 'some_command')
parent_external_job = JobDef('parent_external_job', template,
other_workflow)
other_workflow.add_job(parent_external_job)
external_job = JobDef('external_job', template, other_workflow)
external_job.add_dep(parent_external_job)
other_workflow.add_job(external_job)
self.job1.add_dep(external_job)
def test_to_string(self):
"""Test conversion of job to string."""
self.assertEqual(
"workflow:some_workflow, schedule:WorkflowSchedule(next_run_time=2013-06-04 "
"14:00:00 UTC, recurrence=1 day, 0:00:00, overrun_policy=SKIP, "
"parser_params={}, "
"workflow=some_workflow, email=['some_email'], max_running_instances=3), "
"notify_emails:some_emails, jobs:{'some_job_3': "
"job:some_workflow.some_job_3, deps:[], 'some_job_2': "
"job:some_workflow.some_job_2, deps:[some_workflow.some_job_1], "
"'some_job_1': job:some_workflow.some_job_1, deps:[], "
"'some_job_4': job:some_workflow.some_job_4, "
"deps:[some_workflow.some_job_3]}", str(self.workflow))
def test_get_leaf_jobs(self):
"""Test retrieval of leaf jobs."""
leaf_jobs = self.workflow.get_leaf_jobs()
leaf_jobs.sort()
expected_jobs = [self.job2, self.job4]
expected_jobs.sort()
self.assertEqual(expected_jobs, leaf_jobs)
def test_verify(self):
"""Test workflow verification."""
# TODO(pawel): figure out why putting this at the top level does not
# work.
from pinball_ext.workflow.parser import WorkflowVerificationException
# Raise an exception if the workflow does not have a final job.
self.assertRaises(WorkflowVerificationException, self.workflow.verify)
# Verify that the workflow has a final job and is acyclic.
self._add_final_job()
self.workflow.verify()
# Raise an exception if the workflow has a cycle.
self.job1.add_dep(self.job2)
self.assertRaises(Exception, self.workflow.verify)
@mock.patch('time.time')
def test_get_schedule_token(self, time_mock):
# the value must be a float
time_mock.return_value = 1.0 * self.previous_run_time
self._add_final_job()
token = self.workflow.get_schedule_token()
self.assertEqual('/schedule/workflow/some_workflow', token.name)
schedule = pickle.loads(token.data)
self.assertEqual(self.next_run_time, schedule.next_run_time)
self.assertEqual(24 * 60 * 60, schedule.recurrence_seconds)
self.assertEqual(OverrunPolicy.SKIP, schedule.overrun_policy)
self.assertEqual('some_workflow', schedule.workflow)
def test_get_workflow_tokens(self):
self._add_final_job()
self._add_external_deps()
tokens = self.workflow.get_workflow_tokens()
# There should be 5 (4 local and 1 external) workflow job tokens, 1
# final job token, and 2 top-level job event tokens.
self.assertEqual(5 + 1 + 2, len(tokens))
