def test_command_job_template(self):
"""Test the config produced by a command job template."""
job_template = CommandJobTemplate('some_job', 'some_command')
pb_job = job_template.get_pinball_job([], [], params={
'end_date': '2015-02-25',
})
self.assertEqual('some_command', pb_job.command)
def setUp(self):
# 1370354400 = 06/04/2012 2:00p UTC
self.next_run_time = 1370354400
# 1370289240 = 06/03/2013 7:53p UTC
self.previous_run_time = 1370289240
self.workflow_schedule = WorkflowSchedule(
next_run_time=self.next_run_time,
recurrence_seconds=24 * 60 * 60,
workflow='some_workflow',
emails=['some_email'])
self.workflow = WorkflowDef('some_workflow', self.workflow_schedule,
'some_emails')
self.job1 = JobDef(
'some_job_1', CommandJobTemplate('some_template1',
'some_command1'), self.workflow)
self.workflow.add_job(self.job1)
self.job2 = JobDef(
'some_job_2', CommandJobTemplate('some_template2',
'some_command2'), self.workflow)
self.workflow.add_job(self.job2)
self.job2.add_dep(self.job1)
self.job3 = JobDef(
'some_job_3', CommandJobTemplate('some_template3',
'some_command3'), self.workflow)
self.workflow.add_job(self.job3)
self.job4 = JobDef(
'some_job_4', CommandJobTemplate('some_template4',
'some_command4'), self.workflow)
self.workflow.add_job(self.job4)
self.job4.add_dep(self.job3)
def test_command_job_template(self):
"""Test the config produced by a command job template."""
job_template = CommandJobTemplate('some_job', 'some_command')
pb_job = job_template.get_pinball_job([], [],
params={
'end_date': '2015-02-25',
})
self.assertEqual('some_command', pb_job.command)
def setUp(self):
self.workflow = WorkflowDef('some_workflow', 'some_schedule',
'some_emails')
template = CommandJobTemplate('some_template', 'some_command')
output_template = CommandJobTemplate('output_template',
'output_command',
priority=30.0)
self.job = JobDef('some_job', template, self.workflow)
self.output_job = JobDef('output_job', output_template, self.workflow)
self.job.outputs = [self.output_job]
def _add_external_deps(self):
other_workflow = WorkflowDef('some_other_workflow', 'some_schedule',
'some_emails')
template = CommandJobTemplate('some_template', 'some_command')
parent_external_job = JobDef('parent_external_job', template,
other_workflow)
other_workflow.add_job(parent_external_job)
external_job = JobDef('external_job', template, other_workflow)
external_job.add_dep(parent_external_job)
other_workflow.add_job(external_job)
self.job1.add_dep(external_job)
from datetime import datetime
from datetime import timedelta
from pinball_ext.workflow.config import JobConfig
from pinball_ext.workflow.config import WorkflowConfig
from pinball_ext.workflow.config import ScheduleConfig
from pinball_ext.job_templates import JobTemplate
from pinball_ext.job_templates import CommandJobTemplate
from tutorial.example_repo.job_templates import ExampleEMRJobTemplate
from tutorial.example_repo.job_templates import ExampleQuboleJobTemplate
# A template for a placeholder final job to add to the end of
# workflows.
FINAL_JOB = CommandJobTemplate('final', 'echo success')
WORKFLOWS = {
'tutorial_workflow': WorkflowConfig(
jobs={
'example_python_job':
JobConfig(JobTemplate('ExamplePythonJob'), []),
'example_command_job':
JobConfig(JobTemplate('ExampleCommandJob'), ['example_python_job']),
'example_quoble_hive_job':
JobConfig(ExampleQuboleJobTemplate('ShowTableHiveJob'), ['example_command_job']),
'example_emr_hive_job':
JobConfig(ExampleEMRJobTemplate('RandomUsersHiveJob'), ['example_command_job']),
'example_emr_hadoop_job':
JobConfig(ExampleEMRJobTemplate('EmrWordCount'), ['example_emr_hive_job']),
},
def _add_final_job(self):
template = CommandJobTemplate('some_template', 'some_command')
final_job = JobDef('final_job', template, self.workflow)
self.workflow.add_job(final_job)
final_job.add_dep(self.job2)
final_job.add_dep(self.job4)
3, len(py_workflow_parser.workflows['another_workflow'].jobs))
self.assertEquals(
24 * 60 * 60, py_workflow_parser.workflows['some_workflow'].
schedule.recurrence_seconds)
self.assertEquals(
60 * 60, py_workflow_parser.workflows['some_other_workflow'].
schedule.recurrence_seconds)
self.assertEquals(
60 * 60, py_workflow_parser.workflows['another_workflow'].schedule.
recurrence_seconds)
self.assertEquals(py_workflow_parser.workflows_config_str,
'tests.pinball_ext.workflow.parser_test.WORKFLOWS')
FINAL_JOB_CONFIG = JobConfig(CommandJobTemplate('final', 'true'))
WORKFLOWS = {
'some_workflow':
WorkflowConfig(jobs={
'some_job':
JobConfig(CommandJobTemplate('some_job', 'true'), [])
},
final_job_config=FINAL_JOB_CONFIG,
schedule=ScheduleConfig(recurrence=timedelta(days=1),
reference_timestamp=datetime(
year=2015,
month=2,
day=1,
second=1)),
notify_emails='*****@*****.**'),
'some_other_workflow':
import datetime
import os
from pinball_ext.job.basic_jobs import PythonJob
from pinball_ext.job.basic_jobs import CommandLineJob
from pinball_ext.job_templates import CommandJobTemplate
JOB_IMPORT_DIRS = [os.path.dirname(os.path.realpath(__file__))]
class ExamplePythonJob(PythonJob):
def _setup(self):
print 'Do some setup in example python job!'
def _execute(self):
print 'Current time is %s' % str(datetime.datetime.now())
class ExampleCommandJob(CommandLineJob):
def _setup(self):
print 'Do not need to add args for this command!'
self.arguments = ''
def _get_command(self):
return 'date'
# A template for a placeholder final job to add to the end of
# workflows.
FINAL_JOB = CommandJobTemplate('final', 'echo success')