def test_command_job_template(self): """Test the config produced by a command job template.""" job_template = CommandJobTemplate('some_job', 'some_command') pb_job = job_template.get_pinball_job([], [], params={ 'end_date': '2015-02-25', }) self.assertEqual('some_command', pb_job.command)
def setUp(self): # 1370354400 = 06/04/2012 2:00p UTC self.next_run_time = 1370354400 # 1370289240 = 06/03/2013 7:53p UTC self.previous_run_time = 1370289240 self.workflow_schedule = WorkflowSchedule( next_run_time=self.next_run_time, recurrence_seconds=24 * 60 * 60, workflow='some_workflow', emails=['some_email']) self.workflow = WorkflowDef('some_workflow', self.workflow_schedule, 'some_emails') self.job1 = JobDef( 'some_job_1', CommandJobTemplate('some_template1', 'some_command1'), self.workflow) self.workflow.add_job(self.job1) self.job2 = JobDef( 'some_job_2', CommandJobTemplate('some_template2', 'some_command2'), self.workflow) self.workflow.add_job(self.job2) self.job2.add_dep(self.job1) self.job3 = JobDef( 'some_job_3', CommandJobTemplate('some_template3', 'some_command3'), self.workflow) self.workflow.add_job(self.job3) self.job4 = JobDef( 'some_job_4', CommandJobTemplate('some_template4', 'some_command4'), self.workflow) self.workflow.add_job(self.job4) self.job4.add_dep(self.job3)
def setUp(self): self.workflow = WorkflowDef('some_workflow', 'some_schedule', 'some_emails') template = CommandJobTemplate('some_template', 'some_command') output_template = CommandJobTemplate('output_template', 'output_command', priority=30.0) self.job = JobDef('some_job', template, self.workflow) self.output_job = JobDef('output_job', output_template, self.workflow) self.job.outputs = [self.output_job]
def _add_external_deps(self): other_workflow = WorkflowDef('some_other_workflow', 'some_schedule', 'some_emails') template = CommandJobTemplate('some_template', 'some_command') parent_external_job = JobDef('parent_external_job', template, other_workflow) other_workflow.add_job(parent_external_job) external_job = JobDef('external_job', template, other_workflow) external_job.add_dep(parent_external_job) other_workflow.add_job(external_job) self.job1.add_dep(external_job)
from datetime import datetime from datetime import timedelta from pinball_ext.workflow.config import JobConfig from pinball_ext.workflow.config import WorkflowConfig from pinball_ext.workflow.config import ScheduleConfig from pinball_ext.job_templates import JobTemplate from pinball_ext.job_templates import CommandJobTemplate from tutorial.example_repo.job_templates import ExampleEMRJobTemplate from tutorial.example_repo.job_templates import ExampleQuboleJobTemplate # A template for a placeholder final job to add to the end of # workflows. FINAL_JOB = CommandJobTemplate('final', 'echo success') WORKFLOWS = { 'tutorial_workflow': WorkflowConfig( jobs={ 'example_python_job': JobConfig(JobTemplate('ExamplePythonJob'), []), 'example_command_job': JobConfig(JobTemplate('ExampleCommandJob'), ['example_python_job']), 'example_quoble_hive_job': JobConfig(ExampleQuboleJobTemplate('ShowTableHiveJob'), ['example_command_job']), 'example_emr_hive_job': JobConfig(ExampleEMRJobTemplate('RandomUsersHiveJob'), ['example_command_job']), 'example_emr_hadoop_job': JobConfig(ExampleEMRJobTemplate('EmrWordCount'), ['example_emr_hive_job']), },
def _add_final_job(self): template = CommandJobTemplate('some_template', 'some_command') final_job = JobDef('final_job', template, self.workflow) self.workflow.add_job(final_job) final_job.add_dep(self.job2) final_job.add_dep(self.job4)
3, len(py_workflow_parser.workflows['another_workflow'].jobs)) self.assertEquals( 24 * 60 * 60, py_workflow_parser.workflows['some_workflow']. schedule.recurrence_seconds) self.assertEquals( 60 * 60, py_workflow_parser.workflows['some_other_workflow']. schedule.recurrence_seconds) self.assertEquals( 60 * 60, py_workflow_parser.workflows['another_workflow'].schedule. recurrence_seconds) self.assertEquals(py_workflow_parser.workflows_config_str, 'tests.pinball_ext.workflow.parser_test.WORKFLOWS') FINAL_JOB_CONFIG = JobConfig(CommandJobTemplate('final', 'true')) WORKFLOWS = { 'some_workflow': WorkflowConfig(jobs={ 'some_job': JobConfig(CommandJobTemplate('some_job', 'true'), []) }, final_job_config=FINAL_JOB_CONFIG, schedule=ScheduleConfig(recurrence=timedelta(days=1), reference_timestamp=datetime( year=2015, month=2, day=1, second=1)), notify_emails='*****@*****.**'), 'some_other_workflow':
import datetime import os from pinball_ext.job.basic_jobs import PythonJob from pinball_ext.job.basic_jobs import CommandLineJob from pinball_ext.job_templates import CommandJobTemplate JOB_IMPORT_DIRS = [os.path.dirname(os.path.realpath(__file__))] class ExamplePythonJob(PythonJob): def _setup(self): print 'Do some setup in example python job!' def _execute(self): print 'Current time is %s' % str(datetime.datetime.now()) class ExampleCommandJob(CommandLineJob): def _setup(self): print 'Do not need to add args for this command!' self.arguments = '' def _get_command(self): return 'date' # A template for a placeholder final job to add to the end of # workflows. FINAL_JOB = CommandJobTemplate('final', 'echo success')