def test_customize_command(self):
job = ShellJob(name='some_job',
inputs=['some_input', 'some_other_input'])
some_event = Event(attributes={'some_attr': 'some_value'})
some_other_event = Event(attributes={
'some_attr': 'some_other_value',
'yet_another_attr': 'yet_another_value'})
execution_record = ExecutionRecord(instance=123, start_time=10)
execution_record.events = [some_event, some_other_event]
job.history = [execution_record]
# Empty command.
job.command = ''
self.assertEqual('', job.customize_command())
# Command with no attributes.
job.command = 'some_command'
self.assertEqual('some_command', job.customize_command())
# Command with attributes.
job.command = ('%(non_existent_attr)s %(some_attr)s '
'%(yet_another_attr)s')
self.assertEqual(' some_value,some_other_value yet_another_value',
job.customize_command())
# Command with percentage marks.
job.command = ('%% some_command')
self.assertEqual('% some_command', job.customize_command())
def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock):
file_mock = mock.MagicMock()
open_mock.return_value = file_mock
file_mock.__enter__.return_value = file_mock
s3_key_mock = mock.MagicMock()
get_s3_key_mock.return_value = s3_key_mock
s3_key_mock.__enter__.return_value = s3_key_mock
job = ShellJob(name='some_job',
command="printf \"%s\"" % ('a' * 20000),
emails=['*****@*****.**'],
warn_timeout_sec=10,
abort_timeout_sec=20)
executor = ShellJobExecutor('some_workflow', '123', 'some_job',
job, self._data_builder,
self._emailer)
self.assertTrue(executor.prepare())
self.assertTrue(executor.execute())
file_mock.write.assert_has_calls(
[mock.call('a' * 16384), mock.call('a' * 3616)])
exists_mock.assert_called_once_with(
'/tmp/pinball_job_logs/some_workflow/123')
self.assertEqual(1, len(executor.job.history))
execution_record = executor.job.history[0]
self.assertEqual(0, execution_record.exit_code)
self.assertEqual(2, get_s3_key_mock.call_count)
def test_process_log_line(self):
job = ShellJob(name='some_job',
command="echo ok",
emails=['*****@*****.**'],
warn_timeout_sec=10,
abort_timeout_sec=20)
executor = ShellJobExecutor('some_workflow', '123', 'some_job', job,
self._data_builder, self._emailer)
import time
execution_record = ExecutionRecord(instance=123456,
start_time=time.time())
executor.job.history.append(execution_record)
executor._process_log_line("PINBALL:kv_job_url=j_id1|j_url1\n")
executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n")
executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n")
executor._process_log_line("PINBALL:kill_id=qubole1/123\n")
executor._process_log_line("PINBALL:kill_id=qubole2/456\n")
executor._process_log_line("PINBALL:kill_id=qubole1/123\n")
erp = executor._get_last_execution_record().properties
self.assertEqual(len(erp), 2)
self.assertIn('kv_job_url', erp.keys())
self.assertEqual(type(erp['kv_job_url']), list)
self.assertEqual(len(erp['kv_job_url']), 2)
self.assertEqual(erp['kv_job_url'], ['j_id1|j_url1', 'j_id2|j_url2'])
self.assertIn('kill_id', erp.keys())
self.assertEqual(type(erp['kill_id']), list)
self.assertEqual(len(erp['kill_id']), 2)
self.assertEqual(erp['kill_id'], ['qubole1/123', 'qubole2/456'])
def test_retry(self):
job = ShellJob(name='some_job')
# Empty history.
self.assertFalse(job.retry())
# History with a successful execution.
record = ExecutionRecord(instance=123, exit_code=0)
job.history.append(record)
self.assertRaises(AssertionError, job.retry)
# History with too many failures.
record = ExecutionRecord(instance=1234, exit_code=1)
job.history.append(record)
self.assertFalse(job.retry())
# History without too many failures.
job.max_attempts = 2
self.assertTrue(job.retry())
# History with too many failures in a different instance.
job.history.append(record)
record = ExecutionRecord(instance=12345, exit_code=1)
job.history.append(record)
self.assertTrue(job.retry())
def get_pinball_job(self, inputs, outputs, params=None):
params = params if params else {}
command = self._command % params
max_attempts = (self._max_attempts if self._max_attempts is not None
else 1)
return ShellJob(name=self.name, inputs=inputs, outputs=outputs,
emails=self._emails, max_attempts=max_attempts,
warn_timeout_sec=self._warn_timeout_sec,
abort_timeout_sec=self._abort_timeout_sec,
command=command)
def _get_child_job_token(self):
name = Name(workflow='some_workflow',
instance='12345',
job_state=Name.WAITING_STATE,
job='child_job')
job = ShellJob(name=name.job,
inputs=['parent_job'],
outputs=[],
command='echo child',
emails=['*****@*****.**'])
return Token(name=name.get_job_token_name(), data=pickle.dumps(job))
def test_retry(self):
job = ShellJob(name='some_job')
# Empty history.
self.assertFalse(job.retry())
# History with a successful execution.
record = ExecutionRecord(instance=123, exit_code=0)
job.history.append(record)
self.assertRaises(AssertionError, job.retry)
# History with too many failures.
record = ExecutionRecord(instance=1234, exit_code=1)
job.history.append(record)
self.assertFalse(job.retry())
# History without too many failures.
job.max_attempts = 2
self.assertTrue(job.retry())
# History with too many failures in a different instance.
job.history.append(record)
record = ExecutionRecord(instance=12345, exit_code=1)
job.history.append(record)
self.assertTrue(job.retry())
def _add_active_workflow_tokens(self):
"""Add some active workflow tokens.
The job dependencies form a complete binary tree turned upside down.
I.e., each job has two parents.
"""
self._store = EphemeralStore()
version = 1
for level in range(AnalyzerTestCase._NUM_LEVELS):
jobs_at_level = 2**(AnalyzerTestCase._NUM_LEVELS - level - 1)
for job_index in range(jobs_at_level):
job_name = 'job_%d_%d' % (level, job_index)
event_name = Name(workflow='some_workflow',
instance='123',
job=job_name,
event='some_event')
if level == 0:
inputs = [
Name.WORKFLOW_START_INPUT,
Name.WORKFLOW_START_INPUT + '_prime'
]
event_name.input = Name.WORKFLOW_START_INPUT
else:
inputs = [
'job_%d_%d' % (level - 1, 2 * job_index),
'job_%d_%d' % (level - 1, 2 * job_index + 1)
]
event_name.input = 'job_%d_%d' % (level - 1, 2 * job_index)
if level == AnalyzerTestCase._NUM_LEVELS - 1:
outputs = []
else:
outputs = ['job_%d_%d' % (level + 1, job_index / 2)]
job = ShellJob(name=job_name,
inputs=inputs,
outputs=outputs,
command='some_command')
job.history.append(ExecutionRecord())
name = Name(workflow='some_workflow',
instance='123',
job_state=Name.WAITING_STATE,
job=job_name)
job_token = Token(version=version,
name=name.get_job_token_name(),
priority=10,
data=pickle.dumps(job))
version += 1
event = Event('some_event')
event_token = Token(version=version,
name=event_name.get_event_token_name(),
priority=10,
data=pickle.dumps(event))
self._store.commit_tokens([job_token, event_token])
def get_pinball_job(self, inputs, outputs, params):
assert 'job_repo_dir' in params and 'job_import_dirs_config' in params
# Config job_params
job_params = params.get('job_params', {})
# TODO(csliu): end_date should always be passed from params
if 'end_date' not in params:
yesterday = str(datetime.utcnow().date() - timedelta(days=1))
job_params['end_date'] = yesterday
else:
job_params['end_date'] = params['end_date']
assert job_params, 'job_params should not be empty!'
job_params_text = ','.join(
['%s=%s' % (k, v) for k, v in job_params.iteritems()])
# Config executor for job runner. (Only ClusterJob needs executor)
executor_params = ''
if self._executor:
executor_params = '--executor=%s ' % self._executor
# Executor config.
executor_config = ''
if self._executor_config:
executor_config_text = ','.join(
['%s=%s' % (k, v) for k, v in self._executor_config.iteritems()])
executor_config = '--executor_config="%s" ' % executor_config_text
runner_extra_params = ''
if self.write_lock:
runner_extra_params += '--write_lock=%s ' % self.write_lock
# Construct job runner command line.
job_runner_command = self.command_template % {
'job_repo_dir': params['job_repo_dir'],
'job_import_dirs_config': params['job_import_dirs_config'],
'job_class_name': self._job_class_name,
'job_params': job_params_text,
'executor_params': executor_params,
'executor_config': executor_config,
'runner_extra_params': runner_extra_params
}
return ShellJob(name=self.name, inputs=inputs, outputs=outputs,
command=job_runner_command,
emails=self._emails, max_attempts=self._max_attempts,
warn_timeout_sec=self._warn_timeout_sec,
abort_timeout_sec=self._abort_timeout_sec)
def setUp(self):
self._data_builder = mock.Mock()
self._emailer = mock.Mock()
job = ShellJob(name='some_job',
command='printf "line1\\nline2\\nline3";'
'printf "line1\\nline2" >&2',
emails=['*****@*****.**'],
warn_timeout_sec=10,
abort_timeout_sec=20)
self._executor = ShellJobExecutor('some_workflow', '123', 'some_job',
job, self._data_builder,
self._emailer)
# Set PinballConfig to enable s3 log saver
PinballConfig.S3_LOGS_DIR_PREFIX = 's3n://pinball/tmp/'
PinballConfig.S3_LOGS_DIR = \
PinballConfig.S3_LOGS_DIR_PREFIX \
+ PinballConfig.JOB_LOG_PATH_PREFIX
def _generate_job_token(workflow, instance, job, executions, max_jobs):
if job == 0:
inputs = [Name.WORKFLOW_START_INPUT]
else:
inputs = ['job_%d' % (job - 1)]
if job == max_jobs - 1:
outputs = []
else:
outputs = ['job_%d' % (job + 1)]
shell_job = ShellJob(name='job_%d' % job,
inputs=inputs,
outputs=outputs,
command='some command %d' % job)
for e in range(0, executions):
start_time = 1000000 * workflow + 10000 * instance + 100 * job + e + 1
end_time = start_time + 10 * e + 1
DIR = '/tmp/pinball/logs'
if not os.path.exists(DIR):
os.makedirs(DIR)
LOG_PATTERN = '%s/%%s.%%d.%%s' % DIR
info_log_file = LOG_PATTERN % (job, start_time, 'info')
with open(info_log_file, 'w') as f:
f.write('some info log of execution %d' % e)
error_log_file = LOG_PATTERN % (job, start_time, 'error')
with open(error_log_file, 'w') as f:
f.write('some error log of execution %d' % e)
record = ExecutionRecord(info='some_command %d some_args %d' % (e, e),
instance='instance_%d' % instance,
start_time=start_time,
end_time=end_time,
exit_code=(workflow + instance + e) % 2,
logs={
'info': info_log_file,
'error': error_log_file
})
shell_job.history.append(record)
name = Name(workflow='workflow_%d' % workflow,
instance='instance_%d' % instance,
job_state=Name.WAITING_STATE,
job='job_%d' % job)
return Token(name=name.get_job_token_name(),
version=1000000 * workflow + 10000 * instance + 100 * job,
priority=job,
data=pickle.dumps(shell_job))
def test_customize_command(self):
job = ShellJob(name='some_job',
inputs=['some_input', 'some_other_input'])
some_event = Event(attributes={'some_attr': 'some_value'})
some_other_event = Event(
attributes={
'some_attr': 'some_other_value',
'yet_another_attr': 'yet_another_value'
})
execution_record = ExecutionRecord(instance=123, start_time=10)
execution_record.events = [some_event, some_other_event]
job.history = [execution_record]
# Empty command.
job.command = ''
self.assertEqual('', job.customize_command())
# Command with no attributes.
job.command = 'some_command'
self.assertEqual('some_command', job.customize_command())
# Command with attributes.
job.command = ('%(non_existent_attr)s %(some_attr)s '
'%(yet_another_attr)s')
self.assertEqual(' some_value,some_other_value yet_another_value',
job.customize_command())
# Command with percentage marks.
job.command = ('%% some_command')
self.assertEqual('% some_command', job.customize_command())
