def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock):
file_mock = mock.MagicMock()
open_mock.return_value = file_mock
file_mock.__enter__.return_value = file_mock
s3_key_mock = mock.MagicMock()
get_s3_key_mock.return_value = s3_key_mock
s3_key_mock.__enter__.return_value = s3_key_mock
job = ShellJob(name='some_job',
command="printf \"%s\"" % ('a' * 20000),
emails=['*****@*****.**'],
warn_timeout_sec=10,
abort_timeout_sec=20)
executor = ShellJobExecutor('some_workflow', '123', 'some_job',
job, self._data_builder,
self._emailer)
self.assertTrue(executor.prepare())
self.assertTrue(executor.execute())
file_mock.write.assert_has_calls(
[mock.call('a' * 16384), mock.call('a' * 3616)])
exists_mock.assert_called_once_with(
'/tmp/pinball_job_logs/some_workflow/123')
self.assertEqual(1, len(executor.job.history))
execution_record = executor.job.history[0]
self.assertEqual(0, execution_record.exit_code)
self.assertEqual(2, get_s3_key_mock.call_count)
def test_process_log_line(self):
job = ShellJob(name='some_job',
command="echo ok",
emails=['*****@*****.**'],
warn_timeout_sec=10,
abort_timeout_sec=20)
executor = ShellJobExecutor('some_workflow', '123', 'some_job', job,
self._data_builder, self._emailer)
import time
execution_record = ExecutionRecord(instance=123456,
start_time=time.time())
executor.job.history.append(execution_record)
executor._process_log_line("PINBALL:kv_job_url=j_id1|j_url1\n")
executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n")
executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n")
executor._process_log_line("PINBALL:kill_id=qubole1/123\n")
executor._process_log_line("PINBALL:kill_id=qubole2/456\n")
executor._process_log_line("PINBALL:kill_id=qubole1/123\n")
erp = executor._get_last_execution_record().properties
self.assertEqual(len(erp), 2)
self.assertIn('kv_job_url', erp.keys())
self.assertEqual(type(erp['kv_job_url']), list)
self.assertEqual(len(erp['kv_job_url']), 2)
self.assertEqual(erp['kv_job_url'], ['j_id1|j_url1', 'j_id2|j_url2'])
self.assertIn('kill_id', erp.keys())
self.assertEqual(type(erp['kill_id']), list)
self.assertEqual(len(erp['kill_id']), 2)
self.assertEqual(erp['kill_id'], ['qubole1/123', 'qubole2/456'])
def setUp(self):
self._data_builder = mock.Mock()
self._emailer = mock.Mock()
job = ShellJob(name='some_job',
command='printf "line1\\nline2\\nline3";'
'printf "line1\\nline2" >&2',
emails=['*****@*****.**'],
warn_timeout_sec=10,
abort_timeout_sec=20)
self._executor = ShellJobExecutor('some_workflow', '123', 'some_job',
job, self._data_builder,
self._emailer)
# Set PinballConfig to enable s3 log saver
PinballConfig.S3_LOGS_DIR_PREFIX = 's3n://pinball/tmp/'
PinballConfig.S3_LOGS_DIR = \
PinballConfig.S3_LOGS_DIR_PREFIX \
+ PinballConfig.JOB_LOG_PATH_PREFIX
