def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock): file_mock = mock.MagicMock() open_mock.return_value = file_mock file_mock.__enter__.return_value = file_mock s3_key_mock = mock.MagicMock() get_s3_key_mock.return_value = s3_key_mock s3_key_mock.__enter__.return_value = s3_key_mock job = ShellJob(name='some_job', command="printf \"%s\"" % ('a' * 20000), emails=['*****@*****.**'], warn_timeout_sec=10, abort_timeout_sec=20) executor = ShellJobExecutor('some_workflow', '123', 'some_job', job, self._data_builder, self._emailer) self.assertTrue(executor.prepare()) self.assertTrue(executor.execute()) file_mock.write.assert_has_calls( [mock.call('a' * 16384), mock.call('a' * 3616)]) exists_mock.assert_called_once_with( '/tmp/pinball_job_logs/some_workflow/123') self.assertEqual(1, len(executor.job.history)) execution_record = executor.job.history[0] self.assertEqual(0, execution_record.exit_code) self.assertEqual(2, get_s3_key_mock.call_count)
def test_process_log_line(self): job = ShellJob(name='some_job', command="echo ok", emails=['*****@*****.**'], warn_timeout_sec=10, abort_timeout_sec=20) executor = ShellJobExecutor('some_workflow', '123', 'some_job', job, self._data_builder, self._emailer) import time execution_record = ExecutionRecord(instance=123456, start_time=time.time()) executor.job.history.append(execution_record) executor._process_log_line("PINBALL:kv_job_url=j_id1|j_url1\n") executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n") executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n") executor._process_log_line("PINBALL:kill_id=qubole1/123\n") executor._process_log_line("PINBALL:kill_id=qubole2/456\n") executor._process_log_line("PINBALL:kill_id=qubole1/123\n") erp = executor._get_last_execution_record().properties self.assertEqual(len(erp), 2) self.assertIn('kv_job_url', erp.keys()) self.assertEqual(type(erp['kv_job_url']), list) self.assertEqual(len(erp['kv_job_url']), 2) self.assertEqual(erp['kv_job_url'], ['j_id1|j_url1', 'j_id2|j_url2']) self.assertIn('kill_id', erp.keys()) self.assertEqual(type(erp['kill_id']), list) self.assertEqual(len(erp['kill_id']), 2) self.assertEqual(erp['kill_id'], ['qubole1/123', 'qubole2/456'])
def setUp(self): self._data_builder = mock.Mock() self._emailer = mock.Mock() job = ShellJob(name='some_job', command='printf "line1\\nline2\\nline3";' 'printf "line1\\nline2" >&2', emails=['*****@*****.**'], warn_timeout_sec=10, abort_timeout_sec=20) self._executor = ShellJobExecutor('some_workflow', '123', 'some_job', job, self._data_builder, self._emailer) # Set PinballConfig to enable s3 log saver PinballConfig.S3_LOGS_DIR_PREFIX = 's3n://pinball/tmp/' PinballConfig.S3_LOGS_DIR = \ PinballConfig.S3_LOGS_DIR_PREFIX \ + PinballConfig.JOB_LOG_PATH_PREFIX