Exemplo n.º 1
0
    def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        job = ShellJob(name='some_job',
                       command="printf \"%s\"" % ('a' * 20000),
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                    job, self._data_builder,
                                    self._emailer)

        self.assertTrue(executor.prepare())
        self.assertTrue(executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('a' * 16384), mock.call('a' * 3616)])

        self.assertEqual(1, len(executor.job.history))
        execution_record = executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)
Exemplo n.º 2
0
    def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        job = ShellJob(name='some_job',
                       command="printf \"%s\"" % ('a' * 20000),
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                    job, self._data_builder,
                                    self._emailer)

        self.assertTrue(executor.prepare())
        self.assertTrue(executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('a' * 16384), mock.call('a' * 3616)])

        exists_mock.assert_called_once_with(
            '/tmp/pinball_job_logs/some_workflow/123')

        self.assertEqual(1, len(executor.job.history))
        execution_record = executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)
Exemplo n.º 3
0
class ShellJobExecutorTestCase(unittest.TestCase):
    def setUp(self):
        self._data_builder = mock.Mock()
        self._emailer = mock.Mock()
        job = ShellJob(name='some_job',
                       command='printf "line1\\nline2\\nline3";'
                               'printf "line1\\nline2" >&2',
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        self._executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                          job, self._data_builder,
                                          self._emailer)
        # Set PinballConfig to enable s3 log saver
        PinballConfig.S3_LOGS_DIR_PREFIX = 's3n://pinball/tmp/'
        PinballConfig.S3_LOGS_DIR = \
            PinballConfig.S3_LOGS_DIR_PREFIX \
            + PinballConfig.JOB_LOG_PATH_PREFIX


    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('__builtin__.open')
    def test_token_lost(self, open_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        execution_record = ExecutionRecord(start_time=10)
        self._executor.job.history = [execution_record]

        self.assertFalse(self._executor.prepare())
        file_mock.write.assert_called_once_with('executor failed to renew job '
                                                'ownership on time\n')
        get_s3_key_mock.assert_called_once_with('s3n://pinball/tmp/pinball_job_logs/'
                                                'some_workflow/123/some_job.10.pinlog')

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver.open')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_events(self, open_mock, exists_mock, s3_open_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        exists_mock.return_value = True
        some_event = Event(creator='some_creator')
        some_other_event = Event(creator='some_other_creator')
        self._executor.job.events = [some_event, some_other_event]
        self.assertTrue(self._executor.prepare())
        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual([some_event, some_other_event],
                         execution_record.events)
        self.assertEqual(s3_open_mock.call_count, 2)

    def test_disabled(self):
        self._executor.job.disabled = True
        self.assertTrue(self._executor.prepare())
        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual('DISABLED', execution_record.info)

    @mock.patch('subprocess.Popen')
    def test_execute_cleanup(self, subprocess_mock):

        self._executor.job.cleanup_template = 'cleanup %(job_id)s'
        execution_record = ExecutionRecord()
        execution_record.properties['job_id'] = '123'
        self._executor.job.history = [execution_record]
        self._executor._execute_cleanup()
        env = os.environ.copy()
        env.pop('DJANGO_SETTINGS_MODULE', None)
        subprocess_mock.assert_called_with('cleanup 123',
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE,
                                           shell=True,
                                           env=env,
                                           preexec_fn=os.setsid)

    @mock.patch('time.time')
    def test_check_timeout_noop(self, time_mock):
        execution_record = ExecutionRecord(start_time=10)
        self._executor.job.history = [execution_record]

        time_mock.return_value = 15
        self._executor._check_timeouts()
        self.assertEqual(
            0, self._emailer.send_job_timeout_warning_message.call_count)

        time_mock.return_value = 25
        self._data_builder.get_schedule.return_value = None
        job_execution_data = mock.Mock()
        self._data_builder.get_execution.return_value = job_execution_data
        self._executor._check_timeouts()
        self._data_builder.get_schedule.assert_called_once_with(
            'some_workflow')
        self._data_builder.get_execution.assert_called_once_with(
            'some_workflow', '123', 'some_job', 0)
        self._emailer.send_job_timeout_warning_message.assert_called_once_with(
            ['*****@*****.**'], job_execution_data)

        time_mock.return_value = 35
        self._executor._check_timeouts()
        self.assertTrue(self._executor._aborted)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_execute(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        self.assertTrue(self._executor.prepare())
        self.assertTrue(self._executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('line1\n'), mock.call('line2\n'), mock.call('line3'),
             mock.call('line1\n'), mock.call('line2')],
            any_order=True)
        self.assertEqual(file_mock.write.call_count, 5)

        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.makedirs')
    @mock.patch('__builtin__.open')
    def test_abort(self, open_mock, _, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        self.assertTrue(self._executor.prepare())
        self._executor.abort()
        self.assertFalse(self._executor.execute())

        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual(1, execution_record.exit_code)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        job = ShellJob(name='some_job',
                       command="printf \"%s\"" % ('a' * 20000),
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                    job, self._data_builder,
                                    self._emailer)

        self.assertTrue(executor.prepare())
        self.assertTrue(executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('a' * 16384), mock.call('a' * 3616)])

        self.assertEqual(1, len(executor.job.history))
        execution_record = executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)
Exemplo n.º 4
0
class ShellJobExecutorTestCase(unittest.TestCase):
    def setUp(self):
        self._data_builder = mock.Mock()
        self._emailer = mock.Mock()
        job = ShellJob(name='some_job',
                       command='printf "line1\\nline2\\nline3";'
                               'printf "line1\\nline2" >&2',
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        self._executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                          job, self._data_builder,
                                          self._emailer)
        # Set PinballConfig to enable s3 log saver
        PinballConfig.S3_LOGS_DIR_PREFIX = 's3n://pinball/tmp/'
        PinballConfig.S3_LOGS_DIR = \
            PinballConfig.S3_LOGS_DIR_PREFIX \
            + PinballConfig.JOB_LOG_PATH_PREFIX


    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('__builtin__.open')
    def test_token_lost(self, open_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        execution_record = ExecutionRecord(start_time=10)
        self._executor.job.history = [execution_record]

        self.assertFalse(self._executor.prepare())
        file_mock.write.assert_called_once_with('executor failed to renew job '
                                                'ownership on time\n')
        get_s3_key_mock.assert_called_once_with('s3n://pinball/tmp/pinball_job_logs/'
                                                'some_workflow/123/some_job.10.pinlog')

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver.open')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_events(self, open_mock, exists_mock, s3_open_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        exists_mock.return_value = True
        some_event = Event(creator='some_creator')
        some_other_event = Event(creator='some_other_creator')
        self._executor.job.events = [some_event, some_other_event]
        self.assertTrue(self._executor.prepare())
        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual([some_event, some_other_event],
                         execution_record.events)
        self.assertEqual(s3_open_mock.call_count, 2)

    def test_disabled(self):
        self._executor.job.disabled = True
        self.assertTrue(self._executor.prepare())
        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual('DISABLED', execution_record.info)

    @mock.patch('subprocess.Popen')
    def test_execute_cleanup(self, subprocess_mock):
        self._executor.job.cleanup_template = 'cleanup %(kill_id)s'
        execution_record = ExecutionRecord()
        execution_record.properties['kill_id'] = ['123', '456']
        self._executor.job.history = [execution_record]
        self._executor._execute_cleanup()
        env = os.environ.copy()
        env.pop('DJANGO_SETTINGS_MODULE', None)
        subprocess_mock.assert_called_with('cleanup 123,456',
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE,
                                           shell=True,
                                           env=env,
                                           preexec_fn=os.setsid)

    @mock.patch('time.time')
    def test_check_timeout_noop(self, time_mock):
        execution_record = ExecutionRecord(start_time=10)
        self._executor.job.history = [execution_record]

        time_mock.return_value = 15
        self._executor._check_timeouts()
        self.assertEqual(
            0, self._emailer.send_job_timeout_warning_message.call_count)

        time_mock.return_value = 25
        self._data_builder.get_schedule.return_value = None
        job_execution_data = mock.Mock()
        self._data_builder.get_execution.return_value = job_execution_data
        self._executor._check_timeouts()
        self._data_builder.get_schedule.assert_called_once_with(
            'some_workflow')
        self._data_builder.get_execution.assert_called_once_with(
            'some_workflow', '123', 'some_job', 0)
        self._emailer.send_job_timeout_warning_message.assert_called_once_with(
            ['*****@*****.**'], job_execution_data)

        time_mock.return_value = 35
        self._executor._check_timeouts()
        self.assertTrue(self._executor._aborted)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_execute(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        self.assertTrue(self._executor.prepare())
        self.assertTrue(self._executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('line1\n'), mock.call('line2\n'), mock.call('line3'),
             mock.call('line1\n'), mock.call('line2')],
            any_order=True)
        self.assertEqual(file_mock.write.call_count, 5)

        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.makedirs')
    @mock.patch('__builtin__.open')
    def test_abort(self, open_mock, _, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        self.assertTrue(self._executor.prepare())
        self._executor.abort()
        self.assertFalse(self._executor.execute())

        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual(1, execution_record.exit_code)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        job = ShellJob(name='some_job',
                       command="printf \"%s\"" % ('a' * 20000),
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                    job, self._data_builder,
                                    self._emailer)

        self.assertTrue(executor.prepare())
        self.assertTrue(executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('a' * 16384), mock.call('a' * 3616)])

        self.assertEqual(1, len(executor.job.history))
        execution_record = executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_execute_env_var(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        job_name = 'some_job'
        workflow_name = 'some_workflow'
        instance = '123'

        job = ShellJob(name=job_name,
                       command="echo $PINBALL_WORKFLOW && "
                               "echo $PINBALL_JOB && "
                               "echo $PINBALL_INSTANCE && "
                               "echo $PINBALL_EXECUTION && "
                               "echo $PINBALL_BASE_URL",
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor(workflow_name, instance, job_name,
                                    job, self._data_builder,
                                    self._emailer)

        execution_record = ExecutionRecord(instance=instance,
                                           start_time=time.time())
        execution_record.end_time = time.time()
        execution_record.exit_code = 1
        job.history.append(execution_record)

        self.assertTrue(executor.prepare())
        self.assertTrue(executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call(workflow_name + '\n'),
             mock.call(job_name + '\n'),
             mock.call(instance + '\n'),
             mock.call('1\n')])

        self.assertEqual(len(executor.job.history), 2)
        self.assertEqual(get_s3_key_mock.call_count, len(executor.job.history))
        latest_execution_record = executor.job.history[1]
        self.assertEqual(latest_execution_record.exit_code, 0)

        exists_mock.assert_any_call(
            '/tmp/pinball_job_logs/{wf}/{inst}'.format(
                wf=workflow_name, inst=instance
            ),
        )
        exists_mock.assert_any_call(
            '/tmp/pinball_job_logs/{wf}/{inst}/some_job.{ts}.stdout'.format(
                wf=workflow_name, inst=instance, ts=int(latest_execution_record.start_time)
            ),
        )
        exists_mock.assert_any_call(
            '/tmp/pinball_job_logs/{wf}/{inst}/some_job.{ts}.stderr'.format(
                wf=workflow_name, inst=instance, ts=int(latest_execution_record.start_time)
            ),
        )

    def test_process_log_line(self):
        job = ShellJob(name='some_job',
                       command="echo ok",
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor('some_workflow', '123', 'some_job', job,
                                    self._data_builder,
                                    self._emailer)
        import time
        execution_record = ExecutionRecord(instance=123456,
                                           start_time=time.time())
        executor.job.history.append(execution_record)

        executor._process_log_line("PINBALL:kv_job_url=j_id1|j_url1\n")
        executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n")
        executor._process_log_line("PINBALL:kv_job_url=j_id2|j_url2\n")
        executor._process_log_line("PINBALL:kill_id=qubole1/123\n")
        executor._process_log_line("PINBALL:kill_id=qubole2/456\n")
        executor._process_log_line("PINBALL:kill_id=qubole1/123\n")

        erp = executor._get_last_execution_record().properties
        self.assertEqual(len(erp), 2)

        self.assertIn('kv_job_url', erp.keys())
        self.assertEqual(type(erp['kv_job_url']), list)
        self.assertEqual(len(erp['kv_job_url']), 2)
        self.assertEqual(erp['kv_job_url'], ['j_id1|j_url1', 'j_id2|j_url2'])

        self.assertIn('kill_id', erp.keys())
        self.assertEqual(type(erp['kill_id']), list)
        self.assertEqual(len(erp['kill_id']), 2)
        self.assertEqual(erp['kill_id'], ['qubole1/123', 'qubole2/456'])
Exemplo n.º 5
0
    def test_execute_env_var(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        job_name = 'some_job'
        workflow_name = 'some_workflow'
        instance = '123'

        job = ShellJob(name=job_name,
                       command="echo $PINBALL_WORKFLOW && "
                               "echo $PINBALL_JOB && "
                               "echo $PINBALL_INSTANCE && "
                               "echo $PINBALL_EXECUTION && "
                               "echo $PINBALL_BASE_URL",
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor(workflow_name, instance, job_name,
                                    job, self._data_builder,
                                    self._emailer)

        execution_record = ExecutionRecord(instance=instance,
                                           start_time=time.time())
        execution_record.end_time = time.time()
        execution_record.exit_code = 1
        job.history.append(execution_record)

        self.assertTrue(executor.prepare())
        self.assertTrue(executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call(workflow_name + '\n'),
             mock.call(job_name + '\n'),
             mock.call(instance + '\n'),
             mock.call('1\n')])

        self.assertEqual(len(executor.job.history), 2)
        self.assertEqual(get_s3_key_mock.call_count, len(executor.job.history))
        latest_execution_record = executor.job.history[1]
        self.assertEqual(latest_execution_record.exit_code, 0)

        exists_mock.assert_any_call(
            '/tmp/pinball_job_logs/{wf}/{inst}'.format(
                wf=workflow_name, inst=instance
            ),
        )
        exists_mock.assert_any_call(
            '/tmp/pinball_job_logs/{wf}/{inst}/some_job.{ts}.stdout'.format(
                wf=workflow_name, inst=instance, ts=int(latest_execution_record.start_time)
            ),
        )
        exists_mock.assert_any_call(
            '/tmp/pinball_job_logs/{wf}/{inst}/some_job.{ts}.stderr'.format(
                wf=workflow_name, inst=instance, ts=int(latest_execution_record.start_time)
            ),
        )
Exemplo n.º 6
0
class ShellJobExecutorTestCase(unittest.TestCase):
    def setUp(self):
        self._data_builder = mock.Mock()
        self._emailer = mock.Mock()
        job = ShellJob(name='some_job',
                       command='printf "line1\\nline2\\nline3";'
                               'printf "line1\\nline2" >&2',
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        self._executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                          job, self._data_builder,
                                          self._emailer)
        # Set PinballConfig to enable s3 log saver
        PinballConfig.S3_LOGS_DIR_PREFIX = 's3n://pinball/tmp/'
        PinballConfig.S3_LOGS_DIR = \
            PinballConfig.S3_LOGS_DIR_PREFIX \
            + PinballConfig.JOB_LOG_PATH_PREFIX


    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('__builtin__.open')
    def test_token_lost(self, open_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        execution_record = ExecutionRecord(start_time=10)
        self._executor.job.history = [execution_record]

        self.assertFalse(self._executor.prepare())
        file_mock.write.assert_called_once_with('executor failed to renew job '
                                                'ownership on time\n')
        get_s3_key_mock.assert_called_once_with('s3n://pinball/tmp/pinball_job_logs/'
                                                'some_workflow/123/some_job.10.pinlog')

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver.open')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_events(self, open_mock, exists_mock, s3_open_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        exists_mock.return_value = True
        some_event = Event(creator='some_creator')
        some_other_event = Event(creator='some_other_creator')
        self._executor.job.events = [some_event, some_other_event]
        self.assertTrue(self._executor.prepare())
        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual([some_event, some_other_event],
                         execution_record.events)
        self.assertEqual(s3_open_mock.call_count, 2)

    def test_disabled(self):
        self._executor.job.disabled = True
        self.assertTrue(self._executor.prepare())
        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual('DISABLED', execution_record.info)

    @mock.patch('subprocess.Popen')
    def test_execute_cleanup(self, subprocess_mock):

        self._executor.job.cleanup_template = 'cleanup %(job_id)s'
        execution_record = ExecutionRecord()
        execution_record.properties['job_id'] = '123'
        self._executor.job.history = [execution_record]
        self._executor._execute_cleanup()
        env = os.environ.copy()
        env.pop('DJANGO_SETTINGS_MODULE', None)
        subprocess_mock.assert_called_with('cleanup 123',
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE,
                                           shell=True,
                                           env=env,
                                           preexec_fn=os.setsid)

    @mock.patch('time.time')
    def test_check_timeout_noop(self, time_mock):
        execution_record = ExecutionRecord(start_time=10)
        self._executor.job.history = [execution_record]

        time_mock.return_value = 15
        self._executor._check_timeouts()
        self.assertEqual(
            0, self._emailer.send_job_timeout_warning_message.call_count)

        time_mock.return_value = 25
        self._data_builder.get_schedule.return_value = None
        job_execution_data = mock.Mock()
        self._data_builder.get_execution.return_value = job_execution_data
        self._executor._check_timeouts()
        self._data_builder.get_schedule.assert_called_once_with(
            'some_workflow')
        self._data_builder.get_execution.assert_called_once_with(
            'some_workflow', '123', 'some_job', 0)
        self._emailer.send_job_timeout_warning_message.assert_called_once_with(
            ['*****@*****.**'], job_execution_data)

        time_mock.return_value = 35
        self._executor._check_timeouts()
        self.assertTrue(self._executor._aborted)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_execute(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        self.assertTrue(self._executor.prepare())
        self.assertTrue(self._executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('line1\n'), mock.call('line2\n'), mock.call('line3'),
             mock.call('line1\n'), mock.call('line2')],
            any_order=True)
        self.assertEqual(file_mock.write.call_count, 5)

        exists_mock.assert_called_once_with(
            '/tmp/pinball_job_logs/some_workflow/123')

        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.makedirs')
    @mock.patch('__builtin__.open')
    def test_abort(self, open_mock, _, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        self.assertTrue(self._executor.prepare())
        self._executor.abort()
        self.assertFalse(self._executor.execute())

        self.assertEqual(1, len(self._executor.job.history))
        execution_record = self._executor.job.history[0]
        self.assertEqual(1, execution_record.exit_code)

    @mock.patch('pinball.workflow.log_saver.S3FileLogSaver._get_or_create_s3_key')
    @mock.patch('os.path.exists')
    @mock.patch('__builtin__.open')
    def test_execute_long_line(self, open_mock, exists_mock, get_s3_key_mock):
        file_mock = mock.MagicMock()
        open_mock.return_value = file_mock
        file_mock.__enter__.return_value = file_mock

        s3_key_mock = mock.MagicMock()
        get_s3_key_mock.return_value = s3_key_mock
        s3_key_mock.__enter__.return_value = s3_key_mock

        job = ShellJob(name='some_job',
                       command="printf \"%s\"" % ('a' * 20000),
                       emails=['*****@*****.**'],
                       warn_timeout_sec=10,
                       abort_timeout_sec=20)
        executor = ShellJobExecutor('some_workflow', '123', 'some_job',
                                    job, self._data_builder,
                                    self._emailer)

        self.assertTrue(executor.prepare())
        self.assertTrue(executor.execute())

        file_mock.write.assert_has_calls(
            [mock.call('a' * 16384), mock.call('a' * 3616)])

        exists_mock.assert_called_once_with(
            '/tmp/pinball_job_logs/some_workflow/123')

        self.assertEqual(1, len(executor.job.history))
        execution_record = executor.job.history[0]
        self.assertEqual(0, execution_record.exit_code)

        self.assertEqual(2, get_s3_key_mock.call_count)