示例#1
0
    def testReleaseBatchWithPermanent(self):
        """Test releasing a batch where the spare time is over the threshold.
        """
        mock_consumer = self._create_patch('kale.consumer.Consumer.__init__')
        mock_consumer.return_value = None
        mock_release = self._create_patch(
            'kale.consumer.Consumer.release_messages')
        mock_delete = self._create_patch(
            'kale.consumer.Consumer.delete_messages')
        mock_publish_dlq = self._create_patch(
            'kale.publisher.Publisher.publish_messages_to_dead_letter_queue')
        get_time = self._create_patch('time.time')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()
        worker_inst._incomplete_messages = [
            test_utils.new_mock_message() for i in range(2)
        ]
        worker_inst._successful_messages = [
            test_utils.new_mock_message() for i in range(3)
        ]
        worker_inst._failed_messages = [
            test_utils.new_mock_message() for i in range(4)
        ]
        # Permanent failures should be a subset of failures.
        worker_inst._permanent_failures = worker_inst._failed_messages[:2]

        worker_inst._batch_stop_time = 20
        # _batch_stop_time - get_time > RESET_TIMEOUT_THRESHOLD (20 - 10 > 1)
        get_time.return_value = 10

        releasable_messages = worker_inst._incomplete_messages
        permament_failures = worker_inst._permanent_failures
        deletable_messages = (worker_inst._successful_messages +
                              worker_inst._failed_messages)
        num_deleted, num_released = worker_inst._release_batch()
        mock_release.assert_called_once_with(releasable_messages,
                                             worker_inst._batch_queue.name)
        mock_delete.assert_called_once_with(deletable_messages,
                                            worker_inst._batch_queue.name)
        mock_publish_dlq.assert_called_once_with(
            worker_inst._batch_queue.dlq_name, permament_failures)
        self.assertEqual(num_deleted, len(deletable_messages))
        self.assertEqual(num_released, len(releasable_messages))

        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._failed_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
示例#2
0
    def testBlacklistedTaskNoRetries(self):
        """Test that a blacklisted task raises an exception."""
        setup_env = self._create_patch(
            'kale.task.Task._setup_task_environment')
        pre_run = self._create_patch('kale.task.Task._pre_run')
        run_task = self._create_patch('kale.task.Task.run_task')
        clean_env = self._create_patch(
            'kale.task.Task._clean_task_environment')
        check_blacklist = self._create_patch('kale.task.Task._check_blacklist')
        raised_exc = exceptions.BlacklistedException()
        check_blacklist.side_effect = raised_exc

        mock_message = test_utils.new_mock_message(
            task_class=test_utils.MockTask)
        task_inst = mock_message.task_inst
        task_inst._start_time = 1
        task_args = [1, 'a']

        with self.assertRaises(exceptions.BlacklistedException):
            task_inst.run(*task_args)

        setup_env.assert_called_once_with()
        pre_run.assert_called_once_with(*task_args)
        self.assertFalse(run_task.called)
        clean_env.assert_called_once_with(task_id='mock_task',
                                          task_name='kale.test_utils.MockTask',
                                          exc=raised_exc)

        # Check that task
        permanent_failure = not task_inst.__class__.handle_failure(
            mock_message, raised_exc)
        self.assertTrue(permanent_failure)
示例#3
0
    def testRunBatchTaskExceptionPermanentFailure(self):
        """Test batch with a task exception."""
        mock_consumer = self._create_patch('kale.consumer.Consumer')
        get_time = self._create_patch('time.time')
        mock_failure = self._create_patch(
            'kale.test_utils.FailTask.handle_failure')
        mock_failure.return_value = False

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()

        worker_inst._batch_stop_time = 100
        # _batch_stop_time - (get_time + task.time_limit) > 0
        # (100 - (10 + 60)) > 0)
        get_time.return_value = 10

        message = test_utils.new_mock_message(task_class=test_utils.FailTask)
        message_batch = [message]
        num_messages = len(message_batch)

        worker_inst._run_batch(message_batch)

        fail_msg, fail_exc = mock_failure.call_args[0]
        self.assertEqual(fail_msg, message)
        self.assertTrue(type(fail_exc) == exceptions.TaskException)
        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(1, len(worker_inst._permanent_failures))
        self.assertEqual(num_messages, len(worker_inst._failed_messages))
示例#4
0
    def testRunBatchTaskShouldNotRun(self):
        """Test batch with a task which should not run."""
        mock_consumer = self._create_patch('kale.consumer.Consumer')
        get_time = self._create_patch('time.time')
        mock_republish = self._create_patch(
            'kale.test_utils.ShouldNotRunTask.republish')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()

        worker_inst._batch_stop_time = 100
        # _batch_stop_time - (get_time + task.time_limit) > 0
        # (100 - (10 + 60)) > 0)
        get_time.return_value = 10

        message = test_utils.new_mock_message(
            task_class=test_utils.ShouldNotRunTask)
        message_batch = [message]

        worker_inst._run_batch(message_batch)

        message_republished, failure_count = mock_republish.call_args[0]
        self.assertEqual(message, message_republished)
        self.assertEqual(0, failure_count)
        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
        self.assertEqual(0, len(worker_inst._failed_messages))
示例#5
0
    def testBlacklistedTaskNoRetries(self):
        """Test that a blacklisted task raises an exception."""
        setup_env = self._create_patch(
            'kale.task.Task._setup_task_environment')
        pre_run = self._create_patch('kale.task.Task._pre_run')
        run_task = self._create_patch('kale.task.Task.run_task')
        clean_env = self._create_patch(
            'kale.task.Task._clean_task_environment')
        check_blacklist = self._create_patch('kale.task.Task._check_blacklist')
        raised_exc = exceptions.BlacklistedException()
        check_blacklist.side_effect = raised_exc

        mock_message = test_utils.new_mock_message(
            task_class=test_utils.MockTask)
        task_inst = mock_message.task_inst
        task_inst._start_time = 1
        task_args = [1, 'a']

        with self.assertRaises(exceptions.BlacklistedException):
            task_inst.run(*task_args)

        setup_env.assert_called_once_with()
        pre_run.assert_called_once_with(*task_args)
        self.assertFalse(run_task.called)
        clean_env.assert_called_once_with(
            task_id='mock_task', task_name='kale.test_utils.MockTask',
            exc=raised_exc)

        # Check that task
        permanent_failure = not task_inst.__class__.handle_failure(
            mock_message, raised_exc)
        self.assertTrue(permanent_failure)
示例#6
0
    def testRunBatchNoTimeRemaining(self):
        """Test a batch where there is not enough time remaining."""
        mock_consumer = self._create_patch('kale.consumer.Consumer')
        get_time = self._create_patch('time.time')
        log_info = self._create_patch('kale.worker.logger.info')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()

        worker_inst._batch_stop_time = 50
        # _batch_stop_time - (get_time + task.time_limit) > 0
        # (100 - (10 + 60)) < 0)
        get_time.return_value = 10

        message_batch = [test_utils.new_mock_message()]
        num_messages = len(message_batch)

        worker_inst._run_batch(message_batch)

        self.assertEqual(num_messages, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._failed_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
        self.assertEqual(
            ('Task deferred. Task id: mock_task; Queue: %s; '
             'Time remaining: 40 sec') % worker_inst._batch_queue.name,
            log_info.call_args[0][0])
示例#7
0
    def testRunBatchTaskExceptionPermanentFailure(self):
        """Test batch with a task exception."""
        mock_consumer = self._create_patch('kale.consumer.Consumer')
        get_time = self._create_patch('time.time')
        mock_failure = self._create_patch(
            'kale.test_utils.FailTask.handle_failure')
        mock_failure.return_value = False
        log_info = self._create_patch('kale.worker.logger.info')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()

        worker_inst._batch_stop_time = 100
        # _batch_stop_time - (get_time + task.time_limit) > 0
        # (100 - (10 + 60)) > 0)
        get_time.return_value = 10

        message = test_utils.new_mock_message(task_class=test_utils.FailTask)
        message_batch = [message]
        num_messages = len(message_batch)

        worker_inst._run_batch(message_batch)

        fail_msg, fail_exc = mock_failure.call_args[0]
        self.assertEqual(fail_msg, message)
        self.assertTrue(type(fail_exc) == exceptions.TaskException)
        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(1, len(worker_inst._permanent_failures))
        self.assertEqual(num_messages, len(worker_inst._failed_messages))
        self.assertEqual(
            ('Task failed. Task id: fail_task; Queue: %s; '
             'Time remaining: 90 sec') % worker_inst._batch_queue.name,
            log_info.call_args[0][0])
示例#8
0
    def testReleaseBatchWithPermanent(self):
        """Test releasing a batch where the spare time is over the threshold.
        """
        mock_consumer = self._create_patch('kale.consumer.Consumer.__init__')
        mock_consumer.return_value = None
        mock_release = self._create_patch(
            'kale.consumer.Consumer.release_messages')
        mock_delete = self._create_patch(
            'kale.consumer.Consumer.delete_messages')
        mock_publish_dlq = self._create_patch(
            'kale.publisher.Publisher.publish_messages_to_dead_letter_queue')
        get_time = self._create_patch('time.time')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()
        worker_inst._incomplete_messages = [
            test_utils.new_mock_message() for i in range(2)]
        worker_inst._successful_messages = [
            test_utils.new_mock_message() for i in range(3)]
        worker_inst._failed_messages = [
            test_utils.new_mock_message() for i in range(4)]
        # Permanent failures should be a subset of failures.
        worker_inst._permanent_failures = worker_inst._failed_messages[:2]

        worker_inst._batch_stop_time = 20
        # _batch_stop_time - get_time > RESET_TIMEOUT_THRESHOLD (20 - 10 > 1)
        get_time.return_value = 10

        releasable_messages = worker_inst._incomplete_messages
        permament_failures = worker_inst._permanent_failures
        deletable_messages = (
            worker_inst._successful_messages + worker_inst._failed_messages)
        num_deleted, num_released = worker_inst._release_batch()
        mock_release.assert_called_once_with(
            releasable_messages, worker_inst._batch_queue.name)
        mock_delete.assert_called_once_with(
            deletable_messages, worker_inst._batch_queue.name)
        mock_publish_dlq.assert_called_once_with(
            worker_inst._batch_queue.dlq_name, permament_failures)
        self.assertEqual(num_deleted, len(deletable_messages))
        self.assertEqual(num_released, len(releasable_messages))

        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._failed_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
示例#9
0
    def testReleaseBatchWithNoSuccessfulAndNoTimeLeft(self):
        """Test releasing a batch where the spare time is over the threshold.
        """
        mock_consumer = self._create_patch('kale.consumer.Consumer.__init__')
        mock_consumer.return_value = None
        mock_release = self._create_patch(
            'kale.consumer.Consumer.release_messages')
        mock_delete = self._create_patch(
            'kale.consumer.Consumer.delete_messages')
        mock_publish_dlq = self._create_patch(
            'kale.publisher.Publisher.publish_messages_to_dead_letter_queue')
        get_time = self._create_patch('time.time')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()
        worker_inst._successful_messages = []
        worker_inst._incomplete_messages = [
            test_utils.new_mock_message() for i in range(2)
        ]
        worker_inst._failed_messages = [
            test_utils.new_mock_message() for i in range(4)
        ]

        worker_inst._batch_stop_time = 20
        # _batch_stop_time - get_time > RESET_TIMEOUT_THRESHOLD (20 - 19.5 < 1)
        get_time.return_value = 19.5

        deletable_messages = worker_inst._failed_messages
        num_deleted, num_released = worker_inst._release_batch()
        assert not mock_release.called, ('No messages should have '
                                         'been released.')
        # Failed messages should have been deleted.
        mock_delete.assert_called_once_with(deletable_messages,
                                            worker_inst._batch_queue.name)
        assert not mock_publish_dlq.called, ('No messages should have'
                                             'been moved to dlq.')
        self.assertEqual(num_deleted, len(deletable_messages))
        self.assertEqual(num_released, 0)

        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._failed_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
示例#10
0
    def testReleaseBatchWithNoSuccessfulAndNoTimeLeft(self):
        """Test releasing a batch where the spare time is over the threshold.
        """
        mock_consumer = self._create_patch('kale.consumer.Consumer.__init__')
        mock_consumer.return_value = None
        mock_release = self._create_patch(
            'kale.consumer.Consumer.release_messages')
        mock_delete = self._create_patch(
            'kale.consumer.Consumer.delete_messages')
        mock_publish_dlq = self._create_patch(
            'kale.publisher.Publisher.publish_messages_to_dead_letter_queue')
        get_time = self._create_patch('time.time')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()
        worker_inst._successful_messages = []
        worker_inst._incomplete_messages = [
            test_utils.new_mock_message() for i in range(2)]
        worker_inst._failed_messages = [
            test_utils.new_mock_message() for i in range(4)]

        worker_inst._batch_stop_time = 20
        # _batch_stop_time - get_time > RESET_TIMEOUT_THRESHOLD (20 - 19.5 < 1)
        get_time.return_value = 19.5

        deletable_messages = worker_inst._failed_messages
        num_deleted, num_released = worker_inst._release_batch()
        assert not mock_release.called, ('No messages should have '
                                         'been released.')
        # Failed messages should have been deleted.
        mock_delete.assert_called_once_with(
            deletable_messages, worker_inst._batch_queue.name)
        assert not mock_publish_dlq.called, ('No messages should have'
                                             'been moved to dlq.')
        self.assertEqual(num_deleted, len(deletable_messages))
        self.assertEqual(num_released, 0)

        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._failed_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
示例#11
0
    def testRunIterationWithTasks(self):
        """Test an iteration that has tasks."""
        mock_consumer = self._create_patch('kale.consumer.Consumer.__init__')
        mock_consumer.return_value = None

        fetch_batch = self._create_patch('kale.consumer.Consumer.fetch_batch')
        message = test_utils.new_mock_message()
        fetch_batch.return_value = [message]

        run_batch = self._create_patch('kale.worker.Worker._run_batch')
        run_batch.return_value = (1, 1)

        worker_inst = worker.Worker()
        mock_consumer.assert_called_once_with()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()

        self.assertTrue(worker_inst._run_single_iteration())

        self.assertEqual(fetch_batch.called, 1)
        self.assertTrue(worker_inst._dirty)
        run_batch.assert_called_once_with([message])
示例#12
0
    def testRunIterationWithTasks(self):
        """Test an iteration that has tasks."""
        mock_consumer = self._create_patch('kale.consumer.Consumer.__init__')
        mock_consumer.return_value = None

        fetch_batch = self._create_patch('kale.consumer.Consumer.fetch_batch')
        message = test_utils.new_mock_message()
        fetch_batch.return_value = [message]

        run_batch = self._create_patch('kale.worker.Worker._run_batch')
        run_batch.return_value = (1, 1)

        worker_inst = worker.Worker()
        mock_consumer.assert_called_once_with()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()

        self.assertTrue(worker_inst._run_single_iteration())

        self.assertEqual(fetch_batch.called, 1)
        self.assertTrue(worker_inst._dirty)
        run_batch.assert_called_once_with([message])
示例#13
0
    def testBlacklistedTaskNoRetriesChildProcess(self):
        """Test that a blacklisted task raises an exception in the child process.."""
        check_blacklist = self._create_patch('kale.task.Task._check_blacklist')
        raised_exc = exceptions.BlacklistedException()
        check_blacklist.side_effect = raised_exc

        settings.RUN_TASK_AS_CHILD = True

        mock_message = test_utils.new_mock_message(
            task_class=test_utils.MockTask)
        task_inst = mock_message.task_inst
        task_inst._start_time = 1
        task_args = [1, 'a']

        with self.assertRaises(exceptions.BlacklistedException):
            task_inst.run(*task_args)

        # Check that task
        permanent_failure = not task_inst.__class__.handle_failure(
            mock_message, raised_exc)
        self.assertTrue(permanent_failure)
示例#14
0
    def testRunBatchSuccessful(self):
        """Test a successful batch."""
        mock_consumer = self._create_patch('kale.consumer.Consumer')
        get_time = self._create_patch('time.time')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()

        worker_inst._batch_stop_time = 100
        # _batch_stop_time - (get_time + task.time_limit) > 0
        # (100 - (10 + 60)) > 0)
        get_time.return_value = 10

        message_batch = [test_utils.new_mock_message()]
        num_messages = len(message_batch)

        worker_inst._run_batch(message_batch)

        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(num_messages, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._failed_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
示例#15
0
    def testRunBatchTaskTimeout(self):
        """Test batch with a task timeout."""
        mock_consumer = self._create_patch('kale.consumer.Consumer')
        get_time = self._create_patch('time.time')
        mock_failure = self._create_patch(
            'kale.test_utils.TimeoutTask.handle_failure')
        mock_failure.return_value = True
        log_info = self._create_patch('kale.worker.logger.info')

        worker_inst = worker.Worker()
        worker_inst._batch_queue = worker_inst._queue_selector.get_queue()
        mock_consumer.assert_called_once_with()

        worker_inst._batch_stop_time = 100
        # _batch_stop_time - (get_time + task.time_limit) > 0
        # (100 - (10 + 60)) > 0)
        get_time.return_value = 10

        message = test_utils.new_mock_message(
            task_class=test_utils.TimeoutTask)
        message_batch = [message]
        num_messages = len(message_batch)

        worker_inst._run_batch(message_batch)

        fail_msg, fail_exc = mock_failure.call_args[0]
        self.assertEqual(fail_msg, message)
        self.assertTrue(type(fail_exc) == exceptions.TimeoutException)
        self.assertEqual(0, len(worker_inst._incomplete_messages))
        self.assertEqual(0, len(worker_inst._successful_messages))
        self.assertEqual(0, len(worker_inst._permanent_failures))
        self.assertEqual(num_messages, len(worker_inst._failed_messages))
        self.assertEqual(
            ('Task failed. Task id: fail_task; Queue: %s; '
             'Time remaining: 90 sec') % worker_inst._batch_queue.name,
            log_info.call_args[0][0])