class test_frame_handler:

    @pytest.fixture(autouse=True)
    def setup_conn(self):
        self.conn = Mock(name='connection')
        self.conn.bytes_recv = 0
        self.callback = Mock(name='callback')
        self.g = frame_handler(self.conn, self.callback)

    def test_header(self):
        buf = pack('>HH', 60, 51)
        assert self.g((1, 1, buf))
        self.callback.assert_called_with(1, (60, 51), buf, None)
        assert self.conn.bytes_recv

    def test_header_message_empty_body(self):
        assert not self.g((1, 1, pack('>HH', *spec.Basic.Deliver)))
        self.callback.assert_not_called()

        with pytest.raises(UnexpectedFrame):
            self.g((1, 1, pack('>HH', *spec.Basic.Deliver)))

        m = Message()
        m.properties = {}
        buf = pack('>HxxQ', m.CLASS_ID, 0)
        buf += m._serialize_properties()
        assert self.g((2, 1, buf))

        self.callback.assert_called()
        msg = self.callback.call_args[0][3]
        self.callback.assert_called_with(
            1, msg.frame_method, msg.frame_args, msg,
        )

    def test_header_message_content(self):
        assert not self.g((1, 1, pack('>HH', *spec.Basic.Deliver)))
        self.callback.assert_not_called()

        m = Message()
        m.properties = {}
        buf = pack('>HxxQ', m.CLASS_ID, 16)
        buf += m._serialize_properties()
        assert not self.g((2, 1, buf))
        self.callback.assert_not_called()

        assert not self.g((3, 1, b'thequick'))
        self.callback.assert_not_called()

        assert self.g((3, 1, b'brownfox'))
        self.callback.assert_called()
        msg = self.callback.call_args[0][3]
        self.callback.assert_called_with(
            1, msg.frame_method, msg.frame_args, msg,
        )
        assert msg.body == b'thequickbrownfox'

    def test_heartbeat_frame(self):
        assert not self.g((8, 1, ''))
        self.callback.assert_not_called()
        assert self.conn.bytes_recv
Exemple #2
0
    def test_pool_restart_reload_modules(self):
        consumer = Consumer(self.app)
        consumer.controller = _WC(app=self.app)
        consumer.controller.consumer = consumer
        consumer.controller.pool.restart = Mock()
        consumer.reset_rate_limits = Mock(name="reset_rate_limits()")
        consumer.update_strategies = Mock(name="update_strategies()")
        panel = self.create_panel(consumer=consumer)
        panel.app = self.app
        _import = panel.app.loader.import_from_cwd = Mock()
        _reload = Mock()

        self.app.conf.worker_pool_restarts = True
        with patch.dict(sys.modules, {"foo": None}):
            panel.handle("pool_restart", {"modules": ["foo"], "reload": False, "reloader": _reload})

            consumer.controller.pool.restart.assert_called()
            _reload.assert_not_called()
            _import.assert_not_called()

            _import.reset_mock()
            _reload.reset_mock()
            consumer.controller.pool.restart.reset_mock()

            panel.handle("pool_restart", {"modules": ["foo"], "reload": True, "reloader": _reload})
            consumer.controller.pool.restart.assert_called()
            _reload.assert_called()
            _import.assert_not_called()
Exemple #3
0
    def test_apply_timeout(self):
        self.patching.modules(*gevent_modules)

        class Timeout(Exception):
            value = None

            def __init__(self, value):
                self.__class__.value = value

            def __enter__(self):
                return self

            def __exit__(self, *exc_info):
                pass
        timeout_callback = Mock(name='timeout_callback')
        apply_target = Mock(name='apply_target')
        apply_timeout(
            Mock(), timeout=10, callback=Mock(name='callback'),
            timeout_callback=timeout_callback,
            apply_target=apply_target, Timeout=Timeout,
        )
        assert Timeout.value == 10
        apply_target.assert_called()

        apply_target.side_effect = Timeout(10)
        apply_timeout(
            Mock(), timeout=10, callback=Mock(),
            timeout_callback=timeout_callback,
            apply_target=apply_target, Timeout=Timeout,
        )
        timeout_callback.assert_called_with(False, 10)
Exemple #4
0
 def test_with_prerun_receivers(self):
     on_prerun = Mock()
     signals.task_prerun.connect(on_prerun)
     try:
         self.trace(self.add, (2, 2), {})
         on_prerun.assert_called()
     finally:
         signals.task_prerun.receivers[:] = []
Exemple #5
0
 def test_with_success_receivers(self):
     on_success = Mock()
     signals.task_success.connect(on_success)
     try:
         self.trace(self.add, (2, 2), {})
         on_success.assert_called()
     finally:
         signals.task_success.receivers[:] = []
Exemple #6
0
 def test_callback(self):
     cb = Mock()
     with self.move_context(callback=cb) as (callback, pred, republish):
         pred.return_value = "foo"
         body, message = self.msgpair()
         callback(body, message)
         republish.assert_called()
         cb.assert_called()
Exemple #7
0
    def test_reraise_error(self):
        m = Message('body', channel=Mock())
        callback = Mock(name='callback')
        try:
            raise KeyError('foo')
        except KeyError:
            m.errors.append(sys.exc_info())
        m._reraise_error(callback)
        callback.assert_called()

        with pytest.raises(KeyError):
            m._reraise_error(None)
Exemple #8
0
    def test_migrate(self, app, name='testcelery'):
        connection_kwargs = {
            'transport_options': {'polling_interval': 0.01}
        }
        x = Connection('memory://foo', **connection_kwargs)
        y = Connection('memory://foo', **connection_kwargs)
        # use separate state
        x.default_channel.queues = {}
        y.default_channel.queues = {}

        ex = Exchange(name, 'direct')
        q = Queue(name, exchange=ex, routing_key=name)
        q(x.default_channel).declare()
        Producer(x).publish('foo', exchange=name, routing_key=name)
        Producer(x).publish('bar', exchange=name, routing_key=name)
        Producer(x).publish('baz', exchange=name, routing_key=name)
        assert x.default_channel.queues
        assert not y.default_channel.queues
        migrate_tasks(x, y, accept=['text/plain'], app=app)

        yq = q(y.default_channel)
        assert yq.get().body == ensure_bytes('foo')
        assert yq.get().body == ensure_bytes('bar')
        assert yq.get().body == ensure_bytes('baz')

        Producer(x).publish('foo', exchange=name, routing_key=name)
        callback = Mock()
        migrate_tasks(x, y,
                      callback=callback, accept=['text/plain'], app=app)
        callback.assert_called()
        migrate = Mock()
        Producer(x).publish('baz', exchange=name, routing_key=name)
        migrate_tasks(x, y, callback=callback,
                      migrate=migrate, accept=['text/plain'], app=app)
        migrate.assert_called()

        with patch('kombu.transport.virtual.Channel.queue_declare') as qd:

            def effect(*args, **kwargs):
                if kwargs.get('passive'):
                    raise ChannelError('some channel error')
                return 0, 3, 0
            qd.side_effect = effect
            migrate_tasks(x, y, app=app)

        x = Connection('memory://', **connection_kwargs)
        x.default_channel.queues = {}
        y.default_channel.queues = {}
        callback = Mock()
        migrate_tasks(x, y,
                      callback=callback, accept=['text/plain'], app=app)
        callback.assert_not_called()
Exemple #9
0
    def test_autoretry(self):
        myfun = Mock()

        self.conn.transport.connection_errors = (KeyError,)

        def on_call(*args, **kwargs):
            myfun.side_effect = None
            raise KeyError('foo')

        myfun.side_effect = on_call
        insured = self.conn.autoretry(myfun)
        insured()

        myfun.assert_called()
Exemple #10
0
    def test_migrate(self, app, name="testcelery"):
        x = Connection("memory://foo")
        y = Connection("memory://foo")
        # use separate state
        x.default_channel.queues = {}
        y.default_channel.queues = {}

        ex = Exchange(name, "direct")
        q = Queue(name, exchange=ex, routing_key=name)
        q(x.default_channel).declare()
        Producer(x).publish("foo", exchange=name, routing_key=name)
        Producer(x).publish("bar", exchange=name, routing_key=name)
        Producer(x).publish("baz", exchange=name, routing_key=name)
        assert x.default_channel.queues
        assert not y.default_channel.queues

        migrate_tasks(x, y, accept=["text/plain"], app=app)

        yq = q(y.default_channel)
        assert yq.get().body == ensure_bytes("foo")
        assert yq.get().body == ensure_bytes("bar")
        assert yq.get().body == ensure_bytes("baz")

        Producer(x).publish("foo", exchange=name, routing_key=name)
        callback = Mock()
        migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app)
        callback.assert_called()
        migrate = Mock()
        Producer(x).publish("baz", exchange=name, routing_key=name)
        migrate_tasks(x, y, callback=callback, migrate=migrate, accept=["text/plain"], app=app)
        migrate.assert_called()

        with patch("kombu.transport.virtual.Channel.queue_declare") as qd:

            def effect(*args, **kwargs):
                if kwargs.get("passive"):
                    raise ChannelError("some channel error")
                return 0, 3, 0

            qd.side_effect = effect
            migrate_tasks(x, y, app=app)

        x = Connection("memory://")
        x.default_channel.queues = {}
        y.default_channel.queues = {}
        callback = Mock()
        migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app)
        callback.assert_not_called()
Exemple #11
0
    def test_accept__content_allowed(self):
        conn = Connection('memory://')
        q = Queue('foo', exchange=self.exchange)
        p = conn.Producer()
        p.publish(
            {'complex': object()},
            declare=[q], exchange=self.exchange, serializer='pickle',
        )

        callback = Mock(name='callback')
        with conn.Consumer(queues=[q], accept=['pickle'],
                           callbacks=[callback]):
            conn.drain_events(timeout=1)
        callback.assert_called()
        body, message = callback.call_args[0]
        assert body['complex']
Exemple #12
0
    def test_start(self):
        with patch('celery.contrib.migrate.eventloop') as evloop:
            app = Mock()
            filt = Mock(name='filter')
            conn = Connection('memory://')
            evloop.side_effect = StopFiltering()
            app.amqp.queues = {'foo': Queue('foo'), 'bar': Queue('bar')}
            consumer = app.amqp.TaskConsumer.return_value = Mock(name='consum')
            consumer.queues = list(app.amqp.queues.values())
            consumer.channel = conn.default_channel
            consumer.__enter__ = Mock(name='consumer.__enter__')
            consumer.__exit__ = Mock(name='consumer.__exit__')
            consumer.callbacks = []

            def register_callback(x):
                consumer.callbacks.append(x)
            consumer.register_callback = register_callback

            start_filter(app, conn, filt,
                         queues='foo,bar', ack_messages=True)
            body = {'task': 'add', 'id': 'id'}
            for callback in consumer.callbacks:
                callback(body, Message(body))
            consumer.callbacks[:] = []
            cb = Mock(name='callback=')
            start_filter(app, conn, filt, tasks='add,mul', callback=cb)
            for callback in consumer.callbacks:
                callback(body, Message(body))
            cb.assert_called()

            on_declare_queue = Mock()
            start_filter(app, conn, filt, tasks='add,mul', queues='foo',
                         on_declare_queue=on_declare_queue)
            on_declare_queue.assert_called()
            start_filter(app, conn, filt, queues=['foo', 'bar'])
            consumer.callbacks[:] = []
            state = State()
            start_filter(app, conn, filt,
                         tasks='add,mul', callback=cb, state=state, limit=1)
            stop_filtering_raised = False
            for callback in consumer.callbacks:
                try:
                    callback(body, Message(body))
                except StopFiltering:
                    stop_filtering_raised = True
            assert state.count
            assert stop_filtering_raised
Exemple #13
0
    def test_start(self):
        with patch("celery.contrib.migrate.eventloop") as evloop:
            app = Mock()
            filt = Mock(name="filter")
            conn = Connection("memory://")
            evloop.side_effect = StopFiltering()
            app.amqp.queues = {"foo": Queue("foo"), "bar": Queue("bar")}
            consumer = app.amqp.TaskConsumer.return_value = Mock(name="consum")
            consumer.queues = list(app.amqp.queues.values())
            consumer.channel = conn.default_channel
            consumer.__enter__ = Mock(name="consumer.__enter__")
            consumer.__exit__ = Mock(name="consumer.__exit__")
            consumer.callbacks = []

            def register_callback(x):
                consumer.callbacks.append(x)

            consumer.register_callback = register_callback

            start_filter(app, conn, filt, queues="foo,bar", ack_messages=True)
            body = {"task": "add", "id": "id"}
            for callback in consumer.callbacks:
                callback(body, Message(body))
            consumer.callbacks[:] = []
            cb = Mock(name="callback=")
            start_filter(app, conn, filt, tasks="add,mul", callback=cb)
            for callback in consumer.callbacks:
                callback(body, Message(body))
            cb.assert_called()

            on_declare_queue = Mock()
            start_filter(app, conn, filt, tasks="add,mul", queues="foo", on_declare_queue=on_declare_queue)
            on_declare_queue.assert_called()
            start_filter(app, conn, filt, queues=["foo", "bar"])
            consumer.callbacks[:] = []
            state = State()
            start_filter(app, conn, filt, tasks="add,mul", callback=cb, state=state, limit=1)
            stop_filtering_raised = False
            for callback in consumer.callbacks:
                try:
                    callback(body, Message(body))
                except StopFiltering:
                    stop_filtering_raised = True
            assert state.count
            assert stop_filtering_raised
Exemple #14
0
    def test_process_initializer(self, set_mp_process_title, _signals):
        with mock.restore_logging():
            from celery import signals
            from celery._state import _tls
            from celery.concurrency.prefork import (
                process_initializer, WORKER_SIGRESET, WORKER_SIGIGNORE,
            )
            on_worker_process_init = Mock()
            signals.worker_process_init.connect(on_worker_process_init)

            def Loader(*args, **kwargs):
                loader = Mock(*args, **kwargs)
                loader.conf = {}
                loader.override_backends = {}
                return loader

            with self.Celery(loader=Loader) as app:
                app.conf = AttributeDict(DEFAULTS)
                process_initializer(app, 'awesome.worker.com')
                _signals.ignore.assert_any_call(*WORKER_SIGIGNORE)
                _signals.reset.assert_any_call(*WORKER_SIGRESET)
                assert app.loader.init_worker.call_count
                on_worker_process_init.assert_called()
                assert _tls.current_app is app
                set_mp_process_title.assert_called_with(
                    'celeryd', hostname='awesome.worker.com',
                )

                with patch('celery.app.trace.setup_worker_optimizations') as S:
                    os.environ['FORKED_BY_MULTIPROCESSING'] = '1'
                    try:
                        process_initializer(app, 'luke.worker.com')
                        S.assert_called_with(app, 'luke.worker.com')
                    finally:
                        os.environ.pop('FORKED_BY_MULTIPROCESSING', None)

                os.environ['CELERY_LOG_FILE'] = 'worker%I.log'
                app.log.setup = Mock(name='log_setup')
                try:
                    process_initializer(app, 'luke.worker.com')
                finally:
                    os.environ.pop('CELERY_LOG_FILE', None)
Exemple #15
0
    def test_custom_request_gets_instantiated(self):
        _MyRequest = Mock(name='MyRequest')

        class MyRequest(Request):
            def __init__(self, *args, **kwargs):
                Request.__init__(self, *args, **kwargs)
                _MyRequest()

        class MyTask(Task):
            Request = MyRequest

        @self.app.task(base=MyTask)
        def failed():
            raise AssertionError

        sig = failed.s()
        with self._context(sig) as C:
            task_message_handler = default_strategy(
                failed,
                self.app,
                C.consumer
            )
            task_message_handler(C.message, None, None, None, None)
            _MyRequest.assert_called()
Exemple #16
0
    def test_pool_restart_reload_modules(self):
        consumer = Consumer(self.app)
        consumer.controller = _WC(app=self.app)
        consumer.controller.consumer = consumer
        consumer.controller.pool.restart = Mock()
        consumer.reset_rate_limits = Mock(name='reset_rate_limits()')
        consumer.update_strategies = Mock(name='update_strategies()')
        panel = self.create_panel(consumer=consumer)
        panel.app = self.app
        _import = panel.app.loader.import_from_cwd = Mock()
        _reload = Mock()

        self.app.conf.worker_pool_restarts = True
        with patch.dict(sys.modules, {'foo': None}):
            panel.handle('pool_restart', {
                'modules': ['foo'],
                'reload': False,
                'reloader': _reload,
            })

            consumer.controller.pool.restart.assert_called()
            _reload.assert_not_called()
            _import.assert_not_called()

            _import.reset_mock()
            _reload.reset_mock()
            consumer.controller.pool.restart.reset_mock()

            panel.handle('pool_restart', {
                'modules': ['foo'],
                'reload': True,
                'reloader': _reload,
            })
            consumer.controller.pool.restart.assert_called()
            _reload.assert_called()
            _import.assert_not_called()
Exemple #17
0
    def test_ensure_connection_on_error(self):
        c = Connection('amqp://A;amqp://B')
        with patch('kombu.connection.retry_over_time') as rot:
            c.ensure_connection()
            rot.assert_called()

            args = rot.call_args[0]
            cb = args[4]
            intervals = iter([1, 2, 3, 4, 5])
            assert cb(KeyError(), intervals, 0) == 0
            assert cb(KeyError(), intervals, 1) == 1
            assert cb(KeyError(), intervals, 2) == 0
            assert cb(KeyError(), intervals, 3) == 2
            assert cb(KeyError(), intervals, 4) == 0
            assert cb(KeyError(), intervals, 5) == 3
            assert cb(KeyError(), intervals, 6) == 0
            assert cb(KeyError(), intervals, 7) == 4

            errback = Mock()
            c.ensure_connection(errback=errback)
            args = rot.call_args[0]
            cb = args[4]
            assert cb(KeyError(), intervals, 0) == 0
            errback.assert_called()
Exemple #18
0
def test_reload_from_cwd_custom_reloader():
    reload = Mock()
    reload_from_cwd('foo', reload)
    reload.assert_called()
 def test_apply_target__raises_BaseException(self):
     target = Mock(name='target')
     callback = Mock(name='callback')
     target.side_effect = BaseException()
     apply_target(target, callback=callback)
     callback.assert_called()
Exemple #20
0
 def test_import_from_cwd_custom_imp(self):
     imp = Mock(name='imp')
     self.loader.import_from_cwd('foo', imp=imp)
     imp.assert_called()