Example #1
0
    def test_do_restore_message(self):
        client = Mock(name='client')
        pl1 = {'body': 'BODY'}
        spl1 = dumps(pl1)
        lookup = self.channel._lookup = Mock(name='_lookup')
        lookup.return_value = {'george', 'elaine'}
        self.channel._do_restore_message(
            pl1, 'ex', 'rkey', client,
        )
        client.rpush.assert_has_calls([
            call('george', spl1), call('elaine', spl1),
        ], any_order=True)

        client = Mock(name='client')
        pl2 = {'body': 'BODY2', 'headers': {'x-funny': 1}}
        headers_after = dict(pl2['headers'], redelivered=True)
        spl2 = dumps(dict(pl2, headers=headers_after))
        self.channel._do_restore_message(
            pl2, 'ex', 'rkey', client,
        )
        client.rpush.assert_any_call('george', spl2)
        client.rpush.assert_any_call('elaine', spl2)

        client.rpush.side_effect = KeyError()
        with patch('kombu.transport.redis.crit') as crit:
            self.channel._do_restore_message(
                pl2, 'ex', 'rkey', client,
            )
            crit.assert_called()
Example #2
0
    def test_enter__exit(self):
        P = self.hub.poller = Mock()
        on_close = Mock()
        self.hub.on_close.add(on_close)

        try:
            read_A = Mock()
            read_B = Mock()
            self.hub.add_reader(10, read_A)
            self.hub.add_reader(File(11), read_B)
            write_A = Mock()
            write_B = Mock()
            self.hub.add_writer(20, write_A)
            self.hub.add_writer(File(21), write_B)
            assert self.hub.readers
            assert self.hub.writers
        finally:
            assert self.hub.poller
            self.hub.close()
        assert not self.hub.readers
        assert not self.hub.writers

        P.unregister.assert_has_calls([
            call(10), call(11), call(20), call(21),
        ], any_order=True)

        on_close.assert_called_with(self.hub)
Example #3
0
    def test_get(self, mock_client):
        self.backend.get(b"mykey")

        mock_client.ReadDocument.assert_has_calls(
            [call("dbs/celerydb/colls/celerycol/docs/mykey",
                  {"partitionKey": "mykey"}),
             call().get("value")])
Example #4
0
    def test_pool_restart_import_modules(self, _debug):
        consumer = Consumer(self.app)
        consumer.controller = _WC(app=self.app)
        consumer.controller.consumer = consumer
        consumer.controller.pool.restart = Mock()
        consumer.reset_rate_limits = Mock(name='reset_rate_limits()')
        consumer.update_strategies = Mock(name='update_strategies()')
        panel = self.create_panel(consumer=consumer)
        panel.app = self.app
        assert panel.state.consumer.controller.consumer is consumer
        _import = consumer.controller.app.loader.import_from_cwd = Mock()
        _reload = Mock()

        self.app.conf.worker_pool_restarts = True
        with patch('sys.modules'):
            panel.handle('pool_restart', {
                'modules': ['foo', 'bar'],
                'reloader': _reload,
            })
        consumer.controller.pool.restart.assert_called()
        consumer.reset_rate_limits.assert_called_with()
        consumer.update_strategies.assert_called_with()
        _reload.assert_not_called()
        _import.assert_has_calls([call('bar'), call('foo')], any_order=True)
        assert _import.call_count == 2
Example #5
0
 def test_on_tick(self, sleep):
     on_tick = Mock(name='on_tick')
     t = timer2.Timer(on_tick=on_tick)
     ne = t._next_entry = Mock(name='_next_entry')
     ne.return_value = 3.33
     ne.on_nth_call_do(t._is_shutdown.set, 3)
     t.run()
     sleep.assert_called_with(3.33)
     on_tick.assert_has_calls([call(3.33), call(3.33), call(3.33)])
Example #6
0
    def test_with_uid(self, initgroups, setuid, setgid,
                      getpwuid, parse_gid, parse_uid, getuid, geteuid,
                      getgid, getegid):
        geteuid.return_value = 10
        getuid.return_value = 10

        class pw_struct(object):
            pw_gid = 50001

        def raise_on_second_call(*args, **kwargs):
            setuid.side_effect = OSError()
            setuid.side_effect.errno = errno.EPERM
        setuid.side_effect = raise_on_second_call
        getpwuid.return_value = pw_struct()
        parse_uid.return_value = 5001
        parse_gid.return_value = 5001
        maybe_drop_privileges(uid='user')
        parse_uid.assert_called_with('user')
        getpwuid.assert_called_with(5001)
        setgid.assert_called_with(50001)
        initgroups.assert_called_with(5001, 50001)
        setuid.assert_has_calls([call(5001), call(0)])

        setuid.side_effect = raise_on_second_call

        def to_root_on_second_call(mock, first):
            return_value = [first]

            def on_first_call(*args, **kwargs):
                ret, return_value[0] = return_value[0], 0
                return ret
            mock.side_effect = on_first_call
        to_root_on_second_call(geteuid, 10)
        to_root_on_second_call(getuid, 10)
        with pytest.raises(SecurityError):
            maybe_drop_privileges(uid='user')

        getuid.return_value = getuid.side_effect = None
        geteuid.return_value = geteuid.side_effect = None
        getegid.return_value = 0
        getgid.return_value = 0
        setuid.side_effect = raise_on_second_call
        with pytest.raises(SecurityError):
            maybe_drop_privileges(gid='group')

        getuid.reset_mock()
        geteuid.reset_mock()
        setuid.reset_mock()
        getuid.side_effect = geteuid.side_effect = None

        def raise_on_second_call(*args, **kwargs):
            setuid.side_effect = OSError()
            setuid.side_effect.errno = errno.ENOENT
        setuid.side_effect = raise_on_second_call
        with pytest.raises(OSError):
            maybe_drop_privileges(uid='user')
Example #7
0
    def test_receive_multi(self):
        r = self.app.events.Receiver(Mock(name='connection'))
        r.process = Mock(name='process')
        efm = r.event_from_message = Mock(name='event_from_message')

        def on_efm(*args):
            return args
        efm.side_effect = on_efm
        r._receive([1, 2, 3], Mock())
        r.process.assert_has_calls([call(1), call(2), call(3)])
Example #8
0
def test_close_open_fds(patching):
    _close = patching('os.close')
    fdmax = patching('celery.platforms.get_fdmax')
    with patch('os.closerange', create=True) as closerange:
        fdmax.return_value = 3
        close_open_fds()
        if not closerange.called:
            _close.assert_has_calls([call(2), call(1), call(0)])
            _close.side_effect = OSError()
            _close.side_effect.errno = errno.EBADF
        close_open_fds()
Example #9
0
    def test_mget(self, mock_client):
        keys = [b"mykey1", b"mykey2"]

        self.backend.mget(keys)

        mock_client.ReadDocument.assert_has_calls(
            [call("dbs/celerydb/colls/celerycol/docs/mykey1",
                  {"partitionKey": "mykey1"}),
             call().get("value"),
             call("dbs/celerydb/colls/celerycol/docs/mykey2",
                  {"partitionKey": "mykey2"}),
             call().get("value")])
Example #10
0
    def test_on_chord_part_return(self, restore):
        tasks = [self.create_task() for i in range(10)]

        for i in range(10):
            self.b.on_chord_part_return(tasks[i].request, states.SUCCESS, i)
            assert self.b.client.rpush.call_count
            self.b.client.rpush.reset_mock()
        assert self.b.client.lrange.call_count
        jkey = self.b.get_key_for_group("group_id", ".j")
        tkey = self.b.get_key_for_group("group_id", ".t")
        self.b.client.delete.assert_has_calls([call(jkey), call(tkey)])
        self.b.client.expire.assert_has_calls([call(jkey, 86400), call(tkey, 86400)])
Example #11
0
 def test_set_timeout(self):
     # Checks that context manager sets and reverts timeout properly
     with patch.object(self.t, 'sock') as sock_mock:
         sock_mock.gettimeout.return_value = 3
         with self.t.having_timeout(5) as actual_sock:
             assert actual_sock == self.t.sock
         sock_mock.gettimeout.assert_called()
         sock_mock.settimeout.assert_has_calls(
             [
                 call(5),
                 call(3),
             ]
         )
Example #12
0
 def test_blocking_read__timeout(self):
     self.conn.transport = TCPTransport('localhost:5672')
     sock = self.conn.transport.sock = Mock(name='sock')
     sock.gettimeout.return_value = 1
     self.conn.transport.read_frame = Mock(name='read_frame')
     self.conn.on_inbound_frame = Mock(name='on_inbound_frame')
     self.conn.blocking_read(3)
     sock.gettimeout.assert_called_with()
     sock.settimeout.assert_has_calls([call(3), call(1)])
     self.conn.transport.read_frame.assert_called_with()
     self.conn.on_inbound_frame.assert_called_with(
         self.conn.transport.read_frame(),
     )
     sock.gettimeout.return_value = 3
     self.conn.blocking_read(3)
Example #13
0
    def test_init(self):
        with patch('kombu.asynchronous.http.curl.pycurl') as _pycurl:
            x = self.Client()
            assert x._multi is not None
            assert x._pending is not None
            assert x._free_list is not None
            assert x._fds is not None
            assert x._socket_action == x._multi.socket_action
            assert len(x._curls) == x.max_clients
            assert x._timeout_check_tref

            x._multi.setopt.assert_has_calls([
                call(_pycurl.M_TIMERFUNCTION, x._set_timeout),
                call(_pycurl.M_SOCKETFUNCTION, x._handle_socket),
            ])
Example #14
0
 def test_set_timeout_exception_raised(self):
     # Checks that context manager sets and reverts timeout properly
     # when exception is raised.
     with patch.object(self.t, 'sock') as sock_mock:
         sock_mock.gettimeout.return_value = 3
         with pytest.raises(DummyException):
             with self.t.having_timeout(5) as actual_sock:
                 assert actual_sock == self.t.sock
                 raise DummyException()
         sock_mock.gettimeout.assert_called()
         sock_mock.settimeout.assert_has_calls(
             [
                 call(5),
                 call(3),
             ]
         )
Example #15
0
    def test_on_chord_part_return_no_expiry(self, restore):
        old_expires = self.b.expires
        self.b.expires = None
        tasks = [self.create_task() for i in range(10)]

        for i in range(10):
            self.b.on_chord_part_return(tasks[i].request, states.SUCCESS, i)
            assert self.b.client.rpush.call_count
            self.b.client.rpush.reset_mock()
        assert self.b.client.lrange.call_count
        jkey = self.b.get_key_for_group('group_id', '.j')
        tkey = self.b.get_key_for_group('group_id', '.t')
        self.b.client.delete.assert_has_calls([call(jkey), call(tkey)])
        self.b.client.expire.assert_not_called()

        self.b.expires = old_expires
Example #16
0
    def test_get_many(self):
        for is_dict in True, False:
            self.b.mget_returns_dict = is_dict
            ids = {uuid(): i for i in range(10)}
            for id, i in items(ids):
                self.b.mark_as_done(id, i)
            it = self.b.get_many(list(ids), interval=0.01)
            for i, (got_id, got_state) in enumerate(it):
                assert got_state['result'] == ids[got_id]
            assert i == 9
            assert list(self.b.get_many(list(ids), interval=0.01))

            self.b._cache.clear()
            callback = Mock(name='callback')
            it = self.b.get_many(
                list(ids),
                on_message=callback,
                interval=0.05
            )
            for i, (got_id, got_state) in enumerate(it):
                assert got_state['result'] == ids[got_id]
            assert i == 9
            assert list(
                self.b.get_many(list(ids), interval=0.01)
            )
            callback.assert_has_calls([
                call(ANY) for id in ids
            ])
Example #17
0
    def test_send_buffer_group(self):
        buf_received = [None]
        producer = MockProducer()
        producer.connection = self.app.connection_for_write()
        connection = Mock()
        connection.transport.driver_type = 'amqp'
        eventer = self.app.events.Dispatcher(
            connection, enabled=False,
            buffer_group={'task'}, buffer_limit=2,
        )
        eventer.producer = producer
        eventer.enabled = True
        eventer._publish = Mock(name='_publish')

        def on_eventer_publish(events, *args, **kwargs):
            buf_received[0] = list(events)
        eventer._publish.side_effect = on_eventer_publish
        assert not eventer._group_buffer['task']
        eventer.on_send_buffered = Mock(name='on_send_buffered')
        eventer.send('task-received', uuid=1)
        prev_buffer = eventer._group_buffer['task']
        assert eventer._group_buffer['task']
        eventer.on_send_buffered.assert_called_with()
        eventer.send('task-received', uuid=1)
        assert not eventer._group_buffer['task']
        eventer._publish.assert_has_calls([
            call([], eventer.producer, 'task.multi'),
        ])
        # clear in place
        assert eventer._group_buffer['task'] is prev_buffer
        assert len(buf_received[0]) == 2
        eventer.on_send_buffered = None
        eventer.send('task-received', uuid=1)
Example #18
0
    def test_on_chord_part_return(self, restore):
        tasks = [self.create_task() for i in range(10)]
        self.b.result_consumer.start(tasks[0].request.id)

        for i in range(10):
            self.b.on_chord_part_return(tasks[i].request, states.SUCCESS, i)
            assert self.b.client.rpush.call_count
            self.b.client.rpush.reset_mock()
        assert self.b.client.lrange.call_count
        jkey = self.b.get_key_for_group('group_id', '.j')
        tkey = self.b.get_key_for_group('group_id', '.t')
        self.b.client.delete.assert_has_calls([call(jkey), call(tkey)])
        self.b.client.expire.assert_has_calls([
            call(jkey, 86400), call(tkey, 86400),
        ])
        self.b.result_consumer._pubsub.unsubscribe.assert_has_calls([
            call(self.b.get_key_for_task(task.request.id)) for task in tasks
        ])
Example #19
0
 def test_send_all(self):
     nodes = [Mock(name='n1'), Mock(name='n2')]
     self.cluster.getpids = Mock(name='getpids')
     self.cluster.getpids.return_value = nodes
     self.cluster.send_all(15)
     self.cluster.on_node_signal.assert_has_calls(
         call(node, 'TERM') for node in nodes
     )
     for node in nodes:
         node.send.assert_called_with(15, self.cluster.on_node_signal_dead)
Example #20
0
    def test_close_deletes_autodelete_fanout_queues(self):
        self.channel._fanout_queues = {'foo': ('foo', ''), 'bar': ('bar', '')}
        self.channel.auto_delete_queues = ['foo']
        self.channel.queue_delete = Mock(name='queue_delete')

        client = self.channel.client
        self.channel.close()
        self.channel.queue_delete.assert_has_calls([
            call('foo', client=client),
        ])
Example #21
0
 def test_join_native(self):
     backend = SimpleBackend()
     results = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)]
     ts = self.app.GroupResult(uuid(), results)
     ts.app.backend = backend
     backend.ids = [result.id for result in results]
     res = ts.join_native()
     assert res == list(range(10))
     callback = Mock(name="callback")
     assert not ts.join_native(callback=callback)
     callback.assert_has_calls([call(r.id, i) for i, r in enumerate(ts.results)])
Example #22
0
 def assert_context(self, default, full, get_returns=None, set_effect=None):
     with termsigs(default, full):
         with patch('signal.getsignal') as GET:
             with patch('signal.signal') as SET:
                 GET.return_value = get_returns
                 SET.side_effect = set_effect
                 reset_signals()
                 GET.assert_has_calls([
                     call(signo(sig)) for sig in default
                 ])
                 yield GET, SET
Example #23
0
    def test_register_with_event_loop(self):
        transport = self.connection.transport
        transport.cycle = Mock(name='cycle')
        transport.cycle.fds = {12: 'LISTEN', 13: 'BRPOP'}
        conn = Mock(name='conn')
        loop = Mock(name='loop')
        redis.Transport.register_with_event_loop(transport, conn, loop)
        transport.cycle.on_poll_init.assert_called_with(loop.poller)
        loop.call_repeatedly.assert_called_with(
            10, transport.cycle.maybe_restore_messages,
        )
        loop.on_tick.add.assert_called()
        on_poll_start = loop.on_tick.add.call_args[0][0]

        on_poll_start()
        transport.cycle.on_poll_start.assert_called_with()
        loop.add_reader.assert_has_calls([
            call(12, transport.on_readable, 12),
            call(13, transport.on_readable, 13),
        ])
Example #24
0
    def test_has_queue(self):
        self.channel._create_client = Mock()
        self.channel._create_client.return_value = self.channel.client
        exists = self.channel.client.exists = Mock()
        exists.return_value = True
        assert self.channel._has_queue('foo')
        exists.assert_has_calls([
            call(self.channel._q_for_pri('foo', pri))
            for pri in redis.PRIORITY_STEPS
        ])

        exists.return_value = False
        assert not self.channel._has_queue('foo')
Example #25
0
    def test_pool_restart_import_modules(self, _debug):
        consumer = Consumer(self.app)
        consumer.controller = _WC(app=self.app)
        consumer.controller.consumer = consumer
        consumer.controller.pool.restart = Mock()
        consumer.reset_rate_limits = Mock(name="reset_rate_limits()")
        consumer.update_strategies = Mock(name="update_strategies()")
        panel = self.create_panel(consumer=consumer)
        panel.app = self.app
        assert panel.state.consumer.controller.consumer is consumer
        _import = consumer.controller.app.loader.import_from_cwd = Mock()
        _reload = Mock()

        self.app.conf.worker_pool_restarts = True
        with patch("sys.modules"):
            panel.handle("pool_restart", {"modules": ["foo", "bar"], "reloader": _reload})
        consumer.controller.pool.restart.assert_called()
        consumer.reset_rate_limits.assert_called_with()
        consumer.update_strategies.assert_called_with()
        _reload.assert_not_called()
        _import.assert_has_calls([call("bar"), call("foo")], any_order=True)
        assert _import.call_count == 2
Example #26
0
    def test_delete(self):
        x = self.channel
        x._create_client = Mock()
        x._create_client.return_value = x.client
        delete = x.client.delete = Mock()
        srem = x.client.srem = Mock()

        x._delete('queue', 'exchange', 'routing_key', None)
        delete.assert_has_calls([
            call(x._q_for_pri('queue', pri)) for pri in redis.PRIORITY_STEPS
        ])
        srem.assert_called_with(x.keyprefix_queue % ('exchange',),
                                x.sep.join(['routing_key', '', 'queue']))
Example #27
0
    def test_with_guid(self, initgroups, setuid, setgid,
                       parse_gid, parse_uid):

        def raise_on_second_call(*args, **kwargs):
            setuid.side_effect = OSError()
            setuid.side_effect.errno = errno.EPERM
        setuid.side_effect = raise_on_second_call
        parse_uid.return_value = 5001
        parse_gid.return_value = 50001
        maybe_drop_privileges(uid='user', gid='group')
        parse_uid.assert_called_with('user')
        parse_gid.assert_called_with('group')
        setgid.assert_called_with(50001)
        initgroups.assert_called_with(5001, 50001)
        setuid.assert_has_calls([call(5001), call(0)])

        setuid.side_effect = None
        with pytest.raises(SecurityError):
            maybe_drop_privileges(uid='user', gid='group')
        setuid.side_effect = OSError()
        setuid.side_effect.errno = errno.EINVAL
        with pytest.raises(OSError):
            maybe_drop_privileges(uid='user', gid='group')
Example #28
0
    def test_add_writers(self):
        P = self.hub.poller = Mock()

        write_A = Mock()
        write_B = Mock()
        self.hub.add_writer(20, write_A)
        self.hub.add_writer(File(21), write_B)

        P.register.assert_has_calls([
            call(20, self.hub.WRITE),
            call(21, self.hub.WRITE),
        ], any_order=True)

        assert self.hub.writers[20], (write_A == ())
        assert self.hub.writers[21], (write_B == ())

        self.hub.remove(20)
        assert 20 not in self.hub.writers
        self.hub.remove(File(21))
        assert 21 not in self.hub.writers
        P.unregister.assert_has_calls([
            call(20), call(21),
        ])
Example #29
0
    def test_add_remove_readers(self):
        P = self.hub.poller = Mock()

        read_A = Mock()
        read_B = Mock()
        self.hub.add_reader(10, read_A, 10)
        self.hub.add_reader(File(11), read_B, 11)

        P.register.assert_has_calls([
            call(10, self.hub.READ | self.hub.ERR),
            call(11, self.hub.READ | self.hub.ERR),
        ], any_order=True)

        assert self.hub.readers[10] == (read_A, (10,))
        assert self.hub.readers[11] == (read_B, (11,))

        self.hub.remove(10)
        assert 10 not in self.hub.readers
        self.hub.remove(File(11))
        assert 11 not in self.hub.readers
        P.unregister.assert_has_calls([
            call(10), call(11),
        ])
Example #30
0
    def test_gssapi_step_with_client_name(self):
        with self.fake_gssapi() as gssapi:
            context = Mock()
            context.step.return_value = b'secrets'
            client_name, service_name, credentials = Mock(), Mock(), Mock()
            gssapi.SecurityContext.return_value = context
            gssapi.Credentials.return_value = credentials
            gssapi.Name.side_effect = [client_name, service_name]
            connection = Mock()
            connection.transport.host = 'broker.example.org'
            GSSAPI = sasl._get_gssapi_mechanism()

            mech = GSSAPI(client_name='amqp-client/client.example.org')
            response = mech.start(connection)
            gssapi.Name.assert_has_calls([
                call(b'amqp-client/client.example.org'),
                call(b'*****@*****.**',
                     gssapi.NameType.hostbased_service)])
            gssapi.Credentials.assert_called_with(name=client_name)
            gssapi.SecurityContext.assert_called_with(name=service_name,
                                                      creds=credentials)
            context.step.assert_called_with(None)
            assert response == b'secrets'