Exemple #1
0
        def task_message_handler(message, body, ack, reject, callbacks, **kw):
            if body is None:
                body, headers, decoded, utc = (
                    message.body, message.headers, False, True,
                )
                if not body_can_be_buffer:
                    body = bytes(body) if isinstance(body, buffer_t) else body
            else:
                body, headers, decoded, utc = proto1_to_proto2(message, body)

            request = Req(
                message,
                on_ack=ack, on_reject=reject, app=app, hostname=hostname,
                eventer=eventer, task=task,
                body=body, headers=headers, decoded=decoded, utc=utc,
                connection_errors=connection_errors,
            )
            put_buffer(request)

            if self._tref is None:     # first request starts flush timer.
                self._tref = timer.call_repeatedly(
                    self.flush_interval, flush_buffer,
                )

            if not next(self._count) % self.flush_every:
                flush_buffer()
Exemple #2
0
        def task_message_handler(message, body, ack, reject, callbacks, **kw):
            if body is None:
                body, headers, decoded, utc = (
                    message.body, message.headers, False, True,
                )
                if not body_can_be_buffer:
                    body = bytes(body) if isinstance(body, buffer_t) else body
            else:
                body, headers, decoded, utc = proto1_to_proto2(message, body)

            request = Req(
                message,
                on_ack=ack, on_reject=reject, app=app, hostname=hostname,
                eventer=eventer, task=task,
                body=body, headers=headers, decoded=decoded, utc=utc,
                connection_errors=connection_errors,
            )
            put_buffer(request)

            if self._tref is None:     # first request starts flush timer.
                self._tref = timer.call_repeatedly(
                    self.flush_interval, flush_buffer,
                )

            if not next(self._count) % self.flush_every:
                flush_buffer()
Exemple #3
0
 def test_message(self):
     body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
     assert body == ((1,), {'foo': 'baz'}, {
         'callbacks': None, 'errbacks': None, 'chord': None, 'chain': None,
     })
     assert headers == dict(self.body, group='123')
     assert decoded
     assert not utc
Exemple #4
0
 def test_message(self):
     body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
     self.assertTupleEqual(body, ((1,), {'foo': 'baz'}, {
         'callbacks': None, 'errbacks': None, 'chord': None, 'chain': None,
     }))
     self.assertDictEqual(headers, dict(self.body, group='123'))
     self.assertTrue(decoded)
     self.assertFalse(utc)
 def test_message(self):
     body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
     assert body == ((1,), {'foo': 'baz'}, {
         'callbacks': None, 'errbacks': None, 'chord': None, 'chain': None,
     })
     assert headers == dict(self.body, group='123')
     assert decoded
     assert not utc
Exemple #6
0
 def test_message(self):
     body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
     self.assertTupleEqual(body, ((1, ), {
         'foo': 'baz'
     }, {
         'callbacks': None,
         'errbacks': None,
         'chord': None,
         'chain': None,
     }))
     self.assertDictEqual(headers, dict(self.body, group='123'))
     self.assertTrue(decoded)
     self.assertFalse(utc)
Exemple #7
0
        def task_message_handler(
            message: Message,
            body: Optional[Dict[str, Any]],
            ack: promise,
            reject: promise,
            callbacks: Set,
            **kw: Any,
        ) -> None:
            if body is None and "args" not in message.payload:
                body, headers, decoded, utc = (
                    message.body,
                    message.headers,
                    False,
                    app.uses_utc_timezone(),
                )
            else:
                if "args" in message.payload:
                    body, headers, decoded, utc = hybrid_to_proto2(
                        message, message.payload
                    )
                else:
                    body, headers, decoded, utc = proto1_to_proto2(message, body)

            request = Req(
                message,
                on_ack=ack,
                on_reject=reject,
                app=app,
                hostname=hostname,
                eventer=eventer,
                task=task,
                body=body,
                headers=headers,
                decoded=decoded,
                utc=utc,
                connection_errors=connection_errors,
            )
            put_buffer(request)

            if self._tref is None:  # first request starts flush timer.
                self._tref = timer.call_repeatedly(self.flush_interval, flush_buffer)

            if not next(self._count) % self.flush_every:
                flush_buffer()
Exemple #8
0
class Batches(Task):
    abstract = True

    #: Maximum number of message in buffer.
    flush_every = 10

    #: Timeout in seconds before buffer is flushed anyway.
    flush_interval = 30

    def __init__(self):
        self._buffer = Queue()
        self._count = count(1)
        self._tref = None
        self._pool = None

    def run(self, requests):
        raise NotImplementedError('must implement run(requests)')

    def Strategy(self, task, app, consumer):
        self._pool = consumer.pool
        hostname = consumer.hostname
        eventer = consumer.event_dispatcher
        Req = Request
        connection_errors = consumer.connection_errors
        timer = consumer.timer
        put_buffer = self._buffer.put
        flush_buffer = self._do_flush

        def task_message_handler(message, body, ack, reject, callbacks, **kw):
            if body is None:                                                                                                                        31513 ?        S    125:09 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n [email protected] --app=mai
                body, headers, decoded, utc = (                                                                                                     n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery6.pid
                    message.body, message.headers, False, True,                                                                                     31528 ?        R    128:34 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n [email protected] --app=mai
                )                                                                                                                                   n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery7.pid
                if not body_can_be_buffer:                                                                                                          31543 ?        S    124:32 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n [email protected] --app=mai
                    body = bytes(body) if isinstance(body, buffer_t) else body                                                                      n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery8.pid
            else:                                                                                                                                   26150 ?        S      0:50 /usr/bin/python -m celery worker --without-heartbeat -c 2 --pool=eventlet -n [email protected] --app=main
                body, headers, decoded, utc = proto1_to_proto2(message, body)                                                                        -Q engines --without-gossip --logfile=/home/logs/engines.log --pidfile=/home/logs/pid-engines.pid
                                                                                                                                                    22409 ?        S      0:00 /usr/bin/python -m celery worker --without-heartbeat -c 1 -n [email protected] --app=m
Exemple #9
0
 def test_message_kwargs_not_mapping(self):
     self.body['kwargs'] = (2,)
     with pytest.raises(InvalidTaskError):
         proto1_to_proto2(self.message, self.body)
Exemple #10
0
 def test_message_without_kwargs(self):
     self.body.pop('kwargs')
     body, _, _, _ = proto1_to_proto2(self.message, self.body)
     assert body[:2] == ((1,), {})
Exemple #11
0
 def test_message_without_args(self):
     self.body.pop('args')
     body, _, _, _ = proto1_to_proto2(self.message, self.body)
     assert body[:2] == ((), {'foo': 'baz'})
Exemple #12
0
 def test_message_no_taskset_id(self):
     self.body.pop('taskset')
     assert proto1_to_proto2(self.message, self.body)
Exemple #13
0
 def test_message_kwargs_not_mapping(self):
     self.body['kwargs'] = (2, )
     with pytest.raises(InvalidTaskError):
         proto1_to_proto2(self.message, self.body)
Exemple #14
0
 def test_message_without_kwargs(self):
     self.body.pop('kwargs')
     body, _, _, _ = proto1_to_proto2(self.message, self.body)
     assert body[:2] == ((1, ), {})
Exemple #15
0
 def test_message_without_kwargs(self):
     self.body.pop('kwargs')
     with self.assertRaises(InvalidTaskError):
         proto1_to_proto2(self.message, self.body)
Exemple #16
0
 def test_message_without_kwargs(self):
     self.body.pop('kwargs')
     with pytest.raises(InvalidTaskError):
         proto1_to_proto2(self.message, self.body)
Exemple #17
0
 def test_message_without_args(self):
     self.body.pop('args')
     with self.assertRaises(InvalidTaskError):
         proto1_to_proto2(self.message, self.body)
Exemple #18
0
 def test_message_no_taskset_id(self):
     self.body.pop('taskset')
     assert proto1_to_proto2(self.message, self.body)
 def test_message_without_args(self):
     self.body.pop('args')
     with pytest.raises(InvalidTaskError):
         proto1_to_proto2(self.message, self.body)
Exemple #20
0
 def test_message_without_args(self):
     self.body.pop('args')
     body, _, _, _ = proto1_to_proto2(self.message, self.body)
     assert body[:2] == ((), {'foo': 'baz'})