def task_message_handler(message, body, ack, reject, callbacks, **kw):
if body is None:
body, headers, decoded, utc = (
message.body, message.headers, False, True,
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
request = Req(
message,
on_ack=ack, on_reject=reject, app=app, hostname=hostname,
eventer=eventer, task=task,
body=body, headers=headers, decoded=decoded, utc=utc,
connection_errors=connection_errors,
)
put_buffer(request)
if self._tref is None: # first request starts flush timer.
self._tref = timer.call_repeatedly(
self.flush_interval, flush_buffer,
)
if not next(self._count) % self.flush_every:
flush_buffer()
def task_message_handler(message, body, ack, reject, callbacks, **kw):
if body is None:
body, headers, decoded, utc = (
message.body, message.headers, False, True,
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
request = Req(
message,
on_ack=ack, on_reject=reject, app=app, hostname=hostname,
eventer=eventer, task=task,
body=body, headers=headers, decoded=decoded, utc=utc,
connection_errors=connection_errors,
)
put_buffer(request)
if self._tref is None: # first request starts flush timer.
self._tref = timer.call_repeatedly(
self.flush_interval, flush_buffer,
)
if not next(self._count) % self.flush_every:
flush_buffer()
def test_message(self):
body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
assert body == ((1,), {'foo': 'baz'}, {
'callbacks': None, 'errbacks': None, 'chord': None, 'chain': None,
})
assert headers == dict(self.body, group='123')
assert decoded
assert not utc
def test_message(self):
body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
self.assertTupleEqual(body, ((1,), {'foo': 'baz'}, {
'callbacks': None, 'errbacks': None, 'chord': None, 'chain': None,
}))
self.assertDictEqual(headers, dict(self.body, group='123'))
self.assertTrue(decoded)
self.assertFalse(utc)
def test_message(self):
body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
assert body == ((1,), {'foo': 'baz'}, {
'callbacks': None, 'errbacks': None, 'chord': None, 'chain': None,
})
assert headers == dict(self.body, group='123')
assert decoded
assert not utc
def test_message(self):
body, headers, decoded, utc = proto1_to_proto2(self.message, self.body)
self.assertTupleEqual(body, ((1, ), {
'foo': 'baz'
}, {
'callbacks': None,
'errbacks': None,
'chord': None,
'chain': None,
}))
self.assertDictEqual(headers, dict(self.body, group='123'))
self.assertTrue(decoded)
self.assertFalse(utc)
def task_message_handler(
message: Message,
body: Optional[Dict[str, Any]],
ack: promise,
reject: promise,
callbacks: Set,
**kw: Any,
) -> None:
if body is None and "args" not in message.payload:
body, headers, decoded, utc = (
message.body,
message.headers,
False,
app.uses_utc_timezone(),
)
else:
if "args" in message.payload:
body, headers, decoded, utc = hybrid_to_proto2(
message, message.payload
)
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
request = Req(
message,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
body=body,
headers=headers,
decoded=decoded,
utc=utc,
connection_errors=connection_errors,
)
put_buffer(request)
if self._tref is None: # first request starts flush timer.
self._tref = timer.call_repeatedly(self.flush_interval, flush_buffer)
if not next(self._count) % self.flush_every:
flush_buffer()
class Batches(Task):
abstract = True
#: Maximum number of message in buffer.
flush_every = 10
#: Timeout in seconds before buffer is flushed anyway.
flush_interval = 30
def __init__(self):
self._buffer = Queue()
self._count = count(1)
self._tref = None
self._pool = None
def run(self, requests):
raise NotImplementedError('must implement run(requests)')
def Strategy(self, task, app, consumer):
self._pool = consumer.pool
hostname = consumer.hostname
eventer = consumer.event_dispatcher
Req = Request
connection_errors = consumer.connection_errors
timer = consumer.timer
put_buffer = self._buffer.put
flush_buffer = self._do_flush
def task_message_handler(message, body, ack, reject, callbacks, **kw):
if body is None: 31513 ? S 125:09 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n [email protected] --app=mai
body, headers, decoded, utc = ( n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery6.pid
message.body, message.headers, False, True, 31528 ? R 128:34 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n [email protected] --app=mai
) n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery7.pid
if not body_can_be_buffer: 31543 ? S 124:32 /usr/bin/python -m celery worker --without-heartbeat -c 50 --pool=eventlet -n [email protected] --app=mai
body = bytes(body) if isinstance(body, buffer_t) else body n -Q rss --without-gossip --logfile=/home/logs/rss.log --pidfile=celery8.pid
else: 26150 ? S 0:50 /usr/bin/python -m celery worker --without-heartbeat -c 2 --pool=eventlet -n [email protected] --app=main
body, headers, decoded, utc = proto1_to_proto2(message, body) -Q engines --without-gossip --logfile=/home/logs/engines.log --pidfile=/home/logs/pid-engines.pid
22409 ? S 0:00 /usr/bin/python -m celery worker --without-heartbeat -c 1 -n [email protected] --app=m
def test_message_kwargs_not_mapping(self):
self.body['kwargs'] = (2,)
with pytest.raises(InvalidTaskError):
proto1_to_proto2(self.message, self.body)
def test_message_without_kwargs(self):
self.body.pop('kwargs')
body, _, _, _ = proto1_to_proto2(self.message, self.body)
assert body[:2] == ((1,), {})
def test_message_without_args(self):
self.body.pop('args')
body, _, _, _ = proto1_to_proto2(self.message, self.body)
assert body[:2] == ((), {'foo': 'baz'})
def test_message_no_taskset_id(self):
self.body.pop('taskset')
assert proto1_to_proto2(self.message, self.body)
def test_message_kwargs_not_mapping(self):
self.body['kwargs'] = (2, )
with pytest.raises(InvalidTaskError):
proto1_to_proto2(self.message, self.body)
def test_message_without_kwargs(self):
self.body.pop('kwargs')
body, _, _, _ = proto1_to_proto2(self.message, self.body)
assert body[:2] == ((1, ), {})
def test_message_without_kwargs(self):
self.body.pop('kwargs')
with self.assertRaises(InvalidTaskError):
proto1_to_proto2(self.message, self.body)
def test_message_without_kwargs(self):
self.body.pop('kwargs')
with pytest.raises(InvalidTaskError):
proto1_to_proto2(self.message, self.body)
def test_message_without_args(self):
self.body.pop('args')
with self.assertRaises(InvalidTaskError):
proto1_to_proto2(self.message, self.body)
def test_message_no_taskset_id(self):
self.body.pop('taskset')
assert proto1_to_proto2(self.message, self.body)
def test_message_without_args(self):
self.body.pop('args')
with pytest.raises(InvalidTaskError):
proto1_to_proto2(self.message, self.body)
def test_message_without_args(self):
self.body.pop('args')
body, _, _, _ = proto1_to_proto2(self.message, self.body)
assert body[:2] == ((), {'foo': 'baz'})
