def create_request_cls(self, **kwargs):
return create_request_cls(Request,
self.task,
self.pool,
'foo',
self.eventer,
app=self.app,
**kwargs)
def create_request_cls(self, **kwargs):
return create_request_cls(
Request, self.task, self.pool, 'foo', self.eventer, **kwargs
)
def Strategy(self, task: "Batches", app: Celery, consumer: Consumer) -> Callable:
# See celery.worker.strategy.default for inspiration.
#
# This adds to a buffer at the end, instead of executing the task as
# the default strategy does.
#
# See Batches._do_flush for ETA handling.
self._pool = consumer.pool
hostname = consumer.hostname
connection_errors = consumer.connection_errors
eventer = consumer.event_dispatcher
Request = symbol_by_name(task.Request)
# Celery 5.1 added the app argument to create_request_cls.
if CELERY_VERSION < (5, 1):
Req = create_request_cls(Request, task, consumer.pool, hostname, eventer)
else:
Req = create_request_cls(
Request, task, consumer.pool, hostname, eventer, app=app
)
timer = consumer.timer
put_buffer = self._buffer.put
flush_buffer = self._do_flush
def task_message_handler(
message: Message,
body: Optional[Dict[str, Any]],
ack: promise,
reject: promise,
callbacks: Set,
**kw: Any,
) -> None:
if body is None and "args" not in message.payload:
body, headers, decoded, utc = (
message.body,
message.headers,
False,
app.uses_utc_timezone(),
)
else:
if "args" in message.payload:
body, headers, decoded, utc = hybrid_to_proto2(
message, message.payload
)
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
request = Req(
message,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
body=body,
headers=headers,
decoded=decoded,
utc=utc,
connection_errors=connection_errors,
)
put_buffer(request)
if self._tref is None: # first request starts flush timer.
self._tref = timer.call_repeatedly(self.flush_interval, flush_buffer)
if not next(self._count) % self.flush_every:
flush_buffer()
return task_message_handler
def default(task,
app,
consumer,
info=logger.info,
error=logger.error,
task_reserved=task_reserved,
to_system_tz=timezone.to_system,
bytes=bytes,
buffer_t=buffer_t,
proto1_to_proto2=proto1_to_proto2):
"""Default task execution strategy.
Note:
Strategies are here as an optimization, so sadly
it's not very easy to override.
"""
hostname = consumer.hostname
connection_errors = consumer.connection_errors
_does_info = logger.isEnabledFor(logging.INFO)
# task event related
# (optimized to avoid calling request.send_event)
eventer = consumer.event_dispatcher
events = eventer and eventer.enabled
send_event = eventer.send
task_sends_events = events and task.send_events
call_at = consumer.timer.call_at
apply_eta_task = consumer.apply_eta_task
rate_limits_enabled = not consumer.disable_rate_limits
get_bucket = consumer.task_buckets.__getitem__
handle = consumer.on_task_request
limit_task = consumer._limit_task
limit_post_eta = consumer._limit_post_eta
body_can_be_buffer = consumer.pool.body_can_be_buffer
Request = symbol_by_name(task.Request)
Req = create_request_cls(Request, task, consumer.pool, hostname, eventer)
revoked_tasks = consumer.controller.state.revoked
def task_message_handler(message,
body,
ack,
reject,
callbacks,
to_timestamp=to_timestamp):
if body is None and 'args' not in message.payload:
body, headers, decoded, utc = (
message.body,
message.headers,
False,
app.uses_utc_timezone(),
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
else:
if 'args' in message.payload:
body, headers, decoded, utc = hybrid_to_proto2(
message, message.payload)
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
req = Req(
message,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
connection_errors=connection_errors,
body=body,
headers=headers,
decoded=decoded,
utc=utc,
)
if _does_info:
info('Received task: %s', req)
if (req.expires or req.id in revoked_tasks) and req.revoked():
return
if task_sends_events:
send_event(
'task-received',
uuid=req.id,
name=req.name,
args=req.argsrepr,
kwargs=req.kwargsrepr,
root_id=req.root_id,
parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
bucket = None
eta = None
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, app.timezone)
except (OverflowError, ValueError) as exc:
error("Couldn't convert ETA %r to timestamp: %r. Task: %r",
req.eta,
exc,
req.info(safe=True),
exc_info=True)
req.reject(requeue=False)
if rate_limits_enabled:
bucket = get_bucket(task.name)
if eta and bucket:
consumer.qos.increment_eventually()
return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6)
if eta:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req, ), priority=6)
return task_message_handler
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
return task_message_handler
