def task_message_handler(
message, body, ack, reject, callbacks, to_timestamp=to_timestamp
):
req = Req(
body,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
connection_errors=connection_errors,
message=message,
)
if req.revoked():
return
if _does_info:
info("Received task: %s", req)
if events:
send_event(
"task-received",
uuid=req.id,
name=req.name,
args=safe_repr(req.args),
kwargs=safe_repr(req.kwargs),
retries=req.request_dict.get("retries", 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, timezone.local)
except OverflowError as exc:
error(
"Couldn't convert eta %s to timestamp: %r. Task: %r",
req.eta,
exc,
req.info(safe=True),
exc_info=True,
)
req.acknowledge()
else:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req,), priority=6)
else:
if rate_limits_enabled:
bucket = get_bucket(task.name)
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback() for callback in callbacks]
handle(req)
def task_message_handler(message, body, ack, reject, callbacks,
to_timestamp=to_timestamp):
if body is None:
body, headers, decoded, utc = (
message.body, message.headers, False, app.uses_utc_timezone(),
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
req = Req(
message,
on_ack=ack, on_reject=reject, app=app, hostname=hostname,
eventer=eventer, task=task, connection_errors=connection_errors,
body=body, headers=headers, decoded=decoded, utc=utc,
)
if _does_info:
info('Received task: %s', req)
if (req.expires or req.id in revoked_tasks) and req.revoked():
return
if task_sends_events:
send_event(
'task-received',
uuid=req.id, name=req.name,
args=req.argsrepr, kwargs=req.kwargsrepr,
root_id=req.root_id, parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, app.timezone)
except (OverflowError, ValueError) as exc:
error("Couldn't convert ETA %r to timestamp: %r. Task: %r",
req.eta, exc, req.info(safe=True), exc_info=True)
req.reject(requeue=False)
else:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req,), priority=6)
else:
if rate_limits_enabled:
bucket = get_bucket(task.name)
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
def _do_flush(self) -> None:
logger.debug("Batches: Wake-up to flush buffers...")
ready_requests = []
app = self.app
to_system_tz = timezone.to_system
now = monotonic()
all_requests = list(consume_queue(self._buffer)) + list(
consume_queue(self._pending)
)
for req in all_requests:
# Similar to logic in celery.worker.strategy.default.
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, app.timezone)
except (OverflowError, ValueError) as exc:
logger.error(
"Couldn't convert ETA %r to timestamp: %r. Task: %r",
req.eta,
exc,
req.info(safe=True),
exc_info=True,
)
req.reject(requeue=False)
continue
if eta <= now:
# ETA has elapsed, request is ready.
ready_requests.append(req)
else:
# ETA has not elapsed, add to pending queue.
self._pending.put(req)
else:
# Request does not have an ETA, ready immediately
ready_requests.append(req)
if len(ready_requests) > 0:
logger.debug("Batches: Ready buffer complete: %s", len(ready_requests))
self.flush(ready_requests)
if not ready_requests and self._pending.qsize() == 0:
logger.debug("Batches: Canceling timer: Nothing in buffers.")
if self._tref:
self._tref.cancel() # cancel timer.
self._tref = None
def task_message_handler(message, body, ack, reject, callbacks,
to_timestamp=to_timestamp):
if body is None and 'args' not in message.payload:
body, headers, decoded, utc = (
message.body, message.headers, False, app.uses_utc_timezone(),
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
else:
if 'args' in message.payload:
body, headers, decoded, utc = hybrid_to_proto2(message,
message.payload)
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
req = Req(
message,
on_ack=ack, on_reject=reject, app=app, hostname=hostname,
eventer=eventer, task=task, connection_errors=connection_errors,
body=body, headers=headers, decoded=decoded, utc=utc,
)
if _does_info:
info('Received task: %s', req)
if (req.expires or req.id in revoked_tasks) and req.revoked():
return
signals.task_received.send(sender=consumer, request=req)
if task_sends_events:
send_event(
'task-received',
uuid=req.id, name=req.name,
args=req.argsrepr, kwargs=req.kwargsrepr,
root_id=req.root_id, parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
bucket = None
eta = None
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, app.timezone)
except (OverflowError, ValueError) as exc:
error("Couldn't convert ETA %r to timestamp: %r. Task: %r",
req.eta, exc, req.info(safe=True), exc_info=True)
req.reject(requeue=False)
if rate_limits_enabled:
bucket = get_bucket(task.name)
if eta and bucket:
consumer.qos.increment_eventually()
return call_at(eta, limit_post_eta, (req, bucket, 1),
priority=6)
if eta:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req,), priority=6)
return task_message_handler
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
def test_datetime(self):
assert to_timestamp(datetime.utcnow())
def test_timestamp(self):
assert to_timestamp(3.13) == 3.13
def task_message_handler(message,
body,
ack,
reject,
callbacks,
to_timestamp=to_timestamp):
if body is None and 'args' not in message.payload:
body, headers, decoded, utc = (
message.body,
message.headers,
False,
app.uses_utc_timezone(),
)
octolog.error(
'<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default 1 utc=%s',
utc) # utc=False
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
else:
if 'args' in message.payload:
body, headers, decoded, utc = hybrid_to_proto2(
message, message.payload)
octolog.error(
'<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default 2 utc=%s',
utc)
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
octolog.error(
'<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default 3 utc=%s',
utc)
req = Req(
message,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
connection_errors=connection_errors,
body=body,
headers=headers,
decoded=decoded,
utc=utc,
)
if _does_info:
info('Received task: %s utc=%s', req, utc)
msg = 'Received task: message, ' \
'\n\ton_ack={on_ack}, \n\ton_reject={on_reject}, \n\tapp={app}, \n\thostname={hostname}, \n\t' \
'eventer={eventer}, \n\ttask={task}, \n\tconnection_errors={connection_errors}, \n\t' \
'body={body}, \n\theaders={headers}, \n\tdecoded={decoded}, \n\tutc={utc}'.format(
message, on_ack=ack, on_reject=reject, app=app, hostname=hostname,
eventer=eventer, task=task, connection_errors=connection_errors,
body=body, headers=headers, decoded=decoded, utc=utc)
octolog.error(msg)
if (req.expires or req.id in revoked_tasks) and req.revoked():
return
if task_sends_events:
send_event(
'task-received',
uuid=req.id,
name=req.name,
args=req.argsrepr,
kwargs=req.kwargsrepr,
root_id=req.root_id,
parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
bucket = None
eta = None
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
octolog.error(
"<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default eta %s",
eta)
else:
eta = to_timestamp(req.eta, app.timezone)
octolog.error(
"<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default eta %s",
eta)
except (OverflowError, ValueError) as exc:
error("Couldn't convert ETA %r to timestamp: %r. Task: %r",
req.eta,
exc,
req.info(safe=True),
exc_info=True)
req.reject(requeue=False)
if rate_limits_enabled:
bucket = get_bucket(task.name)
octolog.error(
"<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default bucket %s",
bucket)
if eta and bucket:
consumer.qos.increment_eventually()
octolog.error(
"<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default eta and bucket %s %s",
eta, bucket)
return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6)
if eta:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req, ), priority=6)
return task_message_handler
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
def test_datetime(self):
assert to_timestamp(datetime.utcnow())
def test_timestamp(self):
assert to_timestamp(3.13) is 3.13
def task_message_handler(message,
body,
ack,
reject,
callbacks,
to_timestamp=to_timestamp):
if body is None and 'args' not in message.payload:
body, headers, decoded, utc = (
message.body,
message.headers,
False,
app.uses_utc_timezone(),
)
else:
if 'args' in message.payload:
body, headers, decoded, utc = hybrid_to_proto2(
message, message.payload)
else:
body, headers, decoded, utc = proto1_to_proto2(message, body)
req = Req(
message,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
connection_errors=connection_errors,
body=body,
headers=headers,
decoded=decoded,
utc=utc,
)
if _does_info:
# Similar to `app.trace.info()`, we pass the formatting args as the
# `extra` kwarg for custom log handlers
context = {'id': req.id, 'name': req.name}
info(_app_trace.LOG_RECEIVED, context, extra={'data': context})
if (req.expires or req.id in revoked_tasks) and req.revoked():
return
signals.task_received.send(sender=consumer, request=req)
if task_sends_events:
send_event(
'task-received',
uuid=req.id,
name=req.name,
args=req.argsrepr,
kwargs=req.kwargsrepr,
root_id=req.root_id,
parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
bucket = None
eta = None
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, app.timezone)
except (OverflowError, ValueError) as exc:
error("Couldn't convert ETA %r to timestamp: %r. Task: %r",
req.eta,
exc,
req.info(safe=True),
exc_info=True)
req.reject(requeue=False)
if rate_limits_enabled:
bucket = get_bucket(task.name)
if eta and bucket:
consumer.qos.increment_eventually()
return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6)
if eta:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req, ), priority=6)
return task_message_handler
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
def task_message_handler(message,
body,
ack,
reject,
callbacks,
to_timestamp=to_timestamp):
# print('crawl_task_message_handler %s %s' % (task_name, repr(body)))
body, headers, decoded, utc = (
message.body,
message.headers,
False,
True,
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
req = BaseReq(
message,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
connection_errors=connection_errors,
body=body,
headers=headers,
decoded=decoded,
utc=utc,
)
# if _does_info:
meta = req.task_info()
taskinfo = {'meta': meta}
_info(u'收到任务', extra=taskinfo)
if (req.expires
or req.id in controller_revoked_tasks) and req.revoked():
return
# req_args, req_kwargs, req_embed = req._payload
if task_sends_events:
send_event(
'task-received',
uuid=req.id,
name=req.name,
args=req.argsrepr,
kwargs=req.kwargsrepr,
root_id=req.root_id,
parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
# 保存
# ti = get_task_info(req._args, req._kwargs)
fields = dict(
name=req.name,
# project=req._project, page=req._page, url=req._url,
kwargs=json.dumps(req._kwargs),
# args=req_args, kwargs=req_kwargs,
root_id=req.root_id,
parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
meta=meta)
save_task_status('task-received', req.id, fields)
# 限速
if req._kwargs.get('__limit__'):
try:
key = 'rate:%s' % meta['project']
pending = get_expected_time(key)
# print '----Rate limit pending: %s %r' % (req.id, pending)
if pending > 0:
req.eta = maybe_make_aware(datetime.utcnow() +
timedelta(seconds=pending))
info('Rate Limit [%s.%s] %s', meta['project'],
meta['page'], pending)
except Exception:
error('Rate limit. Task: %r',
req.info(safe=True),
exc_info=True)
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, timezone.local)
except (OverflowError, ValueError):
error("Couldn't convert ETA %r to timestamp. Task: %r",
req.eta,
req.info(safe=True),
exc_info=True)
req.reject(requeue=False)
else:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req, ), priority=6)
else:
if rate_limits_enabled:
bucket = get_bucket(task.name)
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
