def task_message_handler( message, body, ack, reject, callbacks, to_timestamp=to_timestamp ): req = Req( body, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, message=message, ) if req.revoked(): return if _does_info: info("Received task: %s", req) if events: send_event( "task-received", uuid=req.id, name=req.name, args=safe_repr(req.args), kwargs=safe_repr(req.kwargs), retries=req.request_dict.get("retries", 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), ) if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) else: eta = to_timestamp(req.eta, timezone.local) except OverflowError as exc: error( "Couldn't convert eta %s to timestamp: %r. Task: %r", req.eta, exc, req.info(safe=True), exc_info=True, ) req.acknowledge() else: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req,), priority=6) else: if rate_limits_enabled: bucket = get_bucket(task.name) if bucket: return limit_task(req, bucket, 1) task_reserved(req) if callbacks: [callback() for callback in callbacks] handle(req)
def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): if body is None: body, headers, decoded, utc = ( message.body, message.headers, False, app.uses_utc_timezone(), ) if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body else: body, headers, decoded, utc = proto1_to_proto2(message, body) req = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, body=body, headers=headers, decoded=decoded, utc=utc, ) if _does_info: info('Received task: %s', req) if (req.expires or req.id in revoked_tasks) and req.revoked(): return if task_sends_events: send_event( 'task-received', uuid=req.id, name=req.name, args=req.argsrepr, kwargs=req.kwargsrepr, root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), ) if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) else: eta = to_timestamp(req.eta, app.timezone) except (OverflowError, ValueError) as exc: error("Couldn't convert ETA %r to timestamp: %r. Task: %r", req.eta, exc, req.info(safe=True), exc_info=True) req.reject(requeue=False) else: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req,), priority=6) else: if rate_limits_enabled: bucket = get_bucket(task.name) if bucket: return limit_task(req, bucket, 1) task_reserved(req) if callbacks: [callback(req) for callback in callbacks] handle(req)
def _do_flush(self) -> None: logger.debug("Batches: Wake-up to flush buffers...") ready_requests = [] app = self.app to_system_tz = timezone.to_system now = monotonic() all_requests = list(consume_queue(self._buffer)) + list( consume_queue(self._pending) ) for req in all_requests: # Similar to logic in celery.worker.strategy.default. if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) else: eta = to_timestamp(req.eta, app.timezone) except (OverflowError, ValueError) as exc: logger.error( "Couldn't convert ETA %r to timestamp: %r. Task: %r", req.eta, exc, req.info(safe=True), exc_info=True, ) req.reject(requeue=False) continue if eta <= now: # ETA has elapsed, request is ready. ready_requests.append(req) else: # ETA has not elapsed, add to pending queue. self._pending.put(req) else: # Request does not have an ETA, ready immediately ready_requests.append(req) if len(ready_requests) > 0: logger.debug("Batches: Ready buffer complete: %s", len(ready_requests)) self.flush(ready_requests) if not ready_requests and self._pending.qsize() == 0: logger.debug("Batches: Canceling timer: Nothing in buffers.") if self._tref: self._tref.cancel() # cancel timer. self._tref = None
def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): if body is None and 'args' not in message.payload: body, headers, decoded, utc = ( message.body, message.headers, False, app.uses_utc_timezone(), ) if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body else: if 'args' in message.payload: body, headers, decoded, utc = hybrid_to_proto2(message, message.payload) else: body, headers, decoded, utc = proto1_to_proto2(message, body) req = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, body=body, headers=headers, decoded=decoded, utc=utc, ) if _does_info: info('Received task: %s', req) if (req.expires or req.id in revoked_tasks) and req.revoked(): return signals.task_received.send(sender=consumer, request=req) if task_sends_events: send_event( 'task-received', uuid=req.id, name=req.name, args=req.argsrepr, kwargs=req.kwargsrepr, root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), ) bucket = None eta = None if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) else: eta = to_timestamp(req.eta, app.timezone) except (OverflowError, ValueError) as exc: error("Couldn't convert ETA %r to timestamp: %r. Task: %r", req.eta, exc, req.info(safe=True), exc_info=True) req.reject(requeue=False) if rate_limits_enabled: bucket = get_bucket(task.name) if eta and bucket: consumer.qos.increment_eventually() return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6) if eta: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req,), priority=6) return task_message_handler if bucket: return limit_task(req, bucket, 1) task_reserved(req) if callbacks: [callback(req) for callback in callbacks] handle(req)
def test_datetime(self): assert to_timestamp(datetime.utcnow())
def test_timestamp(self): assert to_timestamp(3.13) == 3.13
def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): if body is None and 'args' not in message.payload: body, headers, decoded, utc = ( message.body, message.headers, False, app.uses_utc_timezone(), ) octolog.error( '<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default 1 utc=%s', utc) # utc=False if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body else: if 'args' in message.payload: body, headers, decoded, utc = hybrid_to_proto2( message, message.payload) octolog.error( '<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default 2 utc=%s', utc) else: body, headers, decoded, utc = proto1_to_proto2(message, body) octolog.error( '<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default 3 utc=%s', utc) req = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, body=body, headers=headers, decoded=decoded, utc=utc, ) if _does_info: info('Received task: %s utc=%s', req, utc) msg = 'Received task: message, ' \ '\n\ton_ack={on_ack}, \n\ton_reject={on_reject}, \n\tapp={app}, \n\thostname={hostname}, \n\t' \ 'eventer={eventer}, \n\ttask={task}, \n\tconnection_errors={connection_errors}, \n\t' \ 'body={body}, \n\theaders={headers}, \n\tdecoded={decoded}, \n\tutc={utc}'.format( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, body=body, headers=headers, decoded=decoded, utc=utc) octolog.error(msg) if (req.expires or req.id in revoked_tasks) and req.revoked(): return if task_sends_events: send_event( 'task-received', uuid=req.id, name=req.name, args=req.argsrepr, kwargs=req.kwargsrepr, root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), ) bucket = None eta = None if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) octolog.error( "<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default eta %s", eta) else: eta = to_timestamp(req.eta, app.timezone) octolog.error( "<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default eta %s", eta) except (OverflowError, ValueError) as exc: error("Couldn't convert ETA %r to timestamp: %r. Task: %r", req.eta, exc, req.info(safe=True), exc_info=True) req.reject(requeue=False) if rate_limits_enabled: bucket = get_bucket(task.name) octolog.error( "<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default bucket %s", bucket) if eta and bucket: consumer.qos.increment_eventually() octolog.error( "<=Worker Strategy=> TASK_MESSAGE_HANDLER(): default eta and bucket %s %s", eta, bucket) return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6) if eta: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req, ), priority=6) return task_message_handler if bucket: return limit_task(req, bucket, 1) task_reserved(req) if callbacks: [callback(req) for callback in callbacks] handle(req)
def test_datetime(self): assert to_timestamp(datetime.utcnow())
def test_timestamp(self): assert to_timestamp(3.13) is 3.13
def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): if body is None and 'args' not in message.payload: body, headers, decoded, utc = ( message.body, message.headers, False, app.uses_utc_timezone(), ) else: if 'args' in message.payload: body, headers, decoded, utc = hybrid_to_proto2( message, message.payload) else: body, headers, decoded, utc = proto1_to_proto2(message, body) req = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, body=body, headers=headers, decoded=decoded, utc=utc, ) if _does_info: # Similar to `app.trace.info()`, we pass the formatting args as the # `extra` kwarg for custom log handlers context = {'id': req.id, 'name': req.name} info(_app_trace.LOG_RECEIVED, context, extra={'data': context}) if (req.expires or req.id in revoked_tasks) and req.revoked(): return signals.task_received.send(sender=consumer, request=req) if task_sends_events: send_event( 'task-received', uuid=req.id, name=req.name, args=req.argsrepr, kwargs=req.kwargsrepr, root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), ) bucket = None eta = None if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) else: eta = to_timestamp(req.eta, app.timezone) except (OverflowError, ValueError) as exc: error("Couldn't convert ETA %r to timestamp: %r. Task: %r", req.eta, exc, req.info(safe=True), exc_info=True) req.reject(requeue=False) if rate_limits_enabled: bucket = get_bucket(task.name) if eta and bucket: consumer.qos.increment_eventually() return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6) if eta: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req, ), priority=6) return task_message_handler if bucket: return limit_task(req, bucket, 1) task_reserved(req) if callbacks: [callback(req) for callback in callbacks] handle(req)
def task_message_handler(message, body, ack, reject, callbacks, to_timestamp=to_timestamp): # print('crawl_task_message_handler %s %s' % (task_name, repr(body))) body, headers, decoded, utc = ( message.body, message.headers, False, True, ) if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body req = BaseReq( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, connection_errors=connection_errors, body=body, headers=headers, decoded=decoded, utc=utc, ) # if _does_info: meta = req.task_info() taskinfo = {'meta': meta} _info(u'收到任务', extra=taskinfo) if (req.expires or req.id in controller_revoked_tasks) and req.revoked(): return # req_args, req_kwargs, req_embed = req._payload if task_sends_events: send_event( 'task-received', uuid=req.id, name=req.name, args=req.argsrepr, kwargs=req.kwargsrepr, root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), ) # 保存 # ti = get_task_info(req._args, req._kwargs) fields = dict( name=req.name, # project=req._project, page=req._page, url=req._url, kwargs=json.dumps(req._kwargs), # args=req_args, kwargs=req_kwargs, root_id=req.root_id, parent_id=req.parent_id, retries=req.request_dict.get('retries', 0), eta=req.eta and req.eta.isoformat(), expires=req.expires and req.expires.isoformat(), meta=meta) save_task_status('task-received', req.id, fields) # 限速 if req._kwargs.get('__limit__'): try: key = 'rate:%s' % meta['project'] pending = get_expected_time(key) # print '----Rate limit pending: %s %r' % (req.id, pending) if pending > 0: req.eta = maybe_make_aware(datetime.utcnow() + timedelta(seconds=pending)) info('Rate Limit [%s.%s] %s', meta['project'], meta['page'], pending) except Exception: error('Rate limit. Task: %r', req.info(safe=True), exc_info=True) if req.eta: try: if req.utc: eta = to_timestamp(to_system_tz(req.eta)) else: eta = to_timestamp(req.eta, timezone.local) except (OverflowError, ValueError): error("Couldn't convert ETA %r to timestamp. Task: %r", req.eta, req.info(safe=True), exc_info=True) req.reject(requeue=False) else: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req, ), priority=6) else: if rate_limits_enabled: bucket = get_bucket(task.name) if bucket: return limit_task(req, bucket, 1) task_reserved(req) if callbacks: [callback(req) for callback in callbacks] handle(req)