def _process_message(self, data, message, message_type): handler = self._type_handlers.get(message_type) if handler is None: message.reject_log_error(logger=LOG, errors=(kombu_exc.MessageStateError, )) LOG.warning("Unexpected message type: '%s' in message" " '%s'", message_type, ku.DelayedPretty(message)) else: if isinstance(handler, (tuple, list)): handler, validator = handler try: validator(data) except excp.InvalidFormat as e: message.reject_log_error( logger=LOG, errors=(kombu_exc.MessageStateError, )) LOG.warn( "Message '%s' (%s) was rejected due to it being" " in an invalid format: %s", ku.DelayedPretty(message), message_type, e) return message.ack_log_error(logger=LOG, errors=(kombu_exc.MessageStateError, )) if message.acknowledged: LOG.debug("Message '%s' was acknowledged.", ku.DelayedPretty(message)) handler(data, message) else: message.reject_log_error( logger=LOG, errors=(kombu_exc.MessageStateError, ))
def _on_receive(content, message): LOG.debug("Submitting message '%s' for execution in the" " future to '%s'", ku.DelayedPretty(message), func_name) watch = tt.StopWatch() watch.start() try: self._executor.submit(_on_run, watch, content, message) except RuntimeError: LOG.error("Unable to continue processing message '%s'," " submission to instance executor (with later" " execution by '%s') was unsuccessful", ku.DelayedPretty(message), func_name, exc_info=True)
def _on_run(watch, content, message): LOG.trace( "It took %s seconds to get around to running" " function/method '%s' with" " message '%s'", watch.elapsed(), func_name, ku.DelayedPretty(message)) return func(content, message)
def on_message(self, data, message): """This method is called on incoming messages.""" LOG.debug("Received message '%s'", ku.DelayedPretty(message)) if self._collect_requeue_votes(data, message): self._requeue_log_error(message, errors=(kombu_exc.MessageStateError, )) else: try: message_type = message.properties['type'] except KeyError: message.reject_log_error( logger=LOG, errors=(kombu_exc.MessageStateError, )) LOG.warning( "The 'type' message property is missing" " in message '%s'", ku.DelayedPretty(message)) else: self._process_message(data, message, message_type)
def _process_response(self, response, message): """Process response from remote side.""" LOG.debug("Started processing response message '%s'", ku.DelayedPretty(message)) try: task_uuid = message.properties['correlation_id'] except KeyError: LOG.warning( "The 'correlation_id' message property is" " missing in message '%s'", ku.DelayedPretty(message)) else: request = self._requests_cache.get(task_uuid) if request is not None: response = pr.Response.from_dict(response) LOG.debug( "Extracted response '%s' and matched it to" " request '%s'", response, request) if response.state == pr.RUNNING: request.transition_and_log_error(pr.RUNNING, logger=LOG) elif response.state == pr.EVENT: # Proxy the event + details to the task/request notifier... event_type = response.data['event_type'] details = response.data['details'] request.notifier.notify(event_type, details) elif response.state in (pr.FAILURE, pr.SUCCESS): moved = request.transition_and_log_error(response.state, logger=LOG) if moved: # NOTE(imelnikov): request should not be in the # cache when another thread can see its result and # schedule another request with the same uuid; so # we remove it, then set the result... del self._requests_cache[request.uuid] request.set_result(**response.data) else: LOG.warning("Unexpected response status '%s'", response.state) else: LOG.debug("Request with id='%s' not found", task_uuid)
def _process_response(self, response, message): """Process response from remote side.""" LOG.debug("Started processing response message '%s'", ku.DelayedPretty(message)) try: request_uuid = message.properties['correlation_id'] except KeyError: LOG.warning( "The 'correlation_id' message property is" " missing in message '%s'", ku.DelayedPretty(message)) else: request = self._ongoing_requests.get(request_uuid) if request is not None: response = pr.Response.from_dict(response) LOG.debug( "Extracted response '%s' and matched it to" " request '%s'", response, request) if response.state == pr.RUNNING: request.transition_and_log_error(pr.RUNNING, logger=LOG) elif response.state == pr.EVENT: # Proxy the event + details to the task notifier so # that it shows up in the local process (and activates # any local callbacks...); thus making it look like # the task is running locally (in some regards). event_type = response.data['event_type'] details = response.data['details'] request.task.notifier.notify(event_type, details) elif response.state in (pr.FAILURE, pr.SUCCESS): if request.transition_and_log_error(response.state, logger=LOG): with self._ongoing_requests_lock: del self._ongoing_requests[request.uuid] request.set_result(result=response.data['result']) else: LOG.warning("Unexpected response status '%s'", response.state) else: LOG.debug("Request with id='%s' not found", request_uuid)
def _process_response(self, data, message): """Process notify message sent from remote side.""" LOG.debug("Started processing notify response message '%s'", ku.DelayedPretty(message)) response = pr.Notify(**data) LOG.debug("Extracted notify response '%s'", response) with self._cond: worker, new_or_updated = self._add(response.topic, response.tasks) if new_or_updated: LOG.debug( "Updated worker '%s' (%s total workers are" " currently known)", worker, self._total_workers()) self._cond.notify_all() if new_or_updated: self.notifier.notify(self.WORKER_ARRIVED, {'worker': worker})
def _process_notify(self, notify, message): """Process notify message and reply back.""" LOG.debug("Started processing notify message '%s'", ku.DelayedPretty(message)) try: reply_to = message.properties['reply_to'] except KeyError: LOG.warn( "The 'reply_to' message property is missing" " in received notify message '%s'", ku.DelayedPretty(message), exc_info=True) else: response = pr.Notify(topic=self._topic, tasks=self._endpoints.keys()) try: self._proxy.publish(response, routing_key=reply_to) except Exception: LOG.critical( "Failed to send reply to '%s' with notify" " response '%s'", reply_to, response, exc_info=True)
def process_response(self, data, message): """Process notify message sent from remote side.""" LOG.debug("Started processing notify response message '%s'", ku.DelayedPretty(message)) response = pr.Notify(**data) LOG.debug("Extracted notify response '%s'", response) with self._cond: worker, new_or_updated = self._add(response.topic, response.tasks) if new_or_updated: LOG.debug( "Updated worker '%s' (%s total workers are" " currently known)", worker, self.total_workers) self._cond.notify_all() worker.last_seen = timeutils.now() self._messages_processed += 1
def _process_response(self, response, message): """Process notify message from remote side.""" LOG.debug("Started processing notify message '%s'", ku.DelayedPretty(message)) topic = response['topic'] tasks = response['tasks'] with self._cond: worker, new_or_updated = self._add(topic, tasks) if new_or_updated: LOG.debug("Received notification about worker '%s' (%s" " total workers are currently known)", worker, self._total_workers()) self._cond.notify_all() if self.on_worker is not None and new_or_updated: self.on_worker(worker)
def _requeue_log_error(self, message, errors): # TODO(harlowja): Remove when http://github.com/celery/kombu/pull/372 # is merged and a version is released with this change... try: message.requeue() except errors as exc: # This was taken from how kombu is formatting its messages # when its reject_log_error or ack_log_error functions are # used so that we have a similar error format for requeuing. LOG.critical("Couldn't requeue %r, reason:%r", message.delivery_tag, exc, exc_info=True) else: LOG.debug("Message '%s' was requeued.", ku.DelayedPretty(message))
def _process_request(self, request, message): """Process request message and reply back.""" LOG.debug("Started processing request message '%s'", ku.DelayedPretty(message)) try: # NOTE(skudriashev): parse broker message first to get # the `reply_to` and the `task_uuid` parameters to have # possibility to reply back (if we can't parse, we can't respond # in the first place...). reply_to, task_uuid = self._parse_message(message) except ValueError: LOG.warn("Failed to parse request attributes from message '%s'", ku.DelayedPretty(message), exc_info=True) return else: # prepare reply callback reply_callback = functools.partial(self._reply, True, reply_to, task_uuid) # parse request to get task name, action and action arguments try: bundle = self._parse_request(**request) task_cls, task_name, action, arguments = bundle arguments['task_uuid'] = task_uuid except ValueError: with misc.capture_failure() as failure: LOG.warn("Failed to parse request contents from message '%s'", ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return # get task endpoint try: endpoint = self._endpoints[task_cls] except KeyError: with misc.capture_failure() as failure: LOG.warn( "The '%s' task endpoint does not exist, unable" " to continue processing request message '%s'", task_cls, ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return else: try: handler = getattr(endpoint, action) except AttributeError: with misc.capture_failure() as failure: LOG.warn( "The '%s' handler does not exist on task endpoint" " '%s', unable to continue processing request" " message '%s'", action, endpoint, ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return else: try: task = endpoint.generate(name=task_name) except Exception: with misc.capture_failure() as failure: LOG.warn( "The '%s' task '%s' generation for request" " message '%s' failed", endpoint, action, ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return else: if not reply_callback(state=pr.RUNNING): return # associate *any* events this task emits with a proxy that will # emit them back to the engine... for handling at the engine side # of things... if task.notifier.can_be_registered(nt.Notifier.ANY): task.notifier.register( nt.Notifier.ANY, functools.partial(self._on_event, reply_to, task_uuid)) elif isinstance(task.notifier, nt.RestrictedNotifier): # only proxy the allowable events then... for event_type in task.notifier.events_iter(): task.notifier.register( event_type, functools.partial(self._on_event, reply_to, task_uuid)) # perform the task action try: result = handler(task, **arguments) except Exception: with misc.capture_failure() as failure: LOG.warn( "The '%s' endpoint '%s' execution for request" " message '%s' failed", endpoint, action, ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) else: if isinstance(result, ft.Failure): reply_callback(result=result.to_dict()) else: reply_callback(state=pr.SUCCESS, result=result)