def _process_message(self, data, message, message_type):
handler = self._type_handlers.get(message_type)
if handler is None:
message.reject_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError, ))
LOG.warning("Unexpected message type: '%s' in message"
" '%s'", message_type, ku.DelayedPretty(message))
else:
if isinstance(handler, (tuple, list)):
handler, validator = handler
try:
validator(data)
except excp.InvalidFormat as e:
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError, ))
LOG.warn(
"Message '%s' (%s) was rejected due to it being"
" in an invalid format: %s", ku.DelayedPretty(message),
message_type, e)
return
message.ack_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError, ))
if message.acknowledged:
LOG.debug("Message '%s' was acknowledged.",
ku.DelayedPretty(message))
handler(data, message)
else:
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError, ))
def _on_receive(content, message):
LOG.debug("Submitting message '%s' for execution in the"
" future to '%s'", ku.DelayedPretty(message), func_name)
watch = tt.StopWatch()
watch.start()
try:
self._executor.submit(_on_run, watch, content, message)
except RuntimeError:
LOG.error("Unable to continue processing message '%s',"
" submission to instance executor (with later"
" execution by '%s') was unsuccessful",
ku.DelayedPretty(message), func_name,
exc_info=True)
def _on_run(watch, content, message):
LOG.trace(
"It took %s seconds to get around to running"
" function/method '%s' with"
" message '%s'", watch.elapsed(), func_name,
ku.DelayedPretty(message))
return func(content, message)
def on_message(self, data, message):
"""This method is called on incoming messages."""
LOG.debug("Received message '%s'", ku.DelayedPretty(message))
if self._collect_requeue_votes(data, message):
self._requeue_log_error(message,
errors=(kombu_exc.MessageStateError, ))
else:
try:
message_type = message.properties['type']
except KeyError:
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError, ))
LOG.warning(
"The 'type' message property is missing"
" in message '%s'", ku.DelayedPretty(message))
else:
self._process_message(data, message, message_type)
def _process_response(self, response, message):
"""Process response from remote side."""
LOG.debug("Started processing response message '%s'",
ku.DelayedPretty(message))
try:
task_uuid = message.properties['correlation_id']
except KeyError:
LOG.warning(
"The 'correlation_id' message property is"
" missing in message '%s'", ku.DelayedPretty(message))
else:
request = self._requests_cache.get(task_uuid)
if request is not None:
response = pr.Response.from_dict(response)
LOG.debug(
"Extracted response '%s' and matched it to"
" request '%s'", response, request)
if response.state == pr.RUNNING:
request.transition_and_log_error(pr.RUNNING, logger=LOG)
elif response.state == pr.EVENT:
# Proxy the event + details to the task/request notifier...
event_type = response.data['event_type']
details = response.data['details']
request.notifier.notify(event_type, details)
elif response.state in (pr.FAILURE, pr.SUCCESS):
moved = request.transition_and_log_error(response.state,
logger=LOG)
if moved:
# NOTE(imelnikov): request should not be in the
# cache when another thread can see its result and
# schedule another request with the same uuid; so
# we remove it, then set the result...
del self._requests_cache[request.uuid]
request.set_result(**response.data)
else:
LOG.warning("Unexpected response status '%s'",
response.state)
else:
LOG.debug("Request with id='%s' not found", task_uuid)
def _process_response(self, response, message):
"""Process response from remote side."""
LOG.debug("Started processing response message '%s'",
ku.DelayedPretty(message))
try:
request_uuid = message.properties['correlation_id']
except KeyError:
LOG.warning(
"The 'correlation_id' message property is"
" missing in message '%s'", ku.DelayedPretty(message))
else:
request = self._ongoing_requests.get(request_uuid)
if request is not None:
response = pr.Response.from_dict(response)
LOG.debug(
"Extracted response '%s' and matched it to"
" request '%s'", response, request)
if response.state == pr.RUNNING:
request.transition_and_log_error(pr.RUNNING, logger=LOG)
elif response.state == pr.EVENT:
# Proxy the event + details to the task notifier so
# that it shows up in the local process (and activates
# any local callbacks...); thus making it look like
# the task is running locally (in some regards).
event_type = response.data['event_type']
details = response.data['details']
request.task.notifier.notify(event_type, details)
elif response.state in (pr.FAILURE, pr.SUCCESS):
if request.transition_and_log_error(response.state,
logger=LOG):
with self._ongoing_requests_lock:
del self._ongoing_requests[request.uuid]
request.set_result(result=response.data['result'])
else:
LOG.warning("Unexpected response status '%s'",
response.state)
else:
LOG.debug("Request with id='%s' not found", request_uuid)
def _process_response(self, data, message):
"""Process notify message sent from remote side."""
LOG.debug("Started processing notify response message '%s'",
ku.DelayedPretty(message))
response = pr.Notify(**data)
LOG.debug("Extracted notify response '%s'", response)
with self._cond:
worker, new_or_updated = self._add(response.topic, response.tasks)
if new_or_updated:
LOG.debug(
"Updated worker '%s' (%s total workers are"
" currently known)", worker, self._total_workers())
self._cond.notify_all()
if new_or_updated:
self.notifier.notify(self.WORKER_ARRIVED, {'worker': worker})
def _process_notify(self, notify, message):
"""Process notify message and reply back."""
LOG.debug("Started processing notify message '%s'",
ku.DelayedPretty(message))
try:
reply_to = message.properties['reply_to']
except KeyError:
LOG.warn(
"The 'reply_to' message property is missing"
" in received notify message '%s'",
ku.DelayedPretty(message),
exc_info=True)
else:
response = pr.Notify(topic=self._topic,
tasks=self._endpoints.keys())
try:
self._proxy.publish(response, routing_key=reply_to)
except Exception:
LOG.critical(
"Failed to send reply to '%s' with notify"
" response '%s'",
reply_to,
response,
exc_info=True)
def process_response(self, data, message):
"""Process notify message sent from remote side."""
LOG.debug("Started processing notify response message '%s'",
ku.DelayedPretty(message))
response = pr.Notify(**data)
LOG.debug("Extracted notify response '%s'", response)
with self._cond:
worker, new_or_updated = self._add(response.topic, response.tasks)
if new_or_updated:
LOG.debug(
"Updated worker '%s' (%s total workers are"
" currently known)", worker, self.total_workers)
self._cond.notify_all()
worker.last_seen = timeutils.now()
self._messages_processed += 1
def _process_response(self, response, message):
"""Process notify message from remote side."""
LOG.debug("Started processing notify message '%s'",
ku.DelayedPretty(message))
topic = response['topic']
tasks = response['tasks']
with self._cond:
worker, new_or_updated = self._add(topic, tasks)
if new_or_updated:
LOG.debug("Received notification about worker '%s' (%s"
" total workers are currently known)", worker,
self._total_workers())
self._cond.notify_all()
if self.on_worker is not None and new_or_updated:
self.on_worker(worker)
def _requeue_log_error(self, message, errors):
# TODO(harlowja): Remove when http://github.com/celery/kombu/pull/372
# is merged and a version is released with this change...
try:
message.requeue()
except errors as exc:
# This was taken from how kombu is formatting its messages
# when its reject_log_error or ack_log_error functions are
# used so that we have a similar error format for requeuing.
LOG.critical("Couldn't requeue %r, reason:%r",
message.delivery_tag,
exc,
exc_info=True)
else:
LOG.debug("Message '%s' was requeued.", ku.DelayedPretty(message))
def _process_request(self, request, message):
"""Process request message and reply back."""
LOG.debug("Started processing request message '%s'",
ku.DelayedPretty(message))
try:
# NOTE(skudriashev): parse broker message first to get
# the `reply_to` and the `task_uuid` parameters to have
# possibility to reply back (if we can't parse, we can't respond
# in the first place...).
reply_to, task_uuid = self._parse_message(message)
except ValueError:
LOG.warn("Failed to parse request attributes from message '%s'",
ku.DelayedPretty(message),
exc_info=True)
return
else:
# prepare reply callback
reply_callback = functools.partial(self._reply, True, reply_to,
task_uuid)
# parse request to get task name, action and action arguments
try:
bundle = self._parse_request(**request)
task_cls, task_name, action, arguments = bundle
arguments['task_uuid'] = task_uuid
except ValueError:
with misc.capture_failure() as failure:
LOG.warn("Failed to parse request contents from message '%s'",
ku.DelayedPretty(message),
exc_info=True)
reply_callback(result=failure.to_dict())
return
# get task endpoint
try:
endpoint = self._endpoints[task_cls]
except KeyError:
with misc.capture_failure() as failure:
LOG.warn(
"The '%s' task endpoint does not exist, unable"
" to continue processing request message '%s'",
task_cls,
ku.DelayedPretty(message),
exc_info=True)
reply_callback(result=failure.to_dict())
return
else:
try:
handler = getattr(endpoint, action)
except AttributeError:
with misc.capture_failure() as failure:
LOG.warn(
"The '%s' handler does not exist on task endpoint"
" '%s', unable to continue processing request"
" message '%s'",
action,
endpoint,
ku.DelayedPretty(message),
exc_info=True)
reply_callback(result=failure.to_dict())
return
else:
try:
task = endpoint.generate(name=task_name)
except Exception:
with misc.capture_failure() as failure:
LOG.warn(
"The '%s' task '%s' generation for request"
" message '%s' failed",
endpoint,
action,
ku.DelayedPretty(message),
exc_info=True)
reply_callback(result=failure.to_dict())
return
else:
if not reply_callback(state=pr.RUNNING):
return
# associate *any* events this task emits with a proxy that will
# emit them back to the engine... for handling at the engine side
# of things...
if task.notifier.can_be_registered(nt.Notifier.ANY):
task.notifier.register(
nt.Notifier.ANY,
functools.partial(self._on_event, reply_to, task_uuid))
elif isinstance(task.notifier, nt.RestrictedNotifier):
# only proxy the allowable events then...
for event_type in task.notifier.events_iter():
task.notifier.register(
event_type,
functools.partial(self._on_event, reply_to, task_uuid))
# perform the task action
try:
result = handler(task, **arguments)
except Exception:
with misc.capture_failure() as failure:
LOG.warn(
"The '%s' endpoint '%s' execution for request"
" message '%s' failed",
endpoint,
action,
ku.DelayedPretty(message),
exc_info=True)
reply_callback(result=failure.to_dict())
else:
if isinstance(result, ft.Failure):
reply_callback(result=result.to_dict())
else:
reply_callback(state=pr.SUCCESS, result=result)
