def processMessage(self, message): """ Handles a queue message, can call "acknowledge" on the Queue Consumer class when it is done with the message """ log.debug('processing message.') chan = self.queueConsumer.consumer.p.chan if not getattr(chan, '_flag_qos', False): chan.basic_qos(prefetch_count=ZenActionD.MSGS_TO_PREFETCH) chan._flag_qos = True if message.content.body == self.queueConsumer.MARKER: log.info("Received MARKER sentinel, exiting message loop") self.queueConsumer.acknowledge(message) return try: signal = hydrateQueueMessage(message, self.schema) with self.signal_timer: self.processSignal(signal) log.debug('Done processing signal.') except SchemaException: log.error("Unable to hydrate protobuf %s. " % message.content.body) self.queueConsumer.acknowledge(message) except Exception as e: log.exception(e) # FIXME: Send to an error queue instead of acknowledge. log.error('Acknowledging broken message.') self.queueConsumer.acknowledge(message) else: log.debug('Acknowledging message. (%s)' % signal.message) self.queueConsumer.acknowledge(message)
def processMessage(self, message): try: hydrated = hydrateQueueMessage(message, self._queueSchema) except Exception as e: log.error("Failed to hydrate raw event: %s", e) yield self.queueConsumer.acknowledge(message) else: try: zepRawEvent = self.processor.processMessage(hydrated) if log.isEnabledFor(logging.DEBUG): log.debug("Publishing event: %s", to_dict(zepRawEvent)) yield self.queueConsumer.publishMessage(EXCHANGE_ZEP_ZEN_EVENTS, self._routing_key(zepRawEvent), zepRawEvent, declareExchange=False) yield self.queueConsumer.acknowledge(message) except DropEvent as e: if log.isEnabledFor(logging.DEBUG): log.debug('%s - %s' % (e.message, to_dict(e.event))) yield self.queueConsumer.acknowledge(message) except ProcessingException as e: log.error('%s - %s' % (e.message, to_dict(e.event))) log.exception(e) yield self.queueConsumer.reject(message) except Exception as e: log.exception(e) yield self.queueConsumer.reject(message)
def processMessage(self, message): """ Handles a queue message, can call "acknowledge" on the Queue Consumer class when it is done with the message """ log.debug('processing message.') chan = self.queueConsumer.consumer.p.chan if not getattr(chan, '_flag_qos', False): chan.basic_qos(prefetch_count=ZenActionD.MSGS_TO_PREFETCH) chan._flag_qos = True if message.content.body == self.queueConsumer.MARKER: log.info("Received MARKER sentinel, exiting message loop") self.queueConsumer.acknowledge(message) return try: signal = hydrateQueueMessage(message, self.schema) with self.signal_timer: self.processSignal(signal) log.debug('Done processing signal.') except SchemaException: log.error("Unable to hydrate protobuf %s. " % message.content.body) self.queueConsumer.acknowledge(message) except Exception, e: log.exception(e) # FIXME: Send to an error queue instead of acknowledge. log.error('Acknowledging broken message.') self.queueConsumer.acknowledge(message)
def dump(self, message, schema, queueName, stream): # Make sure we can read it in, then convert back to string routing_key = message.delivery_info['routing_key'] exchange = message.delivery_info['exchange'] proto = hydrateQueueMessage(message, schema) type = proto.DESCRIPTOR.full_name payload = proto.SerializeToString() log.debug(to_json(proto)) self._write_shortstr(stream, type) self._write_shortstr(stream, exchange) self._write_shortstr(stream, routing_key) size = len(payload) stream.write(pack('>I%dsB' % size, size, payload, 0xce))
def _processMessage(self, message): try: proto = hydrateQueueMessage(message, self._queueSchema) try: handler = self._handlers[proto.DESCRIPTOR.full_name] except KeyError: raise Exception('No message handler for "%s"' % proto.DESCRIPTOR.full_name) for publishable in handler(message, proto): yield publishable except SchemaException: # received an invalid message log it and move on log.error("Unable to hydrate protobuf %s with headers %s " % (message.body, message.properties.get('application_headers'))) # we can't process the message so throw it away message.ack()
def dump(self, message, schema, queueName, stream): try: proto = hydrateQueueMessage(message, schema) except SchemaException: log.error("Unable to hydrate protobuf %s with headers %s " % (message.body, message.properties.get('application_headers'))) return self.dumpHeaders(message, stream) self.dumpHeader(HEADER_QUEUE_NAME, queueName, stream) self.dumpCommonHeaders(message, stream) self.dumpHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_JSON, stream) data = to_json(proto, indent=2) self.dumpHeader(HEADER_CONTENT_LENGTH, len(data), stream) stream.write('\n') stream.write(data) stream.write('\n\n')
def dump(self, message, schema, queueName, stream): try: proto = hydrateQueueMessage(message, schema) except SchemaException: log.error( "Unable to hydrate protobuf %s with headers %s " % (message.body, message.properties.get('application_headers'))) return self.dumpHeaders(message, stream) self.dumpHeader(HEADER_QUEUE_NAME, queueName, stream) self.dumpCommonHeaders(message, stream) self.dumpHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_JSON, stream) data = to_json(proto, indent=2) self.dumpHeader(HEADER_CONTENT_LENGTH, len(data), stream) stream.write('\n') stream.write(data) stream.write('\n\n')
def _processMessage(self, message): try: proto = hydrateQueueMessage(message, self._queueSchema) try: handler = self._handlers[proto.DESCRIPTOR.full_name] except KeyError: raise Exception('No message handler for "%s"' % proto.DESCRIPTOR.full_name) for publishable in handler(message, proto): yield publishable except SchemaException: # received an invalid message log it and move on log.error( "Unable to hydrate protobuf %s with headers %s " % (message.body, message.properties.get('application_headers'))) # we can't process the message so throw it away message.ack()
def dump(self, message, schema, queueName, stream): try: # Make sure we can read it in, then convert back to string proto = hydrateQueueMessage(message, schema) except SchemaException: log.error("Unable to hydrate protobuf %s with headers %s " % (message.body, message.properties.get('application_headers'))) return self.dumpHeaders(message, stream) self.dumpHeader(HEADER_QUEUE_NAME, queueName, stream) self.dumpCommonHeaders(message, stream) self.dumpHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_PROTOBUF, stream) self.dumpHeader(HEADER_CONTENT_TRANSFER_ENCODING, 'base64', stream) data = b64encode(proto.SerializeToString()) self.dumpHeader(HEADER_CONTENT_LENGTH, len(data), stream) stream.write('\n') stream.write(data) stream.write('\n\n')
def processMessage(self, message): """ Handles a queue message, can call "acknowledge" on the Queue Consumer class when it is done with the message """ log.debug('processing message.') if message.content.body == self.queueConsumer.MARKER: log.info("Received MARKER sentinel, exiting message loop") self.queueConsumer.acknowledge(message) return try: signal = hydrateQueueMessage(message, self.schema) self.processSignal(signal) log.debug('Done processing signal.') except SchemaException: log.error("Unable to hydrate protobuf %s. " % message.content.body) self.queueConsumer.acknowledge(message) except Exception, e: log.exception(e) # FIXME: Send to an error queue instead of acknowledge. log.error('Acknowledging broken message.') self.queueConsumer.acknowledge(message)
def dump(self, message, schema, queueName, stream): try: # Make sure we can read it in, then convert back to string proto = hydrateQueueMessage(message, schema) except SchemaException: log.error( "Unable to hydrate protobuf %s with headers %s " % (message.body, message.properties.get('application_headers'))) return self.dumpHeaders(message, stream) self.dumpHeader(HEADER_QUEUE_NAME, queueName, stream) self.dumpCommonHeaders(message, stream) self.dumpHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_PROTOBUF, stream) self.dumpHeader(HEADER_CONTENT_TRANSFER_ENCODING, 'base64', stream) data = b64encode(proto.SerializeToString()) self.dumpHeader(HEADER_CONTENT_LENGTH, len(data), stream) stream.write('\n') stream.write(data) stream.write('\n\n')