def test_revive(self): chan = self.connection.channel() p = Producer(chan) chan2 = self.connection.channel() p.revive(chan2) self.assertIs(p.channel, chan2) self.assertIs(p.exchange.channel, chan2)
def test_revive(self): chan = self.connection.channel() p = Producer(chan) chan2 = self.connection.channel() p.revive(chan2) assert p.channel is chan2 assert p.exchange.channel is chan2
class event2amqp(): def __init__(self,host,port,user,password,virtual_host, exchange_name,identifier,maxqueuelength,queue_dump_frequency): self.host = host self.port = port self.user = user self.password = password self.virtual_host = virtual_host self.exchange_name = exchange_name self.identifier = identifier self.maxqueuelength = maxqueuelength self.queue_dump_frequency = queue_dump_frequency self.connection_string = None self.connection = None self.channel = None self.producer = None self.exchange = None self.queue = deque([]) self.tickage = 0 self.load_queue() def create_connection(self): self.connection_string = "amqp://%s:%s@%s:%s/%s" % (self.user,self.password,self.host,self.port,self.virtual_host) try: self.connection = BrokerConnection(self.connection_string) return True except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) return False def connect(self): logger.info("[Canopsis] connection with : %s" % self.connection_string) try: self.connection.connect() if not self.connected(): return False else: self.get_channel() self.get_exchange() self.create_producer() return True except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) return False def disconnect(self): try: if self.connected(): self.connection.release() return True except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) return False def connected(self): try: if self.connection.connected: return True else: return False except: return False def get_channel(self): try: self.channel = self.connection.channel() except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) return False def get_exchange(self): try: self.exchange = Exchange(self.exchange_name , "topic", durable=True, auto_delete=False) except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) return False def create_producer(self): try: self.producer = Producer( channel=self.channel, exchange=self.exchange, routing_key=self.virtual_host ) except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) return False def postmessage(self,message,retry=False): # process enqueud events if possible self.pop_events() if message["source_type"] == "component": key = "%s.%s.%s.%s.%s" % ( message["connector"], message["connector_name"], message["event_type"], message["source_type"], message["component"] ) else: key = "%s.%s.%s.%s.%s[%s]" % ( message["connector"], message["connector_name"], message["event_type"], message["source_type"], message["component"], message["resource"] ) # connection management if not self.connected(): logger.error("[Canopsis] Create connection") self.create_connection() self.connect() # publish message if self.connected(): logger.info("[Canopsis] using routing key %s" % key) logger.info("[Canopsis] sending %s" % str(message)) try: self.producer.revive(self.channel) self.producer.publish(body=message, compression=None, routing_key=key, exchange=self.exchange_name) return True except: logger.error("[Canopsis] Not connected, going to queue messages until connection back") self.queue.append({"key":key,"message":message}) func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) # logger.error(str(traceback.format_exc())) return False else: errmsg="[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % (str(len(self.queue)),str(self.maxqueuelength)) logger.info(errmsg) #enqueue_cano_event(key,message) if len(self.queue) < int(self.maxqueuelength): self.queue.append({"key":key,"message":message}) logger.info("[Canopsis] Queue length : %d" % len(self.queue)) return True else: logger.error("[Canopsis] Maximum retention for event queue %s reached" % str(self.maxqueuelength)) return False def errback(self,exc,interval): logger.warning("Couldn't publish message: %r. Retry in %ds" % (exc, interval)) def pop_events(self): if self.connected(): while len(self.queue) > 0: item = self.queue.pop() try: logger.info("[Canopsis] Pop item from queue [%s] : %s" % (str(len(self.queue)),str(item))) self.producer.revive(self.channel) self.producer.publish(body=item["message"], compression=None, routing_key=item["key"], exchange=self.exchange_name) except: self.queue.append(item) func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func)) return False else: return False def hook_tick(self, brok): self.tickage += 1 # queue retention saving if self.tickage >= int(self.queue_dump_frequency) and len(self.queue) > 0: # flush queue to disk if queue age reach queue_dump_frequency self.save_queue() self.tickage = 0 return True def save_queue(self): retentionfile="%s/canopsis.dat" % os.getcwd() logger.info("[Canopsis] saving to %s" % retentionfile) filehandler = open(retentionfile, 'w') pickle.dump(self.queue, filehandler) filehandler.close() return True def load_queue(self): retentionfile="%s/canopsis.dat" % os.getcwd() logger.info("[Canopsis] loading from %s" % retentionfile) filehandler = open(retentionfile, 'r') try: self.queue = pickle.load(filehandler) except: pass return True
class event2amqp(): def __init__(self, host, port, user, password, virtual_host, exchange_name, identifier, maxqueuelength, queue_dump_frequency): self.host = host self.port = port self.user = user self.password = password self.virtual_host = virtual_host self.exchange_name = exchange_name self.identifier = identifier self.maxqueuelength = maxqueuelength self.queue_dump_frequency = queue_dump_frequency self.connection_string = None self.connection = None self.channel = None self.producer = None self.exchange = None self.queue = deque([]) self.tickage = 0 self.load_queue() def create_connection(self): self.connection_string = "amqp://%s:%s@%s:%s/%s" % ( self.user, self.password, self.host, self.port, self.virtual_host) try: self.connection = BrokerConnection(self.connection_string) return True except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) return False def connect(self): logger.info("[Canopsis] connection with: %s" % self.connection_string) try: self.connection.connect() if not self.connected(): return False else: self.get_channel() self.get_exchange() self.create_producer() return True except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) return False def disconnect(self): try: if self.connected(): self.connection.release() return True except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) return False def connected(self): try: if self.connection.connected: return True else: return False except: return False def get_channel(self): try: self.channel = self.connection.channel() except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) return False def get_exchange(self): try: self.exchange = Exchange(self.exchange_name, "topic", durable=True, auto_delete=False) except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) return False def create_producer(self): try: self.producer = Producer(channel=self.channel, exchange=self.exchange, routing_key=self.virtual_host) except: func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) return False def postmessage(self, message, retry=False): # process enqueud events if possible self.pop_events() if message["source_type"] == "component": key = "%s.%s.%s.%s.%s" % ( message["connector"], message["connector_name"], message["event_type"], message["source_type"], message["component"]) else: key = "%s.%s.%s.%s.%s[%s]" % ( message["connector"], message["connector_name"], message["event_type"], message["source_type"], message["component"], message["resource"]) # connection management if not self.connected(): logger.error("[Canopsis] Create connection") self.create_connection() self.connect() # publish message if self.connected(): logger.debug("[Canopsis] using routing key %s" % key) logger.debug("[Canopsis] sending %s" % str(message)) try: self.producer.revive(self.channel) self.producer.publish(body=message, compression=None, routing_key=key, exchange=self.exchange_name) return True except: logger.error( "[Canopsis] Not connected, going to queue messages until connection back" ) self.queue.append({"key": key, "message": message}) func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) # logger.error(str(traceback.format_exc())) return False else: errmsg = "[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % ( str(len(self.queue)), str(self.maxqueuelength)) logger.error(errmsg) #enqueue_cano_event(key,message) if len(self.queue) < int(self.maxqueuelength): self.queue.append({"key": key, "message": message}) logger.debug("[Canopsis] Queue length: %d" % len(self.queue)) return True else: logger.error( "[Canopsis] Maximum retention for event queue %s reached" % str(self.maxqueuelength)) return False def errback(self, exc, interval): logger.warning("Couldn't publish message: %r. Retry in %ds" % (exc, interval)) def pop_events(self): if self.connected(): while len(self.queue) > 0: item = self.queue.pop() try: logger.debug("[Canopsis] Pop item from queue [%s]: %s" % (str(len(self.queue)), str(item))) self.producer.revive(self.channel) self.producer.publish(body=item["message"], compression=None, routing_key=item["key"], exchange=self.exchange_name) except: self.queue.append(item) func = sys._getframe(1).f_code.co_name error = str(sys.exc_info()[0]) logger.error("[Canopsis] Unexpected error: %s in %s" % (error, func)) return False else: return False def hook_tick(self, brok): self.tickage += 1 # queue retention saving if self.tickage >= int(self.queue_dump_frequency) and len( self.queue) > 0: # flush queue to disk if queue age reach queue_dump_frequency self.save_queue() self.tickage = 0 return True def save_queue(self): retentionfile = "%s/canopsis.dat" % os.getcwd() #:fixme: use path.join logger.info("[Canopsis] saving to %s" % retentionfile) filehandler = open(retentionfile, 'w') pickle.dump(self.queue, filehandler) filehandler.close() return True def load_queue(self): retentionfile = "%s/canopsis.dat" % os.getcwd() logger.info("[Canopsis] loading from %s" % retentionfile) filehandler = open(retentionfile, 'r') try: self.queue = pickle.load(filehandler) except: pass return True
class Worker(object): def __init__(self, queue_name): self.queue_name = queue_name self.serializer = "pickle" self.rabbit_connect() self.poll_messages() def rabbit_connect(self): url = "amqp://{}:{}@{}:5672/".format(config.rabbitmq.username, config.rabbitmq.password, config.rabbitmq.host) self.connection = Connection(url) self.channel = self.connection.channel() self.channel.basic_qos(prefetch_size=0, prefetch_count=1, a_global=False) self.exchange = Exchange("", type="direct", durable=True) self.queue = Queue(name=self.queue_name, exchange=self.exchange, routing_key=self.queue_name) self.queue.maybe_bind(self.connection) self.queue.declare() self.producer = Producer(exchange=self.exchange, channel=self.channel, serializer=self.serializer) self.consumer = Consumer(self.connection, queues=self.queue, callbacks=[self.message_callback], accept=["application/x-python-serialize"]) #self.consumer.qos(prefetch_count = 1) def poll_messages(self): while True: try: self.process_messages() except self.connection.connection_errors: pass def process_messages(self): self.connection = self.renew_connection() while True: try: self.connection.drain_events(timeout=5) except socket.timeout: pass def renew_connection(self): new_connection = self.connection.clone() new_connection.ensure_connection(max_retries=10) self.channel = new_connection.channel() self.channel.basic_qos(prefetch_size=0, prefetch_count=1, a_global=False) self.consumer.revive(self.channel) self.producer.revive(self.channel) self.consumer.consume() return new_connection def message_callback(self, body, message): # Convert body to UTF-8 string body = body.decode('utf-8') # Process message self.process_message(body) # Tell RabbitMQ that we processed the message message.ack() def process_message(self, message): # Generic message processing stub print ("Message from queue '{}': '{}'".format(self.queue_name, message)) def produce_message(self, message): self.producer.publish(message.encode('utf-8'), routing_key=self.queue_name, retry=True, delivery_mode=2)
class MessageBusService(object): _uri: str _connection: Connection _connection_producer: Connection _consuming: bool = False _producer: Producer _producer_reply_to_consumer: Consumer _future: StandardQueue _logger: Optional[logging.Logger] def __init__(self, uri: str, logger: Optional[logging.Logger] = None): self._uri = uri self._logger = logger self._future = StandardQueue() self.connect() def connect(self): self._connection = Connection(self._uri) self._connection.connect() self._connection_producer = self._connection.clone() self._producer = Producer(self._connection_producer) # reply_queue = Queue( channel=self._producer.channel, name="amq.rabbitmq.reply-to", no_ack=True, durable=False, ) self._producer_reply_to_consumer = self._producer.channel.Consumer( queues=[reply_queue], no_ack=True, auto_declare=True, callbacks=[self.on_reply_to_message], accept=["json"], ) self._producer_reply_to_consumer.consume(no_ack=True) def disconnect(self): self._producer.close() self._connection_producer.close() self._connection.close() def start_consuming( self, callback: Callable, queue_name: str, prefetch_count: int = 1, no_ack: bool = False, expires: int = None, callback_ready: Callable = None, ): if self._logger is not None: self._logger.debug("Start consuming queue: %s" % queue_name) self._consuming = True while self._consuming: revived_connection = self._connection.clone() revived_connection.ensure_connection() channel = revived_connection.channel() channel.basic_qos(0, prefetch_count, True) queues = [] queue_obj = Queue( channel=channel, name=queue_name, no_ack=no_ack, durable=False, expires=expires, queue_arguments={"x-max-priority": 255}, ) queue_obj.declare() queues.append(queue_obj) consumer = Consumer( revived_connection, queues, callbacks=[callback], accept=["json"], auto_declare=False, prefetch_count=prefetch_count, ) consumer.revive(channel) consumer.consume() while self._consuming: callback_ready is not None and callback_ready() try: revived_connection.drain_events(timeout=2) except socket.timeout: revived_connection.heartbeat_check() except self._connection.connection_errors + ( AMQPError, ConnectionForced, ConnectionError, ): # pragma: no cover if self._logger is not None: self._logger.exception("Connection error", stack_info=True) break def start_consuming_replies( self, callback: Callable, prefetch_count: int = 1, no_ack: bool = False ): self._consuming = True while self._consuming: revived_connection = self._connection_producer.clone() revived_connection.ensure_connection() while self._consuming: try: revived_connection.drain_events(timeout=2) except socket.timeout: revived_connection.heartbeat_check() except self._connection.connection_errors + ( AMQPError, ConnectionForced, ConnectionError, ): # pragma: no cover if self._logger is not None: self._logger.exception("Connection error", stack_info=True) break def stop_consuming(self): if self._logger is not None: self._logger.debug("Stop consuming...") self._consuming = False def publish( self, body: dict, exchange: str = "", queue_name: str = "", priority: int = None, reply_to: str = None, expiration: int = None, correlation_id: str = None, ): while True: try: self._connection_producer.ensure_connection() self._producer.publish( body=body, exchange=exchange, routing_key=queue_name, priority=priority, reply_to=str(reply_to) if reply_to is not None else None, expiration=expiration if expiration is not None else None, correlation_id=str(correlation_id) if correlation_id is not None else None, ) except self._connection_producer.connection_errors + ( AMQPError, ConnectionForced, ConnectionError, ): # pragma: no cover self._connection_producer = self._connection.clone() self._producer.revive(self._connection_producer) else: break def on_reply_to_message(self, body, message): self._future.put(body) def publish_and_get_reply(self, *args, timeout: int = 1, **kw) -> Optional[dict]: kw['reply_to'] = "amq.rabbitmq.reply-to" self.publish(*args, **kw) try: self._producer_reply_to_consumer.connection.drain_events(timeout=timeout) except socket.timeout: # pragma: no cover return None # return self._future.get(block=False)