def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
assert p.channel is chan2
assert p.exchange.channel is chan2
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
assert p.channel is chan2
assert p.exchange.channel is chan2
class event2amqp():
def __init__(self,host,port,user,password,virtual_host, exchange_name,identifier,maxqueuelength,queue_dump_frequency):
self.host = host
self.port = port
self.user = user
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.identifier = identifier
self.maxqueuelength = maxqueuelength
self.queue_dump_frequency = queue_dump_frequency
self.connection_string = None
self.connection = None
self.channel = None
self.producer = None
self.exchange = None
self.queue = deque([])
self.tickage = 0
self.load_queue()
def create_connection(self):
self.connection_string = "amqp://%s:%s@%s:%s/%s" % (self.user,self.password,self.host,self.port,self.virtual_host)
try:
self.connection = BrokerConnection(self.connection_string)
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def connect(self):
logger.info("[Canopsis] connection with : %s" % self.connection_string)
try:
self.connection.connect()
if not self.connected():
return False
else:
self.get_channel()
self.get_exchange()
self.create_producer()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def disconnect(self):
try:
if self.connected():
self.connection.release()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def connected(self):
try:
if self.connection.connected:
return True
else:
return False
except:
return False
def get_channel(self):
try:
self.channel = self.connection.channel()
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def get_exchange(self):
try:
self.exchange = Exchange(self.exchange_name , "topic", durable=True, auto_delete=False)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def create_producer(self):
try:
self.producer = Producer(
channel=self.channel,
exchange=self.exchange,
routing_key=self.virtual_host
)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def postmessage(self,message,retry=False):
# process enqueud events if possible
self.pop_events()
if message["source_type"] == "component":
key = "%s.%s.%s.%s.%s" % (
message["connector"],
message["connector_name"],
message["event_type"],
message["source_type"],
message["component"]
)
else:
key = "%s.%s.%s.%s.%s[%s]" % (
message["connector"],
message["connector_name"],
message["event_type"],
message["source_type"],
message["component"],
message["resource"]
)
# connection management
if not self.connected():
logger.error("[Canopsis] Create connection")
self.create_connection()
self.connect()
# publish message
if self.connected():
logger.info("[Canopsis] using routing key %s" % key)
logger.info("[Canopsis] sending %s" % str(message))
try:
self.producer.revive(self.channel)
self.producer.publish(body=message, compression=None, routing_key=key, exchange=self.exchange_name)
return True
except:
logger.error("[Canopsis] Not connected, going to queue messages until connection back")
self.queue.append({"key":key,"message":message})
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
# logger.error(str(traceback.format_exc()))
return False
else:
errmsg="[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % (str(len(self.queue)),str(self.maxqueuelength))
logger.info(errmsg)
#enqueue_cano_event(key,message)
if len(self.queue) < int(self.maxqueuelength):
self.queue.append({"key":key,"message":message})
logger.info("[Canopsis] Queue length : %d" % len(self.queue))
return True
else:
logger.error("[Canopsis] Maximum retention for event queue %s reached" % str(self.maxqueuelength))
return False
def errback(self,exc,interval):
logger.warning("Couldn't publish message: %r. Retry in %ds" % (exc, interval))
def pop_events(self):
if self.connected():
while len(self.queue) > 0:
item = self.queue.pop()
try:
logger.info("[Canopsis] Pop item from queue [%s] : %s" % (str(len(self.queue)),str(item)))
self.producer.revive(self.channel)
self.producer.publish(body=item["message"], compression=None, routing_key=item["key"], exchange=self.exchange_name)
except:
self.queue.append(item)
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
else:
return False
def hook_tick(self, brok):
self.tickage += 1
# queue retention saving
if self.tickage >= int(self.queue_dump_frequency) and len(self.queue) > 0:
# flush queue to disk if queue age reach queue_dump_frequency
self.save_queue()
self.tickage = 0
return True
def save_queue(self):
retentionfile="%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] saving to %s" % retentionfile)
filehandler = open(retentionfile, 'w')
pickle.dump(self.queue, filehandler)
filehandler.close()
return True
def load_queue(self):
retentionfile="%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] loading from %s" % retentionfile)
filehandler = open(retentionfile, 'r')
try:
self.queue = pickle.load(filehandler)
except:
pass
return True
class event2amqp():
def __init__(self, host, port, user, password, virtual_host, exchange_name,
identifier, maxqueuelength, queue_dump_frequency):
self.host = host
self.port = port
self.user = user
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.identifier = identifier
self.maxqueuelength = maxqueuelength
self.queue_dump_frequency = queue_dump_frequency
self.connection_string = None
self.connection = None
self.channel = None
self.producer = None
self.exchange = None
self.queue = deque([])
self.tickage = 0
self.load_queue()
def create_connection(self):
self.connection_string = "amqp://%s:%s@%s:%s/%s" % (
self.user, self.password, self.host, self.port, self.virtual_host)
try:
self.connection = BrokerConnection(self.connection_string)
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def connect(self):
logger.info("[Canopsis] connection with: %s" % self.connection_string)
try:
self.connection.connect()
if not self.connected():
return False
else:
self.get_channel()
self.get_exchange()
self.create_producer()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def disconnect(self):
try:
if self.connected():
self.connection.release()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def connected(self):
try:
if self.connection.connected:
return True
else:
return False
except:
return False
def get_channel(self):
try:
self.channel = self.connection.channel()
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def get_exchange(self):
try:
self.exchange = Exchange(self.exchange_name,
"topic",
durable=True,
auto_delete=False)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def create_producer(self):
try:
self.producer = Producer(channel=self.channel,
exchange=self.exchange,
routing_key=self.virtual_host)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def postmessage(self, message, retry=False):
# process enqueud events if possible
self.pop_events()
if message["source_type"] == "component":
key = "%s.%s.%s.%s.%s" % (
message["connector"], message["connector_name"],
message["event_type"], message["source_type"],
message["component"])
else:
key = "%s.%s.%s.%s.%s[%s]" % (
message["connector"], message["connector_name"],
message["event_type"], message["source_type"],
message["component"], message["resource"])
# connection management
if not self.connected():
logger.error("[Canopsis] Create connection")
self.create_connection()
self.connect()
# publish message
if self.connected():
logger.debug("[Canopsis] using routing key %s" % key)
logger.debug("[Canopsis] sending %s" % str(message))
try:
self.producer.revive(self.channel)
self.producer.publish(body=message,
compression=None,
routing_key=key,
exchange=self.exchange_name)
return True
except:
logger.error(
"[Canopsis] Not connected, going to queue messages until connection back"
)
self.queue.append({"key": key, "message": message})
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
# logger.error(str(traceback.format_exc()))
return False
else:
errmsg = "[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % (
str(len(self.queue)), str(self.maxqueuelength))
logger.error(errmsg)
#enqueue_cano_event(key,message)
if len(self.queue) < int(self.maxqueuelength):
self.queue.append({"key": key, "message": message})
logger.debug("[Canopsis] Queue length: %d" % len(self.queue))
return True
else:
logger.error(
"[Canopsis] Maximum retention for event queue %s reached" %
str(self.maxqueuelength))
return False
def errback(self, exc, interval):
logger.warning("Couldn't publish message: %r. Retry in %ds" %
(exc, interval))
def pop_events(self):
if self.connected():
while len(self.queue) > 0:
item = self.queue.pop()
try:
logger.debug("[Canopsis] Pop item from queue [%s]: %s" %
(str(len(self.queue)), str(item)))
self.producer.revive(self.channel)
self.producer.publish(body=item["message"],
compression=None,
routing_key=item["key"],
exchange=self.exchange_name)
except:
self.queue.append(item)
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
else:
return False
def hook_tick(self, brok):
self.tickage += 1
# queue retention saving
if self.tickage >= int(self.queue_dump_frequency) and len(
self.queue) > 0:
# flush queue to disk if queue age reach queue_dump_frequency
self.save_queue()
self.tickage = 0
return True
def save_queue(self):
retentionfile = "%s/canopsis.dat" % os.getcwd() #:fixme: use path.join
logger.info("[Canopsis] saving to %s" % retentionfile)
filehandler = open(retentionfile, 'w')
pickle.dump(self.queue, filehandler)
filehandler.close()
return True
def load_queue(self):
retentionfile = "%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] loading from %s" % retentionfile)
filehandler = open(retentionfile, 'r')
try:
self.queue = pickle.load(filehandler)
except:
pass
return True
class Worker(object):
def __init__(self, queue_name):
self.queue_name = queue_name
self.serializer = "pickle"
self.rabbit_connect()
self.poll_messages()
def rabbit_connect(self):
url = "amqp://{}:{}@{}:5672/".format(config.rabbitmq.username, config.rabbitmq.password, config.rabbitmq.host)
self.connection = Connection(url)
self.channel = self.connection.channel()
self.channel.basic_qos(prefetch_size=0, prefetch_count=1, a_global=False)
self.exchange = Exchange("", type="direct", durable=True)
self.queue = Queue(name=self.queue_name, exchange=self.exchange, routing_key=self.queue_name)
self.queue.maybe_bind(self.connection)
self.queue.declare()
self.producer = Producer(exchange=self.exchange, channel=self.channel, serializer=self.serializer)
self.consumer = Consumer(self.connection, queues=self.queue, callbacks=[self.message_callback], accept=["application/x-python-serialize"])
#self.consumer.qos(prefetch_count = 1)
def poll_messages(self):
while True:
try:
self.process_messages()
except self.connection.connection_errors:
pass
def process_messages(self):
self.connection = self.renew_connection()
while True:
try:
self.connection.drain_events(timeout=5)
except socket.timeout:
pass
def renew_connection(self):
new_connection = self.connection.clone()
new_connection.ensure_connection(max_retries=10)
self.channel = new_connection.channel()
self.channel.basic_qos(prefetch_size=0, prefetch_count=1, a_global=False)
self.consumer.revive(self.channel)
self.producer.revive(self.channel)
self.consumer.consume()
return new_connection
def message_callback(self, body, message):
# Convert body to UTF-8 string
body = body.decode('utf-8')
# Process message
self.process_message(body)
# Tell RabbitMQ that we processed the message
message.ack()
def process_message(self, message):
# Generic message processing stub
print ("Message from queue '{}': '{}'".format(self.queue_name, message))
def produce_message(self, message):
self.producer.publish(message.encode('utf-8'), routing_key=self.queue_name, retry=True, delivery_mode=2)
class MessageBusService(object):
_uri: str
_connection: Connection
_connection_producer: Connection
_consuming: bool = False
_producer: Producer
_producer_reply_to_consumer: Consumer
_future: StandardQueue
_logger: Optional[logging.Logger]
def __init__(self, uri: str, logger: Optional[logging.Logger] = None):
self._uri = uri
self._logger = logger
self._future = StandardQueue()
self.connect()
def connect(self):
self._connection = Connection(self._uri)
self._connection.connect()
self._connection_producer = self._connection.clone()
self._producer = Producer(self._connection_producer)
#
reply_queue = Queue(
channel=self._producer.channel,
name="amq.rabbitmq.reply-to",
no_ack=True,
durable=False,
)
self._producer_reply_to_consumer = self._producer.channel.Consumer(
queues=[reply_queue],
no_ack=True,
auto_declare=True,
callbacks=[self.on_reply_to_message],
accept=["json"],
)
self._producer_reply_to_consumer.consume(no_ack=True)
def disconnect(self):
self._producer.close()
self._connection_producer.close()
self._connection.close()
def start_consuming(
self,
callback: Callable,
queue_name: str,
prefetch_count: int = 1,
no_ack: bool = False,
expires: int = None,
callback_ready: Callable = None,
):
if self._logger is not None:
self._logger.debug("Start consuming queue: %s" % queue_name)
self._consuming = True
while self._consuming:
revived_connection = self._connection.clone()
revived_connection.ensure_connection()
channel = revived_connection.channel()
channel.basic_qos(0, prefetch_count, True)
queues = []
queue_obj = Queue(
channel=channel,
name=queue_name,
no_ack=no_ack,
durable=False,
expires=expires,
queue_arguments={"x-max-priority": 255},
)
queue_obj.declare()
queues.append(queue_obj)
consumer = Consumer(
revived_connection,
queues,
callbacks=[callback],
accept=["json"],
auto_declare=False,
prefetch_count=prefetch_count,
)
consumer.revive(channel)
consumer.consume()
while self._consuming:
callback_ready is not None and callback_ready()
try:
revived_connection.drain_events(timeout=2)
except socket.timeout:
revived_connection.heartbeat_check()
except self._connection.connection_errors + (
AMQPError,
ConnectionForced,
ConnectionError,
): # pragma: no cover
if self._logger is not None:
self._logger.exception("Connection error", stack_info=True)
break
def start_consuming_replies(
self, callback: Callable, prefetch_count: int = 1, no_ack: bool = False
):
self._consuming = True
while self._consuming:
revived_connection = self._connection_producer.clone()
revived_connection.ensure_connection()
while self._consuming:
try:
revived_connection.drain_events(timeout=2)
except socket.timeout:
revived_connection.heartbeat_check()
except self._connection.connection_errors + (
AMQPError,
ConnectionForced,
ConnectionError,
): # pragma: no cover
if self._logger is not None:
self._logger.exception("Connection error", stack_info=True)
break
def stop_consuming(self):
if self._logger is not None:
self._logger.debug("Stop consuming...")
self._consuming = False
def publish(
self,
body: dict,
exchange: str = "",
queue_name: str = "",
priority: int = None,
reply_to: str = None,
expiration: int = None,
correlation_id: str = None,
):
while True:
try:
self._connection_producer.ensure_connection()
self._producer.publish(
body=body,
exchange=exchange,
routing_key=queue_name,
priority=priority,
reply_to=str(reply_to) if reply_to is not None else None,
expiration=expiration if expiration is not None else None,
correlation_id=str(correlation_id)
if correlation_id is not None
else None,
)
except self._connection_producer.connection_errors + (
AMQPError,
ConnectionForced,
ConnectionError,
): # pragma: no cover
self._connection_producer = self._connection.clone()
self._producer.revive(self._connection_producer)
else:
break
def on_reply_to_message(self, body, message):
self._future.put(body)
def publish_and_get_reply(self, *args, timeout: int = 1, **kw) -> Optional[dict]:
kw['reply_to'] = "amq.rabbitmq.reply-to"
self.publish(*args, **kw)
try:
self._producer_reply_to_consumer.connection.drain_events(timeout=timeout)
except socket.timeout: # pragma: no cover
return None
#
return self._future.get(block=False)