def __init__(self, host='localhost', username='******', password='******', queuename='testqueue'):
self.api = AdminAPI(url='http://' + host + ':15672', auth=('guest', 'guest'))
self.host = host
self.cr = pika.PlainCredentials(username=username, password=password)
self.conn = pika.BlockingConnection(pika.ConnectionParameters(self.host, credentials=self.cr))
self.chann = self.conn.channel()
self.queue = self.chann.queue_declare(queuename, durable=True)
self.routing_key = queuename
def get_rabbitmq_stats(self):
if self.__rabbitmq_api == None:
config = Config.get_instance()
url = 'http://%s:%d' % (config.rmq_host, config.rmq_port)
self.__rabbitmq_api = AdminAPI(url=url, auth=(config.rmq_user, config.rmq_password))
overview = self.__rabbitmq_api.overview()
ready = 0
unacked = 0
total = 0
publish_rate = 0
deliver_manual_ack = 0
consumer_ack = 0
disk_read = 0
disk_write = 0
if overview == None:
print('get rabbitmq overview failed')
return None
if 'queue_totals' in overview:
queue_totals = overview['queue_totals']
if 'messages' in queue_totals:
total = queue_totals['messages']
if 'messages_ready' in queue_totals:
ready = queue_totals['messages_ready']
if 'messages_unacknowledged' in queue_totals:
unacked = queue_totals['messages_unacknowledged']
if 'message_stats' in overview:
message_stats = overview['message_stats']
if 'publish_details' in message_stats:
publish_details = message_stats['publish_details']
if 'rate' in publish_details:
publish_rate = publish_details['rate']
if 'deliver_details' in message_stats:
deliver_details = message_stats['deliver_details']
if 'rate' in deliver_details:
deliver_manual_ack = deliver_details['rate']
if 'ack_details' in message_stats:
ack_details = message_stats['ack_details']
if 'rate' in ack_details:
consumer_ack = ack_details['rate']
if 'disk_reads_details' in message_stats:
disk_reads_details = message_stats['disk_reads_details']
if 'rate' in disk_reads_details:
disk_read = disk_reads_details['rate']
if 'disk_writes_details' in message_stats:
disk_writes_details = message_stats['disk_writes_details']
if 'rate' in disk_writes_details:
disk_write = disk_writes_details['rate']
return {
'total': total,
'ready': ready,
'unacked': unacked,
'publish_rate': publish_rate,
'deliver_manual_ack': deliver_manual_ack,
'consumer_ack': consumer_ack,
'disk_read': disk_read,
'disk_write': disk_write
}
def connect_mqtt(self, virtual_host, queue_name):
try:
api = AdminAPI(url='http://' + self.rabbitmq_server + ':' +
str(8080),
auth=(self.rabbitmq_id, self.rabbitmq_pwd))
vhosts_info = api.list_vhosts()
vhost_list = [vhost['name'] for vhost in vhosts_info]
if virtual_host not in vhost_list:
api.create_vhost(virtual_host)
api.create_user_permission(name=self.rabbitmq_id,
vhost=virtual_host)
credentials = pika.PlainCredentials(self.rabbitmq_id,
self.rabbitmq_pwd)
params = pika.ConnectionParameters(self.rabbitmq_server,
self.rabbitmq_port,
virtual_host, credentials)
connection = pika.BlockingConnection(params)
self.channel = connection.channel()
except Exception as ex: # 에러 종류
print('RabbitMQ Connect fail', ex) # ex는 발생한 에러의 이름을 받아오는 변수
return False
try:
self.channel.queue_declare(queue=queue_name)
return True
except Exception as ex: # 에러 종류
print('Declare Queue fail', ex) # ex는 발생한 에러의 이름을 받아오는 변수
return False
def _pre_setup(self):
"""Create RabbitMQ virtual host."""
# Check if we are mixed with the class that makes sense.
if ChannelTestCaseMixin in self.__class__.__mro__:
raise ImproperlyConfigured(
'ChannelTestCase is not allowed as base class for '
'RabbitmqLayerTestCaseMixin')
# Create new virtual host for this test and grant permissions
# to it.
hostname = environ.get('RABBITMQ_HOST', 'localhost')
port = environ.get('RABBITMQ_PORT', '5672')
user = environ.get('RABBITMQ_USER', 'guest')
password = environ.get('RABBITMQ_PASSWORD', 'guest')
management_port = environ.get('RABBITMQ_MANAGEMENT_PORT', '15672')
management_url = 'http://%s:%s' % (hostname, management_port)
self.virtual_host = ''.join(choice(ascii_letters) for i in range(8))
self.amqp_url = 'amqp://%s:%s/%s' % (hostname, port, self.virtual_host)
self.management = AdminAPI(management_url, (user, password))
self.management.create_vhost(self.virtual_host)
self.management.create_user_permission(user, self.virtual_host)
# Substitute Django settings with this virtual host.
if self.local:
layer_class_name = 'asgi_rabbitmq.RabbitmqLocalChannelLayer'
else:
layer_class_name = 'asgi_rabbitmq.RabbitmqChannelLayer'
self._overridden_settings = {
'CHANNEL_LAYERS': {
'default': {
'BACKEND': layer_class_name,
'ROUTING': settings.CHANNEL_LAYERS['default']['ROUTING'],
'CONFIG': settings.CHANNEL_LAYERS['default']['CONFIG'],
'TEST_CONFIG': {
'url': self.amqp_url,
},
},
},
}
self._self_overridden_context = override_settings(
**self._overridden_settings)
self._self_overridden_context.enable()
super(RabbitmqLayerTestCaseMixin, self)._pre_setup()
class RabbitmqLayerTestCaseMixin(object):
"""
TestCase mixin for Django tests.
Allow to test your channels project against real broker. Provide
isolated virtual host for each test.
"""
local = False
def _pre_setup(self):
"""Create RabbitMQ virtual host."""
if ChannelTestCaseMixin in self.__class__.__mro__:
raise ImproperlyConfigured(
'ChannelTestCase is not allowed as base class for '
'RabbitmqLayerTestCaseMixin')
hostname = environ.get('RABBITMQ_HOST', 'localhost')
port = environ.get('RABBITMQ_PORT', '5672')
user = environ.get('RABBITMQ_USER', 'guest')
password = environ.get('RABBITMQ_PASSWORD', 'guest')
management_port = environ.get('RABBITMQ_MANAGEMENT_PORT', '15672')
management_url = 'http://%s:%s' % (hostname, management_port)
self.virtual_host = ''.join(choice(ascii_letters) for i in range(8))
self.amqp_url = 'amqp://%s:%s/%s' % (hostname, port, self.virtual_host)
self.management = AdminAPI(management_url, (user, password))
self.management.create_vhost(self.virtual_host)
self.management.create_user_permission(user, self.virtual_host)
if self.local:
layer_class_name = 'asgi_rabbitmq.RabbitmqLocalChannelLayer'
else:
layer_class_name = 'asgi_rabbitmq.RabbitmqChannelLayer'
self._overridden_settings = {
'CHANNEL_LAYERS': {
'default': {
'BACKEND': layer_class_name,
'ROUTING': settings.CHANNEL_LAYERS['default']['ROUTING'],
'CONFIG': settings.CHANNEL_LAYERS['default']['CONFIG'],
'TEST_CONFIG': {
'url': self.amqp_url,
},
},
},
}
self._self_overridden_context = override_settings(
**self._overridden_settings)
self._self_overridden_context.enable()
super(RabbitmqLayerTestCaseMixin, self)._pre_setup()
def _post_teardown(self):
"""Remove RabbitMQ virtual host."""
super(RabbitmqLayerTestCaseMixin, self)._post_teardown()
self._self_overridden_context.disable()
delattr(self, '_self_overridden_context')
self._overridden_settings = None
self.management.delete_vhost(self.virtual_host)
del self.virtual_host
del self.amqp_url
del self.management
def create_project_user_and_vhost(project_id):
secrets = get_project_secrets(project_id)
hidden_secrets = get_project_hidden_secrets(project_id)
# connect to RabbitMQ management api
rabbit_api = AdminAPI(url=f'http://carrier-rabbit:15672',
auth=(hidden_secrets["rabbit_user"],
hidden_secrets["rabbit_password"]))
# prepare user credentials
user = f"rabbit_user_{project_id}"
password = password_generator()
vhost = f"project_{project_id}_vhost"
# create project user and vhost
rabbit_api.create_vhost(vhost)
rabbit_api.create_user(user, password)
rabbit_api.create_user_permission(user, vhost)
# set project secrets
secrets["rabbit_project_user"] = user
secrets["rabbit_project_password"] = password
secrets["rabbit_project_vhost"] = vhost
set_project_secrets(project_id, secrets)
def initialize_rabbitmq_user(user):
"""
This function is used when a new provider registers in the network.
It adds a new user in RabbitMQ and sets their permissions.
:param user: The newly registered user
"""
# TODO: passwords have a pattern for all users. This needs to be changed.
password = user.username + '_mqtt'
api = AdminAPI(url='http://' + RABBITMQ_HOST + ':' +
RABBITMQ_MANAGEMENT_PORT,
auth=(RABBITMQ_USER, RABBITMQ_PASS))
api.create_user(user.username, password)
# process = subprocess.Popen(['sudo', 'rabbitmqctl', 'add_user', user.username, password],
# stdout=subprocess.PIPE, universal_newlines=True)
# stdout, stderr = process.communicate()
# print(stdout)
permission = "^(" + user.username + ".*|amq.default)$"
api.create_user_permission(user.username, '/', permission, permission,
permission)
class RecSystemsManager:
def __init__(self, host='localhost', username='******', password='******', queuename='testqueue'):
self.api = AdminAPI(url='http://' + host + ':15672', auth=('guest', 'guest'))
self.host = host
self.cr = pika.PlainCredentials(username=username, password=password)
self.conn = pika.BlockingConnection(pika.ConnectionParameters(self.host, credentials=self.cr))
self.chann = self.conn.channel()
self.queue = self.chann.queue_declare(queuename, durable=True)
self.routing_key = queuename
def subscribeSystem(self, teamname, system, passwd):
self.api.create_vhost('second_vhost', tracing=True)
self.api.create_user(teamname, passwd)
self.api.create_user_permission(teamname, 'second_vhost')
# Include SQL Database authentication here
self.chann.queue_declare(teamname + '_' + system + '_' + 'rec_requests', durable=True)
self.chann.queue_declare(teamname + '_' + system + '_' + 'user_assign', durable=True)
self.chann.queue_declare(teamname + '_' + system + '_' + 'news_recs', durable=True)
self.chann.queue_declare(teamname + '_' + system + '_' + 'user_data', durable=True)
self.chann.queue_declare(teamname + '_' + system + '_' + 'news_request', durable=True)
def createQueue(self, name, durable):
self.chann.queue_declare(name, durable)
def __init__(self,
rabbit_server_config,
topicCallback=callback,
subscriptionCallback=callback,
consumerSubscriptions=None,
consumerTopics=None,
consumerSyncTopics=None,
producerSubscriptions=None,
producerTopic=None,
**kwargs):
sender_properties = {
"exchange": kwargs.get("sender_exchange", "RESOURCES_UPDATES"),
"exchange_type": "topic",
"passive": False,
"durable": True,
"auto_delete": False,
"subscription": producerSubscriptions,
"producerTopic": producerTopic
}
receiver_properties = {
"exchange":
kwargs.get("receiver_exchange", "RESOURCES_UPDATES"),
"subscription_exchange":
kwargs.get("subscription_exchange", "RESOURCES_UPDATES"),
"exchange_type":
"topic",
"passive":
False,
"durable":
True,
"auto_delete":
False,
"subscriptions":
consumerSubscriptions,
"queue":
"standard",
"consumerTopics":
consumerTopics,
"consumerSyncTopics":
consumerSyncTopics,
"consumerSyncField":
kwargs.get("consumerSyncField", None),
"queueId":
kwargs.get('queueId', "")
}
rest_api_config = kwargs.get("rest_api_config", {})
if (producerSubscriptions is not None or consumerSubscriptions
is not None) and rest_api_config == {}:
logger.error("rest_api_config is mandatory for susbscriptions")
return
logger.info(
"rabbit server config-{}, rest_api_config-{} ,sender_properties-{} ,receiver_properties-{}"
.format(rabbit_server_config, rest_api_config, sender_properties,
receiver_properties))
self.rabbit_server_config = rabbit_server_config
self.sender_rabbit_server_config = kwargs.get(
"sender_rabbit_server_config", rabbit_server_config)
if self.sender_rabbit_server_config is None:
self.sender_rabbit_server_config = self.rabbit_server_config
self.receiver_rabbit_server_config = kwargs.get(
"receiver_rabbit_server_config", rabbit_server_config)
if self.receiver_rabbit_server_config is None:
self.receiver_rabbit_server_config = self.rabbit_server_config
self.sender_creds = pika.PlainCredentials(
self.sender_rabbit_server_config['user'],
self.sender_rabbit_server_config['password'])
self.receiver_creds = pika.PlainCredentials(
self.receiver_rabbit_server_config['user'],
self.receiver_rabbit_server_config['password'])
self.heartbeat = kwargs.get("heartbeat", 30)
self.start = True
self.subRouteMap = {}
self.subRoutes = []
self.consume_failure = 0
self.sender_properties = sender_properties
self.receiver_properties = receiver_properties
self.rest_api_config = rest_api_config
self.create_hierarchy()
self.topicCallback = topicCallback
self.subscriptionCallback = subscriptionCallback
self.failures = 0
self.testTopic = self.get_queue("") + '-test'
self.sendLock = False
self.receiveLock = False
self.sender_failed_attempts = 0
self.receiver_failed_attempts = 0
self.check_connection_thread = threading.Thread(
target=self.check_connection_state)
self.sender_connection = None
self.receiver_connection = None
self.sender_channel = None
self.receiver_connection_async = None
self.receiver_channel = None
self.receiver_channel_async = None
self.usedRoutingKeys = []
protocol = kwargs.get("protocol", 'http')
try:
adminUrl = "{}://{}:{}".format(
protocol, self.rabbit_server_config['host'],
self.sender_rabbit_server_config['port'])
self.adminApi = AdminAPI(
url=adminUrl,
auth=(self.sender_rabbit_server_config['user'],
self.sender_rabbit_server_config['password']))
except Exception as e:
logger.info("failed to initialize admin api-{}".format(str(e)))
try:
self.init_connections()
except Exception as e:
logger.error(
"some error occurred initializing connection retrying-{}".
format(str(e)))
self.check_connection_thread.start()
if not self.check_connection_thread.is_alive():
self.check_connection_thread.start()
time.sleep(1)
class RabbitMqConnector():
def __init__(self,
rabbit_server_config,
topicCallback=callback,
subscriptionCallback=callback,
consumerSubscriptions=None,
consumerTopics=None,
consumerSyncTopics=None,
producerSubscriptions=None,
producerTopic=None,
**kwargs):
sender_properties = {
"exchange": kwargs.get("sender_exchange", "RESOURCES_UPDATES"),
"exchange_type": "topic",
"passive": False,
"durable": True,
"auto_delete": False,
"subscription": producerSubscriptions,
"producerTopic": producerTopic
}
receiver_properties = {
"exchange":
kwargs.get("receiver_exchange", "RESOURCES_UPDATES"),
"subscription_exchange":
kwargs.get("subscription_exchange", "RESOURCES_UPDATES"),
"exchange_type":
"topic",
"passive":
False,
"durable":
True,
"auto_delete":
False,
"subscriptions":
consumerSubscriptions,
"queue":
"standard",
"consumerTopics":
consumerTopics,
"consumerSyncTopics":
consumerSyncTopics,
"consumerSyncField":
kwargs.get("consumerSyncField", None),
"queueId":
kwargs.get('queueId', "")
}
rest_api_config = kwargs.get("rest_api_config", {})
if (producerSubscriptions is not None or consumerSubscriptions
is not None) and rest_api_config == {}:
logger.error("rest_api_config is mandatory for susbscriptions")
return
logger.info(
"rabbit server config-{}, rest_api_config-{} ,sender_properties-{} ,receiver_properties-{}"
.format(rabbit_server_config, rest_api_config, sender_properties,
receiver_properties))
self.rabbit_server_config = rabbit_server_config
self.sender_rabbit_server_config = kwargs.get(
"sender_rabbit_server_config", rabbit_server_config)
if self.sender_rabbit_server_config is None:
self.sender_rabbit_server_config = self.rabbit_server_config
self.receiver_rabbit_server_config = kwargs.get(
"receiver_rabbit_server_config", rabbit_server_config)
if self.receiver_rabbit_server_config is None:
self.receiver_rabbit_server_config = self.rabbit_server_config
self.sender_creds = pika.PlainCredentials(
self.sender_rabbit_server_config['user'],
self.sender_rabbit_server_config['password'])
self.receiver_creds = pika.PlainCredentials(
self.receiver_rabbit_server_config['user'],
self.receiver_rabbit_server_config['password'])
self.heartbeat = kwargs.get("heartbeat", 30)
self.start = True
self.subRouteMap = {}
self.subRoutes = []
self.consume_failure = 0
self.sender_properties = sender_properties
self.receiver_properties = receiver_properties
self.rest_api_config = rest_api_config
self.create_hierarchy()
self.topicCallback = topicCallback
self.subscriptionCallback = subscriptionCallback
self.failures = 0
self.testTopic = self.get_queue("") + '-test'
self.sendLock = False
self.receiveLock = False
self.sender_failed_attempts = 0
self.receiver_failed_attempts = 0
self.check_connection_thread = threading.Thread(
target=self.check_connection_state)
self.sender_connection = None
self.receiver_connection = None
self.sender_channel = None
self.receiver_connection_async = None
self.receiver_channel = None
self.receiver_channel_async = None
self.usedRoutingKeys = []
protocol = kwargs.get("protocol", 'http')
try:
adminUrl = "{}://{}:{}".format(
protocol, self.rabbit_server_config['host'],
self.sender_rabbit_server_config['port'])
self.adminApi = AdminAPI(
url=adminUrl,
auth=(self.sender_rabbit_server_config['user'],
self.sender_rabbit_server_config['password']))
except Exception as e:
logger.info("failed to initialize admin api-{}".format(str(e)))
try:
self.init_connections()
except Exception as e:
logger.error(
"some error occurred initializing connection retrying-{}".
format(str(e)))
self.check_connection_thread.start()
if not self.check_connection_thread.is_alive():
self.check_connection_thread.start()
time.sleep(1)
def get_all_routingKeys(self, queue, exchange=None):
bindings = self.adminApi.list_bindings()
return [
x['routing_key'] for x in bindings if x['destination'] == queue
]
def init_connections(self):
self.init_sender_connection()
self.init_receiver_connection()
def init_sender_connection(self):
self.sender_connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=self.sender_rabbit_server_config['host'],
credentials=self.sender_creds,
heartbeat=self.heartbeat))
self.sender_channel = self.sender_connection.channel()
self.sender_channel.exchange_declare(
exchange=self.sender_properties["exchange"],
exchange_type=self.sender_properties["exchange_type"],
passive=self.sender_properties["passive"],
durable=self.sender_properties["durable"],
auto_delete=self.sender_properties["auto_delete"])
def get_queue(self, topic):
queueId = str(self.receiver_properties.get("queueId", ""))
return queueId + '-' + topic
def init_receiver_connection(self):
self.receiver_connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=self.receiver_rabbit_server_config['host'],
credentials=self.receiver_creds,
heartbeat=self.heartbeat))
self.receiver_channel = self.receiver_connection.channel()
self.receiver_connection_async = pika.BlockingConnection(
pika.ConnectionParameters(host=self.rabbit_server_config['host'],
credentials=self.receiver_creds,
heartbeat=self.heartbeat))
self.receiver_channel_async = self.receiver_connection_async.channel()
self.sync_receiver_channels = {}
self.receiver_channel.exchange_declare(
exchange=self.receiver_properties["exchange"],
exchange_type=self.receiver_properties["exchange_type"],
passive=self.receiver_properties["passive"],
durable=self.receiver_properties["durable"],
auto_delete=self.receiver_properties["auto_delete"])
self.receiver_channel_async.exchange_declare(
exchange=self.receiver_properties["exchange"],
exchange_type=self.receiver_properties["exchange_type"],
passive=self.receiver_properties["passive"],
durable=self.receiver_properties["durable"],
auto_delete=self.receiver_properties["auto_delete"])
self.receiver_queue_async = None
consumerTopics = self.receiver_properties.get("consumerTopics", None)
subscriptions = self.receiver_properties.get("subscriptions", None)
consumerSyncTopics = self.receiver_properties.get(
"consumerSyncTopics", None)
self.add_subscriptions(subscriptions, consumerTopics,
consumerSyncTopics)
def remove_subscriptions(self, subscriptions=None):
logger.info("removing subscriptions-{}".format(str(subscriptions)))
if self.receiver_properties["subscriptions"] is not None:
for subscription in subscriptions:
self.receiver_properties["subscriptions"].remove(subscription)
self.reinit_receiver()
logger.info("remaining subscriptions-{}".format(
str(self.receiver_properties["subscriptions"])))
def add_subscriptions(self,
subscriptions=None,
consumerTopics=None,
consumerSyncTopics=None):
# If consumers are sync, make a different queue for each consumer
currentSyncKeys = []
if consumerSyncTopics is not None:
for consumerTopic in consumerSyncTopics:
channel = self.receiver_connection.channel()
channel.exchange_declare(
exchange=self.receiver_properties["exchange"],
exchange_type=self.receiver_properties["exchange_type"],
passive=self.receiver_properties["passive"],
durable=self.receiver_properties["durable"],
auto_delete=self.receiver_properties["auto_delete"])
queue_name = self.get_queue(consumerTopic)
channel.queue_declare(queue=queue_name)
currentSyncKeys.append(consumerTopic)
channel.queue_bind(
queue=queue_name,
exchange=self.receiver_properties["exchange"],
routing_key=consumerTopic)
logger.info("listening to sync queue-{}".format(queue_name))
self.sync_receiver_channels[consumerTopic] = channel
self.usedRoutingKeys = []
routingKeys = []
if consumerTopics is not None:
for topic in consumerTopics:
routingKeys.append(topic)
self.usedRoutingKeys.append(topic)
subsKeys = []
if subscriptions is not None:
for subscription in subscriptions:
routingKey = self.getRoutingKey(subscription)
subsKeys.append(routingKey)
self.usedRoutingKeys.append(routingKey)
currentSyncKeys = []
currentSyncKeys = currentSyncKeys + routingKeys + subsKeys
if (len(routingKeys) > 0 or len(subsKeys) > 0):
if not self.receiver_queue_async:
self.receiver_properties["subscriptions"] = subscriptions
self.receiver_queue_async = self.get_queue("") + '-async'
self.receiver_channel_async.queue_declare(
queue=self.receiver_queue_async)
logger.info("subscribing to routes-{}, queue-{}".format(
routingKeys, self.receiver_queue_async))
for routingKey in routingKeys:
self.receiver_channel_async.queue_bind(
queue=self.receiver_queue_async,
exchange=self.receiver_properties["exchange"],
routing_key=routingKey)
logger.info("subscribing to subscriptions-{}, queue-{}".format(
subsKeys, self.receiver_queue_async))
for routingKey in subsKeys:
self.receiver_channel_async.queue_bind(
queue=self.receiver_queue_async,
exchange=self.
receiver_properties["subscription_exchange"],
routing_key=routingKey)
#Cleanup unused keys
try:
alreadyExistingKeys = alreadyExistingKeys = self.get_all_routingKeys(
self.receiver_queue_async)
unusedKeys = [
x for x in alreadyExistingKeys
if x not in currentSyncKeys
]
for key in unusedKeys:
self.receiver_channel_async.queue_unbind(
queue=self.receiver_queue_async,
exchange=self.receiver_properties["exchange"],
routing_key=key)
self.receiver_channel_async.queue_unbind(
queue=self.receiver_queue_async,
exchange=self.
receiver_properties["subscription_exchange"],
routing_key=key)
except Exception as e:
logger.info(
"some exception occurred while removing previous async subscriptions-{}"
.format(str(e)))
self.receiver_channel_async.basic_consume(
queue=self.receiver_queue_async,
on_message_callback=self.receive,
auto_ack=True)
self.consume_thread = threading.Thread(target=self.consume)
if not self.consume_thread.is_alive():
self.consume_thread.start()
time.sleep(2)
else:
subscriptions_hist = self.receiver_properties.get(
"subscriptions", [])
if not subscriptions_hist:
subscriptions_hist = []
subscriptions_hist.extend(subscriptions)
self.receiver_properties["subscriptions"] = subscriptions_hist
self.reinit_receiver()
def reinit_connections(self):
self.reinit_sender()
self.reinit_receiver()
def reinit_sender(self):
time.sleep(5)
logger.info("reinitializing sender connection")
try:
self.sender_connection.close()
self.sender_channel.close()
except Exception as e:
logger.info("failed to reinitialize sender connection")
pass
self.init_sender_connection()
def reinit_receiver(self):
self.receiveLock = True
#time.sleep(5)
logger.info("reinitializing receiver connection")
try:
self.receiver_channel_async.stop_consuming()
except:
logger.info("some error in stopping consuming async")
pass
try:
self.consume_thread.join()
except:
logger.info("some error in closing consumer thread")
pass
try:
self.receiver_connection.close()
self.receiver_channel.close()
except Exception as e:
logger.info("failed to reinitialize connection")
pass
try:
self.receiver_connection_async.close()
self.receiver_channel_async.close()
except Exception as e:
logger.info("failed to reinitialize connection")
pass
try:
self.init_receiver_connection()
except:
pass
self.receiveLock = False
def check_connection_state(self):
while (self.start):
state = 'OK'
try:
time.sleep(self.heartbeat - 5)
logger.info("checking connection state with server")
try:
self.sender_connection.process_data_events()
except:
pass
if self.sender_connection.is_closed or self.sender_channel.is_closed:
logger.info("sender connection closed")
state = 'BROKEN'
logger.info("reinitializing sender connection")
try:
self.reinit_sender()
except:
logger.info("failed to reinitializes sender")
self.sender_failed_attempts += 1
else:
self.sender_failed_attempts = 0
#logger.info("********************************receiveLock********************************--{}".format(self.receiveLock))
if not self.receiveLock:
receiver_status = True
try:
self.receiver_connection.process_data_events()
except:
pass
#logger.info("receiver.connection status*******-{}".format(self.receiver_channel.is_closed))
if self.receiver_connection.is_closed:
receiver_status = False
try:
self.receiver_connection_async.process_data_events()
except:
pass
#logger.info("receiver.connection status*******-{}".format(self.receiver_channel_async.is_closed))
if self.receiver_connection_async.is_closed:
receiver_status = False
if receiver_status == False:
logger.info("receiver connection closed")
state = 'BROKEN'
self.receiver_failed_attempts += 1
logger.info("reinitializing sender connection")
try:
self.reinit_receiver()
except:
logger.info("failed to reinitializes sender")
else:
self.receiver_failed_attempts = 0
except Exception as e:
logger.debug(traceback.format_exc())
state = 'BROKEN'
logger.info("failed to open connections..retrying")
try:
self.reinit_connections()
except:
pass
logger.info("connection state->{}-{}".format(state, time.time()))
def send(self,
message={"message": "ping------pong"},
subscription=None,
producerTopic=None,
showMessage=False):
if producerTopic is None:
producerTopic = self.sender_properties["producerTopic"]
if subscription is None:
subscription = self.sender_properties["subscription"]
if producerTopic is not None:
routingKey = producerTopic
elif subscription is not None:
logger.info("sending to subscription-{}".format(subscription))
routingKey = self.getRoutingKey(subscription)
while (self.sendLock):
logger.info("waiting for send lock to clear")
time.sleep(0.01)
self.sendLock = True
try:
self.sender_channel.basic_publish(
exchange=self.sender_properties["exchange"],
routing_key=routingKey,
body=json.dumps(message))
except Exception as e:
self.sendLock = False
logger.info("failed to send message-{}".format(str(e)))
return False
self.sendLock = False
if routingKey != 'test':
print("=" * 50)
print("Producing Message")
print("Producer topic", routingKey)
print("=" * 50)
if showMessage:
logger.info(
"successfully sent message on routing key-{}, message-{}".
format(routingKey, message))
else:
logger.info(
"successfully sent message on routing key-{}".format(
routingKey))
return True
def send_response(self, props, message, showMessage=False):
while (self.sendLock):
logger.info("waiting for send lock to clear")
time.sleep(0.01)
self.sendLock = True
try:
self.sender_channel.basic_publish(
exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(
correlation_id=props.correlation_id),
body=json.dumps(message))
self.sendLock = False
except Exception as e:
self.sendLock = False
logger.info("failed to send message-{}".format(str(e)))
return False
if showMessage:
logger.info(
"successfully sent message on routing key-{}, message-{}".
format(props.reply_to, message))
else:
logger.info("successfully sent message on routing key-{}".format(
props.reply_to))
return True
def replacecontrolchar(self, text):
for a, b in mapnonprint.items():
if a in text:
logger.warning(
"Json Decode replacecontrolchar:{} with {}".format(a, b))
text = text.replace(a, b)
return text
def receive(self, ch, method, properties, body):
try:
message = body.decode('utf8').replace("'", '"')
try:
message = json.loads(message1)
except Exception as e:
try:
message = json.loads(self.replacecontrolchar(message))
except Exception as e:
logger.info("error loading message to json-{}".format(
str(e)))
routingKey = method.routing_key
logger.info(
"message received from routing key-{}".format(routingKey))
if str(routingKey) not in self.usedRoutingKeys:
logger.info(
"message received from unused route now--skipping!")
#return
if routingKey != self.testTopic:
print("=" * 50)
print("Consuming Message")
print("Consumer topic", routingKey)
print("=" * 50)
if routingKey in self.subRoutes:
self.subscriptionCallback(message, properties, method)
else:
self.subscriptionCallback(message, properties, method)
#self.topicCallback(message,properties,method)
except Exception as e:
logger.info("some exception occurred -{}".format(str(e)))
logger.debug(traceback.format_exc())
def consume(self):
#time.sleep(5)
logger.info("receiver started consuming")
# wait for sometime
try:
self.receiver_channel_async.start_consuming()
except Exception as e:
logger.info(
"some error occured while consuming..async receiver should restart after heartbeat"
.format(str(e)))
logger.debug(traceback.format_exc())
def consume_sync(self, topic):
while self.receiveLock:
logger.info("waiting for receiveLock")
time.sleep(0.1)
try:
method_frame, header_frame, body = self.sync_receiver_channels[
topic].basic_get(self.get_queue(topic), auto_ack=True)
if method_frame:
try:
message = body.decode('utf8').replace("'", '"')
message = json.loads(message1)
except Exception as e:
try:
message = json.loads(self.replacecontrolchar(message))
except Exception as e:
logger.info("error loading message to json-{}".format(
str(e)))
print("=" * 50)
print("Consuming Message")
print("Consumer topic", topic)
print("=" * 50)
return message
return None
except Exception as e:
logger.info("some exception occurred in sync consuming-{}".format(
str(e)))
#print(traceback.format_exc())
logger.debug(traceback.format_exc())
time.sleep(self.heartbeat)
return None
def consume_sync_all(self):
startTime = time.time()
finalMessage = {}
topics = self.receiver_properties['consumerSyncTopics'].copy()
topicLength = len(topics)
syncId = None
syncField = self.receiver_properties['consumerSyncField']
while (len(topics) > 0):
for topic in topics:
out = self.consume_sync(topic)
if out:
finalMessage[topic] = out
topics.remove(topic)
if len(topics) == topicLength:
return None
if (time.time() - startTime) > 10:
logger.info("sync time exceeded, failed to sync")
self.flush_sync_consumer_queues()
return None
if syncField:
for key in finalMessage.keys():
id = finalMessage[key][syncField]
if syncId:
if id != syncId:
logger.info("consumers not in sync..flushing queue")
self.flush_sync_consumer_queues()
return None
else:
syncId = id
return finalMessage
def flush_sync_consumer_queues(self):
for topic in self.receiver_properties['consumerSyncTopics']:
self.sync_receiver_channels[topic].queue_purge(
self.get_queue(topic))
def create_hierarchy(self):
logger.info('resource hierarchy being setup!')
Node.separator = "."
self.organizations = Node("organizations")
self.hubs = Node('hubs', parent=self.organizations)
self.behaviourTypes = Node('behaviourTypes', parent=self.organizations)
self.users = Node('users', parent=self.organizations)
self.artefacts = Node("artefacts", parent=self.organizations)
self.notifications = Node("notificaitons", parent=self.organizations)
self.pipelines = Node("pipelines", parent=self.organizations)
self.charts = Node("charts", parent=self.organizations)
self.recommendations = Node("recommendations",
parent=self.organizations)
self.cameras = Node("cameras", parent=self.hubs)
self.devices = Node("devices", parent=self.hubs)
self.behaviours = Node("behaviours", parent=self.cameras)
self.alerts = Node("alerts", parent=self.behaviours)
self.resourceMap = {
"organization": "organizations",
"hub": "hubs",
"behaviourType": "behaviourTypes",
"user": "******",
"artefact": "artefacts",
"notificaiton": "notificaitons",
"pipeline": "pipelines",
"chart": "charts",
"recommendation": "recommendations",
"camera": "cameras",
"device": "devices",
"behaviour": "behaviours",
"alert": "alerts",
}
def getRoutingKey(self, subscription):
subRouteKeys = self.subRouteMap.keys()
if str(subscription) in subRouteKeys:
return self.subRouteMap[str(subscription)]
sub_keys = subscription.keys()
resourceKeys = self.resourceMap.keys()
resourceKey = None
for key in sub_keys:
if key in resourceKeys:
resourceKey = key
break
if resourceKey is not None:
resource_id = subscription[resourceKey]
resource = self.get_resource(self.resourceMap[resourceKey],
resource_id)
route = self.findRoute(self.resourceMap[resourceKey])
routelist = route.split('.')
#print("routelist",routelist)
finalRoute = ''
for name in routelist:
name = name[:-1]
if name == resourceKey:
finalRoute += str(resource['_id']) + '.'
else:
finalRoute += str(resource[name]) + '.'
topic = subscription.get('topic', None)
if topic is not None:
finalRoute += str(subscription['topic'])
else:
finalRoute = finalRoute[:-1]
self.subRouteMap[str(subscription)] = finalRoute
self.subRoutes.append(finalRoute)
return finalRoute
def findRoute(self, nodeName):
anytreeNodePath = search.find(self.organizations,
lambda node: node.name == nodeName)
#print ("anytree node path:",anytreeNodePath)
try:
if anytreeNodePath:
rt = str(anytreeNodePath).split('(')[1].replace(
"'.", "", 1).replace("')", '', 1)
#print ("designated route:",rt)
return rt
except Exception as anytreeError:
#print ("anytree error occured",anytreeError)
return None
# Import app to get api_config
def get_resource(self, res_type, res_id=None, res_filter=None):
api_config = self.rest_api_config
Veda_auth = HTTPBasicAuth(api_config['VEDA_USER'],
api_config['VEDA_PASSWORD'])
if res_id is not None:
final_url = "{}/{}/{}/{}".format(api_config['VEDA_SERVER_URL'],
api_config['VEDA_API_VERSION'],
res_type, res_id)
elif res_filter is not None:
final_url = "{}/{}/{}?where={}".format(
api_config['VEDA_SERVER_URL'], api_config['VEDA_API_VERSION'],
res_type, json.dumps(res_filter))
else:
final_url = "{}/{}/{}".format(api_config['VEDA_SERVER_URL'],
api_config['VEDA_API_VERSION'],
res_type)
resp = {}
try:
items = []
while True:
logging.info("Fetching cloud resource: {}".format(final_url))
resp = requests.get(final_url, auth=Veda_auth, timeout=10)
resp.raise_for_status()
resp = resp.json()
if "next" in resp["_links"] and "href" in resp["_links"][
"next"]:
route = resp["_links"]["next"]['href']
final_url = "{}/{}/{}".format(
api_config['VEDA_SERVER_URL'],
api_config['VEDA_API_VERSION'], route)
items = items + resp["_items"]
else:
if items:
resp["_items"] = items
break
except Exception as e:
logging.error("get_resource Failed:{}".format(e))
resp = None
if resp is None:
raise RuntimeError(
"Failed to fetch cloud resource of type: {}, id: {} and filter: {}"
.format(res_type, res_id, res_filter))
return resp
def stop(self):
logger.info(
"stopping connector & ignoring all errors..helpless-please don't judge!"
)
self.start = False
try:
self.check_connection_thread.join()
except:
pass
try:
self.sender_connection.close()
except:
pass
try:
self.receiver_channel_async.stop_consuming()
except:
pass
try:
self.receiver_connection.close()
except:
pass
try:
self.receiver_connection_async.close()
except:
pass
try:
self.consume_thread.join()
except:
pass
class PickRmqThread(threading.Thread):
def __init__(self, daemon = True):
super().__init__()
self.setName('pickrmq')
self.setDaemon(daemon)
print('create thread: %s' % self.name)
self.__rabbitmq_api = None
self.__session = requests.Session()
def run(self) -> None:
process = None
while True:
try:
if process == None:
print('rabbitmq not start')
process = get_rmq_process()
if process == None:
time.sleep(5)
continue
if not process.is_running():
self.send_crash(process)
process = None
continue
else:
global capture_bytes
backup_5672_bytes = capture_bytes
# 内存占用, 单位 MB
mem_usage = get_rmq_process_mem_usage(process)
# 磁盘空间占用, 单位 MB
disk_spend = get_file_size(RABBIT_DATA_DIR)
# cpu使用率, 统计间隔5秒, 会导致5秒阻塞
cpu_percent = process.cpu_percent(interval=MONITOR_INTERVAL)
speed = format_speed(cal_speed(backup_5672_bytes, MONITOR_INTERVAL))
# 统计时间
current_time = datetime.datetime.now()
stat_time = time.mktime(current_time.timetuple()) + current_time.microsecond / 1000000.0
data = {
'stat_time': stat_time,
'cpu_usage': cpu_percent,
'mem_usage': mem_usage,
'disk_spend': disk_spend,
'net_speed': speed,
'msg_summary': self.get_rabbitmq_stats()
}
self.send_stat_report(data)
except Exception as err:
print(err)
traceback.print_stack()
def get_rabbitmq_stats(self):
if self.__rabbitmq_api == None:
config = Config.get_instance()
url = 'http://%s:%d' % (config.rmq_host, config.rmq_port)
self.__rabbitmq_api = AdminAPI(url=url, auth=(config.rmq_user, config.rmq_password))
overview = self.__rabbitmq_api.overview()
ready = 0
unacked = 0
total = 0
publish_rate = 0
deliver_manual_ack = 0
consumer_ack = 0
disk_read = 0
disk_write = 0
if overview == None:
print('get rabbitmq overview failed')
return None
if 'queue_totals' in overview:
queue_totals = overview['queue_totals']
if 'messages' in queue_totals:
total = queue_totals['messages']
if 'messages_ready' in queue_totals:
ready = queue_totals['messages_ready']
if 'messages_unacknowledged' in queue_totals:
unacked = queue_totals['messages_unacknowledged']
if 'message_stats' in overview:
message_stats = overview['message_stats']
if 'publish_details' in message_stats:
publish_details = message_stats['publish_details']
if 'rate' in publish_details:
publish_rate = publish_details['rate']
if 'deliver_details' in message_stats:
deliver_details = message_stats['deliver_details']
if 'rate' in deliver_details:
deliver_manual_ack = deliver_details['rate']
if 'ack_details' in message_stats:
ack_details = message_stats['ack_details']
if 'rate' in ack_details:
consumer_ack = ack_details['rate']
if 'disk_reads_details' in message_stats:
disk_reads_details = message_stats['disk_reads_details']
if 'rate' in disk_reads_details:
disk_read = disk_reads_details['rate']
if 'disk_writes_details' in message_stats:
disk_writes_details = message_stats['disk_writes_details']
if 'rate' in disk_writes_details:
disk_write = disk_writes_details['rate']
return {
'total': total,
'ready': ready,
'unacked': unacked,
'publish_rate': publish_rate,
'deliver_manual_ack': deliver_manual_ack,
'consumer_ack': consumer_ack,
'disk_read': disk_read,
'disk_write': disk_write
}
def send_stat_report(self, data):
config = Config.get_instance()
url = 'http://%s:%d/nodes/%d/rabbitmq/resources' % (config.collect_host, config.collect_port, config.node_id)
result = self.__session.post(url=url, data=json.dumps(data))
if result.status_code == 200:
obj = result.json()
if obj['errno'] == 0:
print('add rabbitmq stat report success')
else:
print('add rabbitmq stat report failed: %s' % obj['errstr'])
else:
print('add rabbitmq stat report failed ==>')
print('status => %d, errstr: %s' % (result.status_code, result.text))
def send_crash(self, process):
'''
发送进程崩溃报告
:param process:
:return:
'''
if process == None:
print('process None')
return None
start_time = process.create_time()
crash_time = int(time.mktime(datetime.datetime.now().timetuple()))
pid = process.pid
config = Config.get_instance()
url = 'http://%s:%d/nodes/%d/rabbitmq/crashes' % (config.collect_host, config.collect_port, config.node_id)
data = {
'pid': pid,
'start_time': start_time,
'crash_time': crash_time
}
result = self.__session.post(url=url, data=json.dumps(data))
if result.status_code == 200:
obj = result.json()
if obj['errno'] == 0:
print('add rabbitmq crash report success')
else:
print('add rabbitmq crash report failed: %s' % obj['errstr'])
else:
print('add rabbitmq crash report failed ==>')
print('status => %d, errstr: %s' % (result.status_code, result.text))