def __init__(self, host='localhost', username='******', password='******', queuename='testqueue'): self.api = AdminAPI(url='http://' + host + ':15672', auth=('guest', 'guest')) self.host = host self.cr = pika.PlainCredentials(username=username, password=password) self.conn = pika.BlockingConnection(pika.ConnectionParameters(self.host, credentials=self.cr)) self.chann = self.conn.channel() self.queue = self.chann.queue_declare(queuename, durable=True) self.routing_key = queuename
def get_rabbitmq_stats(self): if self.__rabbitmq_api == None: config = Config.get_instance() url = 'http://%s:%d' % (config.rmq_host, config.rmq_port) self.__rabbitmq_api = AdminAPI(url=url, auth=(config.rmq_user, config.rmq_password)) overview = self.__rabbitmq_api.overview() ready = 0 unacked = 0 total = 0 publish_rate = 0 deliver_manual_ack = 0 consumer_ack = 0 disk_read = 0 disk_write = 0 if overview == None: print('get rabbitmq overview failed') return None if 'queue_totals' in overview: queue_totals = overview['queue_totals'] if 'messages' in queue_totals: total = queue_totals['messages'] if 'messages_ready' in queue_totals: ready = queue_totals['messages_ready'] if 'messages_unacknowledged' in queue_totals: unacked = queue_totals['messages_unacknowledged'] if 'message_stats' in overview: message_stats = overview['message_stats'] if 'publish_details' in message_stats: publish_details = message_stats['publish_details'] if 'rate' in publish_details: publish_rate = publish_details['rate'] if 'deliver_details' in message_stats: deliver_details = message_stats['deliver_details'] if 'rate' in deliver_details: deliver_manual_ack = deliver_details['rate'] if 'ack_details' in message_stats: ack_details = message_stats['ack_details'] if 'rate' in ack_details: consumer_ack = ack_details['rate'] if 'disk_reads_details' in message_stats: disk_reads_details = message_stats['disk_reads_details'] if 'rate' in disk_reads_details: disk_read = disk_reads_details['rate'] if 'disk_writes_details' in message_stats: disk_writes_details = message_stats['disk_writes_details'] if 'rate' in disk_writes_details: disk_write = disk_writes_details['rate'] return { 'total': total, 'ready': ready, 'unacked': unacked, 'publish_rate': publish_rate, 'deliver_manual_ack': deliver_manual_ack, 'consumer_ack': consumer_ack, 'disk_read': disk_read, 'disk_write': disk_write }
def connect_mqtt(self, virtual_host, queue_name): try: api = AdminAPI(url='http://' + self.rabbitmq_server + ':' + str(8080), auth=(self.rabbitmq_id, self.rabbitmq_pwd)) vhosts_info = api.list_vhosts() vhost_list = [vhost['name'] for vhost in vhosts_info] if virtual_host not in vhost_list: api.create_vhost(virtual_host) api.create_user_permission(name=self.rabbitmq_id, vhost=virtual_host) credentials = pika.PlainCredentials(self.rabbitmq_id, self.rabbitmq_pwd) params = pika.ConnectionParameters(self.rabbitmq_server, self.rabbitmq_port, virtual_host, credentials) connection = pika.BlockingConnection(params) self.channel = connection.channel() except Exception as ex: # 에러 종류 print('RabbitMQ Connect fail', ex) # ex는 발생한 에러의 이름을 받아오는 변수 return False try: self.channel.queue_declare(queue=queue_name) return True except Exception as ex: # 에러 종류 print('Declare Queue fail', ex) # ex는 발생한 에러의 이름을 받아오는 변수 return False
def _pre_setup(self): """Create RabbitMQ virtual host.""" # Check if we are mixed with the class that makes sense. if ChannelTestCaseMixin in self.__class__.__mro__: raise ImproperlyConfigured( 'ChannelTestCase is not allowed as base class for ' 'RabbitmqLayerTestCaseMixin') # Create new virtual host for this test and grant permissions # to it. hostname = environ.get('RABBITMQ_HOST', 'localhost') port = environ.get('RABBITMQ_PORT', '5672') user = environ.get('RABBITMQ_USER', 'guest') password = environ.get('RABBITMQ_PASSWORD', 'guest') management_port = environ.get('RABBITMQ_MANAGEMENT_PORT', '15672') management_url = 'http://%s:%s' % (hostname, management_port) self.virtual_host = ''.join(choice(ascii_letters) for i in range(8)) self.amqp_url = 'amqp://%s:%s/%s' % (hostname, port, self.virtual_host) self.management = AdminAPI(management_url, (user, password)) self.management.create_vhost(self.virtual_host) self.management.create_user_permission(user, self.virtual_host) # Substitute Django settings with this virtual host. if self.local: layer_class_name = 'asgi_rabbitmq.RabbitmqLocalChannelLayer' else: layer_class_name = 'asgi_rabbitmq.RabbitmqChannelLayer' self._overridden_settings = { 'CHANNEL_LAYERS': { 'default': { 'BACKEND': layer_class_name, 'ROUTING': settings.CHANNEL_LAYERS['default']['ROUTING'], 'CONFIG': settings.CHANNEL_LAYERS['default']['CONFIG'], 'TEST_CONFIG': { 'url': self.amqp_url, }, }, }, } self._self_overridden_context = override_settings( **self._overridden_settings) self._self_overridden_context.enable() super(RabbitmqLayerTestCaseMixin, self)._pre_setup()
class RabbitmqLayerTestCaseMixin(object): """ TestCase mixin for Django tests. Allow to test your channels project against real broker. Provide isolated virtual host for each test. """ local = False def _pre_setup(self): """Create RabbitMQ virtual host.""" if ChannelTestCaseMixin in self.__class__.__mro__: raise ImproperlyConfigured( 'ChannelTestCase is not allowed as base class for ' 'RabbitmqLayerTestCaseMixin') hostname = environ.get('RABBITMQ_HOST', 'localhost') port = environ.get('RABBITMQ_PORT', '5672') user = environ.get('RABBITMQ_USER', 'guest') password = environ.get('RABBITMQ_PASSWORD', 'guest') management_port = environ.get('RABBITMQ_MANAGEMENT_PORT', '15672') management_url = 'http://%s:%s' % (hostname, management_port) self.virtual_host = ''.join(choice(ascii_letters) for i in range(8)) self.amqp_url = 'amqp://%s:%s/%s' % (hostname, port, self.virtual_host) self.management = AdminAPI(management_url, (user, password)) self.management.create_vhost(self.virtual_host) self.management.create_user_permission(user, self.virtual_host) if self.local: layer_class_name = 'asgi_rabbitmq.RabbitmqLocalChannelLayer' else: layer_class_name = 'asgi_rabbitmq.RabbitmqChannelLayer' self._overridden_settings = { 'CHANNEL_LAYERS': { 'default': { 'BACKEND': layer_class_name, 'ROUTING': settings.CHANNEL_LAYERS['default']['ROUTING'], 'CONFIG': settings.CHANNEL_LAYERS['default']['CONFIG'], 'TEST_CONFIG': { 'url': self.amqp_url, }, }, }, } self._self_overridden_context = override_settings( **self._overridden_settings) self._self_overridden_context.enable() super(RabbitmqLayerTestCaseMixin, self)._pre_setup() def _post_teardown(self): """Remove RabbitMQ virtual host.""" super(RabbitmqLayerTestCaseMixin, self)._post_teardown() self._self_overridden_context.disable() delattr(self, '_self_overridden_context') self._overridden_settings = None self.management.delete_vhost(self.virtual_host) del self.virtual_host del self.amqp_url del self.management
def create_project_user_and_vhost(project_id): secrets = get_project_secrets(project_id) hidden_secrets = get_project_hidden_secrets(project_id) # connect to RabbitMQ management api rabbit_api = AdminAPI(url=f'http://carrier-rabbit:15672', auth=(hidden_secrets["rabbit_user"], hidden_secrets["rabbit_password"])) # prepare user credentials user = f"rabbit_user_{project_id}" password = password_generator() vhost = f"project_{project_id}_vhost" # create project user and vhost rabbit_api.create_vhost(vhost) rabbit_api.create_user(user, password) rabbit_api.create_user_permission(user, vhost) # set project secrets secrets["rabbit_project_user"] = user secrets["rabbit_project_password"] = password secrets["rabbit_project_vhost"] = vhost set_project_secrets(project_id, secrets)
def initialize_rabbitmq_user(user): """ This function is used when a new provider registers in the network. It adds a new user in RabbitMQ and sets their permissions. :param user: The newly registered user """ # TODO: passwords have a pattern for all users. This needs to be changed. password = user.username + '_mqtt' api = AdminAPI(url='http://' + RABBITMQ_HOST + ':' + RABBITMQ_MANAGEMENT_PORT, auth=(RABBITMQ_USER, RABBITMQ_PASS)) api.create_user(user.username, password) # process = subprocess.Popen(['sudo', 'rabbitmqctl', 'add_user', user.username, password], # stdout=subprocess.PIPE, universal_newlines=True) # stdout, stderr = process.communicate() # print(stdout) permission = "^(" + user.username + ".*|amq.default)$" api.create_user_permission(user.username, '/', permission, permission, permission)
class RecSystemsManager: def __init__(self, host='localhost', username='******', password='******', queuename='testqueue'): self.api = AdminAPI(url='http://' + host + ':15672', auth=('guest', 'guest')) self.host = host self.cr = pika.PlainCredentials(username=username, password=password) self.conn = pika.BlockingConnection(pika.ConnectionParameters(self.host, credentials=self.cr)) self.chann = self.conn.channel() self.queue = self.chann.queue_declare(queuename, durable=True) self.routing_key = queuename def subscribeSystem(self, teamname, system, passwd): self.api.create_vhost('second_vhost', tracing=True) self.api.create_user(teamname, passwd) self.api.create_user_permission(teamname, 'second_vhost') # Include SQL Database authentication here self.chann.queue_declare(teamname + '_' + system + '_' + 'rec_requests', durable=True) self.chann.queue_declare(teamname + '_' + system + '_' + 'user_assign', durable=True) self.chann.queue_declare(teamname + '_' + system + '_' + 'news_recs', durable=True) self.chann.queue_declare(teamname + '_' + system + '_' + 'user_data', durable=True) self.chann.queue_declare(teamname + '_' + system + '_' + 'news_request', durable=True) def createQueue(self, name, durable): self.chann.queue_declare(name, durable)
def __init__(self, rabbit_server_config, topicCallback=callback, subscriptionCallback=callback, consumerSubscriptions=None, consumerTopics=None, consumerSyncTopics=None, producerSubscriptions=None, producerTopic=None, **kwargs): sender_properties = { "exchange": kwargs.get("sender_exchange", "RESOURCES_UPDATES"), "exchange_type": "topic", "passive": False, "durable": True, "auto_delete": False, "subscription": producerSubscriptions, "producerTopic": producerTopic } receiver_properties = { "exchange": kwargs.get("receiver_exchange", "RESOURCES_UPDATES"), "subscription_exchange": kwargs.get("subscription_exchange", "RESOURCES_UPDATES"), "exchange_type": "topic", "passive": False, "durable": True, "auto_delete": False, "subscriptions": consumerSubscriptions, "queue": "standard", "consumerTopics": consumerTopics, "consumerSyncTopics": consumerSyncTopics, "consumerSyncField": kwargs.get("consumerSyncField", None), "queueId": kwargs.get('queueId', "") } rest_api_config = kwargs.get("rest_api_config", {}) if (producerSubscriptions is not None or consumerSubscriptions is not None) and rest_api_config == {}: logger.error("rest_api_config is mandatory for susbscriptions") return logger.info( "rabbit server config-{}, rest_api_config-{} ,sender_properties-{} ,receiver_properties-{}" .format(rabbit_server_config, rest_api_config, sender_properties, receiver_properties)) self.rabbit_server_config = rabbit_server_config self.sender_rabbit_server_config = kwargs.get( "sender_rabbit_server_config", rabbit_server_config) if self.sender_rabbit_server_config is None: self.sender_rabbit_server_config = self.rabbit_server_config self.receiver_rabbit_server_config = kwargs.get( "receiver_rabbit_server_config", rabbit_server_config) if self.receiver_rabbit_server_config is None: self.receiver_rabbit_server_config = self.rabbit_server_config self.sender_creds = pika.PlainCredentials( self.sender_rabbit_server_config['user'], self.sender_rabbit_server_config['password']) self.receiver_creds = pika.PlainCredentials( self.receiver_rabbit_server_config['user'], self.receiver_rabbit_server_config['password']) self.heartbeat = kwargs.get("heartbeat", 30) self.start = True self.subRouteMap = {} self.subRoutes = [] self.consume_failure = 0 self.sender_properties = sender_properties self.receiver_properties = receiver_properties self.rest_api_config = rest_api_config self.create_hierarchy() self.topicCallback = topicCallback self.subscriptionCallback = subscriptionCallback self.failures = 0 self.testTopic = self.get_queue("") + '-test' self.sendLock = False self.receiveLock = False self.sender_failed_attempts = 0 self.receiver_failed_attempts = 0 self.check_connection_thread = threading.Thread( target=self.check_connection_state) self.sender_connection = None self.receiver_connection = None self.sender_channel = None self.receiver_connection_async = None self.receiver_channel = None self.receiver_channel_async = None self.usedRoutingKeys = [] protocol = kwargs.get("protocol", 'http') try: adminUrl = "{}://{}:{}".format( protocol, self.rabbit_server_config['host'], self.sender_rabbit_server_config['port']) self.adminApi = AdminAPI( url=adminUrl, auth=(self.sender_rabbit_server_config['user'], self.sender_rabbit_server_config['password'])) except Exception as e: logger.info("failed to initialize admin api-{}".format(str(e))) try: self.init_connections() except Exception as e: logger.error( "some error occurred initializing connection retrying-{}". format(str(e))) self.check_connection_thread.start() if not self.check_connection_thread.is_alive(): self.check_connection_thread.start() time.sleep(1)
class RabbitMqConnector(): def __init__(self, rabbit_server_config, topicCallback=callback, subscriptionCallback=callback, consumerSubscriptions=None, consumerTopics=None, consumerSyncTopics=None, producerSubscriptions=None, producerTopic=None, **kwargs): sender_properties = { "exchange": kwargs.get("sender_exchange", "RESOURCES_UPDATES"), "exchange_type": "topic", "passive": False, "durable": True, "auto_delete": False, "subscription": producerSubscriptions, "producerTopic": producerTopic } receiver_properties = { "exchange": kwargs.get("receiver_exchange", "RESOURCES_UPDATES"), "subscription_exchange": kwargs.get("subscription_exchange", "RESOURCES_UPDATES"), "exchange_type": "topic", "passive": False, "durable": True, "auto_delete": False, "subscriptions": consumerSubscriptions, "queue": "standard", "consumerTopics": consumerTopics, "consumerSyncTopics": consumerSyncTopics, "consumerSyncField": kwargs.get("consumerSyncField", None), "queueId": kwargs.get('queueId', "") } rest_api_config = kwargs.get("rest_api_config", {}) if (producerSubscriptions is not None or consumerSubscriptions is not None) and rest_api_config == {}: logger.error("rest_api_config is mandatory for susbscriptions") return logger.info( "rabbit server config-{}, rest_api_config-{} ,sender_properties-{} ,receiver_properties-{}" .format(rabbit_server_config, rest_api_config, sender_properties, receiver_properties)) self.rabbit_server_config = rabbit_server_config self.sender_rabbit_server_config = kwargs.get( "sender_rabbit_server_config", rabbit_server_config) if self.sender_rabbit_server_config is None: self.sender_rabbit_server_config = self.rabbit_server_config self.receiver_rabbit_server_config = kwargs.get( "receiver_rabbit_server_config", rabbit_server_config) if self.receiver_rabbit_server_config is None: self.receiver_rabbit_server_config = self.rabbit_server_config self.sender_creds = pika.PlainCredentials( self.sender_rabbit_server_config['user'], self.sender_rabbit_server_config['password']) self.receiver_creds = pika.PlainCredentials( self.receiver_rabbit_server_config['user'], self.receiver_rabbit_server_config['password']) self.heartbeat = kwargs.get("heartbeat", 30) self.start = True self.subRouteMap = {} self.subRoutes = [] self.consume_failure = 0 self.sender_properties = sender_properties self.receiver_properties = receiver_properties self.rest_api_config = rest_api_config self.create_hierarchy() self.topicCallback = topicCallback self.subscriptionCallback = subscriptionCallback self.failures = 0 self.testTopic = self.get_queue("") + '-test' self.sendLock = False self.receiveLock = False self.sender_failed_attempts = 0 self.receiver_failed_attempts = 0 self.check_connection_thread = threading.Thread( target=self.check_connection_state) self.sender_connection = None self.receiver_connection = None self.sender_channel = None self.receiver_connection_async = None self.receiver_channel = None self.receiver_channel_async = None self.usedRoutingKeys = [] protocol = kwargs.get("protocol", 'http') try: adminUrl = "{}://{}:{}".format( protocol, self.rabbit_server_config['host'], self.sender_rabbit_server_config['port']) self.adminApi = AdminAPI( url=adminUrl, auth=(self.sender_rabbit_server_config['user'], self.sender_rabbit_server_config['password'])) except Exception as e: logger.info("failed to initialize admin api-{}".format(str(e))) try: self.init_connections() except Exception as e: logger.error( "some error occurred initializing connection retrying-{}". format(str(e))) self.check_connection_thread.start() if not self.check_connection_thread.is_alive(): self.check_connection_thread.start() time.sleep(1) def get_all_routingKeys(self, queue, exchange=None): bindings = self.adminApi.list_bindings() return [ x['routing_key'] for x in bindings if x['destination'] == queue ] def init_connections(self): self.init_sender_connection() self.init_receiver_connection() def init_sender_connection(self): self.sender_connection = pika.BlockingConnection( pika.ConnectionParameters( host=self.sender_rabbit_server_config['host'], credentials=self.sender_creds, heartbeat=self.heartbeat)) self.sender_channel = self.sender_connection.channel() self.sender_channel.exchange_declare( exchange=self.sender_properties["exchange"], exchange_type=self.sender_properties["exchange_type"], passive=self.sender_properties["passive"], durable=self.sender_properties["durable"], auto_delete=self.sender_properties["auto_delete"]) def get_queue(self, topic): queueId = str(self.receiver_properties.get("queueId", "")) return queueId + '-' + topic def init_receiver_connection(self): self.receiver_connection = pika.BlockingConnection( pika.ConnectionParameters( host=self.receiver_rabbit_server_config['host'], credentials=self.receiver_creds, heartbeat=self.heartbeat)) self.receiver_channel = self.receiver_connection.channel() self.receiver_connection_async = pika.BlockingConnection( pika.ConnectionParameters(host=self.rabbit_server_config['host'], credentials=self.receiver_creds, heartbeat=self.heartbeat)) self.receiver_channel_async = self.receiver_connection_async.channel() self.sync_receiver_channels = {} self.receiver_channel.exchange_declare( exchange=self.receiver_properties["exchange"], exchange_type=self.receiver_properties["exchange_type"], passive=self.receiver_properties["passive"], durable=self.receiver_properties["durable"], auto_delete=self.receiver_properties["auto_delete"]) self.receiver_channel_async.exchange_declare( exchange=self.receiver_properties["exchange"], exchange_type=self.receiver_properties["exchange_type"], passive=self.receiver_properties["passive"], durable=self.receiver_properties["durable"], auto_delete=self.receiver_properties["auto_delete"]) self.receiver_queue_async = None consumerTopics = self.receiver_properties.get("consumerTopics", None) subscriptions = self.receiver_properties.get("subscriptions", None) consumerSyncTopics = self.receiver_properties.get( "consumerSyncTopics", None) self.add_subscriptions(subscriptions, consumerTopics, consumerSyncTopics) def remove_subscriptions(self, subscriptions=None): logger.info("removing subscriptions-{}".format(str(subscriptions))) if self.receiver_properties["subscriptions"] is not None: for subscription in subscriptions: self.receiver_properties["subscriptions"].remove(subscription) self.reinit_receiver() logger.info("remaining subscriptions-{}".format( str(self.receiver_properties["subscriptions"]))) def add_subscriptions(self, subscriptions=None, consumerTopics=None, consumerSyncTopics=None): # If consumers are sync, make a different queue for each consumer currentSyncKeys = [] if consumerSyncTopics is not None: for consumerTopic in consumerSyncTopics: channel = self.receiver_connection.channel() channel.exchange_declare( exchange=self.receiver_properties["exchange"], exchange_type=self.receiver_properties["exchange_type"], passive=self.receiver_properties["passive"], durable=self.receiver_properties["durable"], auto_delete=self.receiver_properties["auto_delete"]) queue_name = self.get_queue(consumerTopic) channel.queue_declare(queue=queue_name) currentSyncKeys.append(consumerTopic) channel.queue_bind( queue=queue_name, exchange=self.receiver_properties["exchange"], routing_key=consumerTopic) logger.info("listening to sync queue-{}".format(queue_name)) self.sync_receiver_channels[consumerTopic] = channel self.usedRoutingKeys = [] routingKeys = [] if consumerTopics is not None: for topic in consumerTopics: routingKeys.append(topic) self.usedRoutingKeys.append(topic) subsKeys = [] if subscriptions is not None: for subscription in subscriptions: routingKey = self.getRoutingKey(subscription) subsKeys.append(routingKey) self.usedRoutingKeys.append(routingKey) currentSyncKeys = [] currentSyncKeys = currentSyncKeys + routingKeys + subsKeys if (len(routingKeys) > 0 or len(subsKeys) > 0): if not self.receiver_queue_async: self.receiver_properties["subscriptions"] = subscriptions self.receiver_queue_async = self.get_queue("") + '-async' self.receiver_channel_async.queue_declare( queue=self.receiver_queue_async) logger.info("subscribing to routes-{}, queue-{}".format( routingKeys, self.receiver_queue_async)) for routingKey in routingKeys: self.receiver_channel_async.queue_bind( queue=self.receiver_queue_async, exchange=self.receiver_properties["exchange"], routing_key=routingKey) logger.info("subscribing to subscriptions-{}, queue-{}".format( subsKeys, self.receiver_queue_async)) for routingKey in subsKeys: self.receiver_channel_async.queue_bind( queue=self.receiver_queue_async, exchange=self. receiver_properties["subscription_exchange"], routing_key=routingKey) #Cleanup unused keys try: alreadyExistingKeys = alreadyExistingKeys = self.get_all_routingKeys( self.receiver_queue_async) unusedKeys = [ x for x in alreadyExistingKeys if x not in currentSyncKeys ] for key in unusedKeys: self.receiver_channel_async.queue_unbind( queue=self.receiver_queue_async, exchange=self.receiver_properties["exchange"], routing_key=key) self.receiver_channel_async.queue_unbind( queue=self.receiver_queue_async, exchange=self. receiver_properties["subscription_exchange"], routing_key=key) except Exception as e: logger.info( "some exception occurred while removing previous async subscriptions-{}" .format(str(e))) self.receiver_channel_async.basic_consume( queue=self.receiver_queue_async, on_message_callback=self.receive, auto_ack=True) self.consume_thread = threading.Thread(target=self.consume) if not self.consume_thread.is_alive(): self.consume_thread.start() time.sleep(2) else: subscriptions_hist = self.receiver_properties.get( "subscriptions", []) if not subscriptions_hist: subscriptions_hist = [] subscriptions_hist.extend(subscriptions) self.receiver_properties["subscriptions"] = subscriptions_hist self.reinit_receiver() def reinit_connections(self): self.reinit_sender() self.reinit_receiver() def reinit_sender(self): time.sleep(5) logger.info("reinitializing sender connection") try: self.sender_connection.close() self.sender_channel.close() except Exception as e: logger.info("failed to reinitialize sender connection") pass self.init_sender_connection() def reinit_receiver(self): self.receiveLock = True #time.sleep(5) logger.info("reinitializing receiver connection") try: self.receiver_channel_async.stop_consuming() except: logger.info("some error in stopping consuming async") pass try: self.consume_thread.join() except: logger.info("some error in closing consumer thread") pass try: self.receiver_connection.close() self.receiver_channel.close() except Exception as e: logger.info("failed to reinitialize connection") pass try: self.receiver_connection_async.close() self.receiver_channel_async.close() except Exception as e: logger.info("failed to reinitialize connection") pass try: self.init_receiver_connection() except: pass self.receiveLock = False def check_connection_state(self): while (self.start): state = 'OK' try: time.sleep(self.heartbeat - 5) logger.info("checking connection state with server") try: self.sender_connection.process_data_events() except: pass if self.sender_connection.is_closed or self.sender_channel.is_closed: logger.info("sender connection closed") state = 'BROKEN' logger.info("reinitializing sender connection") try: self.reinit_sender() except: logger.info("failed to reinitializes sender") self.sender_failed_attempts += 1 else: self.sender_failed_attempts = 0 #logger.info("********************************receiveLock********************************--{}".format(self.receiveLock)) if not self.receiveLock: receiver_status = True try: self.receiver_connection.process_data_events() except: pass #logger.info("receiver.connection status*******-{}".format(self.receiver_channel.is_closed)) if self.receiver_connection.is_closed: receiver_status = False try: self.receiver_connection_async.process_data_events() except: pass #logger.info("receiver.connection status*******-{}".format(self.receiver_channel_async.is_closed)) if self.receiver_connection_async.is_closed: receiver_status = False if receiver_status == False: logger.info("receiver connection closed") state = 'BROKEN' self.receiver_failed_attempts += 1 logger.info("reinitializing sender connection") try: self.reinit_receiver() except: logger.info("failed to reinitializes sender") else: self.receiver_failed_attempts = 0 except Exception as e: logger.debug(traceback.format_exc()) state = 'BROKEN' logger.info("failed to open connections..retrying") try: self.reinit_connections() except: pass logger.info("connection state->{}-{}".format(state, time.time())) def send(self, message={"message": "ping------pong"}, subscription=None, producerTopic=None, showMessage=False): if producerTopic is None: producerTopic = self.sender_properties["producerTopic"] if subscription is None: subscription = self.sender_properties["subscription"] if producerTopic is not None: routingKey = producerTopic elif subscription is not None: logger.info("sending to subscription-{}".format(subscription)) routingKey = self.getRoutingKey(subscription) while (self.sendLock): logger.info("waiting for send lock to clear") time.sleep(0.01) self.sendLock = True try: self.sender_channel.basic_publish( exchange=self.sender_properties["exchange"], routing_key=routingKey, body=json.dumps(message)) except Exception as e: self.sendLock = False logger.info("failed to send message-{}".format(str(e))) return False self.sendLock = False if routingKey != 'test': print("=" * 50) print("Producing Message") print("Producer topic", routingKey) print("=" * 50) if showMessage: logger.info( "successfully sent message on routing key-{}, message-{}". format(routingKey, message)) else: logger.info( "successfully sent message on routing key-{}".format( routingKey)) return True def send_response(self, props, message, showMessage=False): while (self.sendLock): logger.info("waiting for send lock to clear") time.sleep(0.01) self.sendLock = True try: self.sender_channel.basic_publish( exchange='', routing_key=props.reply_to, properties=pika.BasicProperties( correlation_id=props.correlation_id), body=json.dumps(message)) self.sendLock = False except Exception as e: self.sendLock = False logger.info("failed to send message-{}".format(str(e))) return False if showMessage: logger.info( "successfully sent message on routing key-{}, message-{}". format(props.reply_to, message)) else: logger.info("successfully sent message on routing key-{}".format( props.reply_to)) return True def replacecontrolchar(self, text): for a, b in mapnonprint.items(): if a in text: logger.warning( "Json Decode replacecontrolchar:{} with {}".format(a, b)) text = text.replace(a, b) return text def receive(self, ch, method, properties, body): try: message = body.decode('utf8').replace("'", '"') try: message = json.loads(message1) except Exception as e: try: message = json.loads(self.replacecontrolchar(message)) except Exception as e: logger.info("error loading message to json-{}".format( str(e))) routingKey = method.routing_key logger.info( "message received from routing key-{}".format(routingKey)) if str(routingKey) not in self.usedRoutingKeys: logger.info( "message received from unused route now--skipping!") #return if routingKey != self.testTopic: print("=" * 50) print("Consuming Message") print("Consumer topic", routingKey) print("=" * 50) if routingKey in self.subRoutes: self.subscriptionCallback(message, properties, method) else: self.subscriptionCallback(message, properties, method) #self.topicCallback(message,properties,method) except Exception as e: logger.info("some exception occurred -{}".format(str(e))) logger.debug(traceback.format_exc()) def consume(self): #time.sleep(5) logger.info("receiver started consuming") # wait for sometime try: self.receiver_channel_async.start_consuming() except Exception as e: logger.info( "some error occured while consuming..async receiver should restart after heartbeat" .format(str(e))) logger.debug(traceback.format_exc()) def consume_sync(self, topic): while self.receiveLock: logger.info("waiting for receiveLock") time.sleep(0.1) try: method_frame, header_frame, body = self.sync_receiver_channels[ topic].basic_get(self.get_queue(topic), auto_ack=True) if method_frame: try: message = body.decode('utf8').replace("'", '"') message = json.loads(message1) except Exception as e: try: message = json.loads(self.replacecontrolchar(message)) except Exception as e: logger.info("error loading message to json-{}".format( str(e))) print("=" * 50) print("Consuming Message") print("Consumer topic", topic) print("=" * 50) return message return None except Exception as e: logger.info("some exception occurred in sync consuming-{}".format( str(e))) #print(traceback.format_exc()) logger.debug(traceback.format_exc()) time.sleep(self.heartbeat) return None def consume_sync_all(self): startTime = time.time() finalMessage = {} topics = self.receiver_properties['consumerSyncTopics'].copy() topicLength = len(topics) syncId = None syncField = self.receiver_properties['consumerSyncField'] while (len(topics) > 0): for topic in topics: out = self.consume_sync(topic) if out: finalMessage[topic] = out topics.remove(topic) if len(topics) == topicLength: return None if (time.time() - startTime) > 10: logger.info("sync time exceeded, failed to sync") self.flush_sync_consumer_queues() return None if syncField: for key in finalMessage.keys(): id = finalMessage[key][syncField] if syncId: if id != syncId: logger.info("consumers not in sync..flushing queue") self.flush_sync_consumer_queues() return None else: syncId = id return finalMessage def flush_sync_consumer_queues(self): for topic in self.receiver_properties['consumerSyncTopics']: self.sync_receiver_channels[topic].queue_purge( self.get_queue(topic)) def create_hierarchy(self): logger.info('resource hierarchy being setup!') Node.separator = "." self.organizations = Node("organizations") self.hubs = Node('hubs', parent=self.organizations) self.behaviourTypes = Node('behaviourTypes', parent=self.organizations) self.users = Node('users', parent=self.organizations) self.artefacts = Node("artefacts", parent=self.organizations) self.notifications = Node("notificaitons", parent=self.organizations) self.pipelines = Node("pipelines", parent=self.organizations) self.charts = Node("charts", parent=self.organizations) self.recommendations = Node("recommendations", parent=self.organizations) self.cameras = Node("cameras", parent=self.hubs) self.devices = Node("devices", parent=self.hubs) self.behaviours = Node("behaviours", parent=self.cameras) self.alerts = Node("alerts", parent=self.behaviours) self.resourceMap = { "organization": "organizations", "hub": "hubs", "behaviourType": "behaviourTypes", "user": "******", "artefact": "artefacts", "notificaiton": "notificaitons", "pipeline": "pipelines", "chart": "charts", "recommendation": "recommendations", "camera": "cameras", "device": "devices", "behaviour": "behaviours", "alert": "alerts", } def getRoutingKey(self, subscription): subRouteKeys = self.subRouteMap.keys() if str(subscription) in subRouteKeys: return self.subRouteMap[str(subscription)] sub_keys = subscription.keys() resourceKeys = self.resourceMap.keys() resourceKey = None for key in sub_keys: if key in resourceKeys: resourceKey = key break if resourceKey is not None: resource_id = subscription[resourceKey] resource = self.get_resource(self.resourceMap[resourceKey], resource_id) route = self.findRoute(self.resourceMap[resourceKey]) routelist = route.split('.') #print("routelist",routelist) finalRoute = '' for name in routelist: name = name[:-1] if name == resourceKey: finalRoute += str(resource['_id']) + '.' else: finalRoute += str(resource[name]) + '.' topic = subscription.get('topic', None) if topic is not None: finalRoute += str(subscription['topic']) else: finalRoute = finalRoute[:-1] self.subRouteMap[str(subscription)] = finalRoute self.subRoutes.append(finalRoute) return finalRoute def findRoute(self, nodeName): anytreeNodePath = search.find(self.organizations, lambda node: node.name == nodeName) #print ("anytree node path:",anytreeNodePath) try: if anytreeNodePath: rt = str(anytreeNodePath).split('(')[1].replace( "'.", "", 1).replace("')", '', 1) #print ("designated route:",rt) return rt except Exception as anytreeError: #print ("anytree error occured",anytreeError) return None # Import app to get api_config def get_resource(self, res_type, res_id=None, res_filter=None): api_config = self.rest_api_config Veda_auth = HTTPBasicAuth(api_config['VEDA_USER'], api_config['VEDA_PASSWORD']) if res_id is not None: final_url = "{}/{}/{}/{}".format(api_config['VEDA_SERVER_URL'], api_config['VEDA_API_VERSION'], res_type, res_id) elif res_filter is not None: final_url = "{}/{}/{}?where={}".format( api_config['VEDA_SERVER_URL'], api_config['VEDA_API_VERSION'], res_type, json.dumps(res_filter)) else: final_url = "{}/{}/{}".format(api_config['VEDA_SERVER_URL'], api_config['VEDA_API_VERSION'], res_type) resp = {} try: items = [] while True: logging.info("Fetching cloud resource: {}".format(final_url)) resp = requests.get(final_url, auth=Veda_auth, timeout=10) resp.raise_for_status() resp = resp.json() if "next" in resp["_links"] and "href" in resp["_links"][ "next"]: route = resp["_links"]["next"]['href'] final_url = "{}/{}/{}".format( api_config['VEDA_SERVER_URL'], api_config['VEDA_API_VERSION'], route) items = items + resp["_items"] else: if items: resp["_items"] = items break except Exception as e: logging.error("get_resource Failed:{}".format(e)) resp = None if resp is None: raise RuntimeError( "Failed to fetch cloud resource of type: {}, id: {} and filter: {}" .format(res_type, res_id, res_filter)) return resp def stop(self): logger.info( "stopping connector & ignoring all errors..helpless-please don't judge!" ) self.start = False try: self.check_connection_thread.join() except: pass try: self.sender_connection.close() except: pass try: self.receiver_channel_async.stop_consuming() except: pass try: self.receiver_connection.close() except: pass try: self.receiver_connection_async.close() except: pass try: self.consume_thread.join() except: pass
class PickRmqThread(threading.Thread): def __init__(self, daemon = True): super().__init__() self.setName('pickrmq') self.setDaemon(daemon) print('create thread: %s' % self.name) self.__rabbitmq_api = None self.__session = requests.Session() def run(self) -> None: process = None while True: try: if process == None: print('rabbitmq not start') process = get_rmq_process() if process == None: time.sleep(5) continue if not process.is_running(): self.send_crash(process) process = None continue else: global capture_bytes backup_5672_bytes = capture_bytes # 内存占用, 单位 MB mem_usage = get_rmq_process_mem_usage(process) # 磁盘空间占用, 单位 MB disk_spend = get_file_size(RABBIT_DATA_DIR) # cpu使用率, 统计间隔5秒, 会导致5秒阻塞 cpu_percent = process.cpu_percent(interval=MONITOR_INTERVAL) speed = format_speed(cal_speed(backup_5672_bytes, MONITOR_INTERVAL)) # 统计时间 current_time = datetime.datetime.now() stat_time = time.mktime(current_time.timetuple()) + current_time.microsecond / 1000000.0 data = { 'stat_time': stat_time, 'cpu_usage': cpu_percent, 'mem_usage': mem_usage, 'disk_spend': disk_spend, 'net_speed': speed, 'msg_summary': self.get_rabbitmq_stats() } self.send_stat_report(data) except Exception as err: print(err) traceback.print_stack() def get_rabbitmq_stats(self): if self.__rabbitmq_api == None: config = Config.get_instance() url = 'http://%s:%d' % (config.rmq_host, config.rmq_port) self.__rabbitmq_api = AdminAPI(url=url, auth=(config.rmq_user, config.rmq_password)) overview = self.__rabbitmq_api.overview() ready = 0 unacked = 0 total = 0 publish_rate = 0 deliver_manual_ack = 0 consumer_ack = 0 disk_read = 0 disk_write = 0 if overview == None: print('get rabbitmq overview failed') return None if 'queue_totals' in overview: queue_totals = overview['queue_totals'] if 'messages' in queue_totals: total = queue_totals['messages'] if 'messages_ready' in queue_totals: ready = queue_totals['messages_ready'] if 'messages_unacknowledged' in queue_totals: unacked = queue_totals['messages_unacknowledged'] if 'message_stats' in overview: message_stats = overview['message_stats'] if 'publish_details' in message_stats: publish_details = message_stats['publish_details'] if 'rate' in publish_details: publish_rate = publish_details['rate'] if 'deliver_details' in message_stats: deliver_details = message_stats['deliver_details'] if 'rate' in deliver_details: deliver_manual_ack = deliver_details['rate'] if 'ack_details' in message_stats: ack_details = message_stats['ack_details'] if 'rate' in ack_details: consumer_ack = ack_details['rate'] if 'disk_reads_details' in message_stats: disk_reads_details = message_stats['disk_reads_details'] if 'rate' in disk_reads_details: disk_read = disk_reads_details['rate'] if 'disk_writes_details' in message_stats: disk_writes_details = message_stats['disk_writes_details'] if 'rate' in disk_writes_details: disk_write = disk_writes_details['rate'] return { 'total': total, 'ready': ready, 'unacked': unacked, 'publish_rate': publish_rate, 'deliver_manual_ack': deliver_manual_ack, 'consumer_ack': consumer_ack, 'disk_read': disk_read, 'disk_write': disk_write } def send_stat_report(self, data): config = Config.get_instance() url = 'http://%s:%d/nodes/%d/rabbitmq/resources' % (config.collect_host, config.collect_port, config.node_id) result = self.__session.post(url=url, data=json.dumps(data)) if result.status_code == 200: obj = result.json() if obj['errno'] == 0: print('add rabbitmq stat report success') else: print('add rabbitmq stat report failed: %s' % obj['errstr']) else: print('add rabbitmq stat report failed ==>') print('status => %d, errstr: %s' % (result.status_code, result.text)) def send_crash(self, process): ''' 发送进程崩溃报告 :param process: :return: ''' if process == None: print('process None') return None start_time = process.create_time() crash_time = int(time.mktime(datetime.datetime.now().timetuple())) pid = process.pid config = Config.get_instance() url = 'http://%s:%d/nodes/%d/rabbitmq/crashes' % (config.collect_host, config.collect_port, config.node_id) data = { 'pid': pid, 'start_time': start_time, 'crash_time': crash_time } result = self.__session.post(url=url, data=json.dumps(data)) if result.status_code == 200: obj = result.json() if obj['errno'] == 0: print('add rabbitmq crash report success') else: print('add rabbitmq crash report failed: %s' % obj['errstr']) else: print('add rabbitmq crash report failed ==>') print('status => %d, errstr: %s' % (result.status_code, result.text))