def get_queue_depths(host, username, password, vhost): cl = Client(host, username, password) depths = {} queues = [q['name'] for q in cl.get_queues(vhost=vhost)] for queue in queues: depths[queue] = cl.get_queue_depth(vhost, queue) return depths
def countMessagesQueue(self, vhost, queue): qtd = 0 cl = Client('{}:{}'.format(self.host, self.web_port), self.user, self.password) try: qtd = cl.get_queue_depth(vhost, queue) except HTTPError as e: logger.error(e) raise e return qtd
def get_queue_depths(host, username, password, vhost): cl = Client(host, username, password) if not cl.is_alive(): raise Exception("Failed to connect to rabbitmq") depths = {} queues = [q["name"] for q in cl.get_queues(vhost=vhost)] for queue in queues: if queue == "aliveness-test": continue depths[queue] = cl.get_queue_depth(vhost, queue) return depths
def get_rabbitmq_queue_length(q): """Fetch queue length from RabbitMQ for a given queue. Used periodically to decide if we want to queue more functions or not. Uses the Management HTTP API of RabbitMQ, since the Celery client doesn not have access to these counts. """ from pyrabbit.api import Client cl = Client(settings.SNOOP_RABBITMQ_HTTP_URL, 'guest', 'guest') return cl.get_queue_depth('/', q)
async def monitor(request): #data loaded for the first time from the function call global refresh db = request.app['db'] await db.collection.drop() if refresh == 0: #Client creation try: client = Client('localhost:15672', 'guest', 'guest') except: return {"result":"Client not created"} #get all the bindings() try: bindings = client.get_bindings() n = len(bindings) except: return {"result":"binding error"} #final list of all the rows result_list = [] for i in range(n): #dict for storing values for each connection try: dict_each = {} vhost_name = bindings[i]['vhost'] dict_each['vhost_name'] = vhost_name dict_each['exchange_name'] = bindings[i]['source'] queue_name = bindings[i]['destination'] dict_each['queue_name'] = queue_name dict_each['queue_size'] = client.get_queue_depth(vhost=vhost_name, name=queue_name) result_list.append(dict_each) except: return {"result" : "data not found"} #insert data in the db try: await db.collection.insert_many(i for i in result_list) except: return {'result': "data not stored in the db"} refresh+=1 return {'result_list': result_list} #data retrieved from the database after refresh if refresh>0: result_list = [] async for document in db.collection.find(): result_list.append(document) return {"result_list" : result_list}
def get_queue_depths(host, username, password, vhost): cl = Client(host, username, password) if not cl.is_alive(): raise Exception("Failed to connect to rabbitmq") depths = {} queues = [q['name'] for q in cl.get_queues(vhost=vhost)] for queue in queues: if queue == "aliveness-test": #pyrabbit continue elif queue.endswith('.pidbox') or queue.startswith('celeryev.'): #celery continue depths[queue] = cl.get_queue_depth(vhost, queue) return depths
def get_queue_depths(host, username, password, vhost): """ Fetches queue depths from rabbitmq instance.""" cl = Client(host, username, password) if not cl.is_alive(): raise Exception("Failed to connect to rabbitmq") depths = {} queues = [q['name'] for q in cl.get_queues(vhost=vhost)] for queue in queues: if queue == "aliveness-test": #pyrabbit continue elif queue.startswith('amq.gen-'): #Anonymous queues continue depths[queue] = cl.get_queue_depth(vhost, queue) return depths
def get_queue_depths(host, username, password, vhost): cl = Client(host, username, password) if not cl.is_alive(): raise Exception("Failed to connect to rabbitmq") depths = {} queues = [q['name'] for q in cl.get_queues(vhost=vhost)] for queue in queues: if queue == "aliveness-test": #pyrabbit continue elif queue.endswith('.pidbox') or queue.startswith( 'celeryev.'): #celery continue depths[queue] = cl.get_queue_depth(vhost, queue) return depths
def rebuild_collection(collection_name): """ Will grab all recs from the database and send them to solr """ # first, fail if we can not monitor queue length before we queue anything u = urlparse(app.conf['OUTPUT_CELERY_BROKER']) rabbitmq = PyRabbitClient(u.hostname + ':' + str(u.port + 10000), u.username, u.password) if not rabbitmq.is_alive('master_pipeline'): logger.error( 'failed to connect to rabbitmq with PyRabbit to monitor queue') sys.exit(1) now = get_date() if collection_name.startswith('http'): solr_urls = [collection_name] else: solr_urls = collection_to_urls(collection_name) logger.info('Sending all records to: %s', ';'.join(solr_urls)) sent = 0 batch = [] _tasks = [] with app.session_scope() as session: # master db only contains valid documents, indexing task will make sure that incomplete docs are rejected for rec in session.query(Records) \ .options(load_only(Records.bibcode, Records.updated, Records.processed)) \ .yield_per(1000): sent += 1 if sent % 1000 == 0: logger.debug('Sending %s records', sent) batch.append(rec.bibcode) if len(batch) > 1000: t = tasks.task_rebuild_index.delay(batch, force=True, update_solr=True, update_metrics=False, update_links=False, ignore_checksums=True, solr_targets=solr_urls, update_timestamps=False) _tasks.append(t) batch = [] if len(batch) > 0: t = tasks.task_rebuild_index.delay(batch, force=True, update_solr=True, update_metrics=False, update_links=False, ignore_checksums=True, solr_targets=solr_urls, update_timestamps=False) _tasks.append(t) logger.info('Done queueing bibcodes for rebuilding collection %s', collection_name) # now wait for queue to empty queue_length = 1 while queue_length > 0: queue_length = rabbitmq.get_queue_depth('master_pipeline', 'rebuild-index') stime = queue_length * 0.1 logger.info( 'Waiting %s for rebuild-collection tasks to finish, queue_length %s, sent %s' % (stime, queue_length, sent)) time.sleep(stime) logger.info('Done rebuilding collection %s, sent %s records', collection_name, sent)
rabbit_queue = os.environ["RABBIT_QUEUE"] marathon_url = os.environ["MARATHON_URL"] marathon_port = os.environ["MARATHON_PORT"] marathon_app = os.environ["MARATHON_APP"] # marathon connections strings url = "http://" + marathon_url + ":" + marathon_port + "/v2/apps" + marathon_app headers = {'content-type': "application/json", 'cache-control': "no-cache"} # connect to rabbit rabbit_connection = Client(rabbit_host + ":" + rabbit_api_port, rabbit_user, rabbit_password) print "connected to rabbit" # get rabbit queue size and figure out amount of tasks needed rabbit_size = rabbit_connection.get_queue_depth(rabbit_vhost, rabbit_queue) workers_needed = int(int(rabbit_size) / int(scale_every_x_waiting_messages)) if workers_needed < int(min_task_size): workers_needed = int(min_task_size) elif workers_needed > int(max_task_size): workers_needed = int(max_task_size) # get current number of tasks response = requests.request("GET", url, headers=headers) app_data = response.json() # if current number is different then required number change it to required number if int(app_data["app"]["instances"]) != int(workers_needed): print "scaling from " + str(app_data["app"]["instances"]) + " to " + str( workers_needed) + " workers" payload = "{\"instances\": " + str(workers_needed) + "}"
def getQueueMessagesCount(rabbit_ip, rabbit_port, rabbit_user, rabbit_password, rabbit_host, queue_name): cl = AdminClient(rabbit_ip + ':1' + rabbit_port, rabbit_user, rabbit_password) messages_cnt = cl.get_queue_depth(vhost=rabbit_host.replace("/", ""), name=queue_name) return messages_cnt