def _get_system_tools(request, cluster_id): data = {} sCmd = ""; data['cluster'] = get_cluster_or_404(id=cluster_id) data['topics'], data['error_zk_topics'] = _get_topics(data['cluster']) data['version'] = settings.HUE_DESKTOP_VERSION try: zk = KazooClient(hosts=data['cluster']['zk_host_ports']) zk.start() data['brokers'], data['error_zk_brokers'] = _get_brokers(zk,data['cluster']['id']) zk.stop() if request.method == 'POST' and request.is_ajax(): sOption = request.POST['txtOption'] sBroker = request.POST['txtBroker'] sBrokers = "" for element in sBroker.split(","): if element <> "": sBrokers += element + ":9092, " sTopic = request.POST['txtTopic'] sTime = request.POST['txtTime'] iWaitTime = request.POST['numWaitTime'] sNumOffset = request.POST['numOffset'] sPartitions = request.POST['txtPartitions'] sCmd = ('/usr/lib/kafka/bin/kafka-run-class.sh %s ' '--broker-list %s ' '--topic %s ' '--time %s ' % (sOption, sBrokers[:-2], sTopic, sTime)) if iWaitTime <> "1000": sCmd += '--max-wait-ms %s ' % (iWaitTime) if sNumOffset <> "1": sCmd += '--offsets %s ' % (sNumOffset) if sPartitions <> "": sCmd += '--partitions %s ' % (sPartitions) output,err = subprocess.Popen([sCmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() data['cmd'] = sCmd data['output'] = output data['error'] = err except NoNodeError: data = {'cluster': get_cluster_or_404(id=cluster_id), 'topics': [], 'brokers': [], 'cmd': "", 'version': settings.HUE_DESKTOP_VERSION, 'error_zk_topics': 1, 'error_zk_brokers': 1 , 'output': "", 'error': -1 } return data
def _get_topology(): topology = CLUSTERS.get() clusters = [] for cluster in topology: zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get()) #zk.add_listener(my_listener) zk.start() brokers = _get_brokers(zk, cluster) consumer_groups = _get_consumer_groups(zk, cluster) consumer_groups_status = {} # 0 = offline, (not 0) = online for consumer_group in consumer_groups: consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get( ) + "/" + consumer_group + "/ids" try: consumers = zk.get_children(consumers_path) except NoNodeError: consumer_groups_status[consumer_group] = 0 # 0 = offline else: consumer_groups_status[consumer_group] = len( consumers) # (not 0) = online c = { 'cluster': get_cluster_or_404(id=cluster), 'brokers': brokers, 'consumer_groups': consumer_groups, 'consumer_groups_status': consumer_groups_status } clusters.append(c) zk.stop() return clusters
def _get_json_type(request, cluster_id, type): data = [] error_brokers = 0 try: cluster = get_cluster_or_404(id=cluster_id) zk = KazooClient(hosts=cluster['zk_host_ports']) zk.start() if type == "broker": brokers, error_brokers = _get_brokers(zk,cluster_id) for broker in brokers: data.append(broker['host']) if type == "topic": topics, error_zk_topics = _get_topics(cluster) for topic in topics: data.append(topic['id']) if type == "metric": data = _get_sections_ini() except KazooException: error_zk_brokers = 1 zk.stop() return JsonResponse(data, safe=False)
def consumer_group(request, cluster_id, group_id): cluster = get_cluster_or_404(id=cluster_id) zk = KazooClient(hosts=cluster['zk_host_ports']) zk.add_listener(my_listener) zk.start() consumer_group = _get_consumer_group(zk=zk,cluster=cluster,group_id=group_id) zk.stop() return render('consumer_group.mako', request, {'cluster': cluster, 'consumer_group':consumer_group})
def _get_topology(): """ Method to get the entire Kafka clusters (defined in hue.ini) topology """ topology = CLUSTERS.get() clusters = [] error = 0 error_brokers = 0 error_consumer_groups = 0 for c in topology: cluster = get_cluster_or_404(c) try: zk = KazooClient(hosts=CLUSTERS[c].ZK_HOST_PORTS.get()) zk.start() brokers, error_brokers = _get_brokers(zk,cluster['id']) consumer_groups, error_consumer_groups = _get_consumer_groups(zk,cluster['id']) consumer_groups_status = {} for consumer_group in consumer_groups: # 0 = offline, (not 0) = online consumers_path = CLUSTERS[c].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids" try: consumers = zk.get_children(consumers_path) except NoNodeError: consumer_groups_status[consumer_group]=0 else: consumer_groups_status[consumer_group]=len(consumers) c = {'cluster':cluster, 'brokers':brokers, 'consumer_groups':consumer_groups, 'consumer_groups_status':consumer_groups_status, 'error_brokers':error_brokers, 'error_consumer_groups':error_consumer_groups, 'error':0} zk.stop() except NoNodeError: c = {'cluster':cluster, 'brokers':[], 'consumer_groups':[], 'consumer_groups_status':[], 'error_brokers':error_brokers, 'error_consumer_groups':error_consumer_groups, 'error':2} except: c = {'cluster':cluster, 'brokers':[], 'consumer_groups':[], 'consumer_groups_status':[], 'error_brokers':error_brokers, 'error_consumer_groups':error_consumer_groups, 'error':1} clusters.append(c) return clusters
def consumer_group(request, cluster_id, group_id): """ Consumers Group detail view. Returns the detailed view of a given consumers group """ cluster = get_cluster_or_404(id=cluster_id) consumer_group = {} error = 0 try: zk = ZooKeeper(cluster['zk_rest_url']) consumer_group = _get_consumer_group(zk=zk,cluster=cluster,group_id=group_id) except ZooKeeper.RESTError: error = 1 return render('consumer_group.mako', request, {'cluster': cluster, 'consumer_group':consumer_group, 'error':error})
def consumer_group(request, cluster_id, group_id): """ Consumers Group detail view. Returns the detailed view of a given consumers group """ cluster = get_cluster_or_404(id=cluster_id) consumer_group = {} error = 0 try: zk = KazooClient(hosts=cluster['zk_host_ports']) zk.start() consumer_group = _get_consumer_group(zk=zk,cluster=cluster,group_id=group_id) except NoNodeError: error = 1 except: error = 2 zk.stop() return render('consumer_group.mako', request, {'cluster': cluster, 'consumer_group':consumer_group, 'error':error})
def _get_json_type(request, cluster_id, type): data = [] try: cluster = get_cluster_or_404(id=cluster_id) if type == "broker": zk = ZooKeeper(cluster['zk_rest_url']) brokers = _get_brokers(zk,cluster) for broker in brokers: data.append(broker['host']) if type == "topic": topics, error_zk_topics = _get_topics(cluster) for topic in topics: data.append(topic['id']) if type == "metric": data = _get_sections_ini() except ZooKeeper.RESTError: error_zk_brokers = 1 return HttpResponse(_get_dumps(data), content_type = "application/json")
def _get_topology(): """ Method to get the entire Kafka clusters (defined in hue.ini) topology """ topology = CLUSTERS.get() clusters = [] for c in topology: cluster = get_cluster_or_404(c) try: zk = ZooKeeper(cluster['zk_rest_url']) brokers = _get_brokers(zk,cluster) consumer_groups = _get_consumer_groups(zk,cluster) consumer_groups_status = {} for consumer_group in consumer_groups: # 0 = offline, (not 0) = online consumer_groups_status[consumer_group] = zk.get(cluster['consumers_path'] + "/" + consumer_group + "/ids")['numChildren'] c = {'cluster':cluster,'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status, 'error':0} except ZooKeeper.RESTError: c = {'cluster':cluster,'brokers':[],'consumer_groups':[],'consumer_groups_status':[], 'error':1} clusters.append(c) return clusters
def _get_topology(): topology = CLUSTERS.get() clusters = [] for cluster in topology: zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get()) zk.add_listener(my_listener) zk.start() brokers = _get_brokers(zk,cluster) consumer_groups = _get_consumer_groups(zk,cluster) consumer_groups_status = {} # 0 = offline, (not 0) = online for consumer_group in consumer_groups: consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids" try: consumers = zk.get_children(consumers_path) except NoNodeError: consumer_groups_status[consumer_group]=0 # 0 = offline else: consumer_groups_status[consumer_group]=len(consumers) # (not 0) = online c = {'cluster':get_cluster_or_404(id=cluster),'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status} clusters.append(c) zk.stop() return clusters
def topics(request, cluster_id): """ Topics view. Returns the topics list of a given cluster """ cluster = get_cluster_or_404(id=cluster_id) topics,error = _get_topics(cluster) return render('topics.mako', request, {'cluster': cluster, 'topics':topics, 'error':error})
def topics(request, cluster_id): cluster = get_cluster_or_404(id=cluster_id) return render('topics.mako', request, { 'cluster': cluster, 'topics': _get_topics(cluster) })
def consumer_groups(request, cluster_id): cluster = get_cluster_or_404(id=cluster_id) return render('consumer_groups.mako', request, {'cluster': cluster, 'consumers_groups':_get_consumers(cluster)})
def cluster(request, cluster_id): c = get_cluster_or_404(id=cluster_id) cluster = _get_cluster_topology(cluster=c) return render('cluster.mako', request, {'cluster':cluster})
def topics(request, cluster_id): cluster = get_cluster_or_404(id=cluster_id) return render('topics.mako', request, {'cluster': cluster, 'topics':_get_topics(cluster)})
def cluster(request, cluster_id): """ Cluster detail view. Returns the cluster detailed topology """ c = get_cluster_or_404(id=cluster_id) cluster = _get_cluster_topology(cluster=c) return render('cluster.mako', request, {'cluster':cluster})
def consumer_groups(request, cluster_id): """ Consumers groups view. Returns the consumers groups list and their info of a given cluster """ cluster = get_cluster_or_404(id=cluster_id) consumers_groups, error = _get_consumers(cluster) return render('consumer_groups.mako', request, {'cluster': cluster, 'consumers_groups':consumers_groups, 'error':error})
def cluster(request, cluster_id): c = get_cluster_or_404(id=cluster_id) cluster = _get_cluster_topology(cluster=c) return render('cluster.mako', request, {'cluster': cluster})
def dashboard(request, cluster_id): aURL = [] aMetrics = [] aOptions = "" sHost = "" sTopic = "" sMetric = "" sMetricComplete = "" sGranularity = "" sDataSource = "" error_brokers = 0 cluster = get_cluster_or_404(id=cluster_id) topics, error_zk_topics = _get_topics(cluster) error_zk_brokers = 0 brokers=[] try: zk = KazooClient(hosts=cluster['zk_host_ports']) zk.start() brokers, error_brokers = _get_brokers(zk,cluster['id']) zk.stop() except NoNodeError: error_zk_brokers = 1 sections = _get_sections_ini() if request.method == 'POST' and request.is_ajax(): sHost = request.POST['txtHost'] sTopic = request.POST['txtTopic'] sMetric = request.POST['txtMetric'] sGranularity = request.POST['txtGranularity'] aOptions = _get_options_ini(sMetric) aMetric = sMetric.split(".") sTopic = "AllTopics" if sTopic == "*" else sTopic + "-" sMetricComplete = aMetric[0] + "." + sTopic + aMetric[1] for element in aOptions.split(","): aMetrics = aMetrics + [sMetricComplete + "." + element] sDataSource = cluster['ganglia_data_source'].replace(" ", "+") for metric in aMetrics: aURL = aURL + [cluster['ganglia_server'] + "/ganglia/graph.php?" + "r=" + sGranularity + "&c=" + sDataSource + "&h=" + sHost + "&m=" + metric + "&json=1"] data = {} data['sMetric'] = sMetricComplete data['sGraphs'] = aOptions data['sGranularity'] = sGranularity data['jsonDumps0'] = _get_dumps(_get_json(aURL[0])) data['jsonDumps1'] = _get_dumps(_get_json(aURL[1])) data['jsonDumps2'] = _get_dumps(_get_json(aURL[2])) data['jsonDumps3'] = _get_dumps(_get_json(aURL[3])) data['jsonDumps4'] = _get_dumps(_get_json(aURL[4])) data['status'] = 0 return JsonResponse(data, safe=False) return render('dashboard.mako', request, {'cluster': cluster, 'topics': topics, 'brokers': brokers, 'metrics': sections, 'error_zk_topics':error_zk_topics, 'error_zk_brokers':error_zk_brokers})
def consumer_groups(request, cluster_id): cluster = get_cluster_or_404(id=cluster_id) return render('consumer_groups.mako', request, { 'cluster': cluster, 'consumers_groups': _get_consumers(cluster) })