Ejemplo n.º 1
0
def _get_system_tools(request, cluster_id):
	data = {}
	sCmd = "";
	data['cluster'] = get_cluster_or_404(id=cluster_id)
	data['topics'], data['error_zk_topics'] = _get_topics(data['cluster'])
	data['version'] = settings.HUE_DESKTOP_VERSION

	try:	
		zk = KazooClient(hosts=data['cluster']['zk_host_ports'])
		zk.start()
		data['brokers'], data['error_zk_brokers'] = _get_brokers(zk,data['cluster']['id'])
		zk.stop()

		if request.method == 'POST' and request.is_ajax():			
			sOption = request.POST['txtOption']
			sBroker = request.POST['txtBroker']
			sBrokers = ""
			for element in sBroker.split(","):
				if element <> "":
					sBrokers += element + ":9092, "
			sTopic = request.POST['txtTopic']
			sTime = request.POST['txtTime']
			iWaitTime = request.POST['numWaitTime']
			sNumOffset = request.POST['numOffset']
			sPartitions = request.POST['txtPartitions']

			sCmd = ('/usr/lib/kafka/bin/kafka-run-class.sh %s ' 
					'--broker-list %s '
					'--topic %s '
					'--time %s ' % (sOption, sBrokers[:-2], sTopic, sTime))

			if iWaitTime <> "1000":
				sCmd += '--max-wait-ms %s ' % (iWaitTime)

			if sNumOffset <> "1":
				sCmd += '--offsets %s ' % (sNumOffset)

			if sPartitions <> "":
				sCmd += '--partitions %s ' % (sPartitions)
			
			output,err = subprocess.Popen([sCmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()		
			data['cmd'] = sCmd
			data['output'] = output
			data['error'] = err

	except NoNodeError:
		data = {'cluster': get_cluster_or_404(id=cluster_id),
				'topics': [], 
				'brokers': [],
				'cmd': "", 
				'version': settings.HUE_DESKTOP_VERSION,
				'error_zk_topics': 1, 
				'error_zk_brokers': 1 ,
				'output': "",
				'error': -1 }

	return data
Ejemplo n.º 2
0
def _get_topology():
    topology = CLUSTERS.get()
    clusters = []
    for cluster in topology:
        zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get())
        #zk.add_listener(my_listener)
        zk.start()
        brokers = _get_brokers(zk, cluster)
        consumer_groups = _get_consumer_groups(zk, cluster)
        consumer_groups_status = {}  # 0 = offline, (not 0) =  online
        for consumer_group in consumer_groups:
            consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get(
            ) + "/" + consumer_group + "/ids"
            try:
                consumers = zk.get_children(consumers_path)
            except NoNodeError:
                consumer_groups_status[consumer_group] = 0  # 0 = offline
            else:
                consumer_groups_status[consumer_group] = len(
                    consumers)  # (not 0) =  online
        c = {
            'cluster': get_cluster_or_404(id=cluster),
            'brokers': brokers,
            'consumer_groups': consumer_groups,
            'consumer_groups_status': consumer_groups_status
        }
        clusters.append(c)
        zk.stop()
    return clusters
Ejemplo n.º 3
0
def _get_json_type(request, cluster_id, type):
	data = []
	error_brokers = 0
	try:	
		cluster = get_cluster_or_404(id=cluster_id)

		zk = KazooClient(hosts=cluster['zk_host_ports'])
		zk.start()

		if type == "broker":			
			brokers, error_brokers = _get_brokers(zk,cluster_id)
			for broker in brokers:
				data.append(broker['host'])
		if type == "topic":
			topics, error_zk_topics = _get_topics(cluster)
			for topic in topics:
				data.append(topic['id'])
		if type == "metric":
			data = _get_sections_ini()
	except KazooException:
		error_zk_brokers = 1

	zk.stop()

	return JsonResponse(data, safe=False)
Ejemplo n.º 4
0
def consumer_group(request, cluster_id, group_id):	
	cluster = get_cluster_or_404(id=cluster_id)
	zk = KazooClient(hosts=cluster['zk_host_ports'])
	zk.add_listener(my_listener)
	zk.start()
	consumer_group = _get_consumer_group(zk=zk,cluster=cluster,group_id=group_id)
	zk.stop()
	return render('consumer_group.mako', request, {'cluster': cluster, 'consumer_group':consumer_group})
Ejemplo n.º 5
0
def _get_topology():
	""" Method to get the entire Kafka clusters (defined in hue.ini) topology """
	topology = CLUSTERS.get()
	clusters = []
	error = 0
	error_brokers = 0
	error_consumer_groups = 0

	for c in topology:
		cluster = get_cluster_or_404(c)
		try:
			zk = KazooClient(hosts=CLUSTERS[c].ZK_HOST_PORTS.get())
			zk.start()
			brokers, error_brokers = _get_brokers(zk,cluster['id'])
			consumer_groups, error_consumer_groups = _get_consumer_groups(zk,cluster['id'])
			consumer_groups_status = {} 

			for consumer_group in consumer_groups:
				# 0 = offline, (not 0) =  online
				consumers_path = CLUSTERS[c].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids"
				try:
					consumers = zk.get_children(consumers_path)
				except NoNodeError:
					consumer_groups_status[consumer_group]=0
				else:
					consumer_groups_status[consumer_group]=len(consumers)
			
			c = {'cluster':cluster,
				'brokers':brokers,
				'consumer_groups':consumer_groups,
				'consumer_groups_status':consumer_groups_status,
				'error_brokers':error_brokers,
				'error_consumer_groups':error_consumer_groups,
				'error':0}
			
			zk.stop()

		except NoNodeError:
			c = {'cluster':cluster,
				'brokers':[],
				'consumer_groups':[],
				'consumer_groups_status':[],
				'error_brokers':error_brokers,
				'error_consumer_groups':error_consumer_groups,
				'error':2}
		except:
			c = {'cluster':cluster,
				'brokers':[],
				'consumer_groups':[],
				'consumer_groups_status':[],
				'error_brokers':error_brokers,
				'error_consumer_groups':error_consumer_groups,
				'error':1}

		clusters.append(c)		
	return clusters
Ejemplo n.º 6
0
def consumer_group(request, cluster_id, group_id):	
	""" Consumers Group detail view. Returns the detailed view of a given consumers group """
	cluster = get_cluster_or_404(id=cluster_id)
	consumer_group = {}
	error = 0
	try:
		zk = ZooKeeper(cluster['zk_rest_url'])
		consumer_group = _get_consumer_group(zk=zk,cluster=cluster,group_id=group_id)
	except ZooKeeper.RESTError:
		error = 1
	return render('consumer_group.mako', request, {'cluster': cluster, 'consumer_group':consumer_group, 'error':error})
Ejemplo n.º 7
0
def consumer_group(request, cluster_id, group_id):	
	""" Consumers Group detail view. Returns the detailed view of a given consumers group """
	cluster = get_cluster_or_404(id=cluster_id)
	consumer_group = {}
	error = 0
	try:
		zk = KazooClient(hosts=cluster['zk_host_ports'])
		zk.start()
		consumer_group = _get_consumer_group(zk=zk,cluster=cluster,group_id=group_id)
	except NoNodeError:
		error = 1
	except:
		error = 2
	zk.stop()

	return render('consumer_group.mako', request, {'cluster': cluster, 'consumer_group':consumer_group, 'error':error})
Ejemplo n.º 8
0
def _get_json_type(request, cluster_id, type):
	data = []
	try:	
		cluster = get_cluster_or_404(id=cluster_id)

		if type == "broker":
			zk = ZooKeeper(cluster['zk_rest_url'])
			brokers = _get_brokers(zk,cluster)
			for broker in brokers:
				data.append(broker['host'])
		if type == "topic":
			topics, error_zk_topics = _get_topics(cluster)
			for topic in topics:
				data.append(topic['id'])
		if type == "metric":
			data = _get_sections_ini()
	except ZooKeeper.RESTError:
		error_zk_brokers = 1

	return HttpResponse(_get_dumps(data), content_type = "application/json")
Ejemplo n.º 9
0
def _get_topology():
	""" Method to get the entire Kafka clusters (defined in hue.ini) topology """
	topology = CLUSTERS.get()
	clusters = []
	for c in topology:
		cluster = get_cluster_or_404(c)
		try:
			zk = ZooKeeper(cluster['zk_rest_url'])
			brokers = _get_brokers(zk,cluster)
			consumer_groups = _get_consumer_groups(zk,cluster)
			consumer_groups_status = {} 
			for consumer_group in consumer_groups:
				# 0 = offline, (not 0) =  online
				consumer_groups_status[consumer_group] = zk.get(cluster['consumers_path'] + "/" + consumer_group + "/ids")['numChildren']
			
			c = {'cluster':cluster,'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status, 'error':0}
			
		except ZooKeeper.RESTError:
			c = {'cluster':cluster,'brokers':[],'consumer_groups':[],'consumer_groups_status':[], 'error':1}
		clusters.append(c)
	return clusters
Ejemplo n.º 10
0
def _get_topology():
	topology = CLUSTERS.get()
	clusters = []
	for cluster in topology:
		zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get())
		zk.add_listener(my_listener)
		zk.start()
		brokers = _get_brokers(zk,cluster)
		consumer_groups = _get_consumer_groups(zk,cluster)
		consumer_groups_status = {} # 0 = offline, (not 0) =  online
		for consumer_group in consumer_groups:
			consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids"
			try:
				consumers = zk.get_children(consumers_path)
			except NoNodeError:
				consumer_groups_status[consumer_group]=0 # 0 = offline
			else:
				consumer_groups_status[consumer_group]=len(consumers) # (not 0) =  online
		c = {'cluster':get_cluster_or_404(id=cluster),'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status}
		clusters.append(c)
		zk.stop()
	return clusters
Ejemplo n.º 11
0
def topics(request, cluster_id):
	""" Topics view. Returns the topics list of a given cluster """
	cluster = get_cluster_or_404(id=cluster_id)
	topics,error = _get_topics(cluster)
	return render('topics.mako', request, {'cluster': cluster, 'topics':topics, 'error':error})
Ejemplo n.º 12
0
def topics(request, cluster_id):
    cluster = get_cluster_or_404(id=cluster_id)
    return render('topics.mako', request, {
        'cluster': cluster,
        'topics': _get_topics(cluster)
    })
Ejemplo n.º 13
0
def consumer_groups(request, cluster_id):	
	cluster = get_cluster_or_404(id=cluster_id)
	return render('consumer_groups.mako', request, {'cluster': cluster, 'consumers_groups':_get_consumers(cluster)})
Ejemplo n.º 14
0
def cluster(request, cluster_id):
	c = get_cluster_or_404(id=cluster_id)
	cluster = _get_cluster_topology(cluster=c)
	return render('cluster.mako', request, {'cluster':cluster})
Ejemplo n.º 15
0
def topics(request, cluster_id):
	cluster = get_cluster_or_404(id=cluster_id)
	return render('topics.mako', request, {'cluster': cluster, 'topics':_get_topics(cluster)})
Ejemplo n.º 16
0
def cluster(request, cluster_id):
	""" Cluster detail view. Returns the cluster detailed topology """
	c = get_cluster_or_404(id=cluster_id)
	cluster = _get_cluster_topology(cluster=c)
	return render('cluster.mako', request, {'cluster':cluster})
Ejemplo n.º 17
0
def consumer_groups(request, cluster_id):	
	""" Consumers groups view. Returns the consumers groups list and their info of a given cluster """
	cluster = get_cluster_or_404(id=cluster_id)
	consumers_groups, error = _get_consumers(cluster)
	return render('consumer_groups.mako', request, {'cluster': cluster, 'consumers_groups':consumers_groups, 'error':error})
Ejemplo n.º 18
0
def cluster(request, cluster_id):
    c = get_cluster_or_404(id=cluster_id)
    cluster = _get_cluster_topology(cluster=c)
    return render('cluster.mako', request, {'cluster': cluster})
Ejemplo n.º 19
0
def dashboard(request, cluster_id):
	aURL = []
	aMetrics = []
	aOptions = ""
	sHost = ""
	sTopic = ""
	sMetric = ""
	sMetricComplete = ""
	sGranularity = ""
	sDataSource = ""
	error_brokers = 0

	cluster = get_cluster_or_404(id=cluster_id)
	topics, error_zk_topics = _get_topics(cluster)
	error_zk_brokers = 0
	brokers=[]

	try:	
		zk = KazooClient(hosts=cluster['zk_host_ports'])
		zk.start()
		brokers, error_brokers = _get_brokers(zk,cluster['id'])
		zk.stop()
	except NoNodeError:
		error_zk_brokers = 1

	sections = _get_sections_ini()
	
	if request.method == 'POST' and request.is_ajax():
		sHost = request.POST['txtHost']
		sTopic = request.POST['txtTopic']	
		sMetric = request.POST['txtMetric']        
		sGranularity = request.POST['txtGranularity']
		aOptions = _get_options_ini(sMetric)
		aMetric = sMetric.split(".")
		sTopic = "AllTopics" if sTopic == "*" else sTopic + "-"
		sMetricComplete = aMetric[0] + "." + sTopic + aMetric[1]
        
		for element in aOptions.split(","):
			aMetrics = aMetrics + [sMetricComplete + "." + element]

		sDataSource = cluster['ganglia_data_source'].replace(" ", "+")

		for metric in aMetrics:
			aURL = aURL + [cluster['ganglia_server'] + "/ganglia/graph.php?" + "r=" + sGranularity + "&c=" + sDataSource + "&h=" + sHost + "&m=" + metric + "&json=1"]

		data = {}
		data['sMetric'] = sMetricComplete
		data['sGraphs'] = aOptions
		data['sGranularity'] = sGranularity
		data['jsonDumps0'] =  _get_dumps(_get_json(aURL[0]))
		data['jsonDumps1'] =  _get_dumps(_get_json(aURL[1]))
		data['jsonDumps2'] =  _get_dumps(_get_json(aURL[2]))
		data['jsonDumps3'] =  _get_dumps(_get_json(aURL[3]))
		data['jsonDumps4'] =  _get_dumps(_get_json(aURL[4]))
		data['status'] = 0

		return JsonResponse(data, safe=False)
    
	return render('dashboard.mako', request, {'cluster': cluster,
												'topics': topics,
												'brokers': brokers,
												'metrics': sections,
												'error_zk_topics':error_zk_topics,
												'error_zk_brokers':error_zk_brokers})
Ejemplo n.º 20
0
def consumer_groups(request, cluster_id):
    cluster = get_cluster_or_404(id=cluster_id)
    return render('consumer_groups.mako', request, {
        'cluster': cluster,
        'consumers_groups': _get_consumers(cluster)
    })