Exemplo n.º 1
0
def _get_topology():
    topology = CLUSTERS.get()
    clusters = []
    for cluster in topology:
        zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get())
        #zk.add_listener(my_listener)
        zk.start()
        brokers = _get_brokers(zk, cluster)
        consumer_groups = _get_consumer_groups(zk, cluster)
        consumer_groups_status = {}  # 0 = offline, (not 0) =  online
        for consumer_group in consumer_groups:
            consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get(
            ) + "/" + consumer_group + "/ids"
            try:
                consumers = zk.get_children(consumers_path)
            except NoNodeError:
                consumer_groups_status[consumer_group] = 0  # 0 = offline
            else:
                consumer_groups_status[consumer_group] = len(
                    consumers)  # (not 0) =  online
        c = {
            'cluster': get_cluster_or_404(id=cluster),
            'brokers': brokers,
            'consumer_groups': consumer_groups,
            'consumer_groups_status': consumer_groups_status
        }
        clusters.append(c)
        zk.stop()
    return clusters
Exemplo n.º 2
0
def _get_topology():
	""" Method to get the entire Kafka clusters (defined in hue.ini) topology """
	topology = CLUSTERS.get()
	clusters = []
	error = 0
	error_brokers = 0
	error_consumer_groups = 0

	for c in topology:
		cluster = get_cluster_or_404(c)
		try:
			zk = KazooClient(hosts=CLUSTERS[c].ZK_HOST_PORTS.get())
			zk.start()
			brokers, error_brokers = _get_brokers(zk,cluster['id'])
			consumer_groups, error_consumer_groups = _get_consumer_groups(zk,cluster['id'])
			consumer_groups_status = {} 

			for consumer_group in consumer_groups:
				# 0 = offline, (not 0) =  online
				consumers_path = CLUSTERS[c].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids"
				try:
					consumers = zk.get_children(consumers_path)
				except NoNodeError:
					consumer_groups_status[consumer_group]=0
				else:
					consumer_groups_status[consumer_group]=len(consumers)
			
			c = {'cluster':cluster,
				'brokers':brokers,
				'consumer_groups':consumer_groups,
				'consumer_groups_status':consumer_groups_status,
				'error_brokers':error_brokers,
				'error_consumer_groups':error_consumer_groups,
				'error':0}
			
			zk.stop()

		except NoNodeError:
			c = {'cluster':cluster,
				'brokers':[],
				'consumer_groups':[],
				'consumer_groups_status':[],
				'error_brokers':error_brokers,
				'error_consumer_groups':error_consumer_groups,
				'error':2}
		except:
			c = {'cluster':cluster,
				'brokers':[],
				'consumer_groups':[],
				'consumer_groups_status':[],
				'error_brokers':error_brokers,
				'error_consumer_groups':error_consumer_groups,
				'error':1}

		clusters.append(c)		
	return clusters
Exemplo n.º 3
0
def get_cluster_or_404(id):
    try:
        name = id
        cluster = CLUSTERS.get()[name]
    except (TypeError, ValueError):
        raise Http404()

    cluster = {
        'id': id,
        'nice_name': id,
        'zk_host_ports': cluster.ZK_HOST_PORTS.get(),
        'zk_rest_url': cluster.ZK_REST_URL.get(),
        'brokers_path': cluster.BROKERS_PATH.get(),
        'consumers_path': cluster.CONSUMERS_PATH.get(),
        'topics_path': cluster.TOPICS_PATH.get(),
    }

    return cluster
Exemplo n.º 4
0
def get_cluster_or_404(id):
  try:
    name = id
    cluster = CLUSTERS.get()[name]
  except (TypeError, ValueError):
    raise Http404()

  cluster = {
    'id': id,
    'nice_name': id,
    'zk_host_ports': cluster.ZK_HOST_PORTS.get(),
    'zk_rest_url': cluster.ZK_REST_URL.get(),
    'brokers_path' : cluster.BROKERS_PATH.get(),
    'consumers_path' : cluster.CONSUMERS_PATH.get(),
    'topics_path' : cluster.TOPICS_PATH.get(),
  }

  return cluster
Exemplo n.º 5
0
def get_cluster_or_404(id):
  """ Get a cluster information from its ID or a 404 Error """
  try:
    name = id
    cluster = CLUSTERS.get()[name]
  except (TypeError, ValueError, KeyError):
    raise Http404()

  cluster = {
    'id': id,
    'nice_name': id,
    'zk_host_ports': cluster.ZK_HOST_PORTS.get(),
    'brokers_path' : cluster.BROKERS_PATH.get(),
    'consumers_path' : cluster.CONSUMERS_PATH.get(),
    'topics_path' : cluster.TOPICS_PATH.get(),
    'ganglia_server' : cluster.GANGLIA_SERVER.get(),
    'ganglia_data_source' : cluster.GANGLIA_DATA_SOURCE.get(),
  }

  return cluster
Exemplo n.º 6
0
def _get_topology():
	""" Method to get the entire Kafka clusters (defined in hue.ini) topology """
	topology = CLUSTERS.get()
	clusters = []
	for c in topology:
		cluster = get_cluster_or_404(c)
		try:
			zk = ZooKeeper(cluster['zk_rest_url'])
			brokers = _get_brokers(zk,cluster)
			consumer_groups = _get_consumer_groups(zk,cluster)
			consumer_groups_status = {} 
			for consumer_group in consumer_groups:
				# 0 = offline, (not 0) =  online
				consumer_groups_status[consumer_group] = zk.get(cluster['consumers_path'] + "/" + consumer_group + "/ids")['numChildren']
			
			c = {'cluster':cluster,'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status, 'error':0}
			
		except ZooKeeper.RESTError:
			c = {'cluster':cluster,'brokers':[],'consumer_groups':[],'consumer_groups_status':[], 'error':1}
		clusters.append(c)
	return clusters
Exemplo n.º 7
0
def _get_topology():
	topology = CLUSTERS.get()
	clusters = []
	for cluster in topology:
		zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get())
		zk.add_listener(my_listener)
		zk.start()
		brokers = _get_brokers(zk,cluster)
		consumer_groups = _get_consumer_groups(zk,cluster)
		consumer_groups_status = {} # 0 = offline, (not 0) =  online
		for consumer_group in consumer_groups:
			consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids"
			try:
				consumers = zk.get_children(consumers_path)
			except NoNodeError:
				consumer_groups_status[consumer_group]=0 # 0 = offline
			else:
				consumer_groups_status[consumer_group]=len(consumers) # (not 0) =  online
		c = {'cluster':get_cluster_or_404(id=cluster),'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status}
		clusters.append(c)
		zk.stop()
	return clusters