def _get_topology():
topology = CLUSTERS.get()
clusters = []
for cluster in topology:
zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get())
#zk.add_listener(my_listener)
zk.start()
brokers = _get_brokers(zk, cluster)
consumer_groups = _get_consumer_groups(zk, cluster)
consumer_groups_status = {} # 0 = offline, (not 0) = online
for consumer_group in consumer_groups:
consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get(
) + "/" + consumer_group + "/ids"
try:
consumers = zk.get_children(consumers_path)
except NoNodeError:
consumer_groups_status[consumer_group] = 0 # 0 = offline
else:
consumer_groups_status[consumer_group] = len(
consumers) # (not 0) = online
c = {
'cluster': get_cluster_or_404(id=cluster),
'brokers': brokers,
'consumer_groups': consumer_groups,
'consumer_groups_status': consumer_groups_status
}
clusters.append(c)
zk.stop()
return clusters
def _get_topology():
""" Method to get the entire Kafka clusters (defined in hue.ini) topology """
topology = CLUSTERS.get()
clusters = []
error = 0
error_brokers = 0
error_consumer_groups = 0
for c in topology:
cluster = get_cluster_or_404(c)
try:
zk = KazooClient(hosts=CLUSTERS[c].ZK_HOST_PORTS.get())
zk.start()
brokers, error_brokers = _get_brokers(zk,cluster['id'])
consumer_groups, error_consumer_groups = _get_consumer_groups(zk,cluster['id'])
consumer_groups_status = {}
for consumer_group in consumer_groups:
# 0 = offline, (not 0) = online
consumers_path = CLUSTERS[c].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids"
try:
consumers = zk.get_children(consumers_path)
except NoNodeError:
consumer_groups_status[consumer_group]=0
else:
consumer_groups_status[consumer_group]=len(consumers)
c = {'cluster':cluster,
'brokers':brokers,
'consumer_groups':consumer_groups,
'consumer_groups_status':consumer_groups_status,
'error_brokers':error_brokers,
'error_consumer_groups':error_consumer_groups,
'error':0}
zk.stop()
except NoNodeError:
c = {'cluster':cluster,
'brokers':[],
'consumer_groups':[],
'consumer_groups_status':[],
'error_brokers':error_brokers,
'error_consumer_groups':error_consumer_groups,
'error':2}
except:
c = {'cluster':cluster,
'brokers':[],
'consumer_groups':[],
'consumer_groups_status':[],
'error_brokers':error_brokers,
'error_consumer_groups':error_consumer_groups,
'error':1}
clusters.append(c)
return clusters
def get_cluster_or_404(id):
try:
name = id
cluster = CLUSTERS.get()[name]
except (TypeError, ValueError):
raise Http404()
cluster = {
'id': id,
'nice_name': id,
'zk_host_ports': cluster.ZK_HOST_PORTS.get(),
'zk_rest_url': cluster.ZK_REST_URL.get(),
'brokers_path': cluster.BROKERS_PATH.get(),
'consumers_path': cluster.CONSUMERS_PATH.get(),
'topics_path': cluster.TOPICS_PATH.get(),
}
return cluster
def get_cluster_or_404(id):
try:
name = id
cluster = CLUSTERS.get()[name]
except (TypeError, ValueError):
raise Http404()
cluster = {
'id': id,
'nice_name': id,
'zk_host_ports': cluster.ZK_HOST_PORTS.get(),
'zk_rest_url': cluster.ZK_REST_URL.get(),
'brokers_path' : cluster.BROKERS_PATH.get(),
'consumers_path' : cluster.CONSUMERS_PATH.get(),
'topics_path' : cluster.TOPICS_PATH.get(),
}
return cluster
def get_cluster_or_404(id):
""" Get a cluster information from its ID or a 404 Error """
try:
name = id
cluster = CLUSTERS.get()[name]
except (TypeError, ValueError, KeyError):
raise Http404()
cluster = {
'id': id,
'nice_name': id,
'zk_host_ports': cluster.ZK_HOST_PORTS.get(),
'brokers_path' : cluster.BROKERS_PATH.get(),
'consumers_path' : cluster.CONSUMERS_PATH.get(),
'topics_path' : cluster.TOPICS_PATH.get(),
'ganglia_server' : cluster.GANGLIA_SERVER.get(),
'ganglia_data_source' : cluster.GANGLIA_DATA_SOURCE.get(),
}
return cluster
def _get_topology():
""" Method to get the entire Kafka clusters (defined in hue.ini) topology """
topology = CLUSTERS.get()
clusters = []
for c in topology:
cluster = get_cluster_or_404(c)
try:
zk = ZooKeeper(cluster['zk_rest_url'])
brokers = _get_brokers(zk,cluster)
consumer_groups = _get_consumer_groups(zk,cluster)
consumer_groups_status = {}
for consumer_group in consumer_groups:
# 0 = offline, (not 0) = online
consumer_groups_status[consumer_group] = zk.get(cluster['consumers_path'] + "/" + consumer_group + "/ids")['numChildren']
c = {'cluster':cluster,'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status, 'error':0}
except ZooKeeper.RESTError:
c = {'cluster':cluster,'brokers':[],'consumer_groups':[],'consumer_groups_status':[], 'error':1}
clusters.append(c)
return clusters
def _get_topology():
topology = CLUSTERS.get()
clusters = []
for cluster in topology:
zk = KazooClient(hosts=CLUSTERS[cluster].ZK_HOST_PORTS.get())
zk.add_listener(my_listener)
zk.start()
brokers = _get_brokers(zk,cluster)
consumer_groups = _get_consumer_groups(zk,cluster)
consumer_groups_status = {} # 0 = offline, (not 0) = online
for consumer_group in consumer_groups:
consumers_path = CLUSTERS[cluster].CONSUMERS_PATH.get() + "/" + consumer_group + "/ids"
try:
consumers = zk.get_children(consumers_path)
except NoNodeError:
consumer_groups_status[consumer_group]=0 # 0 = offline
else:
consumer_groups_status[consumer_group]=len(consumers) # (not 0) = online
c = {'cluster':get_cluster_or_404(id=cluster),'brokers':brokers,'consumer_groups':consumer_groups,'consumer_groups_status':consumer_groups_status}
clusters.append(c)
zk.stop()
return clusters