def init_codis_info(self):
if self.has_init():
return
# start zookeeper client
zk_client = KazooClient(hosts=self.zk_addr)
zk_client.start()
# get codis server information
zk_servers_dir = "/zk/codis/db_%s/servers" % self.product_name
for zk_server in zk_client.get_children(zk_servers_dir):
zk_server_path = '/'.join((zk_servers_dir, zk_server))
for server in zk_client.get_children(zk_server_path):
server_path = '/'.join((zk_server_path, server))
data, stat = zk_client.get(server_path)
server_info = json.loads(data)
group_id = server_info.get('group_id')
server_type = server_info.get('type')
server_addr = server_info.get('addr')
self.add_codis_server(group_id, server_type, server_addr)
# get codis proxy information
zk_proxy_dir = "/zk/codis/db_%s/proxy" % self.product_name
for zk_proxy in zk_client.get_children(zk_proxy_dir):
zk_proxy_path = '/'.join((zk_proxy_dir, zk_proxy))
data, stat = zk_client.get(zk_proxy_path)
proxy_info = json.loads(data)
self.add_proxy(proxy_info['id'], proxy_info['addr'], proxy_info['debug_var_addr'], proxy_info['state'])
self.redis_client.init_connection(self.get_group_info(), self.get_proxy_info())
self.init_done()
return None
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--hosts', required=False, help='zk hosts, example:127.0.0.1:2181')
parser.add_argument('--ip', required=False, help='docker host ip, example:172.17.0.8')
args = parser.parse_args()
ip = args.ip
zk = KazooClient(hosts=args.hosts)
zk.start()
data = zk.get("/hbase/meta-region-server")
data = str(data)
start = data.find("regionserver:") + len("regionserver:")
end = data.find("\\", start)
port = filter(str.isdigit, data[start:end])
cmd = 'socat TCP-LISTEN:' + port + ',fork TCP:' + ip + ':' + port + ' &'
print cmd
os.system(cmd)
data=zk.get("/hbase/master")
data = str(data)
start = data.find("master:") + len("master:")
end = data.find("\\", start)
port = filter(str.isdigit, data[start:end])
cmd = 'socat TCP-LISTEN:' + port + ',fork TCP:' + ip + ':' + port + ' &'
print cmd
os.system(cmd)
def zk_init(server_address, zk_node, zk_address = ''):
#初始化zk节点
zk_cli = KazooClient(hosts = zk_address)
try:
zk_cli.start()
except:
logging.error('zk Init error, can not connect %s' %(str(zk_address)))
return -1
try:
zk_cli.get(zk_node)
except:
logging.warn('can not find zk path %s, creat it' %(str(zk_node)))
zk_cli.ensure_path(zk_node)
if zk_node[-1] != '/':
zk_node += '/'
try:
zk_cli.create(zk_node + server_address, '1', ephemeral=True)
except:
if zk_cli.get(zk_node + server_address):
return 0
else:
logging.error('create zk_node error, can not create node %s' %(str(zk_node) + str(zk_address)))
return -1
return 0
def cluster_state(pathVariable):
try:
zk_host = zk_map[pathVariable]
zk = KazooClient(hosts=zk_host, read_only=True)
zk.start()
if pathVariable.find('kafka') > 0:
nodes = zk.get_children('/brokers/ids')
brokers = ""
for id in nodes:
data, stat = zk.get('/brokers/ids/'+id)
jdata = json.loads(data)
brokers += jdata['host']+"\n"
return 'There are '+str(len(nodes))+\
' brokers running\nids: '+\
','.join(nodes)+'\nbrokers:'+\
brokers+'\nZK:'+zk_host+\
'\nThe cluster looks healthy. ', 200, {'Content-Type': 'text/plain; charset=utf-8'}
else:
data, stat = zk.get('/hbase/master')
start = data.find('bach-')
end = data.find('.bloomberg')
hmaster = data[start:end]
data = zk.get_children('/hbase/rs')
rs = ""
for node in data:
rs += node+"\n"
return "Its a hadoop cluster\n"+\
'hmaster :'+hmaster+\
'\nRegionServers :'+ rs+\
'\nZK: '+zk_host+\
'\nThe cluster looks healthy.', 200, {'Content-Type': 'text/plain; charset=utf-8'}
zk.stop()
except:
return 'Cluster seems down'
class Zookeeper(KeyManager):
def __init__(self, hosts):
self._hosts = hosts
self.zk = KazooClient(hosts=hosts)
self.zk.start()
def get(self, key):
result = self.zk.get(key)[0]
if result == "":
result = []
children = self.zk.get_children(key)
for i in children:
result.append({'name': i, 'value': self.zk.get(os.path.join(key, i))[0]})
return result
else:
return self.zk.get(key)[0]
def set(self, key, data):
try:
self.zk.set(key, data.encode('utf-8'))
except:
self.zk.create(key, data.encode('utf-8'))
def mkdir(self, key):
self.set(key, "")
def close(self):
self.zk.stop()
self.zk.close()
@property
def hosts(self):
return self._hosts
class KafkaInfo(object):
def __init__(self, hosts):
self.zk = KazooClient(hosts)
self.zk.start()
def brokers(self):
broker_ids = self.zk.get_children('/brokers/ids')
return [json.loads(self.zk.get('brokers/ids/'+broker_id)[0])['host'] for broker_id in broker_ids]
def jmxports(self):
broker_ids = self.zk.get_children('/brokers/ids')
return [json.loads(self.zk.get('brokers/ids/'+broker_id)[0])['jmx_port'] for broker_id in broker_ids]
def topics(self):
return self.zk.get_children('/brokers/topics')
def partitions(self, topic):
strs = self.zk.get_children('/brokers/topics/%s/partitions' % topic)
return map(int, strs)
def consumers(self):
return self.zk.get_children('/consumers')
def topics_for_consumer(self, consumer):
return self.zk.get_children('/consumers/%s/offsets' % consumer)
def offset(self, topic, consumer, partition):
(n, _) = self.zk.get('/consumers/%s/offsets/%s/%d' % (consumer, topic, partition))
return int(n)
def check_broker_id_in_zk(broker_id_policy, process, region):
"""
Check endlessly for the Zookeeper Connection.
This function checks endlessly if the broker is still registered in ZK
(we observered running brokers but missing broker id's so we implemented this check)
and if the ZK IP's changed (e.g. due to a node restart). If this happens a Kafka restart is enforced.
"""
from kazoo.client import KazooClient
zk_conn_str = os.getenv('ZOOKEEPER_CONN_STRING')
broker_id_manager = find_out_own_id.get_broker_policy(broker_id_policy)
broker_id = broker_id_manager.get_id(kafka_data_dir)
logging.info("check broker id... {}".format(broker_id))
if not broker_id:
broker_id = wait_for_broker_id(broker_id_manager, kafka_data_dir)
while True:
check_kafka(region)
new_zk_conn_str = generate_zk_conn_str.run(os.getenv('ZOOKEEPER_STACK_NAME'), region)
if zk_conn_str != new_zk_conn_str:
logging.warning("ZooKeeper connection string changed!")
logging.warning("new ZK: " + new_zk_conn_str)
logging.warning("old ZK: " + zk_conn_str)
zk_conn_str = new_zk_conn_str
os.environ['ZOOKEEPER_CONN_STRING'] = zk_conn_str
create_broker_properties(zk_conn_str)
from random import randint
wait_to_stop = randint(1, 10)
logging.info("Waiting " + str(wait_to_stop) + " seconds to stop kafka broker ...")
sleep(wait_to_stop)
process.terminate()
process.wait()
wait_to_restart = randint(10, 20)
logging.info("Waiting " + str(wait_to_restart) + " seconds to restart kafka broker ...")
sleep(wait_to_restart)
logging.info("Restarting kafka broker with new ZooKeeper connection string ...")
process = subprocess.Popen([kafka_dir + "/bin/kafka-server-start.sh",
kafka_dir + "/config/server.properties"])
os.environ['WAIT_FOR_KAFKA'] = 'yes'
continue
zk = KazooClient(hosts=zk_conn_str)
zk.start()
try:
zk.get("/brokers/ids/" + broker_id)
logging.info("I'm still in ZK registered, all good!")
sleep(60)
zk.stop()
except:
logging.warning("I'm not in ZK registered, stopping kafka broker process!")
zk.stop()
process.terminate()
process.wait()
logging.info("Restarting kafka broker ...")
process = subprocess.Popen([kafka_dir + "/bin/kafka-server-start.sh",
kafka_dir + "/config/server.properties"])
os.environ['WAIT_FOR_KAFKA'] = 'yes'
def check_broker_id_in_zk(broker_id, process):
"""
Check endlessly for the Zookeeper Connection.
This function checks endlessly if the broker is still registered in ZK
(we observered running brokers but missing broker id's so we implemented this check)
and if the ZK IP's changed (e.g. due to a node restart). If this happens a Kafka restart is enforced.
"""
import requests
from time import sleep
from kazoo.client import KazooClient
zk_conn_str = os.getenv('ZOOKEEPER_CONN_STRING')
while True:
if os.getenv('WAIT_FOR_KAFKA') != 'no':
ip = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document').json()['privateIp']
wait_for_kafka_startup.run(ip)
os.environ['WAIT_FOR_KAFKA'] = 'no'
new_zk_conn_str = generate_zk_conn_str.run(os.getenv('ZOOKEEPER_STACK_NAME'), region)
if zk_conn_str != new_zk_conn_str:
logging.warning("ZooKeeper connection string changed!")
logging.warning("new ZK: " + new_zk_conn_str)
logging.warning("old ZK: " + zk_conn_str)
zk_conn_str = new_zk_conn_str
os.environ['ZOOKEEPER_CONN_STRING'] = zk_conn_str
create_broker_properties(zk_conn_str)
from random import randint
wait_to_kill = randint(1, 10)
logging.info("Waiting " + str(wait_to_kill) + " seconds to kill kafka broker ...")
sleep(wait_to_kill)
process.kill()
wait_to_restart = randint(10, 20)
logging.info("Waiting " + str(wait_to_restart) + " seconds to restart kafka broker ...")
sleep(wait_to_restart)
logging.info("Restarting kafka broker with new ZooKeeper connection string ...")
process = subprocess.Popen([kafka_dir + "/bin/kafka-server-start.sh",
kafka_dir + "/config/server.properties"])
os.environ['WAIT_FOR_KAFKA'] = 'yes'
continue
zk = KazooClient(hosts=zk_conn_str)
zk.start()
try:
zk.get("/brokers/ids/" + broker_id)
logging.info("I'm still in ZK registered, all good!")
sleep(60)
zk.stop()
except:
logging.warning("I'm not in ZK registered, killing kafka broker process!")
zk.stop()
process.kill()
logging.info("Restarting kafka broker ...")
process = subprocess.Popen([kafka_dir + "/bin/kafka-server-start.sh",
kafka_dir + "/config/server.properties"])
os.environ['WAIT_FOR_KAFKA'] = 'yes'
def _get_topics(cluster):
""" Method to get the topic list of a given cluster """
topic_list = []
error = 0
try:
zk = KazooClient(hosts=cluster['zk_host_ports'])
zk.start()
topics = zk.get_children(cluster['topics_path'])
except NoNodeError:
error = 2
return topic_list, error
except:
error = 1
return topic_list, error
else:
for topic in topics:
t = {'id':topic}
topic_path = cluster['topics_path'] + "/" + topic
data, stat = zk.get(topic_path)
d=json.loads(data)
t['topic_partitions_data']=d['partitions']
partitions_path = topic_path + "/partitions"
try:
partitions = zk.get_children(partitions_path)
t['partitions']=partitions
tpp = {}
p =[]
for partition in partitions:
tps = {}
p.append(partition.encode('ascii'))
partition_path = partitions_path + "/" + partition + "/state"
data, stat = zk.get(partition_path)
d = json.loads(data)
tps['isr'] = d['isr']
tps['leader'] = d['leader']
tpp[partition.encode('ascii')]=tps
t['partitions']=p
t['topic_partitions_states']=tpp
topic_list.append(t)
except NoNodeError:
topic_list = []
error = 2
return topic_list, error
except:
topic_list = []
error = 1
return topic_list, error
zk.stop()
return topic_list, error
def zookeeper_swarm(zk_server_list, path='/swarm'):
path = path + '/docker/swarm/leader'
zk = KazooClient(hosts=zk_server_list)
zk.start()
master, stat = zk.get(path)
zk.stop()
return master.decode('utf-8')
def readAMHostPort(self):
logger.debug("Trying to connect to ZK...")
amHost = ""
amSecuredPort = ""
amUnsecuredPort = ""
zk = None
try:
zk = KazooClient(hosts=self.zk_quorum, read_only=True)
zk.start()
data, stat = zk.get(self.zk_reg_path)
logger.debug("Registry Data: %s" % (data.decode("utf-8")))
sliderRegistry = json.loads(data)
internalAttr = sliderRegistry["internal"]
for internal in internalAttr:
if internal["api"] == "classpath:org.apache.slider.agents.secure":
address0 = internal["addresses"][0]
amUrl = address0["uri"]
amHost = amUrl.split("/")[2].split(":")[0]
amSecuredPort = amUrl.split(":")[2].split("/")[0]
if internal["api"] == "classpath:org.apache.slider.agents.oneway":
address0 = internal["addresses"][0]
amUnsecureUrl = address0["uri"]
amHost = amUnsecureUrl.split("/")[2].split(":")[0]
amUnsecuredPort = amUnsecureUrl.split(":")[2].split("/")[0]
# the ports needs to be utf-8 encoded
amSecuredPort = amSecuredPort.encode('utf8', 'ignore')
amUnsecuredPort = amUnsecuredPort.encode('utf8', 'ignore')
except Exception, e:
# log and let empty strings be returned
logger.error("Could not connect to zk registry at %s in quorum %s. Error: %s" %
(self.zk_reg_path, self.zk_quorum, str(e)))
pass
class ActorAddressBook(object):
def __init__(self, zk_hosts, timeout=60.0):
self.retry = KazooRetry(max_tries=10)
self.zk = KazooClient(hosts=zk_hosts, timeout=timeout)
self.zk.start()
def lookup(self, path):
return self.retry(self._lookup, path)
def _lookup(self, path):
actor_url, stat = self.zk.get(path)
return RemoteActor(actor_url.decode('utf-8'))
def register(self, path, actor_url):
return self.retry(self._register, path, actor_url)
def _register(self, path, actor_url):
self.zk.ensure_path(path)
self.zk.set(path, actor_url.encode('utf-8'))
def delete(self, path):
self.zk.delete(path, recursive=True)
def __del__(self):
self.zk.stop()
def _get_registered_nodes(zk: KazooClient, zk_path: str) -> List[str]:
"""
Return the IPs of nodes that have registered in ZooKeeper.
The ZNode `zk_path` is expected to exist, having been
created during cluster bootstrap.
Args:
zk:
The client to use to communicate with ZooKeeper.
zk_path:
The path of the ZNode to use for node registration.
Returns:
A list of internal IP addresses of nodes that have
previously joined the CockroachDB cluster.
"""
# We call `sync()` before reading the value in order to
# read the latest data written to ZooKeeper.
# See https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#ch_zkGuarantees
log.info("Calling sync() on ZNode `{}`".format(zk_path))
zk.sync(zk_path)
log.info("Loading data from ZNode `{}`".format(zk_path))
data, _ = zk.get(zk_path)
if data:
log.info("Cluster was previously initialized.")
nodes = json.loads(data.decode('ascii'))['nodes'] # type: List[str]
log.info("Found registered nodes: {}".format(nodes))
return nodes
log.info("Found no registered nodes.")
return []
class ZKStore:
def __init__(self, hosts):
self.zk = KazooClient(hosts=hosts)
self.zk.add_listener(listener)
self.zk.start()
def isConnected(self):
if __state__ == 1:
return True
return False
def write(self, path, node, value):
self.zk.ensure_path(path)
if self.zk.exists(path+"/"+node):
self.zk.set(path+"/"+node, value)
else:
self.zk.create(path + "/" + node, value)
def read(self, path):
if self.zk.exists(path):
data, stat = self.zk.get(path)
return data
return None
def main():
if len(sys.argv) < 5:
print(help_msg, "\n")
print(sys.argv[0], 'zookeeper_server broker_uri_1 broker_uri_2 broker_uri_3')
print('Example:', sys.argv[0], 'localhost:2181 socket://localhost:10001/broker1 '
'socket://localhost:10002/broker2 socket://localhost:10003/broker3')
exit()
zk_server = sys.argv[1]
broker_uris = sys.argv[2:5]
shift_cmd = shift_cmd_template.format(*broker_uris)
print('Deploying', shift_cmd)
zk_client = KazooClient(zk_server, timeout=10 * 60)
print('Connecting to Zookeeper at', zk_server)
zk_client.start()
for uri in broker_uris:
broker_status[uri] = False
bid = get_id(uri)
# make sure broker is free
data, stats = zk_client.get(ZK_BROKER_OPS_STATUS_STR.format(bid))
op_status = OpStatus(data.decode('utf-8').upper())
if op_status not in [OpStatus.Null, OpStatus.Finished]:
raise RuntimeError('Cannot start {}, {} is in {} state'.format(shift_cmd, bid, op_status.name))
# update broker's ops status
zk_client.set(ZK_BROKER_OPS_STATUS_STR.format(bid), OpStatus.Null.value.encode('utf-8'))
# write the cmd to the broker's ops
zk_client.set(ZK_BROKER_OPS_STR.format(bid), shift_cmd.encode('utf-8'))
# set watches for this broker's op status
DataWatch(zk_client, ZK_BROKER_OPS_STATUS_STR.format(bid), func=get_broker_op_data_watcher(uri))
print('Waiting for brokers ...')
while not all_done():
time.sleep(1)
def readAMHostPort(self):
amHost = ""
amSecuredPort = ""
zk = None
try:
zk = KazooClient(hosts=self.zk_quorum, read_only=True)
zk.start()
data, stat = zk.get(self.zk_reg_path)
logger.debug("Registry Data: %s" % (data.decode("utf-8")))
sliderRegistry = json.loads(data)
amUrl = sliderRegistry["payload"]["internalView"]["endpoints"]["org.apache.slider.agents"]["address"]
amHost = amUrl.split("/")[2].split(":")[0]
amSecuredPort = amUrl.split(":")[2].split("/")[0]
# the port needs to be utf-8 encoded
amSecuredPort = amSecuredPort.encode('utf8', 'ignore')
except Exception:
# log and let empty strings be returned
logger.error("Could not connect to zk registry at %s in quorum %s" %
(self.zk_reg_path, self.zk_quorum))
pass
finally:
if not zk == None:
zk.stop()
zk.close()
logger.info("AM Host = %s, AM Secured Port = %s" % (amHost, amSecuredPort))
return amHost, amSecuredPort
def get_children_data(ensemble, namespace, read_only=True):
hdfs = cluster.get_hdfs()
if hdfs is None:
raise PopupException(_('No [hdfs] configured in hue.ini.'))
if hdfs.security_enabled:
sasl_server_principal = PRINCIPAL_NAME.get()
else:
sasl_server_principal = None
zk = KazooClient(hosts=ensemble, read_only=read_only, sasl_server_principal=sasl_server_principal)
zk.start()
children_data = []
children = zk.get_children(namespace)
for node in children:
data, stat = zk.get("%s/%s" % (namespace, node))
children_data.append(data)
zk.stop()
return children_data
def run(self):
zk = KazooClient(hosts='%s:%d' % (self.options.host, self.options.port),
read_only=True, timeout=3)
try:
zk.start()
options = vars(self.options)
options.update({
'system.hostname': socket.gethostname()
})
if self.options.regex:
content, stats = zk.get(self.options.file)
options['stats'] = stats
m = re.search(self.options.regex, content, re.MULTILINE | re.DOTALL)
if m:
options.update(m.groupdict())
self.ok(self.options.message.format(**options))
else:
self.critical(self.options.message.format(**options))
elif zk.exists(self.options.file):
self.ok(self.options.message.format(**options))
else:
self.critical(self.options.message.format(**options))
except Exception as ex:
self.critical(ex)
finally:
zk.stop()
def get_alive_master_ip():
zk_conn_str = get_os_env('ZOOKEEPER_CONN_STR')
master_stack_name = get_os_env('MASTER_STACK_NAME')
master_ip = ""
global region
if zk_conn_str != "":
from kazoo.client import KazooClient
zk = KazooClient(hosts=zk_conn_str)
zk.start()
try:
master_ip = zk.get("/spark/leader_election/current_master")[0].decode('utf-8')
zk.stop()
except:
master_ip = ""
zk.stop()
return master_ip
elif master_stack_name != "" and region is not None:
try:
elb = boto3.client('elb', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
master_ips = get_instance_ips(elb, ec2, master_stack_name)
if len(master_ips) != 1:
return "" # shouldn't happen without zookeeper
elif len(master_ips) == 1:
return master_ips[0]
else:
return ""
except:
return ""
else:
return ""
class GetInfo:
def __init__(self):
self.all_info = {}
#self.SERVER_IP_AND_PORT = "127.0.0.1:2181"
self.SERVER_IP_AND_PORT = "172.18.229.251:2181"
self.zk = None
def start_zk(self):
self.zk = KazooClient(hosts=self.SERVER_IP_AND_PORT)
self.zk.start();
def getInfo(self):
children = self.zk.get_children("/monitorData")
node_nums = len(children)
for i in range(node_nums):
data, stat = self.zk.get("/monitorData/" + str(children[i]))
#data2, stat2 = self.zk.get("/monitorDataJustOneTime/" + str(children[i]))
#print json.loads(data2.decode("utf-8"))
#print data2
#data3, stat3 = self.zk.get("/monitorDataProcessInfo/" + str(children[i]))
#print data3
#print str(children[i])
#print json.loads(data.decode("utf-8"))
self.all_info[children[i]] = json.loads(data.decode("utf-8"))
for key in self.all_info[children[i]].keys():
print key
print self.all_info[children[i]][key]
#print self.all_info
return self.all_info
class ZookeeperSession(object):
def __init__(self, locations, name_prefix, root_prefix='/frontera'):
self._zk = KazooClient(hosts=locations)
self._zk.add_listener(self.zookeeper_listener)
self._zk.start()
self.root_prefix = root_prefix
self.znode_path = self._zk.create("%s/%s" % (self.root_prefix, name_prefix),
ephemeral=True,
sequence=True,
makepath=True)
def zookeeper_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
pass
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
pass
else:
# Handle being connected/reconnected to Zookeeper
pass
def set(self, value):
self._zk.set(self.znode_path, value)
def get_workers(self, prefix='', exclude_prefix=''):
for znode_name in self._zk.get_children(self.root_prefix):
if prefix and not znode_name.startswith(prefix):
continue
if exclude_prefix and znode_name.startswith(exclude_prefix):
continue
location, _ = self._zk.get(self.root_prefix+"/"+znode_name)
yield location
class Store(object):
def __init__(self,**kwargs):
self.config = kwargs
self.client = None
def get_client(self):
return self.client
def open(self):
self.client = KazooClient(**self.config)
self.client.add_listener
self.client.start()
def close(self):
self.client.stop()
def read(self,path):
return self.client.get(path)
def write(self,path,value):
base_path = os.path.dirname(path)
self.client.ensure_path(base_path)
self.client.create(path,value)
def overwrite(self,path,value):
self.client.set(path,value)
def exists(self,path):
return self.client.exists(path)
def get_scn_from_zookeeper():
try:
print("Starting the process")
zk = KazooClient(hosts='10.0.57.146:2181,10.0.57.145:2181,10.0.77.195:2181')
print("Connection established")
zk.start()
sum_of_scn_num = 0
if zk.exists("/fk_kafka_cluster1"):
children_list = zk.get_children("/fk_kafka_cluster1/PROPERTYSTORE/")
sorted_list = sorted(children_list)
partion_num_scn_num_dict = {}
for children in sorted_list:
recv_data,stat = zk.get("/fk_kafka_cluster1/PROPERTYSTORE/"+str(children))
data_dict = ast.literal_eval(recv_data)
partition_num = data_dict["id"]
fields = ast.literal_eval(data_dict['simpleFields']['c'])
scn_num = fields["windowScn"]
sum_of_scn_num += scn_num
partion_num_scn_num_dict[partition_num] = scn_num
print "Data fetching from Zookeeper complete"
avg_scn_num = (sum_of_scn_num/len(children_list))
sorted_dict = sorted(partion_num_scn_num_dict.items(),key=operator.itemgetter(1))
return avg_scn_num,sorted_dict
else:
print("Node does not exist")
except Exception as e:
print("Exception occured!",str(e))
def create_from_zookeeper(cls, zkconnect):
log.info("Connecting to zookeeper {0}".format(zkconnect))
try:
zk = KazooClient(zkconnect)
zk.start()
except Exception as e:
raise ZookeeperException("Cannot connect to Zookeeper: {0}".format(e))
# Get broker list
cluster = cls()
add_brokers_from_zk(cluster, zk)
# Get current partition state
log.info("Getting partition list from Zookeeper")
for topic in zk.get_children("/brokers/topics"):
zdata, zstat = zk.get("/brokers/topics/{0}".format(topic))
add_topic_with_replicas(cluster, topic, json.loads(zdata))
if cluster.num_topics() == 0:
raise ZookeeperException("The cluster specified does not have any topics")
log.info("Closing connection to zookeeper")
zk.stop()
zk.close()
return cluster
def cleanup(args):
now = dt.utcnow()
server = '{server}:{port}'.format(server=args.server, port=args.port)
logging.info('Connecting to {}'.format(server))
zk = KazooClient(hosts=server)
zk.start()
for path in args.zk_paths:
zk_path = '{}/{}'.format(args.zk_root_path, path)
nodes = zk.get_children(zk_path)
logging.info("Found {} nodes under {}".format(len(nodes), zk_path))
deleted = 0
for node in nodes:
node_path = '{}/{}'.format(zk_path, node)
data, stat = zk.get(node_path)
last_modified = dt.fromtimestamp(stat.mtime/1000.0)
if ((now - last_modified).days > args.age) or (args.inclusive and (now - last_modified).days >= args.age):
if not args.dry_run:
# Kazoo does not support recursive async deletes
if stat.children_count == 0:
res = zk.delete_async(node_path)
else:
zk.delete(node_path, recursive=True)
deleted += 1
logging.info("Deleted {} nodes".format(deleted))
zk.stop()
def check_broker_id_in_zk(broker_id, process):
import requests
from time import sleep
from kazoo.client import KazooClient
zk_conn_str = os.getenv("ZOOKEEPER_CONN_STRING")
while True:
if os.getenv("WAIT_FOR_KAFKA") != "no":
ip = requests.get("http://169.254.169.254/latest/dynamic/instance-identity/document").json()["privateIp"]
wait_for_kafka_startup.run(ip)
os.environ["WAIT_FOR_KAFKA"] = "no"
new_zk_conn_str = generate_zk_conn_str.run(os.getenv("ZOOKEEPER_STACK_NAME"), region)
if zk_conn_str != new_zk_conn_str:
logging.warning("ZooKeeper connection string changed!")
zk_conn_str = new_zk_conn_str
os.environ["ZOOKEEPER_CONN_STRING"] = zk_conn_str
create_broker_properties(zk_conn_str)
from random import randint
wait_to_restart = randint(1, 20)
logging.info("Waiting " + str(wait_to_restart) + " seconds to restart kafka broker ...")
sleep(wait_to_restart)
process.kill()
logging.info("Restarting kafka broker with new ZooKeeper connection string ...")
process = subprocess.Popen(
[kafka_dir + "/bin/kafka-server-start.sh", kafka_dir + "/config/server.properties"]
)
os.environ["WAIT_FOR_KAFKA"] = "yes"
continue
zk = KazooClient(hosts=zk_conn_str)
zk.start()
try:
zk.get("/brokers/ids/" + broker_id)
logging.info("I'm still in ZK registered, all good!")
sleep(60)
zk.stop()
except:
logging.warning("I'm not in ZK registered, killing kafka broker process!")
zk.stop()
process.kill()
logging.info("Restarting kafka broker ...")
process = subprocess.Popen(
[kafka_dir + "/bin/kafka-server-start.sh", kafka_dir + "/config/server.properties"]
)
os.environ["WAIT_FOR_KAFKA"] = "yes"
class Zookeeper(KeyManager): def __init__(self,hosts): self._hosts = hosts self.zk = KazooClient(hosts=hosts) def get(self,key): return self.zk.get(key)[0] def set(self,key,data): self.zk.set(key, data)
def get_brokers():
zk = KazooClient(hosts=os.environ['ZOOKEEPER'], read_only=True)
zk.start()
broker_list = ""
children = zk.get_children( '/brokers/ids' )
for i in children:
data, stat = zk.get( '/brokers/ids/'+i )
data = json.loads( data )
if broker_list != "":
broker_list += ","
broker_list += data['host'].encode('utf8') + ":" + str(data['port'])
data, stat = zk.get( '/brokers/ids/0' )
zk.stop()
data = json.loads( data )
return broker_list
def get_broker(): zk = KazooClient(hosts='10.10.0.71:2181', read_only=True) zk.start() broker_list = "" children = zk.get_children( '/brokers/ids' ) for i in children: data, stat = zk.get( '/brokers/ids/'+i ) data = json.loads( data ) if broker_list != "": broker_list += "," broker_list += data['host'] + ":" + str(data['port']) data, stat = zk.get( '/brokers/ids/0' ) zk.stop() data = json.loads( data ) return broker_list
def zookeeper_resolve_leader(addresses, path):
"""
Resolve the leader using a znode path. ZooKeeper imposes a total
order on the elements of the queue, guaranteeing that the
oldest element of the queue is the first one. We can
thus return the first address we get from ZooKeeper.
"""
hosts = ",".join(addresses)
try:
zk = KazooClient(hosts=hosts)
zk.start()
except Exception as exception:
raise CLIException("Unable to initialize Zookeeper Client: {error}"
.format(error=exception))
try:
children = zk.get_children(path)
except Exception as exception:
raise CLIException("Unable to get children of {zk_path}: {error}"
.format(zk_path=path, error=exception))
masters = sorted(
# 'json.info' is the prefix for master nodes.
child for child in children if child.startswith("json.info")
)
address = ""
for master in masters:
try:
node_path = "{path}/{node}".format(path=path, node=master)
json_data, _ = zk.get(node_path)
except Exception as exception:
raise CLIException("Unable to get the value of '{node}': {error}"
.format(node=node_path, error=exception))
try:
data = json.loads(json_data)
except Exception as exception:
raise CLIException("Could not load JSON from '{data}': {error}"
.format(data=data, error=str(exception)))
if ("address" in data and "ip" in data["address"] and
"port" in data["address"]):
address = "{ip}:{port}".format(ip=data["address"]["ip"],
port=data["address"]["port"])
break
try:
zk.stop()
except Exception as exception:
raise CLIException("Unable to stop Zookeeper Client: {error}"
.format(error=exception))
if not address:
raise CLIException("Unable to resolve the leading"
" master using ZooKeeper")
return address
class PyZooConn(object):
# connection manager class
def __init__(self):
self.zk = KazooClient(hosts='xxxx.xxxx.xxxx.xxxx:2181')
self.zk.add_listener(my_listener)
self.zk.start()
def get_data(self, param, watch_func=None):
""" 获取节点的信息 """
result = self.zk.get(param, watch=watch_func)
return result
def get_children_data(self, param):
lists = []
if self.zk.exists(param):
children_list = self.get_children_list(param)
for each in children_list:
lists.append(self.get_data(param + '/' + each))
return lists
def get_children_list(self, param):
""" 获取指定节点的所有子节点 """
if self.zk.exists(param):
result = self.zk.get_children(param)
return result
def set_data(self, param, data):
try:
self.zk.set(param, data)
except exceptions.NoNodeError as e:
print '不存在此节点'
def create_node(self, param, data, overlook=True):
""" 创建节点，overlook为是否忽略所创建节点所经过节点是否存在，如不存在会默认沿途创建空的节点, 最后一个节点设置为指定值 """
if self.zk.exists(param):
print '节点已经存在'
else:
if overlook:
try:
self.zk.create(param, data)
except exceptions.NoNodeError as e:
print '节点所在路径不存在'
else:
self.zk.ensure_path(param)
self.set_data(param, data)
def delete_node(self, param):
""" 删除指定节点，并删除此节点的子节点 """
if self.zk.exists(param):
self.zk.delete(param, recursive=False)
def close(self):
self.zk.stop()
def _get_server_properties():
global _api_cache
if _api_cache is None:
_api_cache_lock.acquire()
try:
if _api_cache is None:
if get_sentry_server_ha_has_security():
try:
from zookeeper.conf import CLUSTERS
sasl_server_principal = CLUSTERS.get(
)['default'].PRINCIPAL_NAME.get()
except Exception, e:
LOG.error(
"Could not get principal name from ZooKeeper app config: %s. Using 'zookeeper' as principal name."
% e)
sasl_server_principal = 'zookeeper'
else:
sasl_server_principal = None
zk = KazooClient(hosts=get_sentry_server_ha_zookeeper_quorum(),
read_only=True,
sasl_server_principal=sasl_server_principal)
zk.start()
servers = []
namespace = get_sentry_server_ha_zookeeper_namespace()
children = zk.get_children(
"/%s/sentry-service/sentry-service/" % namespace)
for node in children:
data, stat = zk.get(
"/%s/sentry-service/sentry-service/%s" %
(namespace, node))
server = json.loads(data.decode("utf-8"))
servers.append({
'hostname':
server['address'],
'port':
server['sslPort']
if server['sslPort'] else server['port']
})
zk.stop()
_api_cache = servers
finally:
_api_cache_lock.release()
return _api_cache
def _connect_zookeeper(self, zk_url):
"""
Find the active Swarm master using Zookeeper
"""
aux = zk_url[len('zk://'):]
zk_hosts = aux.split('/')[0]
path = aux[aux.find('/'):] + '/docker/swarm/leader'
zk = KazooClient(hosts=zk_hosts)
zk.start()
master, stat = zk.get(path)
zk.stop()
self._connect_tcp('tcp://' + master.decode('ascii'))
def get_zk_config(zk_path="/mipush/cfg",
hosts='00011:vrs.poodah.kz.gnigatsqwjt'[::-1],
mode='yaml'):
zk = KazooClient(hosts)
zk.start()
data, stat = zk.get(zk_path)
if mode == 'yaml':
return yaml.safe_load(data)
else:
return data
def get_boost_values(zk_boost_path: str, zk: KazooClient) -> BoostValues:
# Default values, non-boost.
end_time: float = 0
boost_factor: float = 1.0
expected_load: float = 0
try:
end_time = float(
zk.get(zk_boost_path + "/end_time")[0].decode("utf-8"))
boost_factor = float(
zk.get(zk_boost_path + "/factor")[0].decode("utf-8"))
expected_load = float(
zk.get(zk_boost_path + "/expected_load")[0].decode("utf-8"))
except NoNodeError:
# If we can't read boost values from zookeeper
return BoostValues(end_time=0, boost_factor=1.0, expected_load=0)
return BoostValues(end_time=end_time,
boost_factor=boost_factor,
expected_load=expected_load)
class DynamicIP(object):
def __init__(self, hosts, watch_node):
self.proxies = dict()
self.watch_node = watch_node
self.zk_client = KazooClient(hosts=hosts)
self.zk_client.start()
self.logger = logging.getLogger(__name__)
def watcher(self, proxy_ids):
current_proxy_ids = set(self.proxies.keys())
newcome_proxy_ids = set(proxy_ids)
# 删除失效的代理
expried_proxy_ids = current_proxy_ids - newcome_proxy_ids
for expried_proxy_id in expried_proxy_ids:
expried_proxy = self.proxies.pop(expried_proxy_id)
# self.logger.info('expried proxy: %s' % expried_proxy)
# 新增代理
new_proxy_ids = newcome_proxy_ids - current_proxy_ids
for new_proxy_id in new_proxy_ids:
try:
ip, stat = self.zk_client.get(self.watch_node + '/' +
new_proxy_id)
data, stat = self.zk_client.get('/adsl_proxy/ip' + '/' + ip)
data = json.loads(data)
self.proxies[new_proxy_id] = data['host']
# self.logger.info('new proxy: %s' % data['host'])
except Exception as e:
pass
def get_proxy(self):
if len(self.proxies):
max_proxy_id = max(self.proxies.keys())
return self.proxies[max_proxy_id]
else:
return None
def run(self):
self.zk_client.ChildrenWatch(self.watch_node, self.watcher)
class ZookeeperClient:
def __init__(self, configuration):
self.host = configuration['host']
self.port = configuration['port']
self.client = KazooClient(hosts='{}:{}'.format(self.host, self.port))
self.client.start()
def create_node(self, path, data):
self.client.create(path, data)
def get_node(self, path):
data, stat = self.client.get(path)
return data
def get_children(self, path):
return self.client.get_children(path)
def get_children_count(self, path):
try:
return len(self.get_children(path))
except NoNodeError:
return -1
@contextmanager
def transaction(self):
transaction = self.client.transaction()
try:
yield transaction
finally:
results = transaction.commit()
if "failure" in results:
logging.error("Zookeeper transaction failed!")
def delete_node(self, path):
self.client.delete(path)
def safe_delete_node(self, path):
try:
self.client.delete(path)
return True
except BadVersionError as e:
logging.warn("Bad Version Error")
return False
except NoNodeError as e:
logging.warn("No Node Error")
return False
def close(self):
self.client.stop()
def main():
try:
nodePath = "/zktest"
host = "127.0.0.1"
port = "2181"
timeout = 100
zkc = KazooClient(hosts=host + ':' + port, timeout=timeout)
zkc.start()
print(zkc.exists(nodePath)) # 输出None
print(not zkc.exists(nodePath)) # 输出True
# 判断节点是否存在
if not zkc.exists(nodePath):
zkc.create(nodePath, b"a test value")
if zkc.exists(nodePath + "/test111"):
print(nodePath + "/test111", "存在")
else:
# 建立节点，成功后返回新节点路径
childrenPath = zkc.create(nodePath + "/test111", b"test111")
print("创建节点：", childrenPath, "成功。")
# 创建临时节点，连接断开则节点自动删除
zkc.create(nodePath + "/test999", b"test999", ephemeral=True)
# 获取节点数据和节点数据，返回2个值，一个是节点数据，一个是节点stat，这是个ZnodeStat对象，它其实是节点属性，一共有12个属性
dataAndStat = zkc.get(nodePath)
data = dataAndStat[0]
print("数据为：", data)
stat = dataAndStat[1]
print("数据版本号为：", stat.version)
print("数据长度为：", stat.data_length)
print("子节点数量：", stat.numChildren)
# 更新节点数据,数据最大为1MB超过则报错，成功后返回 ZnodeStat 对象
stat = zkc.set(nodePath, value=b"test222")
print("数据版本号为：", stat.version)
# 删除节点，后面的参数用于控制是否递归删除，默认是False,但是这样就会有一个问题，如果该节点下有子节点则本次删除失败，你需要先删除
# 它下面的所有子节点才行
if zkc.exists(nodePath + "/test111"):
result = zkc.delete(nodePath + "/test111", recursive=False)
if result:
print("删除节点成功。")
print(nodePath + " 的子节点为：", zkc.get_children(nodePath))
zkc.delete(nodePath + "/test999", recursive=False)
zkc.delete(nodePath, recursive=False)
# zkc.close()
zkc.stop()
except Exception as err:
print(err)
def reader(self):
# l.info("started thread-%s"%j)
conn_start = time.time() * 1000
zkr = KazooClient(hosts=self.zk_server_ip)
zkr.start()
conn_end = time.time() * 1000
conn_diff = conn_end - conn_start
l.info("Let's create arbitrary znodes for reading!")
for nodes in range(5):
zkr.ensure_path("/Hydra/Test/test-%s" % nodes, '')
l.info("Done creating test znodes under /Hydra/Test/")
l.info("About to start reading")
totalread_end = 0
read_time = []
child = zkr.get_children("/Hydra/Test/")
child_len = len(child)
while True:
# time.sleep(0.5)
if self.run_data['test_status'] == 'start':
for r in child:
time.sleep(0.1)
l.info(r)
read_time_start = time.time() * 1000
data, stat = zkr.get("/Hydra/Test/%s" % r)
read_time_end = (time.time() * 1000)
read_time_diff = read_time_end - read_time_start
# l.info(read_time_diff)
totalread_end = totalread_end + read_time_diff
# l.info(totalread_end)
dict_read = {'read': {}, 'total': {}, 'conn': {}}
dict_read['read'][read_time_end] = read_time_diff
# l.info(dict_read)
read_time.append(dict_read['read'])
elif self.run_data['test_status'] == 'stop':
l.info("Stopping reader connection")
zkr.stop()
dict_read['conn'][conn_end] = conn_diff
dict_read['total'][time.time() * 1000] = totalread_end
self.run_data['stats']['read_times'] = read_time
# self.run_data['stats']['total_reads'] = dict_read['total']
# self.run_data['stats']['connection'] = dict_read['conn']
l.info(self.run_data['stats'])
# self.run_data['test_action'] = 'waiting'
self.run_data['test_status'] = 'stopped'
break
l.info("Reader test ended")
# if self.run_data['test_status'] == 'stop':
# zkr.stop()
# l.info("Reader test ended")
# break
return 'ok', None
class Zookeeper_Server:
def __init__(self,host,port):
self.zk = KazooClient(hosts=host+':'+port)
self.zk.start()
#读取zookeeper节点
def get_zk_node(self,nodePath):
#获取节点下的所有子节点
zkNode = self.zk.get_children(nodePath,watch=self.watches())
#获取节点的值
# zkNodeValue = self.zk.get(nodePath,watch=self.watches())
zkNodeValue = self.zk.get(nodePath)
# self.zk.stop()
return zkNode,zkNodeValue
#设置zookeeper节点
def set_zk_node(self,nodePath,setNodeValue):
#设置节点的值
self.zk.set(nodePath,setNodeValue)
# self.zk.stop()
#删除zookeeper节点
def del_zk_node(self,nodePath):
#recursive为True则删除此节点和所有子节点
#recursive为False则当节点有子节点，则抛出NotEmptyError异常
self.zk.delete(nodePath,recursive=True)
# self.zk.stop()
#创建zookeeper节点
def create_zk_node(self,nodePath,nodeValue):
#sequence:若为 True 则在你创建节点名后面增加10位数字
#makepath： 若为 False 父节点不存在时抛 NoNodeError。若为 True 父节点不存在则创建父节点。默认 False
self.zk.create(nodePath,bytes(nodeValue),sequence=False,makepath=True)
# self.zk.stop()
#判断节点是否存在
def jude_node_exists(self,nodePath):
if self.zk.exists(nodePath):
# self.zk.stop()
return True
else:
# self.zk.stop()
return False
#监听事件(只能设置在节点的读取上)
def watches(self):
pass
def zookeeper_swarm(zk_server_list: str, path='/docker'):
"""
Given a Zookeeper server list, find the currently active Swarm master.
:param zk_server_list: Zookeeper server list
:param path: Swarm path in Zookeeper
:return: Swarm master connection string
"""
path += '/docker/swarm/leader'
zk = KazooClient(hosts=zk_server_list)
zk.start()
master, stat = zk.get(path)
zk.stop()
return master
def createCon():
zk = KazooClient(hosts=DDBConfig.HOST)
zk.start()
temp = zk.get(DDBConfig.ZK_KEY)
aws_id = temp[0].split(':::')[0]
aws_key = temp[0].split(':::')[1]
region = DDBConfig.REGION
session = Session(aws_access_key_id=aws_id,
aws_secret_access_key=aws_key,
region_name=region)
DDB.dynamodb = session.resource('dynamodb')
def main(argv):
hdfs_namenode = os.environ['HDFS_NAMENODE']
model_on_hdfs = os.environ['MODEL_ON_HDFS']
ip, port = hdfs_namenode.rsplit(':', 1)
client = Client(ip, int(port), use_trash=False)
dst_dir = os.path.join('/')
for x in client.copyToLocal([model_on_hdfs], dst_dir):
print x
zk_master = os.environ['ZK_MASTER']
logger.info('job_name: {0}, task_index: {1}'.format(
os.environ['JOB_NAME'], os.environ['TASK_INDEX']))
logger.info('command: {0}'.format(os.environ['CMD']))
zk = KazooClient(hosts=zk_master)
zk.start()
logger.info('job uid: {0}'.format(os.environ['UID']))
job_zk_dir = '/' + os.environ['UID']
members = zk.get_children(job_zk_dir + '/member/')
members.sort()
cluster_def = {}
for member in members:
host = zk.get(job_zk_dir + '/member/' + member)[0]
if host != '':
logger.info('{0} running on {1}'.format(member, host))
job_type = member.split('_')[2]
if job_type == 'ps':
cluster_def.setdefault('ps', []).append(host)
elif job_type == 'worker':
cluster_def.setdefault('worker', []).append(host)
else:
logger.error('unkown type: {0}'.format(job_type))
ps = ','.join(cluster_def['ps'])
worker = ','.join(cluster_def['worker'])
my_env = os.environ.copy()
logger.info(my_env)
my_env['PS'] = ps
my_env['WORKER'] = worker
cmd = [os.environ['CMD']]
child = subprocess.Popen(cmd, shell=True, env=my_env)
child.wait()
zk.stop()
class Subscriber:
# instantiate variables and connect to broker
def __init__(self, ip_add, timeout=-1):
self.kill = True
self.count = 0
self.full_add = "tcp://" + str(ip_add)
self.context = zmq.Context()
self.sock_sub = self.context.socket(zmq.SUB)
self.sock_sub.RCVTIMEO = timeout
# PRESIDENT ZNODE ADDRESS
self.home = "/president/pres"
self.zk_driver = KazooClient(hosts='127.0.0.1:2181')
self.zk_driver.start()
data, stat = self.zk_driver.get(self.home)
ports = data.decode('ASCII').split(":")
self.full_add = "tcp://" + str(ip_add) + ":" + ports[1]
self.sock_sub.connect(self.full_add)
def register_sub(self, topics):
topic_list = topics.split(",")
topic_list = [topic.strip() for topic in topics.split(',')]
for topic in topic_list:
#subscribe to topic
self.sock_sub.setsockopt_string(zmq.SUBSCRIBE, topic)
def notify(self, stop=None):
if stop:
while (not stop.is_set()):
# only used for measurements.py
message = self.sock_sub.recv_string()
topic, info = message.split("||")
print("Topic: %s. Message: %s" % (topic, info))
# print("Time received: %.20f" % time.time()) # uncomment for measurements.py purposes
self.count = self.count + 1
else:
while True:
@self.zk_driver.DataWatch(self.home)
def watch_node(data, stat, event):
if event is not None and event.type == "CREATED" and self.kill:
self.kill = False
print("Updated Broker!")
message = self.sock_sub.recv_string()
topic, info = message.split("||")
print("Topic: %s. Message: %s" % (topic, info))
self.count = self.count + 1
def clusterstate(zookeepers, all_hosts, node='clusterstate.json'):
"""
Print clusterstatus.json contents
"""
zk_hosts = parse_zk_hosts(zookeepers, all_hosts=all_hosts)
print('')
# we'll keep track of differences for this node between zookeepers.
# because zookeeper keeps all nodes in-sync, there shouldn't be differences between the
# nodes... but there might be if you are having replication problems.
first_state = None
for host in zk_hosts:
# connect to zookeeper
zk = KazooClient(hosts=host, read_only=True)
try:
zk.start()
except KazooTimeoutError as e:
print('ZK Timeout host: [%s], %s' % (host, e))
continue
# If the node doesn't exist... just let the user know.
if not zk.exists(node):
node_str = style_text(node, BLUE_STYLE, restore=ERROR_STYLE)
zk_str = style_text(host, BLUE_STYLE, restore=ERROR_STYLE)
print(
style_text('No node [%s] on %s' % (node_str, zk_str),
ERROR_STYLE))
continue
print(style_header('Response From: %s [%s]' % (host, node)))
state = bytes.decode(zk.get(node)[0])
if not first_state:
first_state = state
lines_1 = first_state.split('\n')
lines_2 = state.split('\n')
# Print the content of the file, highlighting lines that do not match between hosts.
for idx, line in enumerate(lines_2):
if len(lines_1) - 1 < idx or line != lines_1[idx]:
style = DIFF_STYLE
else:
style = INFO_STYLE
print(style_text(line, style, lpad=4))
zk.stop()
class ZookeeperConn(object):
def __init__(self):
self.ZK_ADDRESS = "10.9.1.176:2181"
self.ZK = KazooClient(self.ZK_ADDRESS)
self.ZK.start()
def get_data(self):
print(self.ZK.get("node"))
def create_node(self, node, data):
self.ZK.create(node, str.encode(data))
def set_data(self, node, value):
self.ZK.set(node, str.encode(data))
def get(self,server='localhost',port=2181,path="/"):
conn=ZookeeperAdmin()
connstr=conn.stat(server,port)
if(connstr=="imok"):
serverport='%s:%s' %(server,port)
zk = KazooClient(serverport,10.0)
zk.start()
data,stat=zk.get(path)
zk.stop()
result='{"data":"%s","czxid":"%s","mzxid":"%s","ctime":"%s","mtime":"%s","version":"%s","cversion":"%s","aversion":"%s","ephemeralOwner":"%s","dataLength":"%s","numChildren":"%s","pzxid":"%s"}' %(data,stat.czxid,stat.mzxid,stat.ctime,stat.mtime,stat.version,stat.cversion,stat.aversion,stat.ephemeralOwner,stat.dataLength,stat.numChildren,stat.pzxid)
return result
else:
result="offline"
return result
def get_redis_info(ip, path):
zk = KazooClient(hosts=ip)
zk.start()
if zk.exists(path):
data = zk.get(path)
else:
raise IOError('please check ZooKeeper path')
json_data = json.loads(data[0])
ip = json_data['Spider_Queue.master.ip'].split(',')[0]
port = json_data['Spider_Queue.master.port'].split(',')[0]
password = json_data["Spider_Queue.master.password"]
zk.stop()
zk.close()
return dict(ip=ip, port=port, password=password)
def modify_mappings():
zk = KazooClient("localhost:2181")
zk.start()
children = zk.get_children("/big_data_assign")
for each in children:
path = "/big_data_assign/" + each
print(path)
c = zk.get(path)
c = c[0]
c = c.decode()
c = eval(c, {'OrderedDict': OrderedDict})
#print("here4",c,type(c))
#for key, value in c.items():
key = list(c.items())[0]
key = key[0]
print(key)
d = {}
d[key] = select_all_keys(each)
d = OrderedDict(d)
modify_server(d, each)
#print(d)
for each in children:
path = "/big_data_assign/" + each
print(path)
c = zk.get(path)
c = c[0]
c = c.decode()
c = eval(c, {'OrderedDict': OrderedDict})
#print("here4",c,type(c))
#for key, value in c.items():
key = list(c.items())[0]
key = key[0]
print(key)
d = {}
d[key] = select_all_keys(each)
d = OrderedDict(d)
register_server_in_others(d)
def main():
"""连接zk"""
zk = KazooClient(hosts=ZK_SERVER, read_only=True)
zk.start()
current_epoch = zk.get(KAFKA_PATH + "controller_epoch")[0]
"""若不存在本地文件，则先创建"""
filename = "controller_epoch"
if not os.path.exists(filename):
f = open(filename, 'w')
try:
f.write(current_epoch)
finally:
f.close()
"""打开本地文件读取相关信息"""
f = open(filename, 'r')
try:
last_epoch = f.read().strip()
finally:
f.close()
"""查看controller epoch是否改变，如果改变则发送报警"""
if last_epoch == current_epoch:
logging.info("last epoch:%s, current epoch: %s", last_epoch,
current_epoch)
else:
controller_info = zk.get(KAFKA_PATH + "controller")[0]
msg = "epoch changed, last epoch:%s, current epoch:%s, current controller:%s" \
% (last_epoch, current_epoch, controller_info)
logging.warn(msg)
# watchalert.sendAlertToGroups("kafka", "controller", msg, msg, "DIP_ALL", True, True, False)
watchalert.sendAlertToUsers("kafka", "controller", msg, msg,
"jianhong1", True, True, False)
f = open(filename, 'w')
try:
f.write(current_epoch)
finally:
f.close()
def zookeeper_command(hosts, command, path):
try:
zk = KazooClient(hosts=hosts)
zk.start()
if command == 'get':
data, stat = zk.get(path)
return data.decode("utf-8")
elif command == 'delete':
zk.delete(path, recursive=True)
return 'Successfully deleted ' + path
else:
return False
zk.stop()
except:
return False
def get_topology(servicename, config=config):
zk = KazooClient(**config)
zk.start()
try:
topology, stat = zk.get(
"/sdn/services/{name}".format(name=servicename))
except Exception as e:
#TODO
pass
zk.stop()
topology = json.loads(topology)
return topology
def initialize_parameters():
# connect to Zookeeper
zk = KazooClient(
hosts="172.16.2.36:2181,172.16.2.27:2181,172.16.1.235:2181",
read_only=True)
zk.start()
data, stat = zk.get("/scooterCloudApp/rabbitmq_user")
print("Version: %s, rabbitmq_user: %s" %
(stat.version, data.decode("utf-8")))
Config.rabbitmq_user = data.decode("utf-8")
data, stat = zk.get("/scooterCloudApp/rabbitmq_pwd")
print("Version: %s, rabbitmq_pwd: %s" %
(stat.version, data.decode("utf-8")))
Config.rabbitmq_pwd = data.decode("utf-8")
data, stat = zk.get("/scooterCloudApp/rabbitmq_address")
print("Version: %s, rabbitmq_address: %s" %
(stat.version, data.decode("utf-8")))
Config.rabbitmq_address = data.decode("utf-8")
data, stat = zk.get("/scooterCloudApp/rabbitmq_port")
print("Version: %s, rabbitmq_port: %s" %
(stat.version, data.decode("utf-8")))
Config.rabbitmq_port = int(data.decode("utf-8"))
data, stat = zk.get("/scooterCloudApp/mysql_address")
print("Version: %s, mysql_address: %s" %
(stat.version, data.decode("utf-8")))
Config.mysql_address = (data.decode("utf-8"))
data, stat = zk.get("/scooterCloudApp/mysql_user")
print("Version: %s, mysql_user: %s" %
(stat.version, data.decode("utf-8")))
Config.mysql_user = (data.decode("utf-8"))
data, stat = zk.get("/scooterCloudApp/mysql_pwd")
print("Version: %s, mysql_pwd: %s" %
(stat.version, data.decode("utf-8")))
Config.mysql_pwd = (data.decode("utf-8"))
data, stat = zk.get("/scooterCloudApp/mysql_db")
print("Version: %s, mysql_db: %s" %
(stat.version, data.decode("utf-8")))
Config.mysql_db = (data.decode("utf-8"))
def get_mysqldb_info(ip, path):
zk = KazooClient(hosts=ip)
zk.start()
if zk.exists(path):
data = zk.get(path)
else:
raise IOError('please check ZooKeeper path')
json_data = json.loads(data[0])
db_ip = json_data["SpiderDB.master.ip"]
db_port = json_data["SpiderDB.master.port"]
db_user = json_data['SpiderDB.master.username']
db_pwd = json_data['SpiderDB.master.password']
zk.stop()
zk.close()
return dict(host=db_ip, port=db_port, username=db_user, password=db_pwd)
def _get_topics(cluster):
zk = KazooClient(hosts=cluster['zk_host_ports'])
#zk.add_listener(my_listener)
zk.start()
topic_list = []
try:
topics = zk.get_children(cluster['topics_path'])
except NoNodeError:
return topic_list
else:
for topic in topics:
t = {'id': topic}
topic_path = cluster['topics_path'] + "/" + topic
data, stat = zk.get(topic_path)
d = json.loads(data)
t['topic_partitions_data'] = d['partitions']
partitions_path = topic_path + "/partitions"
partitions = zk.get_children(partitions_path)
t['partitions'] = partitions
tpp = {}
p = []
for partition in partitions:
tps = {}
p.append(partition.encode('ascii'))
partition_path = partitions_path + "/" + partition + "/state"
data, stat = zk.get(partition_path)
d = json.loads(data)
tps['isr'] = d['isr']
tps['leader'] = d['leader']
tpp[partition.encode('ascii')] = tps
t['partitions'] = p
t['topic_partitions_states'] = tpp
topic_list.append(t)
zk.stop()
return topic_list
def zkHandle(groupname=None):
__zk = KazooClient(hosts=zk_hosts)
__zk.start()
if groupname:
path = ha_path + '/' + groupname
if __zk.exists(path=path):
conf, stat = __zk.get(path=path)
result = conf
else:
result = False
else:
if __zk.exists(path=ha_path):
group_list = __zk.get_children(path=ha_path)
result = group_list
return result
def get_kafka_brokers(zk: KazooClient):
log.info("Getting broker_host:port")
brokers: list = []
broker_ids: list = zk.get_children(constants.BROKER_PATH_IN_ZK)
if broker_ids is not None:
for broker_id in broker_ids:
if broker_id is not None:
broker_info_str = zk.get("{}{}".format(
constants.BROKER_PATH_IN_ZK,
broker_id))[0].decode("utf-8").strip()
broker_info = json.loads(broker_info_str)
brokers.append("{}:{}".format(broker_info["host"],
broker_info["port"]))
return brokers
def getHealthCheckHost(zookeeperInfo, getHostName):
zk = KazooClient(hosts=zookeeperInfo)
zk.start()
for child in zk.get_children('/brokers/ids'):
brokerInfo = zk.get('/brokers/ids/%s' % child)
if getHostName in brokerInfo[0]:
getBrokerID = child
break
zk.stop()
try:
getBrokerID = int(getBrokerID)
return getBrokerID
except:
print 'Not found broker in zookeeper'
sys.exit()
class ZooKepperConnection:
def __init__(self, host, root="/"):
self.zk = KazooClient(hosts='127.0.0.1:2181')
self.zk.start()
self.root = root
def disconnect(self):
self.zk.stop()
def children(self, path):
p = self.root + path
return self.zk.get_children(p)
def raw_data(self, path):
return self.zk.get(self.root + path)
def getTopicLists(zookeeperInfo, brokerID):
zk = KazooClient(hosts=zookeeperInfo)
zk.start()
topicLists = []
for child in zk.get_children('/brokers/topics'):
if '__' not in child:
partitionInfo = zk.get('/brokers/topics/%s' % child)
jsonData = json.loads(partitionInfo[0])
for partitionSeekFor in range(len(jsonData['partitions'])):
getISR = jsonData['partitions']['%s'% partitionSeekFor]
if brokerID in getISR:
topicLists.append(child)
break
return topicLists
zk.stop()
