class ZKStore:
def __init__(self, hosts):
self.zk = KazooClient(hosts=hosts)
self.zk.add_listener(listener)
self.zk.start()
def isConnected(self):
if __state__ == 1:
return True
return False
def write(self, path, node, value):
self.zk.ensure_path(path)
if self.zk.exists(path+"/"+node):
self.zk.set(path+"/"+node, value)
else:
self.zk.create(path + "/" + node, value)
def read(self, path):
if self.zk.exists(path):
data, stat = self.zk.get(path)
return data
return None
class ShellTestCase(unittest.TestCase):
""" base class for all tests """
def setUp(self):
"""
make sure that the prefix dir is empty
"""
self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests")
self.zk_host = os.getenv("ZKSHELL_ZK_HOST", "localhost:2181")
self.username = os.getenv("ZKSHELL_USER", "user")
self.password = os.getenv("ZKSHELL_PASSWD", "user")
self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=")
self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "secret")
self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest")
self.client = KazooClient(self.zk_host, 5)
self.client.start()
self.client.add_auth(self.scheme, self.auth_id)
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.create(self.tests_path, str.encode(""))
self.output = StringIO()
self.shell = Shell([self.zk_host], 5, self.output, setup_readline=False, async=False)
# Create an empty test dir (needed for some tests)
self.temp_dir = tempfile.mkdtemp()
@property
def auth_id(self):
return "%s:%s" % (self.username, self.password)
@property
def auth_digest(self):
return "%s:%s" % (self.username, self.digested_password)
def tearDown(self):
self.output = None
self.shell = None
if os.path.isdir(self.temp_dir):
shutil.rmtree(self.temp_dir)
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.stop()
###
# Helpers.
##
def create_compressed(self, path, value):
"""
ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client
to create a znode with zlib compressed content.
"""
compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value)
self.client.create(path, compressed, makepath=True)
class dzk:
def __init__(self):
self.BasePath = "/my/"
self.zk = KazooClient(hosts='x.24.79.51:2181,x.24.79.53:2181',retry_max_delay=2000)
self.zk.start()
self.zk.add_listener(self.listener)
def listener(state):
if state == KazooState.LOST:
self.zk.start()
elif state == KazooState.SUSPENDED:
print "*******listener saw KazooState.LOST"
else:
print "*******listener saw KazooState.CONNECT"
def getIpHost(self):
self.myname = socket.getfqdn(socket.gethostname())
myip = socket.gethostbyname(self.myname)
return myip
def register(self):
ip = self.getIpHost()
if ip:
NODE = self.BasePath + ip
print "register:",NODE
else:
print "[ERROR:] %s does not exist " %(NODE)
sys.exit(2)
if not self.zk.exists(NODE):
self.zk.ensure_path(NODE)
def getData(self):
ip = self.getIpHost()
if ip:
NODE = self.BasePath + ip
else:
print "[ERROR:] %s does not exist " %(NODE)
if self.zk.exists(NODE):
data, stat = self.zk.get(NODE)
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
def monitor(self):
pass
def heartbeat(self):
pass
def role(self):
pass
def command(self):
pass
class ZKClient():
def __init__(self, hosts, auth_data=None):
if auth_data:
auth_data = [('digest', auth_data)]
self.zk = KazooClient(hosts=hosts, auth_data=auth_data)
self.zk.start()
def __del__(self):
self.zk.stop()
def exist(self, zoo_path):
ret = self.zk.exists(zoo_path)
if ret:
print True
else:
print False
return ret
def mv(self, zoo_path_src, zoo_path_dst):
if self.zk.exists(zoo_path_dst):
print '%s exists! Please delete it manually'
return False
if self.cp(zoo_path_src, zoo_path_dst):
self.rmr(zoo_path_src)
else:
self.rmr(zoo_path_dst)
def cp(self, zoo_path_src, zoo_path_dst):
if not self.zk.exists(zoo_path_src):
print '%s does not exist!' % zoo_path_src
return False
src_name = filter(lambda x:x, zoo_path_src.split('/'))[-1]
try:
children = self.zk.get_children(zoo_path_dst)
if children:
zoo_path_dst = zoo_path_dst + '/' + src_name
except NoNodeError, e:
pass
children = self.zk.get_children(zoo_path_src)
if not children:
buf = self.__get_zk_file(zoo_path_src)
src_name = filter(lambda x:x, zoo_path_src.split('/'))[-1]
try:
dst_is_dir = self.zk.get_children(zoo_path_dst)
if dst_is_dir:
dst_path = zoo_path_dst + '/' + src_name
else:
dst_path = zoo_path_dst
self.put_str(dst_path, buf)
except NoNodeError, e:
self.create_long(zoo_path_dst)
self.put_str(zoo_path_dst, buf)
def main():
'''
A manual configuration file pusher for the crawlers. This will update
Zookeeper with the contents of the file specified in the args.
'''
import argparse
from kazoo.client import KazooClient
parser = argparse.ArgumentParser(
description="Crawler config file pusher to Zookeeper")
parser.add_argument('-f', '--file', action='store', required=True,
help="The yaml file to use")
parser.add_argument('-i', '--id', action='store', default="all",
help="The crawler id to use in zookeeper")
parser.add_argument('-p', '--path', action='store',
default="/scrapy-cluster/crawler/",
help="The zookeeper path to use")
parser.add_argument('-w', '--wipe', action='store_const', const=True,
help="Remove the current config")
parser.add_argument('-z', '--zoo-keeper', action='store', required=True,
help="The Zookeeper connection <host>:<port>")
args = vars(parser.parse_args())
filename = args['file']
id = args['id']
wipe = args['wipe']
zoo = args['zoo_keeper']
path = args['path']
zk = KazooClient(hosts=zoo)
zk.start()
# ensure path exists
zk.ensure_path(path)
bytes = open(filename, 'rb').read()
if zk.exists(path):
# push the conf file
if not zk.exists(path + id) and not wipe:
print "creaing conf node"
zk.create(path + id, bytes)
elif not wipe:
print "updating conf file"
zk.set(path + id, bytes)
if wipe:
zk.set(path + id, None)
zk.stop()
def random_nodes(number_of_nodes_to_return, exclude=None, nodes=None):
""" Selects a group of nodes from the pool of registered nodes
Arguments:
number_of_nodes_to_return: The total number of nodes to be returned
to the caller of the function
exclude (optional): A list of nodes that will be excluded from the
results
nodes (optional): A list of nodes to process, if None then the
specified zookeeper will be contacted and the registered brokers used
:return:
"""
if not nodes:
zk = KazooClient(hosts=zookeeper_connection_string, read_only=True)
zk.start()
try:
if zk.exists('/brokers/ids'):
ids = zk.get_children('/brokers/ids')
finally:
zk.stop()
else:
ids = nodes
if exclude:
ids = [x for x in ids if x not in exclude]
return random.sample(ids, number_of_nodes_to_return)
def get_scn_from_zookeeper():
try:
print("Starting the process")
zk = KazooClient(hosts='10.0.57.146:2181,10.0.57.145:2181,10.0.77.195:2181')
print("Connection established")
zk.start()
sum_of_scn_num = 0
if zk.exists("/fk_kafka_cluster1"):
children_list = zk.get_children("/fk_kafka_cluster1/PROPERTYSTORE/")
sorted_list = sorted(children_list)
partion_num_scn_num_dict = {}
for children in sorted_list:
recv_data,stat = zk.get("/fk_kafka_cluster1/PROPERTYSTORE/"+str(children))
data_dict = ast.literal_eval(recv_data)
partition_num = data_dict["id"]
fields = ast.literal_eval(data_dict['simpleFields']['c'])
scn_num = fields["windowScn"]
sum_of_scn_num += scn_num
partion_num_scn_num_dict[partition_num] = scn_num
print "Data fetching from Zookeeper complete"
avg_scn_num = (sum_of_scn_num/len(children_list))
sorted_dict = sorted(partion_num_scn_num_dict.items(),key=operator.itemgetter(1))
return avg_scn_num,sorted_dict
else:
print("Node does not exist")
except Exception as e:
print("Exception occured!",str(e))
def init_zk(namespace=''):
"""Initialize Zookeeper Instance."""
hosts = os.getenv('ZOOKEEPER_CONN_STRING', 'localhost:2181')
timeout = 600
timeout_treshold = time() + timeout
while True:
try:
# check if namespace exists
zk = KazooClient(hosts,
read_only=False)
zk.start()
if zk.exists(namespace) is None:
zk.create(namespace)
zk.stop
zk.close
zk = KazooClient(hosts + namespace,
read_only=False)
zk.start()
if type(zk) is not KazooClient:
logging.error("can't connect to Zookeeper ...")
exit(1)
return zk
except KazooTimeoutError:
if time() > timeout_treshold:
logging.error("can't connect to Zookeeper ...")
exit(1)
else:
logging.warning("can't connect to Zookeeper, " +
"trying again since timeout " +
"is not reached yet ...")
class Store(object):
def __init__(self,**kwargs):
self.config = kwargs
self.client = None
def get_client(self):
return self.client
def open(self):
self.client = KazooClient(**self.config)
self.client.add_listener
self.client.start()
def close(self):
self.client.stop()
def read(self,path):
return self.client.get(path)
def write(self,path,value):
base_path = os.path.dirname(path)
self.client.ensure_path(base_path)
self.client.create(path,value)
def overwrite(self,path,value):
self.client.set(path,value)
def exists(self,path):
return self.client.exists(path)
def run(self):
zk = KazooClient(hosts='%s:%d' % (self.options.host, self.options.port),
read_only=True, timeout=3)
try:
zk.start()
options = vars(self.options)
options.update({
'system.hostname': socket.gethostname()
})
if self.options.regex:
content, stats = zk.get(self.options.file)
options['stats'] = stats
m = re.search(self.options.regex, content, re.MULTILINE | re.DOTALL)
if m:
options.update(m.groupdict())
self.ok(self.options.message.format(**options))
else:
self.critical(self.options.message.format(**options))
elif zk.exists(self.options.file):
self.ok(self.options.message.format(**options))
else:
self.critical(self.options.message.format(**options))
except Exception as ex:
self.critical(ex)
finally:
zk.stop()
def validate_zookeeper_fault_tolerance():
"""
Validate that we can still connect to zookeeper instance 2 to read the node
"""
zk2 = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2182'))
zk2.start()
assert zk2.exists("/my/zookeeper_errorinjection/"), "zookeeper_errorinjection node not found"
zk2.stop()
def zookeeper_ephemeral_node(name):
zk = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2181'))
zk.start()
zk.create("/my/zookeeper_test/node1", b"process1 running", ephemeral=True)
#At 10 validate that ephemeral node exist that is the process is still running
time.sleep(10)
assert zk.exists("/my/zookeeper_test/node1"), "process node is not found at 10 s when it is still running"
time.sleep(20)
zk.stop()
class CockRoach(object):
def __init__(self, zkHost, stale_max_days=30, assume_yes=False, preview=False):
self.ConsumerGroups = []
self.zk_client = KazooClient(hosts=zkHost)
self.zk_client.start()
self.stale_max_days = stale_max_days
self.assume_yes = assume_yes
self.preview = preview
if self.zk_client.exists("/consumers"):
for cg_name in self.zk_client.get_children("/consumers"):
self.ConsumerGroups.append(ConsumerGroup(cg_name, \
self.zk_client))
def get_stale_cgroups(self, display):
"""
get_stale_cgroups returns ConsumerGroups
that were not used for stale_max_days
"""
ret = []
for consumergroup in self.ConsumerGroups:
delta = datetime.now() - consumergroup.last_seen().mtime
if delta.days > self.stale_max_days:
if display:
print "Stale: %s" % (consumergroup)
ret.append(consumergroup)
return ret
def delete_stale_cgroups(self):
""" Delete consumer groups that are considered stale"""
stale_cgroups = self.get_stale_cgroups(display=False)
for stale_cg in stale_cgroups:
print stale_cg
if self.assume_yes is False:
confirm = raw_input("Delete?")
else:
confirm = "Y"
if confirm == "Y":
self.delete_cgroup(stale_cg)
def delete_cgroup(self, consumergroup):
"""Deletes a consumer Group"""
print "Deleting %s" % (consumergroup.gid)
if self.preview is False:
self.zk_client.delete("/consumers/%s" % (consumergroup.gid), version=-1, recursive=True)
print "executed"
else:
print "pass"
def __str__(self):
ret = ""
for consumer in self.ConsumerGroups:
ret += "%s" % (consumer)
return ret
class Pool(object):
def __init__(self, name):
self.name = name
self.zk = None
self.path = "/carousel/{}".format(name)
# Pool meta-data
self.meta = None
self.meta_stat = None
self.resources = set()
def on_resources_change(self, res):
self.resources = set(res)
def create(self, metadata={}):
# Create the base pool path with metadata
self.zk.create(self.path, str.encode(json.dumps(metadata)), makepath=True)
for path in ["resources", "nodes", "leaders"]:
self.zk.create(os.path.join(self.path, path))
self.load()
def connect(self, *hosts, timeout=4):
self.zk = KazooClient(",".join(hosts), timeout=timeout)
self.zk.start()
def load(self):
# Check whether the pool exists
if not self.zk.exists(self.path):
raise PoolException("Pool with name {} does not exist!".format(self.name))
# Next load the pool meta-data
self.meta, self.meta_stat = self.zk.get(self.path)
self.meta = json.loads(self.meta.decode())
# Finally, we need to keep track of all resources in this pool
ChildrenWatch(self.zk, os.path.join(self.path, "resources"), self.on_resources_change)
def disconnect(self):
self.zk.stop()
def get_node(self):
return Node(self)
def get_resource(self, name):
return Resource(self, name)
def __eq__(self, other):
return other.name == self.name
def test(self, wat):
print(wat)
def validate_zookeeper_process_tracking():
"""
Verify if process register node correctly with zookeeper and zookeeper deletes it when process terminates
"""
zk = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2181'))
zk.start()
#At 60 validate that process has terminated by looking at the ephemeral node
time.sleep(60)
assert not zk.exists("/my/zookeeper_test/node1"), "process node not found at 60 s when it should have terminated"
zk.stop()
class ZookeeperContext:
def __init__(self, zookeeper_url, test_id, node_id):
self.assertNotNone(zookeeper_url)
self.assertNotNone(test_id)
self.assertNotNone(node_id)
self.zookeeper_url = zookeeper_url
self.test_id = test_id
self.node_id = node_id
self.zk_path = '/bolt/testing/%s/%s' % (self.test_id, self.node_id)
logger.info("Initialized decorator: %s " % self.__str__())
self.__connect_zookeeper()
def assertNotNone(self,thing):
if thing == None:
logger.fatal("Invalid key in initializer")
def __connect_zookeeper(self):
try:
logger.info('Attempting to connect to: %s' % self.zookeeper_url)
self.zk = KazooClient(hosts=self.zookeeper_url)
self.zk.start(timeout=10)
logger.info('initialized zookeeper')
except Exception as exception:
logger.error('Failed to connect to zk')
logger.fatal(exception)
def __str__(self):
return json.dumps(self, default=lambda o: o.__dict__, indent=4)
def __del__(self):
self.zk.stop()
def publish(self, key, data):
self.assertNotNone(key)
self.assertNotNone(data)
path = self.zk_path + "/" + key
logger.info("Creating data in: %s" % path)
try:
bytes = pickle.dumps(data);
if not self.zk.exists(path):
logger.info("Path does not exist in zk, creating... %s" % path)
self.zk.create(path, value=bytes, makepath=True)
else:
self.zk.set(path, value=bytes)
except Exception as exception:
logger.error('Error setting data in ' % path)
logger.fatal(exception)
def test_zip_response(pub_node, host = '127.0.0.1', port = 2181):
zookeeper = KazooClient('%s:%s' % (host, port,))
zookeeper.start()
node_value = {
'status' : 'ok',
'update_time' : time.time(),
}
if zookeeper.exists('/test/to_zip_result/%s' % pub_node) is not None:
zookeeper.set('/test/to_zip_result/%s' % pub_node, json.dumps(node_value))
else:
print 'Not found node `/test/to_zip_result/%s`' % pub_node
def main():
zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0)
zk.add_listener(my_listener)
zk.start()
if zk.exists("/ELECTION") == None:
zk.ensure_path("/ELECTION")
c = 1
node_pathes = []
while c < 10:
c += 1
node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True)
node_pathes.append(node_path)
my_path = random.choice(node_pathes)
my_path = my_path.replace("/ELECTION/", "")
# print "print my_path", my_path
children = zk.get_children("/ELECTION/", watch=election_child_watcher)
get_next_path = False
prev_path = None
for child_path in sorted(children):
if child_path == my_path:
break
prev_path = child_path
# I'm the leader
if prev_path == None:
print "OK I'm leader don't have to watch"
return
# fires twice, once on creation ignore
@zk.DataWatch("/ELECTION/" + prev_path)
def watch_node(data, stat):
# only watch for first change
if stat.version == 1:
print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
print "setting watch on " + prev_path
print "my", my_path
zk.set("/ELECTION/" + prev_path, b"some data")
print "boom. watch triggered?"
# time.sleep(10)
print "bye"
zk.stop()
def test_leave_abc_behind():
obj = type('',(),{})
obj.flg = False
config=skelton.Config()
config.check_health=lambda:3
config.become_active=lambda:0
config.become_standby_from_active=lambda:-1
z = zha.ZHA(config)
trigger_zha(z)
zk = KazooClient(hosts="127.0.0.1:2181")
zk.start()
assert zk.exists("/zha-abc")
zk.stop()
time.sleep(10)
def reset(bpath, new_val):
zk = KazooClient(hosts=args.zkhost)
zk.start()
if zk.exists(bpath): # /offsets
for child in zk.get_children(bpath): # offsets/topic
c = '{0}/{1}'.format(bpath, child)
print('Topic: {0}'.format(c))
for c2 in zk.get_children(c): # offsets/topic/partition
c2 = '{0}/{1}'.format(c, c2)
print('Set {0} to {1}'.format(c2, new_val))
zk.set(c2, new_val)
else:
print('Path <{0}> not exists'.format(bpath))
zk.stop()
def number_of_nodes():
""" Will return the total number of nodes registered in the cluster
Return:
int: Total count of nodes that are a part of the cluster
"""
zk = KazooClient(hosts=zookeeper_connection_string, read_only=True)
zk.start()
try:
if zk.exists('/brokers/ids'):
ids = zk.get_children('/brokers/ids')
return len(ids)
finally:
zk.stop()
def put_topology(topology, config=config):
zk = KazooClient(**config)
zk.start()
for service, links in topology.iteritems():
if zk.exists("/sdn/services/{name}".format(name=service)):
zk.set("/sdn/services/{name}".format(name=service), json.dumps(links))
else:
zk.ensure_path("/sdn/services")
zk.create("/sdn/services/{name}".format(name=service), json.dumps(links))
ret_check = zk.get_children("/sdn/services")
zk.stop()
return ret_check
def test_init_servers(num, host = '127.0.0.1', port = 2181):
zookeeper = KazooClient('%s:%s' % (host, port,))
zookeeper.start()
try:
node = '/test/server_list'
if zookeeper.exists(node) is None:
zookeeper.create(node, json.dumps({'update_time' : time.time()}), makepath = True)
except kazoo.exceptions.NodeExistsError:
pass
v = 1
while v <= num:
try:
node = '/test/server_list/s%s' % v
if zookeeper.exists(node) is None:
zookeeper.create(node, json.dumps({
'update_time' : time.time(),
'server_name' : 's%s' % v,
'server_id' : v,
}), makepath = True)
except kazoo.exceptions.NodeExistsError:
pass
v += 1
def test_pub_response(pub_node, server_list, host = '127.0.0.1', port = 2181):
zookeeper = KazooClient('%s:%s' % (host, port,))
zookeeper.start()
for s in server_list:
if isinstance(s, int) is True:
continue
try:
node = '/test/to_pub_result/%s/%s' % (pub_node, s, )
if zookeeper.exists(node) is None:
zookeeper.create(node, json.dumps({
'update_time' : time.time(),
'status' : 'ok'
}), makepath = True)
except kazoo.exceptions.NodeExistsError:
pass
class ZkServiceRegister:
def __init__(self, zk_address, zk_timeout):
self.__zkClient = KazooClient(hosts=zk_address, timeout=zk_timeout, read_only=False)
self.__zkListener = ZkServiceRegisterListener(self.__zkClient)
self.__zkClient.add_listener(self.__zkListener)
self.__zkClient.start()
def register(self, path, host, port, weight=DEFAULT_HOST_WEIGHT):
try:
if not self.__zkClient.exists(path):
self.__zkClient.ensure_path(path)
except Exception, e:
print e.message
reg_path = path + '/' + host + ':' + str(port) + ':' + str(weight)
if self.__zkClient.exists(reg_path):
self.__zkClient.delete(reg_path)
self.__zkClient.create(reg_path, value='', ephemeral=True)
class ConfigFlags(StorageBase):
""" A configuration manager using ZooKeeper. For setting flags
on all instances for a given product. This will default to
using a locally stored cache if ZooKeeper fails to respond.
Note: current live flags are limited to 1MB total.
Set initial values in config.ini file as
flags.foo = bar
to set "foo" flag to value bar.
"""
#TODO:
# * Break flags into separate elements instead of single JSON?
# * Add fake dict to hold settings if ZK not installed/avaliable.
localFlags = {}
version = None
def __init__(self, config, **kw):
try:
if 'Configurator' in type(config).__name__:
config = config.get_settings()
conf = config.get('flags.zk.settings')
if conf is not None:
conf = dict(json.loads(conf))
self.zk = KazooClient(conf)
else:
self.zk = KazooClient()
# get a copy of the local flags.
self.zk_path = config.get('flags.zk.path',
'/general/config')
self.zk.start()
node = self.zk.exists(self.zk_path)
if node is None:
# Virgin install, set from the config values.
self._init_zk(config)
self.zk.add_listener(self._zk_listener)
self._refreshCache(config=config)
except Exception, e:
warnings.warn("Could not connect to ZooKeeper %s" % repr(e))
def get_zookeeper_master_ip(zkurl, zkpath):
logger.info("Connecting to:%s" % zkurl)
zk = KazooClient(hosts=zkurl)
ip = ""
try:
logger.debug("Starting zk connection")
zk.start()
if not zk.exists(zkpath):
logger.error('Path on zk doesn\'t exist: ' + zkpath)
return ip
logger.debug("Serializing TopologyMetadata() from %s" % zkpath)
data, stat = zk.get(zkpath + "/masterip")
logger.debug("Status of 'getting' %s/masterip: %s" % (zkpath, str(stat)))
ip = str(data)
except Exception as e:
logger.exception(e)
finally:
logger.debug("Closing zk connection")
zk.stop()
return ip
class ZkServiceProvider:
def __init__(self, zk_address, zk_timeout, connection):
self.__service_dict = {}
self.__zk_address = zk_address
self.__zkClient = KazooClient(hosts=zk_address, timeout=zk_timeout, read_only=True)
self.__zkClient.start()
self.__zkListener = ZkServiceProviderListener(self.__zkClient)
self.__zkClient.add_listener(self.__zkListener)
self.__connection = connection
def register_service(self, service, zk_path, client_cls):
self.__service_dict[service] = (zk_path, client_cls)
result = self._register_watcher(service, zk_path, client_cls)
return result
def _register_watcher(self, service, zk_path, client_cls):
@self.__zkClient.ChildrenWatch(zk_path)
def child_changed(data):
print '+++++++++++++++' + service + ' child changed.++++++++++++++++++'
print data
hosts = data
self.__connection.update_service(service, hosts)
isExists = self.__zkClient.exists(zk_path)
if not isExists:
return False
try:
hosts = self.__zkClient.get_children(zk_path)
except NoNodeError:
print 'no node for the path of ' + zk_path
return False
except:
print 'other exceptions.'
return False
self.__connection.update_service(service, hosts)
return True
def stop(self):
self.__zkClient.stop()
class Zookeeper(object):
def __init__(self, host):
self.zk = KazooClient(hosts=host, connection_retry=True, read_only=True)
self.zk.start()
def stop(self):
self.zk.stop()
def get_path_instance(self, path):
self.hosts = [] # store server list
if self.zk.exists(path):
if self.zk.get_children(path) > 0:
for node in self.zk.get_children(path):
node_path = path + '/' + node
data, stat = self.zk.get(node_path)
host = data.split(':')[0]
self.hosts.append(host)
else:
raise Exception("path is not exsits")
return self.hosts
def main():
(options, args) = parse_options(sys.argv[1:])
data = options.znode_data_size * b"D"
s = KazooClient(options.server)
s.start()
if s.exists(options.root_znode):
children = s.get_children(options.root_znode)
print("delete old entries: %d" % len(children))
for child in children:
s.delete("%s/%s" % (options.root_znode, child))
else:
s.create(options.root_znode, "kazoo root znode")
evaluation(s, options.root_znode, data, options)
s.stop()
print("Performance test complete")
class Backend:
def __init__(self):
self._logger = logging.getLogger('gunicorn.error')
self._zk = KazooClient(hosts=f'{os.getenv("ZOOKEEPER_HOST")}:2181')
self._hdfsClient = HdfsClient(os.getenv("HADOOP_NAMENODE_HOST"))
self._active = False
self._target_id = None
self._zk_node_path = None
self._range = None
self._trie = None
scheduler = BackgroundScheduler(timezone="UTC")
scheduler.add_job(self._attempt_to_join_any, 'interval', minutes=1)
scheduler.start()
def start(self):
self._zk.start()
datawatch_next_target = DataWatch(client=self._zk,
path=ZK_NEXT_TARGET,
func=self._on_next_target_changed)
datawatch_current_target = DataWatch(
client=self._zk,
path=ZK_CURRENT_TARGET,
func=self._on_current_target_changed)
def stop(self):
self._zk.stop()
def top_phrases_for_prefix(self, prefix):
if (not self._active):
raise NodeInactiveError(
"This backend node is not active. Consult zookeeper for the most recent active nodes"
)
return self._trie.top_phrases_for_prefix(prefix)
def _on_next_target_changed(self, data, stat, event=None):
self._logger.info("_on_next_target_changed Data is %s" % data)
if (data is None):
return
current_target_id = self._zk.get(ZK_CURRENT_TARGET)[0].decode()
next_target_id = data.decode()
self._deactivate_if_not_used(current_target_id, next_target_id)
success = self._attempt_to_join_target(next_target_id)
def _on_current_target_changed(self, data, stat):
self._logger.info("_on_current_target_changed Data is %s" % data)
if (data is None):
return
current_target_id = data.decode()
next_target_id = self._zk.get(ZK_NEXT_TARGET)[0].decode()
self._deactivate_if_not_used(current_target_id, next_target_id)
def _deactivate_if_not_used(self, current_target_id, next_target_id):
if (self._active and self._target_id
and current_target_id != self._target_id
and next_target_id != self._target_id):
self._logger.info(
f'Deactivating {self._target_id}, {current_target_id}, {next_target_id}'
)
if (self._zk.exists(self._zk_node_path)):
self._zk.delete(self._zk_node_path)
self._active = False
self._target_id = None
self._trie = None
self._zk_node_path = None
def _attempt_to_join_any(self):
self._logger.debug("Attempting to join any")
if (self._active):
return
if (self._zk.exists(ZK_CURRENT_TARGET) is not None):
target_id = self._zk.get(ZK_CURRENT_TARGET)[0].decode()
if (self._attempt_to_join_target(target_id)):
return
if (self._zk.exists(ZK_NEXT_TARGET) is not None):
target_id = self._zk.get(ZK_NEXT_TARGET)[0].decode()
if (self._attempt_to_join_target(target_id)):
return
def _attempt_to_join_target(self, target_id):
if (self._active):
return False
if (not target_id or
self._zk.exists(f'/phrases/distributor/{target_id}') is None):
return False
self._logger.info(f'Attempting to join {target_id}')
partitions = self._zk.get_children(
f'/phrases/distributor/{target_id}/partitions')
for partition in partitions:
nodes_path = f'/phrases/distributor/{target_id}/partitions/{partition}/nodes/'
self._zk.sync(nodes_path)
nodes = self._zk.get_children(nodes_path)
if (len(nodes) >= NUMBER_NODES_PER_PARTITION):
self._logger.info(
f'Cannot join {nodes_path}, it has enough nodes {nodes}; current number of nodes {len(nodes)} >= {NUMBER_NODES_PER_PARTITION}'
)
continue # No more nodes needed here
created_node_path = self._zk.create(nodes_path,
value=b'',
ephemeral=True,
sequence=True)
self._zk_node_path = created_node_path
try:
created_node_name = created_node_path.split('/')[-1]
self._zk.sync(nodes_path)
nodes = self._zk.get_children(nodes_path)
nodes.sort()
if (nodes and int(created_node_name) > int(
nodes[0:NUMBER_NODES_PER_PARTITION][-1])):
# The node was not able to join the partition
self._logger.info(
f'Cannot join {nodes_path}, it has already filled up {nodes}; created_node_name: {created_node_name}'
)
self._zk.delete(created_node_path)
continue
if (not self._load_trie_and_activate(target_id, partition)):
self._logger.error(
f'Error while loading and activating trie for {nodes_path}, target_id: {target_id}, partition: {partition}'
)
self._zk.delete(created_node_path)
continue
# Finishes the initialization
self._zk.set(created_node_path, socket.gethostname().encode())
return True
except:
self._zk.delete(created_node_path)
return False
def _load_trie_and_activate(self, target_id, partition):
trie_data_hdfs_path = f'/phrases/distributor/{target_id}/partitions/{partition}/trie_data_hdfs_path'
trie = self._load_trie(self._zk.get(trie_data_hdfs_path)[0].decode())
if (trie):
self._trie = trie
self._active = True
self._target_id = target_id
self._logger.info(
f'Now ACTIVE and loaded trie for partition {partition} and target_id {target_id}'
)
return True
else:
return False
def _load_trie(self, trie_hdfs_path):
local_path = 'trie.dat'
self._hdfsClient.download(trie_hdfs_path, local_path)
with open(local_path, 'rb') as f:
return pickle.load(f)
zk = KazooClient(hosts='zoo:2181')
def zk_listener(state):
if (state == KazooState.LOST):
logging.warning("Zookeeper connection lost")
elif (state == KazooState.SUSPENDED):
logging.warning("Zookeeper connection suspended")
else:
logging.info("Zookeeper connected")
zk.add_listener(zk_listener)
zk.start()
if (zk.exists("/Workers")):
zk.delete("/Workers", recursive=True)
@zk.ChildrenWatch("/Workers/", send_event=True)
def watch_children(children, event):
print("Children are now: %s" % children)
if (event == None):
pass
elif (event.type is DELETED):
print("Slave deleted")
timer_start_flag = False
timer_started_flag = False
read_request_count = 0
class Proxy:
"""Implementation of the proxy"""
def __init__(self):
# Use XPUB/XSUB to get multiple contents from different publishers
self.context = zmq.Context()
self.xsubsocket = self.context.socket(zmq.XSUB)
self.xpubsocket = self.context.socket(zmq.XPUB)
self.xpubsocket.setsockopt(zmq.XPUB_VERBOSE, 1)
self.xpubsocket.send_multipart([b'\x01', b'10001'])
# Now we are going to create a poller
self.poller = zmq.Poller()
self.poller.register(self.xsubsocket, zmq.POLLIN)
self.poller.register(self.xpubsocket, zmq.POLLIN)
self.global_url = 0
self.global_port = 0
self.newSub = False
self.topic_info_queue = [
] # the content queue for different topic (zipcode)
self.topicInd = 0
self.zip_list = [
] # the ziplist to keep track with the zipcodes received
self.zk_object = KazooClient(hosts='127.0.0.1:2181')
self.zk_object.start()
self.path = '/home/'
znode1 = self.path + "broker1"
if self.zk_object.exists(znode1):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode1, to_bytes("5555,5556"))
# Print the version of a node and its data
znode2 = self.path + "broker2"
if self.zk_object.exists(znode2):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode2, to_bytes("5557,5558"))
znode3 = self.path + "broker3"
if self.zk_object.exists(znode3):
pass
else:
# Ensure a path, create if necessaryto_bytes
self.zk_object.ensure_path(self.path)
# Create a node with data
self.zk_object.create(znode3, to_bytes("5553,5554"))
'''
elect a leader and put that node under the path of /leader/ and send the port number to the publisher and subscriber
'''
self.election = self.zk_object.Election(self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode(
'latin-1') # the leader here is a pair of address
self.leader = str(self.leader)
address = self.leader.split(",")
tem = address[0]
tem = tem.split("'")[1]
address[0] = tem
tem = address[1]
tem = tem.split("'")[0]
address[1] = tem
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
print("Current elected broker: ", pub_addr + "," + sub_addr)
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.watch_dir = self.path + self.leader # use Datawatch
self.leader_path = "/leader/"
self.leader_node = self.leader_path + "node"
if self.zk_object.exists(self.leader_node):
pass
else:
# Ensure a path, create if necessary
self.zk_object.ensure_path(self.leader_path)
# Create a node with data
self.zk_object.create(self.leader_node, ephemeral=True)
self.zk_object.set(self.leader_node, to_bytes(self.leader))
def sendToSubscriber(self):
events = dict(self.poller.poll(10000))
if self.xsubsocket in events:
msg = self.xsubsocket.recv_multipart()
content = msg[0]
content = str(content)
content = content.split("'")[1]
#print("content here is {}".format(content))
zipcode, temperature, relhumidity, pub_time = content.split(
" ")[:4]
cur_msg = [zipcode, temperature, relhumidity, pub_time]
if zipcode not in self.zip_list: # a new topic just come from a new publisher
self.zip_list.append(zipcode)
topic_info = cur_msg
self.topic_info_queue.append(topic_info)
else:
zipInd = self.zip_list.index(zipcode)
self.topic_info_queue[zipInd] = cur_msg
self.xpubsocket.send_string(
"%i %i %i %i " % (int(cur_msg[0]), int(
cur_msg[1]), int(cur_msg[2]), int(cur_msg[3])))
if self.xpubsocket in events: # a subscriber comes here
msg = self.xpubsocket.recv_multipart()
self.xsubsocket.send_multipart(msg)
def schedule(self):
while True:
@self.zk_object.DataWatch(self.watch_dir)
def watch_node(data, stat, event):
if event != None:
print(event.type)
if event.type == "DELETED": # redo a election
self.election = self.zk_object.Election(
self.path, "leader")
leader_list = self.election.contenders()
self.leader = leader_list[-1].encode('latin-1')
self.zk_object.set(self.leader_node, self.leader)
self.xsubsocket.unbind(self.current_pub)
self.xpubsocket.unbind(self.current_sub)
address = self.leader.split(",")
pub_addr = "tcp://*:" + address[0]
sub_addr = "tcp://*:" + address[1]
self.xsubsocket.bind(pub_addr)
self.xpubsocket.bind(sub_addr)
self.election.run(self.sendToSubscriber)
def close(self):
""" This method closes the PyZMQ socket. """
self.xsubsocket.close(0)
self.xpubsocket.close(0)
class zkHander(object):
'''zookeeper系列操作'''
def __init__(self):
zk_hosts = GetConf().GetZKHosts()
self.zk = KazooClient(hosts=zk_hosts)
self.zk.start()
def Get(self, path):
value, stat = self.zk.get(path=path)
return value
def GetChildren(self, path):
return self.zk.get_children(path=path)
def Exists(self, path):
return self.zk.exists(path=path)
def Create(self, **kwargs):
return self.zk.create(
path=kwargs["path"],
value=kwargs["value"],
sequence=kwargs['seq'],
makepath=(kwargs['mp'] if 'mp' in kwargs else False))
def Set(self, path, value):
return self.zk.set(path=path, value=value)
def init_node(self, object):
node_list = {
'mysql': [
'lock', 'white', 'meta/host', 'meta/group', 'meta/router',
'online-list', 'master', 'task', 'haproxy', 'watch-down',
'addition', 'addition/replication', 'addition/region',
'readbinlog-status', 'slavedown', 'online-clients',
'execute-gtid'
]
}
for server in node_list:
for i in node_list[server]:
node_path = '{}/{}'.format(server, i)
state = self.zk.exists(path=node_path)
if state is None:
object(node_path)
def InitNode(self):
'''初始化node'''
@self.init_node
def _init_node(node_path):
self.zk.ensure_path(node_path)
def SetReadBinlog(self, master_host, value):
'''创建binlog信息，用于宕机记录读取的点'''
_path = '/mysql/readbinlog-status/{}'.format(
master_host.replace('.', '-'))
self.Create(path=_path, value=value,
seq=False) if not self.Exists(_path) else None
def SetExecuteGtid(self, master_host, value):
'''创建gtid信息，用于切换记录执行到的gtid'''
_path = '/mysql/execute-gtid/{}'.format(master_host.replace('.', '-'))
self.Create(path=_path, value=value,
seq=False) if not self.Exists(_path) else None
def CheckOnlineClient(self, host):
'''检查mysql客户端程序是否在线'''
_path = '{}/{}'.format(GetConf().GetOnlineClientPath(), Replace(host))
return self.Exists(_path)
def CreateDownTask(self, groupname, addition=None, region=None):
task_path = '{}/{}'.format(GetConf().GetTaskPath(), groupname)
if self.Exists(task_path) is None:
time.sleep(random.uniform(0, 1))
try:
if addition:
task_path += '_' + region
self.Create(path=task_path,
value=str([groupname, region, 'append',
'dow']),
seq=False) #跨区域同步主机宕机
else:
self.Create(path=task_path,
value=str([groupname, 'down']),
seq=False)
except:
Logging(msg='Existing outage task for {}'.format(groupname),
level='info')
else:
Logging(msg='Existing outage task for {}'.format(groupname),
level='info')
def CreateWatch(self,
host,
addition=None,
region=None,
region_for_groupname=None):
'''创建watch,触发时写入task节点'''
online_host_path = GetConf().GetOnlinePath()
_group_name = self.GetMeta(type='host', name=host)
group_name = eval(
_group_name)['group'] if _group_name else region_for_groupname
online_state = self.zk.exists('{}/{}'.format(online_host_path, host))
Logging(msg='master watch :{}'.format(host), level='info')
if online_state is not None:
@self.zk.DataWatch('{}/{}'.format(online_host_path, host))
def my_func(data, stat):
if data is None:
self.CreateDownTask(group_name,
addition=addition,
region=region)
Logging(msg='master({}) has been down!'.format(host),
level='error')
self.zk.stop()
sys.exit()
else:
_name = group_name + '_' + region if region else group_name
Logging(msg="this master {} node not exists".format(host),
level='error')
state = self.Exists('{}/{}'.format(GetConf().GetWatchDown(),
_name))
self.Create(path='{}/{}'.format(GetConf().GetWatchDown(), _name),
value="master not online",
seq=False) if state is None else None
self.zk.stop()
def __checklock(self, taskname):
"""循环检查lock是否存在，当不存在时对新master建立监听"""
import time
path = '{}/{}'.format(GetConf().GetLockPath(), taskname)
while True:
state = self.Exists(path)
if state is None:
return True
break
time.sleep(0.5)
def CreateLockWatch(self, taskname):
'''用于任务在其他节点执行，本节点对锁监听，当删除时获取新master启动watch'''
path = '{}/{}'.format(GetConf().GetLockPath(), taskname)
@self.zk.DataWatch(path)
def my_func(data, stat):
if data is None:
masterhost = self.GetMasterMeta(taskname)
self.CreateWatch(masterhost)
sys.exit()
def CreateChildrenWatch(self, path, func):
'''任务列表监听'''
state = self.zk.exists(path=path)
if state is not None:
@self.zk.ChildrenWatch(path)
def my_func(data):
if data:
data.sort()
func(data[0])
def GetWhite(self, groupname):
'''check 白名单'''
white_list = self.GetChildren(path='/mysql/white')
return True if groupname in white_list else False
def GetMasterGroupHosts(self):
'''获取当前master列表,启动检查是否有现有的master，如果有直接监控'''
group_hosts_path = GetConf().GetMasterPath()
master_list = self.GetChildren(group_hosts_path)
return master_list if master_list is not None else False
def GetAddHost(self):
'''获取是否需要添加的集群，该节点下记录的为集群组名称，不能并发'''
Lock_root = GetConf().GetLockPath()
add_host_path = Lock_root + '/add-host'
state = self.Exists(add_host_path)
return self.Get(add_host_path) if state is not None else False
def GetMeta(self, **kwargs):
'''获取元数据信息'''
if kwargs['type'] == 'group':
node_path = '{}/{}'.format(GetConf().GetMetaGroup(),
kwargs['name'])
return self.Get(
node_path) if self.Exists(node_path) != None else False
elif kwargs['type'] == 'host':
node_path = '{}/{}'.format(GetConf().GetMetaHost(), kwargs['name'])
return self.Get(
node_path) if self.Exists(node_path) != None else False
else:
Logging(msg="type is error ,only 【group ,host】", level='error')
raise "type is error ,only 【group ,host】"
def CreateMasterMeta(self, name, host):
'''创建master信息节点，用于新集群加入'''
node_path = '{}/{}'.format(GetConf().GetMasterPath(), name)
return self.Create(
path=node_path, value=host,
seq=False) if self.Exists(node_path) is None else False
def AlterMasterMeta(self, name, host):
'''修改master信息节点，用于切换过后修改元数据'''
node_path = '{}/{}'.format(GetConf().GetMasterPath(), name)
return self.Set(node_path,
host) if self.Exists(node_path) is None else False
def GetMasterMeta(self, groupname):
'''获取master元数据信息'''
node_path = '{}/{}'.format(GetConf().GetMasterPath(), groupname)
return self.Get(node_path) if self.Exists(
node_path) is not None else False
def GetOnlineHostAll(self):
'''获取在线的节点'''
node_path = GetConf().GetOnlinePath()
return self.GetChildren(node_path)
def GetOnlineState(self, host):
'''获取单个节点在线状态'''
node_path = '{}/{}'.format(GetConf().GetOnlinePath(), host)
return True if self.Exists(path=node_path) is not None else False
def SetHaproxyMeta(self, group, reads, master, type=None):
'''加入及master变动设置haproxy配置信息'''
node_path = '{}/{}'.format(GetConf().GetHaproxy(), group)
if type is None:
meta_path = GetConf().GetMetaGroup()
if reads is None:
hosts = self.Get('{}/{}'.format(meta_path, group))
reads = hosts.split(',')
_reads = []
for host in reads:
host_meta = self.GetMeta(type='host', name=host)
host_port = '%s:%d' % (host.replace(
'-', '.'), int(eval(host_meta)['port']))
_reads.append(host_port)
master_host_meta = self.GetMeta(type='host', name=master)
_master = '%s:%d' % (master.replace(
'-', '.'), int(eval(master_host_meta)['port']))
value = '{"read":"%s","write":"%s"}' % (_reads, _master)
state = self.Exists(node_path)
if state is None:
self.Create(path=node_path, value=value, seq=False)
else:
self.Set(path=node_path, value=value)
return True
else:
value = '{"read":"%s","write":"%s"}' % (reads, master)
self.Set(path=node_path, value=value)
return True
def GetTaskList(self):
'''获取任务列表'''
task_path = GetConf().GetTaskPath()
task_list = self.GetChildren(path=task_path)
return task_list if task_list is not None else False
def GetTaskContent(self, taskname):
'''获取任务内容'''
task_path = GetConf().GetTaskPath()
task_value = self.Get('{}/{}'.format(task_path, taskname))
return task_value if task_value is not None else False
def DeleteTask(self, taskname):
'''删除任务'''
task_path = GetConf().GetTaskPath()
self.zk.delete('{}/{}'.format(task_path, taskname)) if self.Exists(
'{}/{}'.format(task_path, taskname)) else None
def DeleteWhite(self, groupname):
node_path = GetConf().GetWhitePath()
self.zk.delete('{}/{}'.format(node_path, groupname)) if self.Exists(
'{}/{}'.format(node_path, groupname)) else None
def DeleteWatchDown(self, groupname):
'''清除心跳丢失的节点'''
watch_down_path = GetConf().GetWatchDown()
self.zk.delete('{}/{}'.format(
watch_down_path, groupname)) if self.Exists('{}/{}'.format(
watch_down_path, groupname)) else None
def SetMasterHost(self, groupname, masterhost):
'''写入master信息'''
masterhost_path = '{}/{}'.format(GetConf().GetMasterPath(), groupname)
self.Set(masterhost_path, masterhost) if self.Exists(
masterhost_path) is not None else self.Create(
path=masterhost_path, value=masterhost, seq=False)
def OnlineExists(self, host):
'''判断是否在线'''
path = GetConf().GetOnlinePath()
return True if self.Exists('{}/{}'.format(path, host)) else False
def GetRouter(self, groupname):
'''获取路由所在'''
path = GetConf().GetRouter()
return self.Get('{}/{}'.format(path, groupname)) if self.Exists(
'{}/{}'.format(path, groupname)) else False
def SetWatchDown(self, groupname, value):
path = GetConf().GetWatchDown()
self.Create(path='{}/{}_send'.format(path, groupname),
value=value,
seq=False) if self.Exists('{}/{}_send'.format(
path, groupname)) is None else None
def SetLockTask(self, taskname):
'''创建锁文件，用户多点运行server控制任务'''
path = GetConf().GetLockPath()
if self.Exists(path='{}/{}'.format(path, taskname)) is None:
try:
time.sleep(random.uniform(0, 1))
self.zk.create(path='{}/{}'.format(path, taskname),
value=b'',
ephemeral=False)
return True
except Exception, e:
Logging(msg=traceback.format_exc(), level='error')
return False
else:
children = zk.get("/worker/master", watch=master_function)
#zk.delete("/worker/slave", recursive=True)
#zk.delete("/worker", recursive=True)
cmd = "cat /proc/self/cgroup | grep 'docker' | sed 's/^.*\///' | tail -n1"
cid = subprocess.check_output(cmd,shell=True)
cid = cid.decode("utf-8")
cid=cid[0:12]
client2 = docker.APIClient()
pid = client2.inspect_container(cid)['State']['Pid']
print("---MASTER PID", pid)
print("---CID", cid)
zk.ensure_path("/worker")
if zk.exists("/worker/master"):
print("Master exists")
else:
data1 = "I am master CID : "+cid+" PID : "+str(pid)
data1 = data1.encode()
zk.create("/worker/master", data1,ephemeral = True)
children = zk.get("/worker/master", watch=master_function)
app = Flask(__name__)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+str(pid)+'.db'
db = SQLAlchemy(app)
class user_details(db.Model):
from kazoo.client import KazooClient
import logging
logging.basicConfig()
zk = KazooClient(hosts='zookeeper:2181')
zk.start()
zk_path = "/kafka/replicator/test_kafka/checkpoint"
if zk.exists(zk_path):
data, stat = zk.get(zk_path)
print(data)
class Zookeeper:
def __init__(self, hosts, max_merge_seq):
print('create a zookeeper object')
self.zk = ""
self.IsConn = False
self.Hosts = hosts
self.MAX_MERGE_FILE_SEQUENCE = max_merge_seq
self.filename = ''
self.pattern = ''
self.process_path = ''
def connect(self):
"""
connect to zookeeper
:return:zookeeper object
"""
print('try connect to zookeeper')
self.zk = KazooClient(self.Hosts)
try:
self.zk.start()
except Exception as e:
print("connect zookeeper failed, err:%s" % e)
sys.exit()
self.IsConn = True
print('connect zookeeper success')
return self.zk
def get_node(self, node_path):
"""
获取空闲的process_id
:return: process_id
"""
self.connect()
self.process_path = node_path
node_list = []
if not (self.zk.exists(node_path)):
logging.error('zookeeper process node path: %s not exist' %
node_path)
sys.exit()
childs = self.zk.get_children(node_path)
# len = 0
p1 = re.compile(r"^process")
for c in childs:
if re.findall(p1, c):
node_list.append(c)
node_list = sorted(node_list)
if len(node_list) <= 0:
print("no process id in zookeeper process path")
sys.exit()
get_times = 0
while 1:
for node in node_list:
lock_flag = False
node_name = '%s/%s' % (node_path, node)
n_child = self.zk.get_children(node_name)
if len(n_child) > 0:
for n in n_child:
if n == 'lock':
lock_flag = True
if lock_flag:
continue
lock_node = "%s/%s" % (node_name, 'lock')
self.zk.create(lock_node, ephemeral=True)
# process_id = ''.join(node.split('_')[1:])
print('get process_id :%s from zookeeper ' % node)
return node
get_times += 1
print("no free process id in zookeeper")
if get_times >= 3:
print(
"get process id faild three times, please check zookeeper process id, exit"
)
sys.exit()
def lock(self, lock):
"""
lock the free node
:param lock:
:return:
"""
self.zk.create(lock, ephemeral=True)
def check_exists(self, node_path):
return self.zk.exists(node_path)
def get_config(self, config_path, config_node):
"""
generate config files based on node's information
:param config_path:
:param config_node:
:return:
"""
data, stat = self.zk.get(config_node)
with open(config_path + "config.ini", 'w') as f:
f.writelines(data.decode())
def get_node_value(self, zk_node):
"""
获取zookeeper的节点信息
:param zk_node:
:return: data:node的value
stat:node的状态信息
"""
data, stat = self.zk.get(zk_node)
return data, stat
def set_node_value(self, zk_node, data):
"""
设置zookeeper节点的value
:param zk_node:
:param data:
:return:
"""
return self.zk.set(zk_node, value=data)
def delete_node(self, zk_node):
"""
删除某一节点
:param zk_node:
:return:
"""
self.zk.delete(zk_node)
def create_node(self, node, flag=False):
"""
创建zookeeper节点
:param node:
:param flag:
:return:
"""
try:
self.zk.create(node, ephemeral=flag)
except Exception as e:
logging.info("create zookeeper node:%s failed, err:%s" % (node, e))
print(node, e)
return False
return True
def cp(self, src, dest):
"""
copy the local file to zookeeper
:param src:local file
:param dest:zookeeper node
:return:
"""
if not os.path.isfile(src):
print("%s: `%s': Local file does not exist" % ('cp', src))
sys.exit()
file_size = os.path.getsize(src)
if file_size > 1048576:
print("%s: `%s': Local file maximum limit of 1M" % ('cp', src))
sys.exit()
self.connect()
if self.zk.exists(dest):
print("%s: `%s': Zookeeper exists" % ('cp', dest))
sys.exit()
with open(src, 'rb') as file:
data = file.read()
self.zk.create(dest)
self.zk.set(dest, value=data)
def zk_get_merge_fn(self, process_path, work_node, cur_seq, filename_pool):
"""
获取filename_pool下的序号，记录redo
:param process_path
:param work_node
:param cur_seq:
:param filename_pool:
:return: zk_seq:
0: 返回0代表未到合并时间点
1: 返回1代表没有抢占到filename_pool
next_child:返回获取到的filename_pool节点
"""
if not self.zk.exists(filename_pool):
logging.error('no filename_pool in zookeeper')
sys.exit()
childs = self.zk.get_children(filename_pool)
if not childs:
logging.error('the zookeeper filename_pool is empty')
sys.exit()
# zk_fn_seq = childs[0]
childs = sorted(childs)
redo_info = []
for child in childs:
file_date, zk_seq, prov = child.split('.')
zk_fs = ("%s%s" % (file_date, zk_seq))
zk_fs = re.sub("[A-Za-z.]", "", zk_fs)
if int(zk_fs) > int(cur_seq):
logging.info('zk_seq:%s > cur_seq:%s, wait...' %
(zk_fs, cur_seq))
return 0
zk_seq = int(zk_seq) + 1
if zk_seq > self.MAX_MERGE_FILE_SEQUENCE:
zk_seq = 0
file_date = datetime.datetime.strptime(file_date, '%Y%m%d')
next_time = file_date + datetime.timedelta(days=1)
file_date = ('%s%02d%02d' %
(next_time.year, next_time.month, next_time.day))
zk_seq = "%03d" % zk_seq
next_child = '%s.%s.%s' % (file_date, zk_seq, prov)
# 创建一次事务，删除旧的序号并创建新的序号，保证原子性
transaction_request = self.zk.transaction()
transaction_request.delete("%s/%s" % (filename_pool, child))
transaction_request.create("%s/%s" % (filename_pool, next_child))
redo_seq = ",".join([file_date, zk_seq, prov])
redo_info.append("filenamepool:" + redo_seq)
redo_node = process_path + "/" + work_node + "/" + "redo"
self.create_node(redo_node)
self.set_node_value(redo_node, ";".join(redo_info).encode("utf-8"))
results = transaction_request.commit()
if results[0] is True and results[1] == (
"%s/%s" % (filename_pool, next_child)):
return next_child
else:
continue
return 1
# 为了方便观察而设置的阻塞时间
sleep = 1 if len(sys.argv) < 2 else int(sys.argv[1])
# 是否使用锁 "yes"/"no"
is_lock = "yes" if len(sys.argv) > 2 and sys.argv[2] == "yes" else "no"
# zookeeper 地址
hosts = "127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183"
# 在这个节点上做自增操作
path = "/data"
# 启动 kazoo
zk = KazooClient(hosts=hosts)
zk.start()
# 初始化数据节点
exists = zk.exists(path)
if exists is None:
zk.create(path, b"0")
# 执行次数
times = 0
while True:
times += 1
work(zk, path, sleep, is_lock, times)
if times > 10 // sleep:
break
print("该节点当前的值:", zk.get(path)[0])
zk.stop()
app.config['ZK_HOST'] = environ.get('ZK_HOST')
STORAGE_HOST = []
for i in range(4):
STORAGE_HOST.append(environ.get('STORAGE_HOST_' + str(i)))
MAX_RETRIES = 3
db = MongoEngine(app)
jwt = JWTManager(app)
zk_app = KazooClient(hosts=app.config['ZK_HOST'])
zk_app.start()
zk_app.ensure_path("/app")
if not zk_app.exists('/app/1'):
zk_app.create('/app/1', b"5000", ephemeral=True)
def zk_get_storage_children(zk_client):
retries = 0
if not zk_client.exists('/storage'):
return []
while len(
zk_client.get_children('/storage')) < 2 and retries < MAX_RETRIES:
retries += 1
time.sleep(1)
# if retries == MAX_RETRIES:
# return []
class ValidatorDetector:
def __init__(self):
# 获取Client实例。
self.zk = KazooClient(
hosts="192.168.21.145:2181,192.168.21.144:2181,192.168.21.141:2181",
timeout=2)
# 设置节点监听和数值监听。
self.my_watch1 = ChildrenWatch(
client=self.zk, path="/",
func=self.validator_watcher_fun) # func= 设置监听函数，这个监听会一直有效。
self.my_watch2 = DataWatch(client=self.zk,
path="/app7",
func=self.data_watch_fun)
# 初始化连接到zookeeper
self.zk.start()
def validator_watcher_fun(self, children):
"""
监听函数
:param children: 返回的字节点
:return:
"""
print("The children now are:", children)
# self.zk.get_children("/", )
def data_watch_fun(self, data, stat, event):
"""监听函数"""
print("The Node data is Change", data, stat, event)
def create_node(self):
"""创建节点"""
self.zk.create('/app-python', b'python_create_node', makepath=True)
def is_exist(self, path):
"""判断节点是否存在"""
print("Yes") if (self.zk.exists(path) is not None) else print("No")
# if self.zk.exists(path) is not None:
# print("Yes")
# else:
# print("No")
def getData(self, path):
"""获取节点上的数据"""
try:
ret = self.zk.get(path) # 返回一个元组，第一个元素是value,第二个是Stat
print("%s : " % path, ret)
except:
print("%s : 无数据 " % path)
def setData(self, path, value):
"""更新数据"""
try:
# 传入的value必须转为byte[]
self.zk.set(path, bytes(value, encoding='utf8'))
ret = self.zk.get(path)
print("更新数据成功。", str(ret[0], encoding='utf8'))
except Exception as ex:
print("更新数据失败。", str(ex))
def delete(self, path):
"""删除节点"""
try:
if self.zk.delete(path):
print("删除 %s 成功" % path)
except Exception as ex:
print("删除 %s 失败!" % path, str(ex))
status = 0
write_stat = 0
write_ret=dict()
#SETTING UP INITIAL ZOOKEEPER STRUCTURE
mpath = '/master'
spath = '/slaves'
sspath = '/slaves/slave'
zk.delete(mpath, recursive=True)
zk.delete(spath, recursive=True)
if zk.exists(mpath):
print("We have a master.")
else:
zk.create(mpath,b'')
if zk.exists(spath):
print("We have slaves.")
else:
zk.create(spath, b'')
mdata = ''
for key in master_id:
mdata = key+'pid'+str(master_id[key])
sdata='All the slaves.'.encode()
class ZkClient:
def __init__(self, hosts, ports, on_failure_action=None):
server = ''
for host, port in zip(hosts, ports):
server = server + '%s:%s,' % (host, port)
server = server[:len(server) - 1]
self.zk = KazooClient(server)
if on_failure_action is not None:
if on_failure_action == 'RECONN':
self.zk.add_listener(self.__zk_status_listener__)
self.__service_node = ""
self.zk.start()
def get_children(self, node):
if self.zk.exists(node):
return self.zk.get_children(node)
def create_node(self, node_name, data):
self.zk.ensure_path(node_name)
def update_node(self, node_name, data):
self.zk.set(node_name, data)
def delete_node(self, node_name):
self.zk.delete(node_name)
def read_data(self, node):
if self.zk.exists(node):
return self.zk.get(node)
def register_service(self, service_type, service_id):
"""
向Zookeeper注册远程服务
:param service_type: 服务类型
:param service_id: 服务ID
:return:
"""
path = ConstData.ServicePath % (service_type, service_id)
self.__service_node = path
self.zk.create(path,
str(ServiceStatusEnum.Idle).encode("UTF-8"),
ephemeral=True,
makepath=True)
def create_task(self, major_task_id, sub_task_id):
todopath = ConstData.TodoTaskPath % (major_task_id, sub_task_id)
self.zk.create(todopath, makepath=True)
def require_task(self, major_task_id, sub_task_id, service_id) -> bool:
'''
向zk请求一个任务
首先向特定的任务类型节点下请求一个临时节点，如果没有报错，则表示任务请求成功，否则任务请求失败
service_id指定了执行任务的服务
'''
todo_path = ConstData.TodoTaskPath % (major_task_id, sub_task_id)
path = ConstData.InProgressPath % (major_task_id, sub_task_id)
result = False
try:
# 首先确认待办中存在对应的节点
if self.zk.exists(todo_path):
self.zk.create(path,
service_id.encode('UTF-8'),
ephemeral=True,
makepath=True)
try:
self.zk.set(self.__service_node,
str(ServiceStatusEnum.Running).encode("UTF-8"))
result = True
except:
# 如果更改状态出错，应该放弃该任务
self.zk.delete(path)
result = False
# 任务节点不存在，则任务已经被执行了
else:
result = False
except Exception as e:
print(e)
# 出现异常，任务请求失败
result = False
return result
def finish_task(self, major_task_id, task_id):
todo_path = ConstData.TodoTaskPath % (major_task_id, task_id)
path = ConstData.InProgressPath % (major_task_id, task_id)
self.zk.delete(todo_path)
self.zk.delete(path)
def task_execute_error(self, task_type, task_id):
path = ConstData.InProgressPath % (task_type, task_id)
self.zk.delete(path)
def __zk_status_listener__(self, state):
if state == KazooState.LOST:
self.zk.start()
def start_service(self):
self.zk.set(self.__service_node,
str(ServiceStatusEnum.Idle).encode("utf-8"))
def running_service(self):
self.zk.set(self.__service_node,
str(ServiceStatusEnum.Running).encode("utf-8"))
def stop_service(self):
# 停止服务其实只是把节点状态设置为了服务下线
self.zk.set(self.__service_node,
str(ServiceStatusEnum.Offline).encode("utf-8"))
def idle_service(self):
self.start_service()
zk = KazooClient(hosts='zoo:2181', timeout=1.0)
zk.start(timeout=1)
cmd = "cat /proc/self/cgroup | grep 'docker' | sed 's/^.*\///' | tail -n1"
cid = subprocess.check_output(cmd, shell=True)
cid = cid.decode("utf-8")
cid = cid[0:12]
client2 = docker.APIClient()
pid = client2.inspect_container(cid)['State']['Pid']
print("---PID", pid)
print("---CID", cid)
zk.ensure_path("/worker")
if zk.exists("/worker/slave"):
print("Slave exists")
else:
zk.create("/worker/slave", b"hi")
if zk.exists("/worker/master"):
present = 0
print("Master exists")
ms = "/worker/master"
data, stat = zk.get(ms)
data = data.decode("utf-8")
ind = data.find('PID')
pid_master = data[ind + 5:len(data) + 1]
pid_master = int(pid_master)
print("HELLO FROM MASTERKK - ", pid_master)
data, stat = zk.get("/worker/master")
log_dir = '/home/shweta/workspace/research/dag-placement/log/msr'
processing_intervals = [1, 5, 10, 15, 20, 25, 30]
vertex_bbb = 'bbb-a702'
collector_bbb = 'bbb-ed97'
for proc in processing_intervals:
for sel in [.5, 1]:
for rate in range(1, 21):
print('Running experiment for proc:%d, sel:%.1f and rate:%d\n' %
(proc, sel, rate))
#start zk
zk = KazooClient(zk_connector)
zk.start()
#set runid
if not zk.exists('/dom%d/runid' % (domain_id)):
zk.ensure_path('/dom%d/runid' % (domain_id))
zk.set('/dom%d/runid' % (domain_id), b'%d' % (rate))
#make sub-dirs
if not os.path.exists('%s/p%d/s%.1f/r%d' %
(log_dir, proc, sel, rate)):
os.makedirs('%s/p%d/s%.1f/r%d' % (log_dir, proc, sel, rate))
if not os.path.exists('%s/p%d/s%.1f/r%d/graphs' %
(log_dir, proc, sel, rate)):
os.makedirs('%s/p%d/s%.1f/r%d/graphs' %
(log_dir, proc, sel, rate))
with open(
'%s/p%d/s%.1f/r%d/graphs/g1.txt' %
(log_dir, proc, sel, rate), 'w') as f:
class ZookeeperClient(object):
def __init__(self, hosts=None, read_only=True):
self.hosts = hosts if hosts else ENSEMBLE.get()
self.read_only = read_only
hdfs = cluster.get_hdfs()
if hdfs is None:
raise ZookeeperConfigurationException(
'No [hdfs] configured in hue.ini.')
if hdfs.security_enabled:
self.sasl_server_principal = PRINCIPAL_NAME.get()
else:
self.sasl_server_principal = None
self.zk = KazooClient(hosts=self.hosts,
read_only=self.read_only,
sasl_server_principal=self.sasl_server_principal)
def start(self):
"""Start the zookeeper session."""
self.zk.start()
def stop(self):
"""Stop the zookeeper session, but leaves the socket open."""
self.zk.stop()
def close(self):
"""Closes a stopped zookeeper socket."""
self.zk.close()
def get_children_data(self, namespace):
children = self.zk.get_children(namespace)
children_data = []
for node in children:
data, stat = self.zk.get("%s/%s" % (namespace, node))
children_data.append(data)
return children_data
def path_exists(self, namespace):
return self.zk.exists(namespace) is not None
def set(self, path, value, version=-1):
return self.zk.set(path, value, version)
def copy_path(self, namespace, filepath):
if self.read_only:
raise ReadOnlyClientException(
'Cannot execute copy_path when read_only is set to True.')
self.zk.ensure_path(namespace)
for dir, subdirs, files in os.walk(filepath):
path = dir.replace(filepath, '').strip('/')
if path:
node_path = '%s/%s' % (namespace, path)
self.zk.create(path=node_path, value='', makepath=True)
for filename in files:
node_path = '%s/%s/%s' % (namespace, path, filename)
with open(os.path.join(dir, filename), 'r') as f:
file_content = f.read()
self.zk.create(path=node_path,
value=file_content,
makepath=True)
def delete_path(self, namespace):
if self.read_only:
raise ReadOnlyClientException(
'Cannot execute delete_path when read_only is set to True.')
self.zk.delete(namespace, recursive=True)
def __enter__(self):
"""Start a zookeeper session and return a `with` context."""
self.zk.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Stops and closes zookeeper session at the end of the `with` context."""
try:
self.stop()
finally:
self.close()
class WebWindow(object):
def setupUi(self, Window):
Window.setObjectName("Window")
self.centralwidget = QtWidgets.QWidget(Window)
self.centralwidget.setObjectName("centralwidget")
self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget)
self.verticalLayout.setContentsMargins(0, 0, 0, 0)
self.verticalLayout.setObjectName("verticalLayout")
self.zk = None
self.webView = QtWebKitWidgets.QWebView(self.centralwidget)
self.webView.setObjectName("webView")
self.webView.setRenderHint(QPainter.Antialiasing, True)
self.webView.setRenderHint(QPainter.TextAntialiasing, True)
self.webView.setRenderHint(QPainter.SmoothPixmapTransform, True)
self.webView.setRenderHint(QPainter.HighQualityAntialiasing, True)
self.webView.setPage(WebPage())
frame = self.webView.page().mainFrame()
frame.javaScriptWindowObjectCleared.connect(self.initJsComm)
self.verticalLayout.addWidget(self.webView)
Window.setCentralWidget(self.centralwidget)
self.retranslateUi(Window)
QtCore.QMetaObject.connectSlotsByName(Window)
self.loadDefaultAcl()
def retranslateUi(self, Window):
_translate = QtCore.QCoreApplication.translate
Window.setWindowTitle(_translate("Window", "Zookeeper GUI"))
def loadLocalFile(self, filename):
localdir = os.path.abspath(sys.path[0])
if os.path.isfile(localdir):
localdir = os.path.dirname(localdir)
self.webView.setUrl( QtCore.QUrl.fromLocalFile(localdir+'/'+filename) )
def getCfgVar(self, name):
localdir = os.path.abspath(sys.path[0])
if os.path.isfile(localdir):
localdir = os.path.dirname(localdir)
cfg = {}
obj = None
try:
obj = open(localdir+'/cfg.json','r')
cfg = json.loads(obj.read())
except Exception as e:
logging.info(str(e))
finally:
if obj is not None:
obj.close()
if name in cfg:
return str(cfg[name])
return ''
def setCfgVar(self, name, value):
localdir = os.path.abspath(sys.path[0])
if os.path.isfile(localdir):
localdir = os.path.dirname(localdir)
cfg = {}
obj = None
try:
obj = open(localdir+'/cfg.json','r')
cfg = json.loads(obj.read())
except Exception as e:
pass
finally:
if obj is not None:
obj.close()
cfg[name] = value
obj = None
try:
obj = open(localdir+'/cfg.json','w')
obj.truncate()
obj.write(json.dumps(cfg))
except Exception as e:
logging.info(str(e))
finally:
if obj is not None:
obj.close()
def makeDigestCred(self, user, plain_pass):
m = hashlib.sha1( bytes(user,'utf8') + b':' + bytes(plain_pass,'utf8') ).digest()
return user+':'+base64.b64encode(m).strip().decode('utf8')
def initJsComm(self):
frame = self.webView.page().mainFrame()
frame.addToJavaScriptWindowObject('py',self)
@pyqtSlot(str)
def jsSetWinTitle(self, title):
_translate = QtCore.QCoreApplication.translate
self.setWindowTitle(_translate("Window", title))
@pyqtSlot(str, result=str)
def jsCheckYaml(self, s):
try:
a = yaml.load(s)
except Exception as e:
return str(e)
if a is None:
return 'Failed'
return ''
@pyqtSlot(str,result=str)
def jsGetCfg(self, name):
return self.getCfgVar(name)
@pyqtSlot(str,str)
def jsSetCfg(self, name, value):
self.setCfgVar(name, value)
def loadDefaultAcl(self):
self.updateDefaultAclCache( self.getCfgVar('defaultacl') )
def updateDefaultAclCache(self, list_str):
if list_str is not None and len(list_str)>0:
cache = json.loads(list_str)
self.default_acl_plain = []
self.default_acl = []
for one in cache:
if(one['scheme']=='world'):
self.default_acl_plain.append(one)
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') )
self.default_acl.append(acl)
elif(one['scheme']=='digest'):
self.default_acl_plain.append(one)
if 'id' in one:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) )
else:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) )
self.default_acl.append(acl)
elif(one['scheme']=='ip'):
self.default_acl_plain.append(one)
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) )
self.default_acl.append(acl)
else:
self.default_acl_plain = []
self.default_acl = None
@pyqtSlot(str,result=str)
def jsSetDefaultAcl(self, list_str):
self.updateDefaultAclCache(list_str)
self.setCfgVar('defaultacl', json.dumps(self.default_acl_plain))
@pyqtSlot(result=str)
def jsGetDefaultAcl(self):
return json.dumps(self.default_acl_plain)
@pyqtSlot()
def jsGetZk(self):
return self.zk
@pyqtSlot(result=int)
def jsZkIsConnected(self):
if self.zk is not None:
return int(self.zk.state=='CONNECTED')
return 0
@pyqtSlot(str, str, result=str)
def jsZkConnect(self,host, auth_list_str):
try:
if self.zk is not None:
#self.zk.remove_listener(self.onZkStateChange)
self.zk.stop()
self.zk.close()
self.zk = KazooClient(hosts=host)
#self.zk.add_listener(self.onZkStateChange)
self.zk.start(15)
auth_list = json.loads(auth_list_str)
for one in auth_list:
cred = self.makeDigestCred(one['user'], one['pass'])
self.zk.add_auth('digest', one['user']+':'+one['pass'])
except Exception as e:
logging.error("jsZkConnect, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
#def onZkStateChange(self,state):
# frame = self.webView.page().mainFrame()
# frame.evaluateJavaScript("onPyZkStateChange('"+state+"')")
@pyqtSlot(str, result=QVariant)
def jsZkGetChildren(self, path):
try:
logging.info("jsZkGetChildren, path="+path)
children = self.zk.get_children(path)
except NoNodeError:
logging.error("jsZkGetChildren, NoNodeError")
return QVariant({"err":"node not exists"})
except Exception as e:
logging.error("jsZkGetChildren, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return QVariant({"err":str(e)+', traceback='+str(strlist)})
return QVariant({"err":"", "children":children})
@pyqtSlot(str, result=QVariant)
def jsZkGet(self, path):
try:
logging.info("jsZkGet, path="+path)
ret = self.zk.get(path)
except NoNodeError:
logging.error("jsZkGet, NoNodeError")
return QVariant({"err":"node not exists"})
except Exception as e:
logging.error("jsZkGet, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return QVariant({"err":str(e)+', traceback='+str(strlist)})
ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].ctime/1000))
mtime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].mtime/1000))
stat = {'ctime':ctime,'mtime':mtime,'version':ret[1].version}
data = ''
if ret[0] is not None:
data = ret[0].decode('utf8')
else:
logging.info('jsZkGet data None, path='+path)
return QVariant({"err":"", "data":data, "stat":QVariant(stat)})
@pyqtSlot(str, str, int, result=str)
def jsZkSet(self, path, data, ver):
try:
logging.info("jsZkSet, path="+path+',ver='+str(ver))
self.zk.set(path, bytes(data, 'utf8'),ver)
except NoNodeError as e:
logging.error("jsZkSet, NoNodeError")
return "node not exists"
except BadVersionError as e:
logging.error("jsZkSet, BadVersionError")
return "bad version"
except Exception as e:
logging.error("jsZkSet, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, result=QVariant)
def jsZkGetAcl(self, path):
try:
logging.info("jsZkGetAcl, path="+path)
ret = self.zk.get_acls(path)
except NoNodeError as e:
logging.error("jsZkGetAcl, NoNodeError")
return QVariant({"err":"node not exists"})
except Exception as e:
logging.error("jsZkGetAcl, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return QVariant({"err":str(e)+', traceback='+str(strlist)})
lst = []
for acl in ret[0]:
dacl = {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id}
lst.append(QVariant(dacl))
stat = {'ctime':ret[1].ctime,'mtime':ret[1].mtime,'version':ret[1].version}
return QVariant({"err":"", "acl_list":QVariant(lst), "stat":QVariant(stat)})
@pyqtSlot(str, str, result=str)
def jsZkSetAcl(self, path, list_str):
try:
acl_list = None
if list_str is not None and len(list_str)>0:
cache = json.loads(list_str)
acl_list = []
for one in cache:
if(one['scheme']=='world'):
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') )
acl_list.append(acl)
elif(one['scheme']=='digest'):
if 'id' in one:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) )
else:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) )
acl_list.append(acl)
elif(one['scheme']=='ip'):
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) )
acl_list.append(acl)
self.zk.set_acls(path, acl_list)
except NoNodeError as e:
logging.error("jsZkSetAcl, NoNodeError")
return "node not exists"
except InvalidACLError as e:
logging.error("jsZkSetAcl, InvalidACLError")
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return "invalid acl, traceback: "+str(strlist)
except BadVersionError as e:
logging.error("jsZkSetAcl, BadVersionError")
return "bad version error"
except Exception as e:
logging.error("jsZkSetAcl, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str,str,int,int,result=str)
def jsZkCreate(self, path, data, ephem, seq):
try:
logging.info("jsZkCreate, path="+path)
self.zk.create(path=path, value=bytes(data,'utf8'), ephemeral=bool(ephem), sequence=bool(seq))
if self.default_acl is not None and len(self.default_acl)>0:
self.zk.set_acls(path, self.default_acl)
except NoNodeError as e:
logging.error("jsZkCreate, NoNodeError")
return "node not exists"
except NodeExistsError as e:
logging.error("jsZkCreate, NodeExistsError")
return "node already exists"
except NoChildrenForEphemeralsError as e:
logging.error("jsZkCreate, NoChildrenForEphemeralsError")
return "ephemeral node can not have child"
except Exception as e:
logging.error("jsZkCreate, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, int, int, result=str)
def jsZkDelete(self, path, ver, recursive):
try:
logging.info("jsZkDelete, path="+path+',ver='+str(ver)+', recursive='+str(recursive))
self.zk.delete(path, ver, bool(recursive))
except NoNodeError as e:
logging.error("jsZkDelete, NoNodeError")
return "node not exists"
except BadVersionError as e:
logging.error("jsZkDelete, BadVersionError")
return "bad version"
except NotEmptyError as e:
logging.error("jsZkDelete, NotEmptyError")
return "node not empty"
except Exception as e:
logging.error("jsZkDelete, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, str, int, int, result=str)
def jsZkCopy(self, dest_path, ori_path, max_depth, children_only):
logging.info("jsZkCopy, dest_path="+dest_path+", ori_path="+ori_path+", children_only="+str(children_only))
#copy node first
if children_only==0:
try:
ori_data = self.zk.get(ori_path)
if self.zk.exists(dest_path) is None:
self.zk.create(dest_path, ori_data[0], acl=self.default_acl)
else:
self.zk.set(dest_path, ori_data[0])
except NoNodeError as e:
logging.error("jsZkCopy, node, NoNodeError, ori_path="+ori_path+', dest_path='+dest_path)
return "node not exists"
except Exception as e:
logging.error("jsZkCopy, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
#copy children
path = ''
try:
max_depth -= 1
path = ori_path
ori_children = self.zk.get_children(ori_path)
for child in ori_children:
path = ori_path+'/'+child
ret = self.jsZkCopy(dest_path+'/'+child, ori_path+'/'+child, max_depth, 0)
if isinstance(ret, QVariant):
return ret
elif len(ret)>0:
return ret
except NoNodeError as e:
logging.error("jsZkCopy, child, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkCopy, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
'''
@pyqtSlot(str, str, int, result=str)
def jsZkCopyChildren(self, dest_path, ori_path, max_depth):
path = ''
try:
max_depth -= 1;
logging.info("jsZkCopyChildren, dest_path="+dest_path+", ori_path="+ori_path)
path = ori_path
ori_children = self.zk.get_children(ori_path)
path = dest_path
dest_children = self.zk.get_children(dest_path)
for child in ori_children:
if child in dest_children:
return 'child ['+child+'] is found in both path'
for child in ori_children:
path = ori_path+'/'+child
data = self.zk.get(path)[0]
path = dest_path+'/'+child
self.zk.create(path, data, acl=self.default_acl)
if max_depth>0:
ret = self.jsZkCopyChildren(dest_path+'/'+child, ori_path+'/'+child, max_depth)
if len(ret)>0:
return ret
except NoNodeError as e:
logging.info("jsZkCopyChildren, NoNodeError")
return "node not exists, path="+path
except ZookeeperError as e:
logging.info("jsZkCopyChildren, ZookeeperError")
return str(e)+', path='+path
return ''
'''
@pyqtSlot(str, int, result=str)
def jsZkDeleteChildren(self, main_path, max_depth):
path = ''
try:
max_depth -= 1
path = main_path
logging.info("jsZkDeleteChildren, path="+main_path)
children = self.zk.get_children(path)
for child in children:
path = main_path+'/'+child
if max_depth>0:
ret = self.jsZkDeleteChildren(path, max_depth)
if len(ret)>0:
return ret
self.zk.delete(path)
except NoNodeError as e:
logging.error("jsZkDeleteChildren, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkDeleteChildren, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, str, int, int, result=str)
def jsZkExport(self, local_dir, main_path, max_depth, without_acl):
path = ''
try:
max_depth -= 1
path = main_path
logging.info("jsZkExport, path="+main_path+' to local dir '+local_dir)
data = self.zk.get(main_path)
p = pathlib.Path(local_dir)
if not p.exists():
p.mkdir(parents=True)
elif not p.is_dir():
return 'local '+local_dir+' exists, but not a directory'
for child in p.iterdir():
return 'local path '+local_dir+' is not empty, clear it first'
p = pathlib.Path(local_dir+'/____data')
p.touch()
obj = open(str(p),'wb')
try:
if data[0] is not None:
obj.write(data[0])
finally:
obj.close()
if not without_acl:
ret = self.zk.get_acls(path)
lst = []
if ret is not None:
for acl in ret[0]:
lst.append( {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id} )
p = pathlib.Path(local_dir+'/____acl')
p.touch()
obj = open(str(p),'w')
try:
obj.write(json.dumps(lst))
finally:
obj.close()
children = self.zk.get_children(path)
if children is not None:
for child in children:
if child=='zookeeper':
continue
path = main_path+'/'+child
if max_depth>0:
ret = self.jsZkExport(local_dir+'/'+child, path, max_depth, without_acl)
if len(ret)>0:
return ret
except NoNodeError as e:
logging.error("jsZkExport, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkExport, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, str, int, int, result=str)
def jsZkImport(self, local_dir, main_path, max_depth, without_acl):
path = ''
try:
max_depth -= 1
path = main_path
logging.info("jsZkImport, path="+main_path+' from local dir '+local_dir)
obj = open(local_dir+'/____data', 'rb')
if self.zk.exists(path) is None:
self.zk.create(path, obj.read(), acl=self.default_acl)
else:
self.zk.set(path, obj.read())
if not without_acl:
obj = open(local_dir+'/____acl', 'r')
acl_list = None
list_str = obj.read()
if list_str is not None and len(list_str)>0:
cache = json.loads(list_str)
acl_list = []
for one in cache:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id(one['scheme'], one['id']) )
acl_list.append(acl)
self.zk.set_acls(path, acl_list)
p = pathlib.Path(local_dir)
for child in p.iterdir():
if not child.is_dir():
continue
if child.name=='zookeeper':
continue
ret = self.jsZkImport(str(child), path+'/'+child.name, max_depth, without_acl)
if len(ret)>0:
return ret
except NoNodeError as e:
logging.error("jsZkImport, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkImport, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
# 发布精确任务
# zk
elif jdata[0]['Agreement'] == '5':
urls = jdata[0]['Content'].split(', ')
print 'urls', urls
url = str(urls[0])
keyword = urls[1]
priority = int(urls[2])
task_type = info_zk[url][1] + '_' + keyword
ready_list = self.get_ready_node(priority)
for i in range(priority):
zk.set("/command/" + ready_list[i], value=task_type)
node_map[ready_list[i]][2] += 1 # task_num++
if zk.exists("/signal/" + task_type) != None:
temp_list = zk.get_children("/signal/" + task_type)
for i in temp_list:
zk.delete('/signal/' + task_type + '/' + str(i))
else:
zk.create("/signal/" + task_type)
zk.create("/signal/" + task_type + '/start')
for ready_node in ready_list:
sql = "insert into task values('" + url + "', '" + keyword + "','" + info[
url][1] + "'," + int(ready_node) + ",now(),'5')"
nsql = "update info set tasknum=tasknum+1 where id='" + ready_node + "'"
# 发布任务后在amount表中插入20条空数据，防止出现时间倒序
tsql = "insert into amount values('" + url + "','" + keyword + "',0,'" + info[
url][1] + "',now())"
for i in range(20):
class KafkaMonitor(object):
def __init__(self,
queue,
kf_ip_port='localhost',
zk_ip_port='localhost',
sleep_time=10):
# 连接 kafka
self.kafka_hosts = kf_ip_port
self.broker = SimpleClient(hosts=self.kafka_hosts)
# 连接zookeeper
self.zookeepers_hosts = zk_ip_port
self.zk = KazooClient(hosts=self.zookeepers_hosts, read_only=True)
# 数据存放
self.queue = queue
# 时间间隔
self.sleep_time = sleep_time - 1
def get_group(self):
"""获取zookeepers下的group"""
group_name = self.zk.get_children('/consumers')
return group_name
def get_topics(self, group):
"""group下的topic"""
try:
topics = self.zk.get_children("/consumers/%s/owners" % group)
return topics
except Exception as e:
logging.error('get group topics failed! %s' % e)
def get_logsize(self, topic):
return get_logsize(self.broker, topic)
def get_partition_offset(self, group, topic, partition):
path = "/consumers/%s/offsets/%s/%s" % (group, topic, partition)
if self.zk.exists(path):
data, stat = self.zk.get(path)
return data
else:
logger.error('get group:%s topic:%s partition:%s offset failed!' %
(group, topic, partition))
return None
def get_offset(self, group, topic):
"""topic下消费量"""
try:
offset = {}
# 获取 partitions
path = "/consumers/%s/offsets/%s" % (group, topic)
partitions = sorted(self.zk.get_children(path))
for partition in partitions:
data = self.get_partition_offset(group, topic, partition)
offset[int(partition)] = int(data)
return offset
except Exception as e:
logger.error('get group:%s topic:%s offset failed! %s' %
(group, topic, e))
return None
def get_group_data(self, group, topics):
"""获取一个 group 的所有 topic 的数据"""
try:
group_data = {}
for topic in topics:
# 获取 topic log_size 值
log_size = self.get_logsize(topic)
# 获取 topic offset 值
offset = self.get_offset(group, topic)
# topic 内的数据字典
topic_data = dict(topic_name=topic,
logsize=log_size,
offset=offset,
group_name=group)
key = '%s_%s' % (group, topic)
group_data[key] = topic_data
return group_data
except Exception as e:
logger.error('get group:%s offset failed! %s' % (group, e))
return None
def get_data(self):
try:
# 获取 groups_name 列表
groups_name = self.get_group()
# 定义本次循环 data 字典
data = {}
for group in groups_name:
# 获取 group 内的 topics 列表
topics = self.get_topics(group)
if not topics:
continue
# 获取 group 数据列表
group_data = self.get_group_data(group, topics)
if group_data:
# group 数据加入到本次循环的data 字典
data = dict(data, **group_data)
return data
except Exception as e:
logger.error('get offset failed! %s' % e)
time.sleep(3)
def run(self):
self.zk.start()
while True:
t0 = time.clock()
data = self.get_data()
self.queue.put(data)
# 执行间隔时间
t = time.clock() - t0
# 睡眠时间减去执行时间 保证间隔时间相等
sleep_time = self.sleep_time - t
if sleep_time < 0:
sleep_time = 0
time.sleep(sleep_time)
# Handle being disconnected from Zookeeper
logging.warning("suspended")
else:
# Handle being connected/reconnected to Zookeeper
logging.info("zookeeper is connected")
zk.add_listener(my_listener)
try:
zk.start()
except:
time.sleep(10)
finally:
zk.start()
if (zk.exists("/Workers")):
zk.delete("/Workers", recursive=True)
if (zk.exists("/Slave")):
zk.delete("/Slave", recursive=True)
zk.ensure_path("/Slave/")
@zk.ChildrenWatch("/Workers/", send_event=True)
def watch(children, event):
print("in zookeeper event")
if event == None:
pass
elif event.type is DELETED:
print("slave deleted")
class ZookeeperAgent2(object):
def __init__(self,
ip=None,
port=None,
serviceType=None,
path=None,
monitorPort=None):
self._ip = ip #'10.66.170.3'
self._port = port #'22182'
self._serviceType = serviceType #'/workers'
self._path = path #'/workers/10.66.170.3'
self._monitorPort = monitorPort #'
self.zk_node = self._path + self._monitorPort
try:
self.zk = KazooClient(hosts=self._ip + ':' + self._port,
timeout=500,
read_only=True)
self.zk.add_listener(self.conn_state_watcher)
self.zk.start()
except Exception as ex:
self.init_ret = False
self.err_str = "Create KazooClient failed! Exception: %s" % str(ex)
logging.error(self.err_str)
def conn_state_watcher(self, state):
if state == KazooState.CONNECTED:
print("Now connected")
if self.zk_node is None:
print("create method has not been called")
return
# info_keeper = InfoKeeper(self)
# info_keeper.start()
elif state == KazooState.LOST:
print("Now lost")
else:
print("Now suspended")
def main(self):
if self.zk.exists(self._path):
print(self._path + " already exists")
else:
self.zk.create(self._path)
while True:
time.sleep(10)
self.check_port(self._ip, int(self._monitorPort))
def check_port(self, ip, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5)
host = (ip, port)
print(host)
r = s.connect_ex(host)
if r == 0:
if self.zk.exists(self.zk_node):
self.zk.delete(self.zk_node)
self.zk.create(self.zk_node, b"ocr/30%", ephemeral=True)
#print(self.zk.commit())
else:
self.zk.delete(self.zk_node)
s.close()
return 1
class Broker_Driver ():
# constructor
def __init__ (self, args):
self.name = args.name
self.zkIPAddr = args.zkIPAddr # ZK server IP address
self.zkPort = args.zkPort # ZK server port num
self.cond = args.cond # used as barrier condition
self.ppath = args.zkPpath # refers to the parent znode path
self.topicPath = args.topicPath # refers to the parent znode path
self.brokerAddressPort = args.port # indicating if barrier has reached
self.zk = None # session handle to the server
# override the run method which is invoked by start
def init_broker (self):
# It is here that we connect with the zookeeper server
# and see if the condition has been met
try:
# Instantiate zookeeper client object
# right now only one host; it could be the ensemble
hosts = self.zkIPAddr + str (":") + str (self.zkPort)
self.zk = KazooClient (hosts)
except:
print("Unexpected error in ClientApp::run", sys.exc_info()[0])
raise
def run_broker(self):
try:
# First, open connection to zookeeper
self.zk.start ()
if (self.zk.exists (self.ppath)) :
# if parent path already exists, set first_flat 'False'
first_flag=False
else:
# if parent path does not exist, create the Parent Path and set first_flag 'True'
# if first_flag is True then delete the tables' contents(publisher, subscriber)
print("Create the znode Path: "+self.ppath)
self.zk.create (self.ppath, value=self.name.encode())
first_flag=True
# set the broker_path so that children node can be created
broker_path = self.ppath + str("/") + "guid-n_"
broker_path = self.zk.create (broker_path, value=self.name.encode(), ephemeral=True, sequence=True)
# Topic Path Creation if not exists
if ( self.zk.exists (self.topicPath)) :
print("Path("+self.topicPath+") already exists")
else :
# if topic path does not exist, create the Topic Path
print("Create the Topic Path: "+self.topicPath)
self.zk.create (self.topicPath, value=b"topic")
except:
print("Unexpected error in ClientApp::run", sys.exc_info()[0])
raise
# Get the Host name and IP of the server
host_name = socket.gethostname()
host_ip = socket.gethostbyname(host_name)
search_list = []
mydb = db_connect()
mycursor = mydb.cursor()
#print("First_flag: "+str(first_flag))
# Table Initialization if first_flag is True
if first_flag == True:
#print("delete execution")
mycursor.execute("DELETE FROM publisher")
mycursor.execute("DELETE FROM subscriber")
mycursor.execute("DELETE FROM brokerrecord")
mydb.commit()
# Prepare our context and sockets
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
bindAddress = "tcp://*:"+self.brokerAddressPort
frontend.bind(bindAddress)
brokeraddress = "tcp://"+host_ip+":"+self.brokerAddressPort
# store the BrokerAddress to the children node
self.zk.set (broker_path, str(brokeraddress).encode ())
brokeraddress = "tcp://"+host_ip+":"+self.brokerAddressPort
print("Broker Zookeeper path: "+broker_path)
print("Broker Address: "+brokeraddress)
# Initialize poll set
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
# Initialize message_type Variable
# message_type
# '1' : publisher Registration(to Mysql DB Table-publisher)
# '2' : Subscriber Registration(to Mysql DB Table-subscriber)
# '3' : Publish message through Broker
message_type = "0"
# Switch messages between sockets
while True:
print("\nWaiting for the message to be received..")
socks = dict(poller.poll())
if socks.get(frontend) == zmq.POLLIN:
message = frontend.recv_multipart()
message_type = message[2].decode()
socket_id = message[3].decode()
topic = message[4].decode()
print("\n\n\nReceived msg_type: "+message_type+", socket_id: "+socket_id+", topic: "+topic)
if message_type == "1" : # request to register the publisher
append_publisher(topic, socket_id)
# DB Insert to the table 'publisher'
sql = "INSERT INTO publisher VALUES (%s, %s)"
val = (topic, socket_id)
mycursor.execute(sql, val)
mydb.commit()
print("\nInsert into publisher("+topic+","+socket_id+")")
#print "SQL execution: "+sql
# Register the publisher's topic to the Zookeeper
# set the topic_path so that children node can be created
topic_path = self.topicPath + str("/") + topic
if (self.zk.exists (topic_path)) :
# if topic already exists then add new
print("Topic("+topic_path+") already exists");
else:
# if parent path does not exist, create the Parent Path and set first_flag 'True'
# if first_flag is True then delete the tables' contents(publisher, subscriber)
print("Create the topic Path: "+topic_path)
self.zk.create (topic_path, value=topic.encode())
# set the broker_path so that children node can be created
topic_path = topic_path + str("/") + topic+"_"
topic_path = self.zk.create (topic_path, value=socket_id.encode(), ephemeral=True, sequence=True)
# send bakc the reply
#frontend.send_multipart([socket_id.encode(),b"",b"Thank you 1"])
# return the created topic_path
frontend.send_multipart([socket_id.encode(),b"",topic_path.encode()])
elif message_type == "2" : # request to register the subscriber
append_subscriber(topic, socket_id)
# DB Insert to the table 'subscriber'
mycursor = mydb.cursor()
sql = "INSERT INTO subscriber VALUES (%s, %s)"
val = (topic, socket_id)
mycursor.execute(sql, val)
mydb.commit()
print("\nInsert into subscriber("+topic+","+socket_id+")")
# send bakc the reply
frontend.send_multipart([socket_id.encode(),b"",b"Thank you 2"])
elif message_type == "3" :
#search_list = find_brute(subscriber, topic)
contents = str(message[5])
print("\nBroker publishing the message("+contents+") to the subscribers")
mycursor = mydb.cursor()
sql = "SELECT * FROM subscriber WHERE topic='"+topic+"'"
#print(sql)
mycursor.execute(sql)
myresult = mycursor.fetchall()
#print "Len of myresult"+str(len(myresult))
if len(myresult)!=0 : # Add the searched DB result to the search_list
for x in myresult:
search_list.append([x[0],x[1]])
# send the return Messages to the subscribers
for i in range(len(search_list)):
return_socket_id = "Wait"+search_list[i][1] #Make the socket_id by adding "Wait" to the front
#print("return topic: "+topic+", return socket: "+return_socket_id)
frontend.send_multipart([return_socket_id.encode(),b"",contents.encode()])
frontend.send_multipart([socket_id.encode(),b"",b"Thank you 3"])
# Initialize the search_list
search_list = []
else:
print("\nNo matched subscriber to receive the contents")
frontend.send_multipart([socket_id.encode(),b"",b"Thank you 3"])
class zookeeper:
def __init__(self, address):
self.address = address
self.zk = KazooClient(address)
self.zk.start()
self.arcus_cache_map = {}
self.arcus_node_map = {}
self.force = False
self.meta = ('', None)
self.meta_mtime = None
def __repr__(self):
repr = '[ZooKeeper: %s] %s, %s' % (self.address, self.meta[0],
str(self.meta[1]))
for code, cache in self.arcus_cache_map.items():
repr = '%s\n\n%s' % (repr, cache)
return repr
def set_force(self):
self.force = True
def zk_read(self, path):
data, stat = self.zk.get(path)
children = self.zk.get_children(path)
return data, stat, children
def zk_children(self, path, watch=None):
if watch != None:
return self.zk.get_children(path, watch=watch)
else:
return self.zk.get_children(path)
def zk_children_if_exists(self, path, watch=None):
if self.zk_exists(path) == False:
return []
return self.zk_children(path, watch)
def zk_exists(self, path):
if self.zk.exists(path) == None:
return False
return True
def zk_create(self, path, value):
try:
self.zk.create(path, bytes(value, 'utf-8'))
except NodeExistsError:
if self.force == False:
raise NodeExistsError
def zk_delete(self, path):
try:
self.zk.delete(path)
except NoNodeError:
if self.force == False:
raise NoNodeError
def zk_delete_tree(self, path):
try:
self.zk.delete(path, recursive=True)
except NoNodeError:
if self.force == False:
raise NoNodeError
def zk_update(self, path, value):
try:
self.zk.set(path, bytes(value, 'utf-8'))
except NoNodeError:
if self.force == False:
raise NoNodeError
def get_arcus_cache_list(self):
children = self.zk_children_if_exists('/arcus/cache_list/')
children += self.zk_children_if_exists('/arcus_repl/cache_list/')
return children
def get_arcus_node_of_code(self, code, server):
# repl case
children = self.zk_children_if_exists('/arcus_repl/cache_list/' + code)
children += self.zk_children_if_exists('/arcus/cache_list/' + code)
ret = []
for child in children:
tmp = child.split('^', 2) # remove repl info
if len(tmp) == 3:
child = tmp[2]
addr, name = child.split('-', 1)
ip, port = addr.split(':', 1)
if server != '' and (server != ip and server != name):
continue # skip this
node = arcus_node(ip, port)
node.name = name
ret.append(node)
return ret
def get_arcus_node_of_server(self, addr):
ip = socket.gethostbyname(addr)
children = self.zk_children_if_exists(
'/arcus_repl/cache_server_mapping/')
children += self.zk_children_if_exists('/arcus/cache_server_mapping/')
ret = []
for child in children:
l = len(ip)
if child[:l] == ip:
code = self.zk_children_if_exists(
'/arcus_repl/cache_server_mapping/' + child)
if len(code) == 0:
code = self.zk_children_if_exists(
'/arcus/cache_server_mapping/' + child)
code = code[0]
tmp = code.split('^') # remove repl info
code = tmp[0]
try:
ip, port = child.split(':')
except ValueError:
print('No port defined in cache_server_mapping: %s' %
child)
continue
node = arcus_node(ip, port)
node.code = code
ret.append(node)
return ret
def _get_arcus_node(self, child, results):
code = self.zk_children_if_exists('/arcus_repl/cache_server_mapping/' +
child)
if len(code) == 0:
code = self.zk_children_if_exists('/arcus/cache_server_mapping/' +
child)
if len(code) == 0:
print('no childrens in cache_server_mapping error: %s' % child)
print(code)
return
code = code[0]
tmp = code.split('^') # remove repl info
code = tmp[0]
try:
ip, port = child.split(':')
except ValueError:
print('No port defined in cache_server_mapping: %s' % child)
ip = child
port = '0'
node = arcus_node(ip, port)
node.code = code
results.append(node)
def get_arcus_node_all(self):
children = self.zk_children_if_exists(
'/arcus_repl/cache_server_mapping/')
children += self.zk_children_if_exists('/arcus/cache_server_mapping/')
ret = []
threads = []
#print(children)
for child in children:
th = threading.Thread(target=self._get_arcus_node,
args=(child, ret))
th.start()
threads.append(th)
for th in threads:
th.join()
return ret
def _get_arcus_meta(self, child, results):
data, stat, children = self.zk_read('/arcus/meta/' + child)
results[child] = [data.decode('utf-8'), stat]
def get_arcus_meta_all(self):
if self.zk.exists('/arcus/meta') == None:
self.zk.create('/arcus/meta', b'arcus meta info')
children = self.zk.get_children('/arcus/meta')
print('# children')
print(children)
threads = []
ret = {}
#print(children)
for child in children:
th = threading.Thread(target=self._get_arcus_meta,
args=(child, ret))
th.start()
threads.append(th)
for th in threads:
th.join()
return ret
def _match_code_and_nodes(self, code, cache, meta):
#repl case
children = self.zk_children_if_exists('/arcus_repl/cache_list/' + code)
children += self.zk_children_if_exists('/arcus/cache_list/' + code)
for child in children:
tmp = child.split('^', 2) # remove repl info
if len(tmp) == 3:
child = tmp[2]
addr, name = child.split('-')
try:
node = self.arcus_node_map[addr]
except KeyError:
print('%s of %s is not defined in cache_server_mapping' %
(addr, code))
ip, port = addr.split(':')
node = arcus_node(ip, port)
node.noport = True
node.active = True
cache.active_node.append(node)
for node in cache.node:
if node.active == False:
cache.dead_node.append(node)
if code in meta:
cache.meta = meta[code]
def load_all(self):
codes = self.get_arcus_cache_list()
for code in codes:
cache = arcus_cache(self.address, code)
self.arcus_cache_map[code] = cache
print('# get_arcus_node_all()')
nodes = self.get_arcus_node_all()
print('# done')
for node in nodes:
self.arcus_node_map[node.ip + ":" + node.port] = node
self.arcus_cache_map[node.code].node.append(node)
# meta info
print('# get_arcus_meta_all()')
meta = self.get_arcus_meta_all()
print('# done')
print('# match code & nodes')
threads = []
for code, cache in self.arcus_cache_map.items():
th = threading.Thread(target=self._match_code_and_nodes,
args=(code, cache, meta))
th.start()
threads.append(th)
for th in threads:
th.join()
print('#done')
if 'zookeeper' in meta:
self.meta = meta['zookeeper']
def _callback(self, event):
child_list = self.zk.get_children(event.path)
cloud = os.path.basename(event.path)
cache = self.arcus_cache_map[cloud]
event_list = {'created': [], 'deleted': []}
current = {}
print('##### active node')
print(cache.active_node)
children = []
for child in child_list:
addr = child.split('-')[0]
children.append(addr)
print('#### children')
print(children)
for node in cache.active_node:
current[node.ip + ':' + node.port] = True
print('##### current')
print(current)
for node in cache.active_node:
addr = node.ip + ':' + node.port
if addr not in children:
event_list['deleted'].append(addr)
cache.active_node.remove(node)
for child in children:
if child not in current:
event_list['created'].append(child)
ip, port = child.split(':')
node = arcus_node(ip, port)
cache.active_node.append(node)
print('####### result')
print(cache.active_node)
self.callback(event, event_list)
children = self.zk.get_children(event.path, watch=self._callback)
def watch(self, callback):
self.callback = callback
for code, cache in self.arcus_cache_map.items():
children = self.zk_children_if_exists('/arcus/cache_list/' + code,
watch=self._callback)
children += self.zk_children_if_exists('/arcus_repl/cache_list/' +
code,
watch=self._callback)
alphadict={"@": "atr" , "?":"qm" , "#":"hash" ,"$":"dollar" , "%":"perc"}
from socket import *
serverPort = 45786
serverSocket = socket(AF_INET,SOCK_STREAM)
serverSocket.bind(('',serverPort))
serverSocket.listen(1)
temp_var1=[]
port_list=[]
name_list=[]
port_list1=[]
name_list1=[]
lis_mixed=[]
list_mixed=[]
change=" "
bk_dict={}
while zk.exists("/masternode") or zk.exists("/masternode1") :
if zk.exists("/specnode") :
zk.set("/specnode","45786 special")
else :
zk.create("/specnode", "45786 special", ephemeral=True)
if zk.exists("/zspec") and change==" " :
t=zk.get("/zspec")
bk_dict=ast.literal_eval(t[0])
change="!!!"
if len(bk_dict)>=len(alphadict):
alphadict=bk_dict
print(alphadict)
else :
pass
connectionSocket, addr = serverSocket.accept()
class ZooKeeper(object):
'''
Class implementing the ZooKeeper interface.
This class uses the facade design pattern to keep common interaction
with the ZooKeeper API simple and consistent for the caller, and
limits coupling between objects. It allows for more complex interactions
by providing direct access to the client connection when needed (though
that is discouraged). It also provides for a convenient entry point for
testing only ZooKeeper interactions.
'''
log = logging.getLogger("zuul.zk.ZooKeeper")
REQUEST_ROOT = '/nodepool/requests'
REQUEST_LOCK_ROOT = "/nodepool/requests-lock"
NODE_ROOT = '/nodepool/nodes'
# Log zookeeper retry every 10 seconds
retry_log_rate = 10
def __init__(self):
'''
Initialize the ZooKeeper object.
'''
self.client = None
self._became_lost = False
self._last_retry_log = 0
def _dictToStr(self, data):
return json.dumps(data).encode('utf8')
def _strToDict(self, data):
return json.loads(data.decode('utf8'))
def _connection_listener(self, state):
'''
Listener method for Kazoo connection state changes.
.. warning:: This method must not block.
'''
if state == KazooState.LOST:
self.log.debug("ZooKeeper connection: LOST")
self._became_lost = True
elif state == KazooState.SUSPENDED:
self.log.debug("ZooKeeper connection: SUSPENDED")
else:
self.log.debug("ZooKeeper connection: CONNECTED")
@property
def connected(self):
return self.client.state == KazooState.CONNECTED
@property
def suspended(self):
return self.client.state == KazooState.SUSPENDED
@property
def lost(self):
return self.client.state == KazooState.LOST
@property
def didLoseConnection(self):
return self._became_lost
def resetLostFlag(self):
self._became_lost = False
def logConnectionRetryEvent(self):
now = time.monotonic()
if now - self._last_retry_log >= self.retry_log_rate:
self.log.warning("Retrying zookeeper connection")
self._last_retry_log = now
def connect(self, hosts, read_only=False, timeout=10.0):
'''
Establish a connection with ZooKeeper cluster.
Convenience method if a pre-existing ZooKeeper connection is not
supplied to the ZooKeeper object at instantiation time.
:param str hosts: Comma-separated list of hosts to connect to (e.g.
127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
:param bool read_only: If True, establishes a read-only connection.
:param float timeout: The ZooKeeper session timeout, in
seconds (default: 10.0).
'''
if self.client is None:
self.client = KazooClient(hosts=hosts,
read_only=read_only,
timeout=timeout)
self.client.add_listener(self._connection_listener)
# Manually retry initial connection attempt
while True:
try:
self.client.start(1)
break
except KazooTimeoutError:
self.logConnectionRetryEvent()
def disconnect(self):
'''
Close the ZooKeeper cluster connection.
You should call this method if you used connect() to establish a
cluster connection.
'''
if self.client is not None and self.client.connected:
self.client.stop()
self.client.close()
self.client = None
def resetHosts(self, hosts):
'''
Reset the ZooKeeper cluster connection host list.
:param str hosts: Comma-separated list of hosts to connect to (e.g.
127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
'''
if self.client is not None:
self.client.set_hosts(hosts=hosts)
def submitNodeRequest(self, node_request, watcher):
'''
Submit a request for nodes to Nodepool.
:param NodeRequest node_request: A NodeRequest with the
contents of the request.
:param callable watcher: A callable object that will be
invoked each time the request is updated. It is called
with two arguments: (node_request, deleted) where
node_request is the same argument passed to this method,
and deleted is a boolean which is True if the node no
longer exists (notably, this will happen on disconnection
from ZooKeeper). The watcher should return False when
further updates are no longer necessary.
'''
node_request.created_time = time.time()
data = node_request.toDict()
path = '{}/{:0>3}-'.format(self.REQUEST_ROOT, node_request.priority)
path = self.client.create(path,
self._dictToStr(data),
makepath=True,
sequence=True,
ephemeral=True)
reqid = path.split("/")[-1]
node_request.id = reqid
def callback(data, stat):
if data:
self.updateNodeRequest(node_request, data)
deleted = (data is None) # data *are* none
return watcher(node_request, deleted)
self.client.DataWatch(path, callback)
def deleteNodeRequest(self, node_request):
'''
Delete a request for nodes.
:param NodeRequest node_request: A NodeRequest with the
contents of the request.
'''
path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
try:
self.client.delete(path)
except kze.NoNodeError:
pass
def nodeRequestExists(self, node_request):
'''
See if a NodeRequest exists in ZooKeeper.
:param NodeRequest node_request: A NodeRequest to verify.
:returns: True if the request exists, False otherwise.
'''
path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
if self.client.exists(path):
return True
return False
def storeNodeRequest(self, node_request):
'''Store the node request.
The request is expected to already exist and is updated in its
entirety.
:param NodeRequest node_request: The request to update.
'''
path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
self.client.set(path, self._dictToStr(node_request.toDict()))
def updateNodeRequest(self, node_request, data=None):
'''Refresh an existing node request.
:param NodeRequest node_request: The request to update.
:param dict data: The data to use; query ZK if absent.
'''
if data is None:
path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
data, stat = self.client.get(path)
data = self._strToDict(data)
request_nodes = list(node_request.nodeset.getNodes())
for i, nodeid in enumerate(data.get('nodes', [])):
request_nodes[i].id = nodeid
self.updateNode(request_nodes[i])
node_request.updateFromDict(data)
def storeNode(self, node):
'''Store the node.
The node is expected to already exist and is updated in its
entirety.
:param Node node: The node to update.
'''
path = '%s/%s' % (self.NODE_ROOT, node.id)
self.client.set(path, self._dictToStr(node.toDict()))
def updateNode(self, node):
'''Refresh an existing node.
:param Node node: The node to update.
'''
node_path = '%s/%s' % (self.NODE_ROOT, node.id)
node_data, node_stat = self.client.get(node_path)
node_data = self._strToDict(node_data)
node.updateFromDict(node_data)
def lockNode(self, node, blocking=True, timeout=None):
'''
Lock a node.
This should be called as soon as a request is fulfilled and
the lock held for as long as the node is in-use. It can be
used by nodepool to detect if Zuul has gone offline and the
node should be reclaimed.
:param Node node: The node which should be locked.
'''
lock_path = '%s/%s/lock' % (self.NODE_ROOT, node.id)
try:
lock = Lock(self.client, lock_path)
have_lock = lock.acquire(blocking, timeout)
except kze.LockTimeout:
raise LockException("Timeout trying to acquire lock %s" %
lock_path)
# If we aren't blocking, it's possible we didn't get the lock
# because someone else has it.
if not have_lock:
raise LockException("Did not get lock on %s" % lock_path)
node.lock = lock
def unlockNode(self, node):
'''
Unlock a node.
The node must already have been locked.
:param Node node: The node which should be unlocked.
'''
if node.lock is None:
raise LockException("Node %s does not hold a lock" % (node, ))
node.lock.release()
node.lock = None
def lockNodeRequest(self, request, blocking=True, timeout=None):
'''
Lock a node request.
This will set the `lock` attribute of the request object when the
lock is successfully acquired.
:param NodeRequest request: The request to lock.
:param bool blocking: Whether or not to block on trying to
acquire the lock
:param int timeout: When blocking, how long to wait for the lock
to get acquired. None, the default, waits forever.
:raises: TimeoutException if we failed to acquire the lock when
blocking with a timeout. ZKLockException if we are not blocking
and could not get the lock, or a lock is already held.
'''
path = "%s/%s" % (self.REQUEST_LOCK_ROOT, request.id)
try:
lock = Lock(self.client, path)
have_lock = lock.acquire(blocking, timeout)
except kze.LockTimeout:
raise LockException("Timeout trying to acquire lock %s" % path)
except kze.NoNodeError:
have_lock = False
self.log.error("Request not found for locking: %s", request)
# If we aren't blocking, it's possible we didn't get the lock
# because someone else has it.
if not have_lock:
raise LockException("Did not get lock on %s" % path)
request.lock = lock
self.updateNodeRequest(request)
def unlockNodeRequest(self, request):
'''
Unlock a node request.
The request must already have been locked.
:param NodeRequest request: The request to unlock.
:raises: ZKLockException if the request is not currently locked.
'''
if request.lock is None:
raise LockException("Request %s does not hold a lock" % request)
request.lock.release()
request.lock = None
def heldNodeCount(self, autohold_key):
'''
Count the number of nodes being held for the given tenant/project/job.
:param set autohold_key: A set with the tenant/project/job names.
'''
identifier = " ".join(autohold_key)
try:
nodes = self.client.get_children(self.NODE_ROOT)
except kze.NoNodeError:
return 0
count = 0
for nodeid in nodes:
node_path = '%s/%s' % (self.NODE_ROOT, nodeid)
try:
node_data, node_stat = self.client.get(node_path)
except kze.NoNodeError:
# Node got removed on us. Just ignore.
continue
if not node_data:
self.log.warning("Node ID %s has no data", nodeid)
continue
node_data = self._strToDict(node_data)
if (node_data['state'] == zuul.model.STATE_HOLD
and node_data.get('hold_job') == identifier):
count += 1
return count
# Copy of nodepool/zk.py begins here
NODE_ROOT = "/nodepool/nodes"
LAUNCHER_ROOT = "/nodepool/launchers"
def _bytesToDict(self, data):
return json.loads(data.decode('utf8'))
def _launcherPath(self, launcher):
return "%s/%s" % (self.LAUNCHER_ROOT, launcher)
def _nodePath(self, node):
return "%s/%s" % (self.NODE_ROOT, node)
def getRegisteredLaunchers(self):
'''
Get a list of all launchers that have registered with ZooKeeper.
:returns: A list of Launcher objects, or empty list if none are found.
'''
try:
launcher_ids = self.client.get_children(self.LAUNCHER_ROOT)
except kze.NoNodeError:
return []
objs = []
for launcher in launcher_ids:
path = self._launcherPath(launcher)
try:
data, _ = self.client.get(path)
except kze.NoNodeError:
# launcher disappeared
continue
objs.append(Launcher.fromDict(self._bytesToDict(data)))
return objs
def getNodes(self):
'''
Get the current list of all nodes.
:returns: A list of nodes.
'''
try:
return self.client.get_children(self.NODE_ROOT)
except kze.NoNodeError:
return []
def getNode(self, node):
'''
Get the data for a specific node.
:param str node: The node ID.
:returns: The node data, or None if the node was not found.
'''
path = self._nodePath(node)
try:
data, stat = self.client.get(path)
except kze.NoNodeError:
return None
if not data:
return None
d = self._bytesToDict(data)
d['id'] = node
return d
def nodeIterator(self):
'''
Utility generator method for iterating through all nodes.
'''
for node_id in self.getNodes():
node = self.getNode(node_id)
if node:
yield node
]
cdeploy.migrator.main()
sys.argv = argv_bak
print "setting up zookeeper"
wait_for('zookeeper', 2181, 30)
from kazoo.client import KazooClient
SERVICE_ENDPOINTS = CONFIG['host_ports']
print 'connecting to zookeeper'
zk = KazooClient(hosts='zookeeper:2181')
zk.start()
for service, paths in SERVICE_ENDPOINTS.items():
for path in paths:
full_path = '/services/cluster_local/' + service + '/' + path
if not zk.exists(full_path):
zk.create(full_path, makepath=True)
zk.stop()
print 'setup complete'
elif cmd == 'START':
if 'mysql' in DEPENDS_ON:
wait_for('mysql', 3306, 30)
wait_for('zookeeper', 2181, 30)
if 'cassandra' in DEPENDS_ON:
wait_for('cassandra', 9042, 30)
run_py('-m', NAME, '--nodaemon', '--flagfile=/home/app/development-flags')
elif cmd == 'START_DEBUG':
# TODO: argparse/dedupe
if 'mysql' in DEPENDS_ON:
wait_for('mysql', 3306, 30)
wait_for('zookeeper', 2181, 30)
class KazooCommandProxy():
def __init__(self, module):
self.module = module
self.zk = KazooClient(module.params['hosts'])
def absent(self):
return self._absent(self.module.params['name'])
def exists(self, znode):
return self.zk.exists(znode)
def list(self):
children = self.zk.get_children(self.module.params['name'])
return True, {
'count': len(children),
'items': children,
'msg': 'Retrieved znodes in path.',
'znode': self.module.params['name']
}
def present(self):
return self._present(self.module.params['name'],
self.module.params['value'])
def get(self):
return self._get(self.module.params['name'])
def shutdown(self):
self.zk.stop()
self.zk.close()
def start(self):
self.zk.start()
def wait(self):
return self._wait(self.module.params['name'],
self.module.params['timeout'])
def _absent(self, znode):
if self.exists(znode):
self.zk.delete(znode, recursive=self.module.params['recursive'])
return True, {'changed': True, 'msg': 'The znode was deleted.'}
else:
return True, {'changed': False, 'msg': 'The znode does not exist.'}
def _get(self, path):
if self.exists(path):
value, zstat = self.zk.get(path)
stat_dict = {}
for i in dir(zstat):
if not i.startswith('_'):
attr = getattr(zstat, i)
if isinstance(attr, (int, str)):
stat_dict[i] = attr
result = True, {
'msg': 'The node was retrieved.',
'znode': path,
'value': value,
'stat': stat_dict
}
else:
result = False, {'msg': 'The requested node does not exist.'}
return result
def _present(self, path, value):
if self.exists(path):
(current_value, zstat) = self.zk.get(path)
if value != current_value:
self.zk.set(path, value)
return True, {
'changed': True,
'msg': 'Updated the znode value.',
'znode': path,
'value': value
}
else:
return True, {
'changed': False,
'msg': 'No changes were necessary.',
'znode': path,
'value': value
}
else:
self.zk.create(path, value, makepath=True)
return True, {
'changed': True,
'msg': 'Created a new znode.',
'znode': path,
'value': value
}
def _wait(self, path, timeout, interval=5):
lim = time.time() + timeout
while time.time() < lim:
if self.exists(path):
return True, {
'msg': 'The node appeared before the configured timeout.',
'znode': path,
'timeout': timeout
}
else:
time.sleep(interval)
return False, {
'msg': 'The node did not appear before the operation timed out.',
'timeout': timeout,
'znode': path
}
# -*- coding: utf-8 -*-
import time
import glob
import sys
import os
from kazoo.client import KazooClient
import logging
logging.basicConfig()
if __name__ == "__main__":
print("begin.")
zk = KazooClient(hosts='172.10.3.111:2181')
zk.start()
mypath = "/my/favorite"
#zk.ensure_path(mypath)
if zk.exists(mypath):
print(mypath, "exist")
data, stat = zk.get(mypath)
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
else:
print(mypath, "not exist")
zk.stop()
print("exit.")
def do_operate(arglist):
def print_usage():
print 'Usage:'
print '\t%s add --target Cluster1' % sys.argv[0]
print '\t%s delete --target Cluster1' % sys.argv[0]
print '\t%s root --target Cluster1' % sys.argv[0]
print ''
def parse_op_args(arglist):
global target
try:
if len(arglist) <= 0:
print_usage()
sys.exit(0)
opts, args = getopt.getopt(arglist, "", ["help", "target="])
for opt, arg in opts:
if opt in ("--target"):
target = str(arg)
elif opt in ("--help"):
print_usage()
sys.exit(0)
else:
log.error('Invalid arg: %s' % opt)
print_usage()
sys.exit(-1)
if not target:
log.error("target host/cluster must be set with --target")
print_usage()
sys.exit(-1)
except getopt.GetoptError as err:
log.error("%s" % err)
print_usage()
sys.exit(-1)
if len(arglist) <= 0:
print_usage()
sys.exit(0)
subcmd = arglist[0]
parse_op_args(arglist[1:])
log.info('Running operate %s target = %s' % (subcmd, target))
zk_client = None
try:
clusternode = target
conf_ret = read_conf(clusternode)
if conf_ret != 0:
raise Exception('Invalid target param, clusternode:%s!' %
clusternode)
# create node
zk_client = KazooClient(hosts=zk_servers)
zk_client.start()
zk_node = zk_path + clusternode
exist_flag = zk_client.exists(zk_node)
if subcmd == 'root':
zk_node = zk_path
subcmd = 'delete'
if subcmd == 'add':
if not exist_flag:
zk_client.ensure_path(zk_node)
for host in cluster_iplist:
new_node = zk_node + '/' + host
if not zk_client.exists(new_node):
zk_client.create(new_node, b'')
# check host node
children = zk_client.get_children(zk_node)
if len(children) != len(cluster_iplist):
raise Exception('Create cluster:%s node failed!' % clusternode)
log.info('create cluster:%s node success!' % clusternode)
elif subcmd == 'delete':
if exist_flag:
zk_client.delete(zk_node, recursive=True)
log.info('Delete cluster:%s node success!' % clusternode)
# if is_check: # TODO
# zk_client.delete(zk_path,recursive=True)
# log.info('Delete zk_path:%s node success!' % clusternode)
else:
log.info('cluster:%s node not exist!' % clusternode)
else:
raise Exception('Invalid op command %s' % subcmd)
finally:
if zk_client:
zk_client.stop()
class QACBox(object):
def __init__(self, type, name):
self._Type = type
self._Name = "%s%s" % (type, str(name))
self._VName = HOST['name']
self._Path = "/%s/%s/%s" % (self._Type, self._VName, self._Name)
self._ZKHost = ZKSERVERS['hosts']
self._IP = HOST['ip']
self._Port = HOST['port']
self._ID = int(name)
self._ZK = None
self._Chatbot = None
self._Conn = None
self._KGname = None
logger.debug('create a %s box named %s in VM %s.' % (self._Type, self._Name, self._VName))
def connectZK(self):
self._ZK = KazooClient(hosts=self._ZKHost)
logger.info('%s is connecting ZK server.' % self._Path)
def getType(self):
return self._Type
def getName(self):
return self._Name
def getVName(self):
return self._VName
def setZK(self,zk):
self._ZK = zk
def getZK(self):
return self._ZK
def startZK(self):
self._ZK.start()
logger.debug('start one connection with ZK server by a %s box named %s in VM %s' % (self._Type, self._Name, self._VName))
def stopZK(self):
self._ZK.stop()
logger.info('stop connection with ZK server by a %s box named %s in VM %s' % (self._Type, self._Name, self._VName))
def InitialBBOXNode(self):
address = "{\"Target\":\"Null\",\"Add\":\"%s:%s/%s\",\"status\":\"0\",\"update_time\":\"%f\"}" % (self._IP, self._Port,self._ID,time.time())
address = address.encode('utf-8')
if self._ZK.exists(self._Path):
self._ZK.delete(self._Path, recursive=True)
self._ZK.create(self._Path, address, None, ephemeral=False, sequence=False, makepath=True)
logger.info('create a B box node: %s, data: %s' % (self._Path, address.decode("utf-8")))
vmknode = "/%s/%s" % (HOST['name'], CBOX['Bk'])
if self._ZK.exists(vmknode):
self._ZK.delete(vmknode, recursive=True)
self._ZK.create(vmknode, None, None, ephemeral=False, sequence=False, makepath=True)
logger.info('create a VM/k node: %s, not data. ' % vmknode)
def updateselfZKBBox(self, status):
# updating 'oneabox' B tpye node data in ZK Server
#oneabox_path = "/%s/%s/%s" % (self._Type, self._VName, oneabox)
oneabox_path = self._Path
data, _ = self._ZK.get(oneabox_path)
oneabox_address = "{\"Target\":\"%s\",\"Add\":\"%s\",\"status\":\"%s\",\"update_time\":\"%f\"}" % (eval(data.decode("utf-8"))['Target'], eval(data.decode("utf-8"))['Add'],str(status),time.time())
oneabox_address = oneabox_address.encode('utf-8')
self._ZK.set(oneabox_path, oneabox_address)
logger.info('success update B Box node %s with data %s.' % (oneabox_path, oneabox_address))
def updateselfZKBBoxTarget(self, target, status):
# updating 'oneabox' B tpye node data in ZK Server
#oneabox_path = "/%s/%s/%s" % (self._Type, self._VName, oneabox)
oneabox_path = self._Path
data, _ = self._ZK.get(oneabox_path)
oneabox_address = "{\"Target\":\"%s\",\"Add\":\"%s\",\"status\":\"%s\",\"update_time\":\"%f\"}" % (target, eval(data.decode("utf-8"))['Add'],str(status),time.time())
oneabox_address = oneabox_address.encode('utf-8')
self._ZK.set(oneabox_path, oneabox_address)
logger.info('success update B')
def stop(self):
self.stopZK()
def initcbot(self, kbname, onlyread=True):
try:
self._Chatbot = ChatBot(self._Name,
storage_adapter=CHATTERBOT['storage_adapter'],
# logic_adapters=[{
# 'import_path': 'chatterbot.logic.BestMatch'
# }],
filters=['chatterbot.filters.RepetitiveResponseFilter'],
database_uri=KGDATABASES['database_uri'],
database='ai_%s' % kbname,
read_only=onlyread,)
self._KGname = kbname
except Exception:
logger.info('Failure to initialize Chatterbot.', exc_info=True)
def get_response(self, inputstatement):
return self._Chatbot.get_response(inputstatement)
def get_similar(self):
# Note that get_similar must be called after get_response called, or return None
return self._Chatbot.similar
# def preprocess(self,sentence):
# if ISFENCI:
# return fenci.seg_sentence(sentence)
# else:
# return sentence
def preprocess(self, sentence, companyid=None):
if ISFENCI:
if ISSYMS:
return fenci.symp_sentence(sentence, companyid)
else:
return sentence
else:
return sentence
# -*- coding: utf-8 -*-
######################################################################
# #
# 创建时间：2019年09月16日 #
# 创 建 者：wyl #
# 功能：写入hive测试脚本 #
# #
######################################################################
import sys, time, datetime
import traceback
from kazoo.client import KazooClient
import hdfs
nameNodeA = [b'phmnn1.bigdata.com', 'phmnn1.bigdata.com']
nameNodeB = [b'phmnn2.bigdata.com', 'phmnn2.bigdata.com']
zooQuorum = '10.73.95.19:2181,10.73.95.20:2181,10.73.95.21:2181'
path = '/hadoop-ha/PHMBIGDATA/ActiveBreadCrumb'
nameNodeADevelop = [b'spark.bigdevelop.com', 'spark.bigdevelop.com']
nameNodeBDevelop = [b'nnredis.bigdevelop.com', 'nnredis.bigdevelop.com']
zooQuorumDevelop = 'datanode1.bigdevelop.com:2181,schedule.bigdevelop.com:2181,datanode2.bigdevelop.com:2181'
pathDevelop = '/hadoop-ha/hanamenode/ActiveBreadCrumb'
zk = KazooClient(hosts=zooQuorum)
zk.start()
if zk.exists(path):
data = zk.get(path)
print(data)