class ZookeeperHostSource(HostSource):
def __init__(self, config):
self.zk = KazooClient(config.hosts.zookeeper.connection_string)
self.zk.start()
credentials = ":".join(
(config.hosts.zookeeper.username, config.hosts.zookeeper.password))
self.zk.add_auth("digest", credentials)
self.retry = KazooRetry(max_tries=3)
def get_all_hosts(self):
try:
return self.retry(self.zk.get_children, "/server")
except KazooException as e:
raise HostLookupError("zk host enumeration failed: %r", e)
def should_host_be_alive(self, host_name):
try:
host_root = "/server/" + host_name
state = self.retry(self.zk.get, host_root + "/state")[0]
if state in ("kicking", "unhealthy"):
return False
is_autoscaled = self.retry(self.zk.exists, host_root + "/asg")
is_running = self.retry(self.zk.exists, host_root + "/running")
return not is_autoscaled or is_running
except NoNodeException:
return False
except KazooException as e:
raise HostLookupError("zk host aliveness check failed: %r", e)
def shut_down(self):
self.zk.stop()
class ZookeeperHostSource(HostSource):
def __init__(self, config):
self.zk = KazooClient(config.hosts.zookeeper.connection_string)
self.zk.start()
credentials = ":".join((config.hosts.zookeeper.username,
config.hosts.zookeeper.password))
self.zk.add_auth("digest", credentials)
self.retry = KazooRetry(max_tries=3)
def get_all_hosts(self):
try:
return self.retry(self.zk.get_children, "/server")
except KazooException as e:
raise HostLookupError("zk host enumeration failed: %r", e)
def should_host_be_alive(self, host_name):
try:
host_root = "/server/" + host_name
state = self.retry(self.zk.get, host_root + "/state")[0]
if state in ("kicking", "unhealthy"):
return False
is_autoscaled = self.retry(self.zk.exists, host_root + "/asg")
is_running = self.retry(self.zk.exists, host_root + "/running")
return not is_autoscaled or is_running
except NoNodeException:
return False
except KazooException as e:
raise HostLookupError("zk host aliveness check failed: %r", e)
def shut_down(self):
self.zk.stop()
def init_hierarchy(hosts, hierarchy, users, auth):
zkcli = KazooClient(hosts)
zkcli.start()
scheme, name, passw = auth
zkcli.add_auth(scheme, name + ':' + passw)
def _init_hierarchy(hierarchy, parent_path):
if len(hierarchy) == 0:
return
for node, attr_children in hierarchy.items():
val = attr_children.get('__val__', {})
val = utfjson.dump(val)
acl = attr_children.get('__acl__')
path = _init_node(zkcli, parent_path, node, val, acl, users)
children = {
k: v
for k, v in attr_children.items()
if k not in ('__val__', '__acl__')
}
_init_hierarchy(children, path)
_init_hierarchy(hierarchy, '/')
close_zk(zkcli)
def _loop_acquire(self, n, ident):
zk = KazooClient(hosts='127.0.0.1:21811')
zk.start()
scheme, name, passw = zk_test_auth
zk.add_auth(scheme, name + ':' + passw)
for ii in range(n):
l = zkutil.ZKLock('foo_name', zkclient=zk)
with l:
self.total += 1
self.counter += 1
self.assertTrue(self.counter == 1)
time.sleep(0.01)
self.counter -= 1
dd("id={ident:0>2} n={ii:0>2} got and released lock: {holder}".format(
ident=ident,
ii=ii,
holder=l.lock_holder))
zk.stop()
def get_zk():
# create : acl=CREATOR_ALL_ACL
# zk = KazooClient(hosts=zk_host, auth_data=[(scheme, credential)])
zk = KazooClient(hosts=zk_host)
zk.start()
zk.add_auth(scheme, credential)
return zk
def init_hierarchy(hosts, hierarchy, users, auth):
zkcli = KazooClient(hosts)
zkcli.start()
scheme, name, passw = auth
zkcli.add_auth(scheme, name + ':' + passw)
def _init_hierarchy(hierarchy, parent_path):
if len(hierarchy) == 0:
return
for node, attr_children in hierarchy.items():
val = attr_children.get('__val__', {})
val = utfjson.dump(val)
acl = attr_children.get('__acl__')
path = _init_node(zkcli, parent_path, node, val, acl, users)
children = {k: v
for k, v in attr_children.items()
if k not in ('__val__', '__acl__')
}
_init_hierarchy(children, path)
_init_hierarchy(hierarchy, '/')
close_zk(zkcli)
class ShellTestCase(unittest.TestCase):
""" base class for all tests """
def setUp(self):
"""
make sure that the prefix dir is empty
"""
self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests")
self.zk_host = os.getenv("ZKSHELL_ZK_HOST", "localhost:2181")
self.username = os.getenv("ZKSHELL_USER", "user")
self.password = os.getenv("ZKSHELL_PASSWD", "user")
self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=")
self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "secret")
self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest")
self.client = KazooClient(self.zk_host, 5)
self.client.start()
self.client.add_auth(self.scheme, self.auth_id)
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.create(self.tests_path, str.encode(""))
self.output = StringIO()
self.shell = Shell([self.zk_host], 5, self.output, setup_readline=False, async=False)
# Create an empty test dir (needed for some tests)
self.temp_dir = tempfile.mkdtemp()
@property
def auth_id(self):
return "%s:%s" % (self.username, self.password)
@property
def auth_digest(self):
return "%s:%s" % (self.username, self.digested_password)
def tearDown(self):
self.output = None
self.shell = None
if os.path.isdir(self.temp_dir):
shutil.rmtree(self.temp_dir)
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.stop()
###
# Helpers.
##
def create_compressed(self, path, value):
"""
ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client
to create a znode with zlib compressed content.
"""
compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value)
self.client.create(path, compressed, makepath=True)
def auth_kazoo(username, password):
auth_client = KazooClient(hosts=ZOO_HOSTS)
try:
auth_client.start(timeout=5)
auth_client.add_auth('digest', '{username}:{password}'.format(username=username, password=password))
yield auth_client
finally:
auth_client.stop()
def make_owning_zkclient(hosts, auth):
zkclient = KazooClient(hosts=hosts)
zkclient.start()
if auth is not None:
scheme, name, passw = auth
zkclient.add_auth(scheme, name + ':' + passw)
return zkclient
def make_owning_zkclient(hosts, auth):
zkclient = KazooClient(hosts=hosts)
zkclient.start()
if auth is not None:
scheme, name, passw = auth
zkclient.add_auth(scheme, name + ':' + passw)
return zkclient
def zk_client(host, scheme, credential):
""" returns a connected (and possibly authenticated) ZK client """
if not re.match(r".*:\d+$", host):
host = "%s:%d" % (host, DEFAULT_ZK_PORT)
client = KazooClient(hosts=host)
client.start()
if scheme != "":
client.add_auth(scheme, credential)
return client
def get_client(args):
client = KazooClient(connection_retry=3,
command_retry=3,
timeout=1,
hosts=args.hosts,
use_ssl=args.secure,
certfile=args.cert,
keyfile=args.key,
ca=args.ca)
client.start()
if (args.user and args.password):
client.add_auth('digest', f'{args.user}:{args.password}')
return client
def zk_client(host, scheme, credential):
""" returns a connected (and possibly authenticated) ZK client """
if not re.match(r".*:\d+$", host):
host = "%s:%d" % (host, DEFAULT_ZK_PORT)
client = KazooClient(hosts=host)
client.start()
if scheme != "":
client.add_auth(scheme, credential)
return client
class ZKWrap(object):
def __init__(self):
self.zk_hosts = settings.ZK_HOSTS
self.zk_root = settings.ZK_ROOT
self.zk_timeout = settings.ZK_TIMEOUT
self.zk_servers = settings.ZK_SERVERS
self.zk_clients = settings.ZK_CLIENTS
self.zk_configs = settings.ZK_CONFIGS
self.zk_statistics = settings.ZK_STATISTICS
self.zk_username = settings.ZK_USERNAME
self.zk_password = settings.ZK_PASSWORD
self.zk_flush = time.time()
self.lock = threading.Lock()
self.resource = {}
self.zk_harpc_acl = make_digest_acl(username=self.zk_username, password=self.zk_password, all=True)
self.zk_anyone_acl = make_acl(scheme='world', credential='anyone', read=True)
def children_callback(self, children):
print '****', children
def get_children(self, path):
return self.zk.get_children(path)
def get(self, path):
return self.zk.get(path)
def get_acls(self, path):
return self.zk.get_acls(path)
def load_zk(self):
self.init_connect()
def get_resource(self):
return self.resource
def init_connect(self):
try:
self.zk = KazooClient(hosts=self.zk_hosts, timeout=self.zk_timeout)
self.zk.start(timeout=self.zk_timeout)
self.zk.add_auth(scheme='digest', credential=self.zk_username + ':' + self.zk_password)
logger.info("connect to zk servers")
except Exception, e:
logger.error('manage.ZKWrap.init_connect:' + getExceptionDesc(e))
return False
def get_zk():
global _zk
if _zk is None:
_zk = KazooClient(
app.config['ZK_CONNECTION_STRING'],
default_acl=(
# grants read permissions to anyone.
make_acl('world', 'anyone', read=True),
# grants all permissions to the creator of the node.
make_acl('auth', '', all=True)
)
)
_zk.start()
_zk.add_auth('digest', jones_credential)
_zk.DataWatch('/services', func=ensure_root)
return _zk
def connect_to_zookeeper(hostlist, credentials):
"""Create a connection to the ZooKeeper ensemble.
If authentication credentials are provided (as a two-tuple: username,
password), we will ensure that they are provided to the server whenever we
establish a connection.
"""
client = KazooClient(hostlist, timeout=5, max_retries=3)
# convenient helper function for making credentials
client.make_acl = functools.partial(make_digest_acl, *credentials)
client.start()
client.add_auth("digest", ":".join(credentials))
return client
def connect_to_zookeeper(hostlist, credentials):
"""Create a connection to the ZooKeeper ensemble.
If authentication credentials are provided (as a two-tuple: username,
password), we will ensure that they are provided to the server whenever we
establish a connection.
"""
client = KazooClient(hostlist,
timeout=5,
max_retries=3)
# convenient helper function for making credentials
client.make_acl = functools.partial(make_digest_acl, *credentials)
client.connect()
client.add_auth("digest", ":".join(credentials))
return client
class ShellTestCase(unittest.TestCase):
""" base class for all tests """
@classmethod
def setUpClass(cls):
get_global_cluster().start()
def setUp(self):
"""
make sure that the prefix dir is empty
"""
self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests")
self.zk_hosts = ",".join(server.address
for server in get_global_cluster())
self.username = os.getenv("ZKSHELL_USER", "user")
self.password = os.getenv("ZKSHELL_PASSWD", "user")
self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD",
"F46PeTVYeItL6aAyygIVQ9OaaeY=")
self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "test")
self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest")
self.client = KazooClient(self.zk_hosts, 5)
self.client.start()
self.client.add_auth(self.scheme, self.auth_id)
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.create(self.tests_path, str.encode(""))
self.output = XStringIO()
self.shell = Shell([self.zk_hosts],
5,
self.output,
setup_readline=False,
asynchronous=False)
# Create an empty test dir (needed for some tests)
self.temp_dir = tempfile.mkdtemp()
@property
def auth_id(self):
return "%s:%s" % (self.username, self.password)
@property
def auth_digest(self):
return "%s:%s" % (self.username, self.digested_password)
def tearDown(self):
if self.output is not None:
self.output.close()
self.output = None
if self.shell is not None:
self.shell._disconnect()
self.shell = None
if os.path.isdir(self.temp_dir):
shutil.rmtree(self.temp_dir)
if self.client is not None:
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.stop()
self.client.close()
self.client = None
###
# Helpers.
##
def create_compressed(self, path, value):
"""
ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client
to create a znode with zlib compressed content.
"""
compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value)
self.client.create(path, compressed, makepath=True)
def test_init_hierarchy(self):
auth = ('digest', 'aa', 'pw_aa')
hosts = '127.0.0.1:21811'
users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'}
hierarchy = {
'node1':
{
'__val__': 'node1_val',
'__acl__': {'aa': 'cdrwa', 'bb': 'rw'},
'node11':
{
'__val__': 'node11_val',
'__acl__': {'aa': 'cdrwa', 'cc': 'r'},
},
'node12':
{
'__val__': 'node12_val',
'node121': {'__val__': 'node121_val'}
},
'node13':
{
'__acl__': {'aa': 'cdrwa'}
}
},
'node2':
{
'__val__': 'node2_val',
'node21': {'__val__': 'node21_val'},
'node22': {'__acl__': {'aa': 'rwa'}}
},
'node3':
{
'__acl__': {'aa': 'carw', 'cc': 'r'},
'node31': {'node311': {'node3111': {}, 'node3112': {}}}
}
}
zkutil.init_hierarchy(hosts, hierarchy, users, auth)
zkcli = KazooClient(hosts)
zkcli.start()
zkcli.add_auth('digest', 'aa:pw_aa')
expected_nodes = (
('/node1', '"node1_val"', [('digest', 'aa', 'cdrwa'),
('digest', 'bb', 'rw')], set(['node11', 'node12', 'node13'])),
('/node1/node11', '"node11_val"',
[('digest', 'aa', 'cdrwa'), ('digest', 'cc', 'r')], set([])),
('/node1/node12', '"node12_val"',
[('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set(['node121'])),
('/node1/node12/node121', '"node121_val"',
[('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set([])),
('/node1/node13', '{}', [('digest', 'aa', 'cdrwa')], set([])),
('/node2', '"node2_val"',
[('world', 'anyone', 'cdrwa')], set(['node21', 'node22'])),
('/node2/node21', '"node21_val"',
[('world', 'anyone', 'cdrwa')], set([])),
('/node2/node22', '{}', [('digest', 'aa', 'rwa')], set([])),
('/node3', '{}', [('digest', 'aa', 'rwca'),
('digest', 'cc', 'r')], set(['node31'])),
('/node3/node31', '{}', [('digest', 'aa', 'rwca'),
('digest', 'cc', 'r')], set(['node311'])),
('/node3/node31/node311', '{}',
[('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set(['node3111', 'node3112'])),
('/node3/node31/node311/node3111', '{}',
[('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set([])),
('/node3/node31/node311/node3112', '{}',
[('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set([])),
)
for node, val, acl, children in expected_nodes:
actual_acl = zkutil.parse_kazoo_acl(zkcli.get_acls(node)[0])
self.assertEqual(val, zkcli.get(node)[0])
self.assertEqual(acl, actual_acl)
self.assertEqual(children, set(zkcli.get_children(node)))
zkcli.stop()
sentry = Sentry(app)
jones_credential = make_digest_acl_credential(
'Jones', app.config['ZK_DIGEST_PASSWORD']
)
zk = KazooClient(
app.config['ZK_CONNECTION_STRING'],
default_acl=(
# grants read permissions to anyone.
make_acl('world', 'anyone', read=True),
# grants all permissions to the creator of the node.
make_acl('auth', '', all=True)
)
)
zk.connect()
zk.add_auth('digest', jones_credential)
zk.ensure_path('/services')
def request_wants(t):
types = ['text/plain', 'text/html', 'application/json']
assert t in types
best = request.accept_mimetypes \
.best_match(types)
return best == t
@app.template_filter()
def as_json(d, indent=None):
return Markup(json.dumps(d, indent=indent))
def main():
module = AnsibleModule(argument_spec=MODULE_ARGUMENTS)
host_a = module.params.get('host', None)
authmethod_a = module.params.get('authmethod', None)
type_a = module.params.get('type', None)
dir_a = module.params.get('dir', None)
acl_a = module.params.get('acl', None)
value_a = module.params.get('value', '')
# return error if library is not installed
if not KAZOO_IMPORTED:
module.fail_json(msg='Kazoo python library is not installed. Use "pip install kazoo" before running script')
# try connect to zookeeper server
try:
client = KazooClient(host_a)
client.start()
except:
module.fail_json(msg='Could not connect to the zookeeper server')
# add auth method to client
if authmethod_a is not None:
try:
method, username, password = [param.replace(':','') for param in authmethod_a.split(':')]
if not client.add_auth(method, username + ':' + password):
module.fail_json(msg='Auth failed')
except AuthFailedError:
module.fail_json(msg='Could not auth as specified user')
except:
module.fail_json(msg='Param authmethod is not correct')
# make sure one of setting acls is also used in authmethod and have admin access or is set to world:anyone
# access can be empty so this acl will be deleted
if acl_a is not None:
acl_list = generate_acl_list(module, acl_a)
else:
acl_list = None
# failing if path
if match('^(/[^ /]+)+$', dir_a) is None:
module.fail_json(msg='Specified directory does not match zookeeper format')
changed = False
# create and update acl for znode
if type_a == 'create':
# check if znode parent exists
if not client.exists(get_parent(dir_a)):
module.fail_json(msg='Znode {0} parent doesnt exists'.format(dir_a))
changed = create_dir(module, client, dir_a, acl_list, value_a if value_a is not None else '')
elif type_a == 'create_r':
path = [a.replace('/', '') for a in dir_a.split('/') if a is not '' and a is not None]
dir = ''
for id in range(0, len(path)):
dir += '/{0}'.format(path[id])
changed = create_dir(module, client, dir, acl_list) or changed
# selected type does not match options
else:
module.fail_json(msg='Unknown option: {0}'.format(type_a))
client.stop()
client.close()
module.exit_json(changed=changed, msg='Znode and acls match')
def test_export_hierarchy(self):
auth = ('digest', 'aa', 'pw_aa')
hosts = '127.0.0.1:21811'
users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'}
hierarchy = {
'node0': {
'__val__': '',
'__acl__': {'aa': 'cdrwa', 'bb': 'rw'}},
'node1': {
'__val__': 'node1_val',
'__acl__': {'aa': 'cdrwa', 'bb': 'rw'},
'node11': {
'__val__': 'node11_val',
'__acl__': {'aa': 'cdrwa', 'cc': 'r'},
},
'node12': {
'__val__': 'node12_val',
'node121': {'__val__': 'node121_val'}
},
'node13': {
'__acl__': {'aa': 'cdrwa'}
}
},
'node2': {
'__val__': 'node2_val',
'node21': {'__val__': 'node21_val'},
'node22': {'__acl__': {'aa': 'rwa'}}
},
'node3': {
'__acl__': {'aa': 'carw', 'cc': 'r'},
'node31': {
'node311': {
'node3111': {},
'node3112': {}
}
}
}
}
zkutil.zkutil.init_hierarchy(hosts, hierarchy, users, auth)
zkcli = KazooClient(hosts)
zkcli.start()
zkcli.add_auth('digest', 'aa:pw_aa')
invalid_zkpath_cases = ('a', 'a/', 'a/b')
for zkpath in invalid_zkpath_cases:
with self.assertRaises(zkutil.zkutil.ZkPathError):
zkutil.zkutil.export_hierarchy(zkcli, zkpath)
valid_cases = (
('/', {'__acl__': {u'anyone': 'cdrwa'},
'__val__': '',
u'node0': {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '""'},
u'node1': {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '"node1_val"',
u'node11': {
'__acl__': {u'aa': 'cdrwa', u'cc': 'r'},
'__val__': '"node11_val"'},
u'node12': {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '"node12_val"',
u'node121': {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '"node121_val"'}},
u'node13': {
'__acl__': {u'aa': 'cdrwa'},
'__val__': '{}'}},
u'node2': {
'__acl__': {u'anyone': 'cdrwa'},
'__val__': '"node2_val"',
u'node21': {
'__acl__': {u'anyone': 'cdrwa'},
'__val__': '"node21_val"'},
u'node22': {'__acl__': {u'aa': 'rwa'}, '__val__': '{}'}},
u'node3': {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}',
u'node31': {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}',
u'node311': {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}',
u'node3111': {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}'},
u'node3112': {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}'}}}},
u'zookeeper': {
'__acl__': {u'anyone': 'cdrwa'},
'__val__': '',
u'quota': {
'__acl__': {u'anyone': 'cdrwa'},
'__val__': ''}}}),
('/node0', {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '""'}),
('/node1', {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '"node1_val"',
u'node11': {
'__acl__': {u'aa': 'cdrwa', u'cc': 'r'},
'__val__': '"node11_val"'},
u'node12': {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '"node12_val"',
u'node121': {
'__acl__': {u'aa': 'cdrwa', u'bb': 'rw'},
'__val__': '"node121_val"'}},
u'node13': {
'__acl__': {u'aa': 'cdrwa'},
'__val__': '{}'}}),
('/node1/node11', {
'__acl__': {u'aa': 'cdrwa', u'cc': 'r'},
'__val__': '"node11_val"'}),
('/node2', {
'__acl__': {u'anyone': 'cdrwa'},
'__val__': '"node2_val"',
u'node21': {
'__acl__': {u'anyone': 'cdrwa'},
'__val__': '"node21_val"'},
u'node22': {
'__acl__': {u'aa': 'rwa'},
'__val__': '{}'}}),
('/node3', {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}',
u'node31': {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}',
u'node311': {
'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}',
u'node3111': {'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}'},
u'node3112': {'__acl__': {u'aa': 'rwca', u'cc': 'r'},
'__val__': '{}'}}}}))
for path, expected_rst in valid_cases:
rst = zkutil.export_hierarchy(zkcli, path)
self.assertEqual(rst, expected_rst)
zkcli.stop()
app.config.from_envvar('JONES_SETTINGS', silent=True)
if 'SENTRY_DSN' in app.config:
sentry = Sentry(app)
jones_credential = make_digest_acl_credential('Jones',
app.config['ZK_DIGEST_PASSWORD'])
zk = KazooClient(
app.config['ZK_CONNECTION_STRING'],
default_acl=(
# grants read permissions to anyone.
make_acl('world', 'anyone', read=True),
# grants all permissions to the creator of the node.
make_acl('auth', '', all=True)))
zk.start()
zk.add_auth('digest', jones_credential)
@zk.DataWatch('/services')
def ensure_root(data, stat):
if not data:
zk.ensure_path('/services')
def request_wants(t):
types = ['text/plain', 'text/html', 'application/json']
assert t in types
best = request.accept_mimetypes \
.best_match(types)
return best == t
class ZkOperation(object):
def __init__(self, zk_hosts, zk_root):
self.zk = KazooClient(zk_hosts)
self.root = zk_root
self.tasks = set()
self.event = threading.Event()
def start(self):
if self.zk.exists:
self.zk.start()
self.zk.add_auth('digest', 'publish:publish')
if self.zk.connected:
self.zk.ensure_path(self.root)
def is_job_exist(self, job_id):
if job_id == '':
raise Exception('job_id is ""')
node = self.root + '/jobs/' + job_id
return self.zk.exists(node)
def check_task_status(self, path):
if path == '':
raise Exception('path is ""')
node = self.root + path
data, _ = self.zk.get(node)
return data['Status']
def _is_exist(self, node):
if self.zk.connected and self.zk.exists(node):
if self.zk.exists(node):
return True
else:
return False
else:
logger.error('zk not connected or node is exists')
return False
def _create_node(self, node, value=None):
if value is None:
value = ''
value = json.dumps(value)
if self.zk.connected and not self.zk.exists(node):
self.zk.create(node, makepath=True, value=value.encode())
return True
else:
logger.error('zk not connected or node is exists')
return False
def _update_node(self, node, value):
if self.zk.connected and self.zk.exists(node):
tx = self.zk.transaction()
tx.set_data(node, json.dumps(value).encode())
tx.commit()
return True
else:
logger.error(
'update node failed: zk not connected or node is not exists')
return False
def _get_node(self, node):
if self.zk.connected and self.zk.exists(node):
node_value, _ = self.zk.get(node)
if node_value:
return json.loads(node_value.decode())
else:
return {}
else:
logger.error('zk not connected or node is exists')
return False
def _delete_node(self, node):
if self.zk.connected:
if not self.zk.exists(node):
return True
else:
self.zk.delete(node, recursive=True)
return True
else:
logger.error('zk not connected')
return False
# is exist
def is_exist_signal(self, job_id):
node = '/{}/signal/{}'.format(self.root, job_id)
return self._is_exist(node)
# CREATE
def create_new_job(self, job_id, job_value=None):
if job_value is None:
job_value = ''
if job_id != '':
node = self.root + '/jobs/' + job_id
ret = self._create_node(node, job_value)
return ret
else:
logger.error('job_id is null')
return False
def create_new_target(self, job_id, target, target_value):
node = '/{}/jobs/{}/targets/{}'.format(self.root, job_id, target)
ret = self._create_node(node, target_value)
return ret
def create_new_task(self, job_id, target, task):
node = '/{}/jobs/{}/targets/{}/tasks/{}'.format(
self.root, job_id, target, task['task_id'])
ret = self._create_node(node, task)
return ret
def create_job_signal(self, job_id):
node = '/{}/signal/{}'.format(self.root, job_id)
ret = self._create_node(node, uuid.uuid4().hex)
return ret
# GET
def get_job_info(self, job_id):
job_node = '{}/jobs/{}'.format(self.root, job_id)
job_value, _ = self.zk.get(job_node)
job_info = json.loads(job_value.decode())
return job_info
def get_target_info(self, job_id, target):
target_node = '{}/jobs/{}/targets/{}'.format(self.root, job_id, target)
target_value, _ = self.zk.get(target_node)
target_info = json.loads(target_value.decode())
return target_info
def get_task_info(self, job_id, target, task_id):
task_node = '{}/jobs/{}/targets/{}/tasks/{}'.format(
self.root, job_id, target, task_id)
task_value, _ = self.zk.get(task_node)
task_info = json.loads(task_value.decode())
return task_info
# UPDATE
def update_job_status(self, job_id, task):
if job_id != '' and task is not None:
node = self.root + '/signal/' + job_id
else:
raise Exception('job_id is ""')
if self.zk.connected and self.is_job_exist(job_id):
tx = self.zk.transaction()
tx.set_data(node, task.encode())
tx.commit()
def handler_task(self, job_id, task_id, task_name, task_message, status):
# 为不必传回target, 遍历任务节点
if not self.is_job_exist(job_id):
logger.error("can not find this jobid: {}".format(job_id))
return False
job_node = "{}/jobs/{}/targets".format(self.root, job_id)
for target in self.zk.get_children(job_node):
target_node = "{}/{}/tasks".format(job_node, target)
for task in self.zk.get_children(target_node):
if task == task_id:
task_node = "{}/{}".format(target_node, task)
task_value, _ = self.zk.get(task_node)
new_task_value = json.loads(task_value.decode())
new_task_value['status'] = status
tx = self.zk.transaction()
tx.set_data(task_node, json.dumps(new_task_value).encode())
tx.commit()
task_value, _ = self.zk.get(task_node)
return True
logger.error("can not find this taskid: {} in {}".format(
task_id, job_id))
return False
def get_target_by_taskid(self, job_id, task_id):
if self.is_job_exist(job_id):
node = "{}/jobs/{}/targets".format(self.root, job_id)
for target in self.zk.get_children(node):
path = '{}/{}/tasks'.format(node, target)
for taskid in self.zk.get_children(path):
if taskid == task_id:
return target
return False
else:
logger.error("job is not exist: job_id={}".format(job_id))
def send_signal(self, job_id):
node = '{}/signal/{}'.format(self.root, job_id)
logger.info("send singal: {}".format(job_id))
tx = self.zk.transaction()
tx.set_data(node, uuid.uuid4().bytes)
tx.commit()
# DELETE
def delete_job(self, job_id):
node = "{}/jobs/{}".format(self.root, job_id)
logger.info("delete job: job_id={}".format(job_id))
self._delete_node(node)
def delete_signal(self, job_id):
node = '{}/signal/{}'.format(self.root, job_id)
logger.info("delete singal: {}".format(job_id))
self._delete_node(node)
def delete_target(self, job_id, target):
target_node = '{}/jobs/{}/targets/{}'.format(self.root, job_id, target)
logger.info("delete target: job_id={}, target={}".format(
job_id, target))
self._delete_node(target_node)
def delete_task(self, job_id, target, task_id):
task_node = '{}/jobs/{}/targets/{}/tasks/{}'.format(
self.root, job_id, target, task_id)
logger.info("delete task: job_id ={}, target={}, task_id={}".format(
job_id, target, task_id))
self._delete_node(task_node)
#################################
# CALLBACK
## exsit CALLBACK
def is_exist_callback(self, callback_node):
node = "{}/callback/{}".format(self.root, callback_node)
if self.zk.exists(node):
return True
else:
return False
## INIT CALLBACK
def init_callback_by_jobid(self, job_id):
node = "{}/callback/{}".format(self.root, job_id)
job_callback_value = {
"job_id": job_id,
"status": JobStatus.init.value,
"messages": ""
}
callback = {
"callback_level": CallbackLevel.job.value,
"callback_info": job_callback_value
}
ret = self._create_node(node, callback)
return ret
def init_callback_by_target(self, job_id, target):
node = "{}/callback/{}".format(self.root, job_id + "_" + target)
target_callback_value = {
"job_id": job_id,
"target": target,
"status": TargetStatus.init.value,
"messages": ""
}
callback = {
"callback_level": CallbackLevel.target.value,
"callback_info": target_callback_value
}
ret = self._create_node(node, callback)
return ret
def init_callback_by_taskid(self, job_id, target, task_id, task_name):
node = "{}/callback/{}".format(self.root, task_id)
taskid_callback_value = {
"job_id": job_id,
"target": target,
"task_name": task_name,
"status": JobStatus.init.value,
"messages": "",
}
callback = {
"callback_level": CallbackLevel.task.value,
"callback_info": taskid_callback_value
}
ret = self._create_node(node, callback)
return ret
## GET CALLBACK
def get_callback_info(self, callback):
node = "{}/callback/{}".format(self.root, callback)
if self.zk.exists(node):
node_value = self._get_node(node)
return node_value
else:
return False
## UPDATE CALLBACK
def update_callback_by_jobid(self, job_id, status, messages=None):
node = "{}/callback/{}".format(self.root, job_id)
if not self.zk.exists(node):
return False
node_value = self._get_node(node)
node_value["callback_info"]["status"] = status
if messages is not None:
node_value["callback_info"]["messages"] = messages
ret = self._update_node(node, node_value)
return ret
def update_callback_by_target(self, job_id, target, status, messages=None):
node = "{}/callback/{}".format(self.root, job_id + "_" + target)
if not self.zk.exists(node):
return False
node_value = self._get_node(node)
node_value["callback_info"]["status"] = status
if messages is not None:
node_value["callback_info"]["messages"] = messages
ret = self._update_node(node, node_value)
return ret
def update_callback_by_taskid(self, job_id, taskid, status, messages=None):
node = "{}/callback/{}".format(self.root, taskid)
if not self.zk.exists(node):
return False
node_value = self._get_node(node)
node_value["callback_info"]["status"] = status
if messages is not None:
node_value["callback_info"]["messages"] = messages
ret = self._update_node(node, node_value)
return ret
## DELETE CALLBACK
def delete_callback_node(self, callback):
node = "{}/callback/{}".format(self.root, callback)
ret = self._delete_node(node)
if ret:
logger.info(
"delete callback node success: callback={}".format(node))
else:
logger.error("delete callback node fail: callback={}".format(node))
return ret
class TestZKLock(unittest.TestCase):
@classmethod
def setUpClass(cls):
utdocker.pull_image(zk_test_tag)
def setUp(self):
config.zk_acl = zk_test_acl
config.zk_auth = zk_test_auth
self.counter = 0
self.running = True
utdocker.create_network()
utdocker.start_container(
zk_test_name,
zk_test_tag,
env={
"ZOO_MY_ID": 1,
"ZOO_SERVERS": "server.1=0.0.0.0:2888:3888",
},
port_bindings={
2181: 21811,
}
)
self.zk = KazooClient(hosts='127.0.0.1:21811')
self.zk.start()
scheme, name, passw = zk_test_auth
self.zk.add_auth(scheme, name + ':' + passw)
# create lock base dir
acl = zkutil.make_kazoo_digest_acl(zk_test_acl)
self.zk.create('lock/', acl=acl)
self.lck = zkutil.ZKLock('foo_name', zkclient=self.zk)
def tearDown(self):
self.zk.stop()
utdocker.remove_container(zk_test_name)
def _on_conn_change(self, state):
self.lsn_count += 1
def test_bounded_listener(self):
# ensure that adding a bounded listener(self.on_xxx) is ok
self.lsn_count = 0
self.zk.add_listener(self._on_conn_change)
self.zk.add_listener(self._on_conn_change)
self.zk.stop()
self.assertEqual(1, self.lsn_count)
def _loop_acquire(self, n, ident):
zk = KazooClient(hosts='127.0.0.1:21811')
zk.start()
scheme, name, passw = zk_test_auth
zk.add_auth(scheme, name + ':' + passw)
for ii in range(n):
l = zkutil.ZKLock('foo_name', zkclient=zk)
with l:
self.total += 1
self.counter += 1
self.assertTrue(self.counter == 1)
time.sleep(0.01)
self.counter -= 1
dd("id={ident:0>2} n={ii:0>2} got and released lock: {holder}".format(
ident=ident,
ii=ii,
holder=l.lock_holder))
zk.stop()
def test_concurrent(self):
self.running = True
self.total = 0
n_repeat = 40
n_thread = 5
ths = []
for ii in range(n_thread):
t = threadutil.start_daemon_thread(self._loop_acquire, args=(n_repeat, ii,))
ths.append(t)
for th in ths:
th.join()
self.running = False
self.assertEqual(n_repeat * n_thread, self.total)
def test_persistent(self):
l = zkutil.ZKLock('foo_name', ephemeral=False, on_lost=lambda: True)
try:
with l:
l.zkclient.stop()
except ConnectionClosedError:
pass
self.assertRaises(zkutil.LockTimeout, self.lck.acquire, timeout=0.2)
def test_timeout(self):
l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
with l1:
with ututil.Timer() as t:
self.assertRaises(zkutil.LockTimeout, l2.acquire, timeout=0.2)
self.assertAlmostEqual(0.2, t.spent(), places=1)
with ututil.Timer() as t:
self.assertRaises(zkutil.LockTimeout, l2.acquire, timeout=-1)
self.assertAlmostEqual(0.0, t.spent(), delta=0.01)
try:
l2.acquire(timeout=-1)
except zkutil.LockTimeout:
self.fail('timeout<0 should could acquire')
def test_lock_holder(self):
a = zkutil.ZKLock('foo_name', on_lost=lambda: True)
b = zkutil.ZKLock('foo_name', on_lost=lambda: True)
with a:
self.assertEqual((a.identifier, 0), a.lock_holder)
val, zstate = self.zk.get(a.lock_path)
self.assertEqual((val, zstate.version), a.lock_holder)
locked, holder, ver = b.try_acquire()
self.assertFalse(locked)
self.assertEqual((a.identifier, 0), (holder, ver))
self.assertEqual((val, zstate.version), (holder, ver))
def test_watch_acquire(self):
a = zkutil.ZKLock('foo', on_lost=lambda: True)
b = zkutil.ZKLock('foo', on_lost=lambda: True)
# no one locked
n = 0
for holder, ver in a.acquire_loop():
n += 1
self.assertEqual(0, n, 'acquired directly')
# watch node change
it = b.acquire_loop()
holder, ver = it.next()
self.assertEqual((a.identifier, 0), (holder, ver))
self.zk.set(a.lock_path, 'xx')
holder, ver = it.next()
self.assertEqual(('xx', 1), (holder, ver), 'watched node change')
a.release()
try:
holder, ver = it.next()
self.fail('should not have next yield')
except StopIteration:
pass
self.assertTrue(b.is_locked())
def test_try_lock(self):
l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
with l1:
with ututil.Timer() as t:
locked, holder, ver = l2.try_acquire()
self.assertFalse(locked)
self.assertEqual(l1.identifier, holder)
self.assertGreaterEqual(ver, 0)
self.assertAlmostEqual(0.0, t.spent(), delta=0.05)
with ututil.Timer() as t:
locked, holder, ver = l2.try_acquire()
self.assertTrue(locked)
self.assertEqual(l2.identifier, holder)
self.assertEqual(ver, 0)
self.assertAlmostEqual(0.0, t.spent(), delta=0.05)
def test_try_release(self):
l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
released, holder, ver = l1.try_release()
self.assertEqual((True, l1.identifier, -1), (released, holder, ver))
with l2:
released, holder, ver = l1.try_release()
self.assertEqual((False, l2.identifier, 0), (released, holder, ver))
released, holder, ver = l2.try_release()
self.assertEqual((True, l2.identifier, 0), (released, holder, ver))
def test_zk_lost(self):
sess = {'acquired': True}
def watch(state):
dd('zk node state changed to: ', state)
sess['acquired'] = False
self.zk.add_listener(watch)
# test zk close
l = zkutil.ZKLock('foo_name', zkclient=self.zk)
l.acquire()
self.zk.stop()
time.sleep(0.1)
self.assertFalse(sess['acquired'])
# test node delete
sess['acquired'] = True
self.zk.start()
self.zk.add_listener(watch)
l = zkutil.ZKLock('foo_name', zkclient=self.zk)
with l:
time.sleep(0.1)
self.zk.delete(l.zkconf.lock('foo_name'))
time.sleep(0.1)
self.assertFalse(sess['acquired'])
def test_node_change_after_acquired(self):
sess = {'acquired': True}
def on_lost():
sess['acquired'] = False
l = zkutil.ZKLock('foo_name',
zkclient=self.zk,
on_lost=on_lost)
with l:
sess['acquired'] = True
self.zk.delete(l.zkconf.lock('foo_name'))
time.sleep(0.1)
self.assertFalse(sess['acquired'])
l = zkutil.ZKLock('foo_name',
zkclient=self.zk,
on_lost=on_lost)
with l:
sess['acquired'] = True
self.zk.set(l.zkconf.lock('foo_name'), 'xxx')
time.sleep(0.1)
self.assertFalse(sess['acquired'])
def test_node_change_after_released(self):
sess = {'acquired': True}
def on_lost():
sess['acquired'] = False
l = zkutil.ZKLock('foo_name',
zkclient=self.zk,
on_lost=on_lost)
with l:
sess['acquired'] = True
time.sleep(0.1)
self.assertTrue(sess['acquired'])
def test_is_locked(self):
l = zkutil.ZKLock('foo_name', zkclient=self.zk)
with l:
pass
self.assertFalse(l.is_locked())
l = zkutil.ZKLock('foo_name', zkclient=self.zk)
l.acquire()
self.assertTrue(l.is_locked())
l.try_release()
self.assertFalse(l.is_locked())
def test_conn_lost_when_blocking_acquiring(self):
l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
th = threadutil.start_daemon(target=self.zk.stop, after=0.5)
with l2:
try:
self.lck.acquire(timeout=1)
self.fail('expected connection error')
except ConnectionClosedError:
pass
th.join()
def test_internal_zkclient(self):
sess = {'acquired': True}
def on_lost():
sess['acquired'] = False
# There must be a listener specified to watch connection issue
self.assertRaises(ValueError, zkutil.ZKLock, 'foo_name')
l = zkutil.ZKLock('foo_name', on_lost=on_lost)
with l:
self.zk.delete(l.zkconf.lock('foo_name'))
time.sleep(0.1)
self.assertFalse(sess['acquired'])
def test_acl(self):
with self.lck:
acls, zstat = self.zk.get_acls(self.lck.lock_path)
dd(acls)
self.assertEqual(1, len(acls))
acl = acls[0]
expected = zkutil.perm_to_long(zk_test_acl[0][2], lower=False)
self.assertEqual(set(expected), set(acl.acl_list))
self.assertEqual('digest', acl.id.scheme)
self.assertEqual(zk_test_acl[0][0], acl.id.id.split(':')[0])
def test_config(self):
old = (config.zk_acl, config.zk_auth, config.zk_node_id)
config.zk_acl = (('foo', 'bar', 'cd'),
('xp', '123', 'cdrwa'))
config.zk_auth = ('digest', 'xp', '123')
config.zk_node_id = 'abc'
l = zkutil.ZKLock('foo_name', on_lost=lambda: True)
dd(l.zkconf.acl())
def _check_ac(ac):
self.assertEqual('digest', ac.id.scheme)
self.assertEqual('foo', ac.id.id.split(':')[0])
self.assertEqual(set(['CREATE', 'DELETE']), set(ac.acl_list))
_check_ac(l.zkconf.kazoo_digest_acl()[0])
with l:
# should have created lock node
data, zstate = self.zk.get(l.lock_path)
dd(data)
self.assertEqual('abc', data.split('-')[0])
acls, zstate = self.zk.get_acls(l.lock_path)
dd(acls)
_check_ac(acls[0])
(config.zk_acl, config.zk_auth, config.zk_node_id) = old
def test_hosts(self):
l = zkutil.ZKLock('foo_name',
zkconf=dict(
hosts='127.0.0.1:21811',
),
on_lost=lambda: True)
with l:
self.assertEqual('127.0.0.1:21811', l._hosts)
def test_specify_identifier(self):
a = zkutil.ZKLock('foo_name',
zkconf=dict(
hosts='127.0.0.1:21811',
),
identifier='faked',
on_lost=lambda: True)
b = zkutil.ZKLock('foo_name',
zkconf=dict(
hosts='127.0.0.1:21811',
),
identifier='faked',
on_lost=lambda: True)
a.acquire()
b.acquire()
dd('a and b has the same identifier thus they both can acquire the lock')
def test_release_listener_removed(self):
self.lck.release()
self.assertNotIn(self.lck.on_connection_change, self.zk.state_listeners)
def test_release_owning_client_stopped(self):
l = zkutil.ZKLock('foo_name',
zkconf=dict(
hosts='127.0.0.1:21811',
),
on_lost=lambda: True)
l.release()
self.assertTrue(l.zkclient._stopped.is_set())
class ZooKeeper(object):
def __init__(self, hosts, user=None, password=None):
self._zookeeper = KazooClient(hosts=hosts)
self._zookeeper.start()
if user and password:
self._zookeeper.add_auth('digest', '{}:{}'.format(user, password))
def stop(self):
if self._zookeeper:
self._zookeeper.stop()
self._zookeeper.close()
self._zookeeper = None
def list(self, path):
try:
return self._zookeeper.get_children(path)
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except NoAuthError:
raise ZooKeeperException("No access to list node: {}".format(path))
def get(self, path):
try:
value, _ = self._zookeeper.get(path)
if value:
value = value.decode('utf-8')
else:
value = ""
return value
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except NoAuthError:
raise ZooKeeperException("No access to get node: {}".format(path))
def set(self, path, data):
try:
self._zookeeper.set(path, data.encode())
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except NoAuthError:
raise ZooKeeperException("No access to set data on node: {}".format(path))
def create(self, path, data=None, ephemeral=False, sequence=False, makepath=False):
if data:
data = data.encode()
else:
data = b""
try:
self._zookeeper.create(path,
value=data,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except NodeExistsError:
raise ZooKeeperException("Node already exists: {}".format(path))
except NoAuthError:
raise ZooKeeperException("No access to create node: {}".format(path))
def delete(self, path, recursive=False):
try:
self._zookeeper.delete(path, recursive=recursive)
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except NotEmptyError:
raise ZooKeeperException("Node contains sub-nodes")
except NoAuthError:
raise ZooKeeperException("No access to delete node: {}".format(path))
def stat(self, path):
try:
_, stat = self._zookeeper.get(path)
return stat
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
def get_acl(self, path):
try:
acl, _ = self._zookeeper.get_acls(path)
return acl
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
def add_acl(self, path, permissions, scheme, id):
perms = get_permissions(permissions)
if scheme == "digest":
username, password = id.split(":")
acl = make_digest_acl(username, password, **perms)
else:
acl = make_acl(scheme, id, **perms)
current_acls = self.get_acl(path)
current_acls.append(acl)
try:
self._zookeeper.set_acls(path, current_acls)
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except InvalidACLError as exc:
raise ZooKeeperException("Invalid ACL format: {}".format(str(exc)))
except NoAuthError:
raise ZooKeeperException("No access to add acl on node: {}".format(path))
def delete_acl(self, path, index):
current_acls = self.get_acl(path)
deleted = current_acls.pop(index)
try:
self._zookeeper.set_acls(path, current_acls)
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except NoAuthError:
raise ZooKeeperException("No access to delete acl from node: {}".format(path))
return deleted
class Zooconf:
__zkcon = None
__serviceConfig = None
__fsList = None
def __init__(self):
sys.stderr.write("Fileservicepython start initialization at " +
str(int(round(time.time() * 1000))))
sys.stderr.flush()
self.__initConfProperties()
self.__zooConnect()
self.__publishService()
self.__initFsWatches()
def __initConfProperties(self):
sys.stderr.write(dirname(dirname(abspath(__file__))))
sys.stderr.flush()
self.__serviceConfig = dict()
with open(
dirname(dirname(abspath(__file__))) + "/config.properties",
'r') as f:
for line in f:
line = line.rstrip()
if "=" not in line:
continue #skips blanks and comments w/o =
if line.startswith("#"):
continue #skips comments which contain =
k, v = line.split("=", 1)
self.__serviceConfig[k] = v
f.close()
def __zooConnect(self):
sys.stderr.write("Start zooconnect")
sys.stderr.flush()
config = self.getServiceConfig()
self.__zkcon = KazooClient(hosts=config["ZOOKEEPER_HOST"])
self.__zkcon.start()
digest_auth = "%s:%s" % (config["ZOOKEEPER_USER"],
config["ZOOKEEPER_PASSWORD"])
self.__zkcon.add_auth("digest", digest_auth)
def __initFsWatches(self):
# lists are supposed to be thread safe in python
self.__fsList = []
@self.__zkcon.ChildrenWatch("/plh414python/fileservices")
def watch_children(children):
self.__fsList = []
sys.stderr.write("Children are now: %s" % children)
sys.stderr.flush()
for child in children:
self.__fsList.append(child)
# Above function called immediately, and from then on
def __publishService(self):
from kazoo.security import make_digest_acl
config = self.getServiceConfig()
acl = make_digest_acl(config["ZOOKEEPER_USER"],
config["ZOOKEEPER_PASSWORD"],
all=True)
dataJsonDict = {}
dataJsonDict['SERVERHOSTNAME'] = config["SERVERHOSTNAME"]
dataJsonDict['SERVER_PORT'] = config["SERVER_PORT"]
dataJsonDict['SERVER_SCHEME'] = config["SERVER_SCHEME"]
dataJsonDict['CONTEXT'] = config["CONTEXT"]
self.__zkcon.set("/plh414python/directoryservice/",
json.JSONEncoder().encode(dataJsonDict).encode(), -1)
def getAvailableFs(self):
return self.__fsList
def getZooConnection(self):
return self.__zkcon
def getServiceConfig(self):
return self.__serviceConfig
from kazoo.client import KazooClient
zk = KazooClient(hosts='127.0.0.1:2181')
zk.start()
print zk.get_children("/")
try:
zk.add_auth("digest", "jamesbond:password123")
print zk.get("/area51")
except:
print "Auth error"
zk.stop()
class Zooconf:
connection = None
webService = None
applicationService = None
storageServicesList = None
authenticationServiceList = None
status = None
def initialize(self):
self.__zooConnect()
self.__publishService()
self.__initAuthenticationServiceWatches()
self.__initStorageServiceWatches()
self.heartbeat()
def __zooConnect(self):
print("Connecting to ZooKeeper")
self.connection = KazooClient(hosts=settings.ZOOKEEPER_HOST)
self.connection.start()
digest_auth = "%s:%s" % (settings.ZOOKEEPER_USER,
settings.ZOOKEEPER_PASSWORD)
self.connection.add_auth("digest", digest_auth)
def __publishService(self):
acl = make_digest_acl(settings.ZOOKEEPER_USER,
settings.ZOOKEEPER_PASSWORD,
all=True)
dataJsonDict = {
'SERVER_HOSTNAME': settings.SERVER_HOSTNAME,
'SERVER_PORT': settings.SERVER_PORT,
'SHARED_KEY_BASE_64': settings.SHARED_KEY_BASE_64,
'CHILDREN': []
}
if settings.ZOOKEEPER_PATH_TO_NODE == 'Auth/':
dataJsonDict['AUTH_SYSTEM'] = settings.AUTH_SYSTEM
if self.connection.exists(path=settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE +
settings.ZOOKEEPER_NODE_ID):
self.connection.set(
path=settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE + settings.ZOOKEEPER_NODE_ID,
value=json.JSONEncoder().encode(dataJsonDict).encode())
else:
self.connection.create_async(
path=settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE + settings.ZOOKEEPER_NODE_ID,
value=json.JSONEncoder().encode(dataJsonDict).encode(),
ephemeral=settings.ZOOKEEPER_NODE_EPHIMERAL)
if settings.ZOOKEEPER_PATH_TO_NODE != '':
data, stat = self.connection.get(settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE)
dataJsonDict = json.loads(data.decode("utf-8"))
if settings.ZOOKEEPER_NODE_ID not in dataJsonDict['CHILDREN']:
dataJsonDict['CHILDREN'].append(settings.ZOOKEEPER_NODE_ID)
self.connection.set(
path=settings.ZOOKEEPER_ROOT + settings.ZOOKEEPER_PATH_TO_NODE,
value=json.JSONEncoder().encode(dataJsonDict).encode())
def __initStorageServiceWatches(self):
self.storageServicesList = []
@self.connection.ChildrenWatch(settings.ZOOKEEPER_ROOT +
"StorageServices")
def watch_children(children):
print(self.readTree())
if settings.ZOOKEEPER_PATH_TO_NODE == "StorageServices/":
node = {
'SERVER_HOSTNAME': settings.SERVER_HOSTNAME,
'SERVER_PORT': settings.SERVER_PORT,
'CHILDREN': []
}
if node not in self.storageServicesList:
self.__publishService()
def __initAuthenticationServiceWatches(self):
self.authenticationServiceList = []
@self.connection.ChildrenWatch(settings.ZOOKEEPER_ROOT + "Auth")
def watch_children(children):
print(self.readTree())
if settings.ZOOKEEPER_PATH_TO_NODE == "Auth/":
node = {
'SERVER_HOSTNAME': settings.SERVER_HOSTNAME,
'SERVER_PORT': settings.SERVER_PORT,
'SHARED_KEY_BASE_64': settings.SHARED_KEY_BASE_64,
'CHILDREN': [],
'AUTH_SYSTEM': settings.AUTH_SYSTEM
}
if node not in self.authenticationServiceList:
self.__publishService()
def getAvailableFs(self):
return self.storageServicesList
def getZooConnection(self):
return self.connection
def readTree(self):
result = "{"
rootChildren = self.connection.get_children(settings.ZOOKEEPER_ROOT)
for child in rootChildren:
data, stat = self.connection.get(settings.ZOOKEEPER_ROOT + child)
result += '"' + child + '": ' + data.decode("utf-8") + ","
try:
grandchildren = self.connection.get_children(
settings.ZOOKEEPER_ROOT + child)
for grandchild in grandchildren:
data, stat = self.connection.get(settings.ZOOKEEPER_ROOT +
child + '/' + grandchild)
result += '"' + grandchild + '": ' + data.decode(
"utf-8") + ","
data, stat = self.connection.get(settings.ZOOKEEPER_ROOT +
child + '/')
dataJsonDict = json.loads(data.decode("utf-8"))
dataJsonDict['CHILDREN'] = grandchildren
self.connection.set(
path=settings.ZOOKEEPER_ROOT + child,
value=json.JSONEncoder().encode(dataJsonDict).encode())
except Exception:
pass
result = result[:-1] + '}'
self.status = json.loads(result)
self.initZkTree()
return self.status
def getStatus(self):
try:
return self.readTree()
except Exception:
self.__zooConnect()
return self.getStatus()
def getNodeData(self, node):
try:
return self.status[node]
except Exception:
return {}
def initZkTree(self):
serviceData = self.getNodeData('WebService')
self.webService = serviceData['SERVER_HOSTNAME'] + ':' + serviceData[
'SERVER_PORT'] + '/'
serviceData = self.getNodeData('ApplicationService')
self.applicationService = serviceData[
'SERVER_HOSTNAME'] + ':' + serviceData['SERVER_PORT'] + '/'
serviceData = self.getNodeData('Auth')
self.authenticationServiceList = []
for authService in serviceData['CHILDREN']:
authServiceData = self.getNodeData(authService)
if authServiceData != {}:
baseUrl = authServiceData['SERVER_HOSTNAME'] + (
(':' + authServiceData['SERVER_PORT'])
if authServiceData['SERVER_PORT'] != '' else '') + '/'
loginUrl = baseUrl + 'login?system=' + authServiceData[
'AUTH_SYSTEM'] + '&callback='
registerUrl = baseUrl + 'register?system=' + authServiceData[
'AUTH_SYSTEM'] + '&callback='
authServiceDict = {
'name': authService,
'url': baseUrl,
'loginUrl': loginUrl,
'registerUrl': registerUrl,
'sharedKey': authServiceData['SHARED_KEY_BASE_64'],
'system': authServiceData['AUTH_SYSTEM']
}
self.authenticationServiceList.append(authServiceDict)
serviceData = self.getNodeData('StorageServices')
self.storageServicesList = []
for storageService in serviceData['CHILDREN']:
storageServiceData = self.getNodeData(storageService)
if storageServiceData != {}:
storageServiceDict = {
'name':
storageService,
'url':
storageServiceData['SERVER_HOSTNAME'] + ':' +
storageServiceData['SERVER_PORT'] + '/'
}
self.storageServicesList.append(storageServiceDict)
def heartbeat(self):
threading.Timer(300.0, self.heartbeat).start()
print("Heartbeat")
print(str(self.getStatus()))
def test_init_hierarchy(self):
auth = ('digest', 'aa', 'pw_aa')
hosts = '127.0.0.1:21811'
users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'}
hierarchy = {
'node1': {
'__val__': 'node1_val',
'__acl__': {
'aa': 'cdrwa',
'bb': 'rw'
},
'node11': {
'__val__': 'node11_val',
'__acl__': {
'aa': 'cdrwa',
'cc': 'r'
},
},
'node12': {
'__val__': 'node12_val',
'node121': {
'__val__': 'node121_val'
}
},
'node13': {
'__acl__': {
'aa': 'cdrwa'
}
}
},
'node2': {
'__val__': 'node2_val',
'node21': {
'__val__': 'node21_val'
},
'node22': {
'__acl__': {
'aa': 'rwa'
}
}
},
'node3': {
'__acl__': {
'aa': 'carw',
'cc': 'r'
},
'node31': {
'node311': {
'node3111': {},
'node3112': {}
}
}
}
}
zkutil.init_hierarchy(hosts, hierarchy, users, auth)
zkcli = KazooClient(hosts)
zkcli.start()
zkcli.add_auth('digest', 'aa:pw_aa')
expected_nodes = (
('/node1', '"node1_val"', [('digest', 'aa', 'cdrwa'),
('digest', 'bb', 'rw')],
set(['node11', 'node12', 'node13'])),
('/node1/node11', '"node11_val"', [('digest', 'aa', 'cdrwa'),
('digest', 'cc', 'r')],
set([])),
('/node1/node12', '"node12_val"', [('digest', 'aa', 'cdrwa'),
('digest', 'bb', 'rw')],
set(['node121'])),
('/node1/node12/node121', '"node121_val"',
[('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set([])),
('/node1/node13', '{}', [('digest', 'aa', 'cdrwa')], set([])),
('/node2', '"node2_val"', [('world', 'anyone', 'cdrwa')],
set(['node21', 'node22'])),
('/node2/node21', '"node21_val"', [('world', 'anyone', 'cdrwa')],
set([])),
('/node2/node22', '{}', [('digest', 'aa', 'rwa')], set([])),
('/node3', '{}', [('digest', 'aa', 'rwca'),
('digest', 'cc', 'r')], set(['node31'])),
('/node3/node31', '{}', [('digest', 'aa', 'rwca'),
('digest', 'cc', 'r')], set(['node311'])),
('/node3/node31/node311', '{}', [('digest', 'aa', 'rwca'),
('digest', 'cc', 'r')],
set(['node3111', 'node3112'])),
('/node3/node31/node311/node3111', '{}', [('digest', 'aa', 'rwca'),
('digest', 'cc', 'r')],
set([])),
('/node3/node31/node311/node3112', '{}', [('digest', 'aa', 'rwca'),
('digest', 'cc', 'r')],
set([])),
)
for node, val, acl, children in expected_nodes:
actual_acl = zkutil.parse_kazoo_acl(zkcli.get_acls(node)[0])
self.assertEqual(val, zkcli.get(node)[0])
self.assertEqual(acl, actual_acl)
self.assertEqual(children, set(zkcli.get_children(node)))
zkcli.stop()
# READ ZK Data
print("Reading unprotected ZK Data back in.")
zk = KazooClient(hosts=zk_hosts)
zk.start()
for zk_path in all_host_paths:
data, stat = zk.get(zk_path)
print("{} {}{}{}".format("ZK Node version: is ", stat.version,
". ZK Data sample ",
data.decode("utf-8").split(' ')[:3]))
zk.stop()
# Get Bouncer Data
print("Reading digest protected /bouncer/data/data.json data from zk")
zk = KazooClient(hosts=zk_hosts)
zk.start()
zk.add_auth('digest', 'super:secret')
data, stat = zk.get('/bouncer/datastore/data.json')
# print("{} {}{}{}".format("ZK Version: is ",stat.version, ".\nBouncer ZK Data is \n", data.decode("utf-8")))
print("{} {} {}".format("Size of bouncer config is ~ ", len(data),
"bytes.\n"))
zk.stop()
bouncer_json = json.loads(data.decode("utf-8"))
# Iterate through the bouncer_json dict
user_count = 0
for user in bouncer_json['users']:
print('Bouncer User = ' + user)
user_count = user_count + 1
group_count = 0
class Zooconf:
__zkcon = None
__serviceConfig = None
def __init__(self):
sys.stderr.write("Fileservicepython start initialization at " +
str(int(round(time.time() * 1000))))
sys.stderr.flush()
self.__initConfProperties()
self.__zooConnect()
self.__publishService()
def __initConfProperties(self):
sys.stderr.write(dirname(dirname(abspath(__file__))))
sys.stderr.flush()
self.__serviceConfig = dict()
with open(
dirname(dirname(abspath(__file__))) + "/config.properties",
'r') as f:
for line in f:
line = line.rstrip()
if "=" not in line:
continue #skips blanks and comments w/o =
if line.startswith("#"):
continue #skips comments which contain =
k, v = line.split("=", 1)
self.__serviceConfig[k] = v
f.close()
def __zooConnect(self):
sys.stderr.write("Start zooconnect")
sys.stderr.flush()
config = self.getServiceConfig()
self.__zkcon = KazooClient(hosts=config["ZOOKEEPER_HOST"])
self.__zkcon.start()
digest_auth = "%s:%s" % (config["ZOOKEEPER_USER"],
config["ZOOKEEPER_PASSWORD"])
self.__zkcon.add_auth("digest", digest_auth)
def __publishService(self):
from kazoo.security import make_digest_acl
config = self.getServiceConfig()
acl = make_digest_acl(config["ZOOKEEPER_USER"],
config["ZOOKEEPER_PASSWORD"],
all=True)
dataJsonDict = {}
dataJsonDict['SERVERHOSTNAME'] = config["SERVERHOSTNAME"]
dataJsonDict['SERVER_PORT'] = config["SERVER_PORT"]
dataJsonDict['SERVER_SCHEME'] = config["SERVER_SCHEME"]
dataJsonDict['HMACKEY'] = config["HMACKEY"]
dataJsonDict['CONTEXT'] = config["CONTEXT"]
self.__zkcon.create("/plh414python/fileservices/" + config["ID"],
json.JSONEncoder().encode(dataJsonDict).encode(),
[acl],
ephemeral=True)
def getZooConnection(self):
return self.__zkcon
def getServiceConfig(self):
return self.__serviceConfig
class ZkClient:
def __init__(self, url):
self.upCont = 0
self.addCont = 0
self.url = url
try:
connection_retry = KazooRetry(max_tries=3, delay=0.1)
command_retry = KazooRetry(max_tries=3, delay=0.1)
self.zk = KazooClient(hosts=url, timeout=10.0,
connection_retry=connection_retry,
command_retry=command_retry,
logger=None)
if self.zk:
self.zk.start(timeout=60)
log.info("连接zk服务器%s成功", url)
# 设置ACL
self.zk.add_auth("digest", "SmartESC:startesc123")
except Exception as e:
raise e
def close(self):
if self.zk:
self.zk.stop()
self.zk.close()
log.info("断开zk连接...")
def writeData(self, path, data):
try:
self.zk.set(path, data)
except Exception as e:
log.exception("数据写入错误")
raise e
def set_zookeeper_data(self, data={}):
"""
父节点的数据不存在时，则创建父节点
data数据中需要包含路径
:param data:
:return:
"""
for path in data.keys():
try:
if self.zk.exists(path):
self.zk.set(path, data[path])
self.upCont += 1
else:
self.zk.create(path, data[path], makepath=True)
self.addCont += 1
# log.info("路径:%s",path)
except Exception as e:
log.exception("数据%s写入错误", path)
raise e
def set_service_data(self, data={}):
"""
父节点的数据不存在时，不创建父节点
data中包含zookeeper的路径，
:param data:
:return:
"""
for path in data.keys():
try:
if self.zk.exists(path):
self.zk.set(path, data[path])
self.upCont += 1
else:
self.zk.create(path, data[path])
self.addCont += 1
# log.info("路径:%s",path)
except Exception as e:
log.exception("数据%s写入错误", path)
raise e
def get_onecfg_service(self,sCode):
data={}
try:
meta=self.zk.get_children("/metadata")
for st in meta:
if sCode in st:
path="/metadata/%s"%(st)
data[path],stat=self.zk.get(path)
cons = self.zk.get_children("/metadata/consumer")
for st in cons:
if sCode in st:
path = "/metadata/consumer/%s" % (st)
data[path], stat = self.zk.get(path)
prvd = self.zk.get_children("/metadata/provider")
for st in prvd:
if sCode in st:
path = "/metadata/provider/%s" % (st)
data[path], stat = self.zk.get(path)
return data
except Exception as e:
log.exception("zk数据读取错误:")
raise e
def get_cfg_service(self,sCodeList=[]):
"""
在zookeeper中查找服务,获取zookeeper中服务的数据
:param sCodeList:
:return:
"""
data = {}
for sCode in sCodeList:
try:
meta = self.zk.get_children("/metadata")
for st in meta:
if sCode in st:
path = "/metadata/%s" % (st)
data[path], stat = self.zk.get(path)
cons = self.zk.get_children("/metadata/consumer")
for st in cons:
if sCode in st:
path = "/metadata/consumer/%s" % (st)
data[path], stat = self.zk.get(path)
prvd = self.zk.get_children("/metadata/provider")
for st in prvd:
if sCode in st:
path = "/metadata/provider/%s" % (st)
data[path], stat = self.zk.get(path)
except Exception as e:
log.exception("zk数据读取错误:")
raise e
return data
def get_cfg_service_by_service_code(self, service_code):
"""
在zookeeper中查找服务,获取zookeeper中服务的数据
:param service_code:
:return:
"""
data = {}
try:
meta = self.zk.get_children("/metadata")
for st in meta:
if service_code in st:
path = "/metadata/%s" % (st)
data[path], stat = self.zk.get(path)
cons = self.zk.get_children("/metadata/consumer")
for st in cons:
if service_code in st:
path = "/metadata/consumer/%s" % (st)
data[path], stat = self.zk.get(path)
prvd = self.zk.get_children("/metadata/provider")
for st in prvd:
if service_code in st:
path = "/metadata/provider/%s" % (st)
data[path], stat = self.zk.get(path)
except Exception as e:
log.exception("zk数据读取错误:")
raise e
return data
def get_cfg_service_auth_by_system_name(self, system_name):
"""
在zookeeper中查找服务的授权信息
:param system_name: 系统名称
:return:
"""
data = {}
new_path = ''
try:
meta = self.zk.get_children("/configs")
for file_name in meta:
if system_name in file_name:
path = "/configs/%s" % (file_name)
new_path = path+"/consumer/authCtrl/"+system_name
if self.zk.exists(new_path):
data[new_path], stat = self.zk.get(new_path)
except Exception as e:
log.exception("zk数据读取授权数据错误:%s", new_path)
raise e
return data
def get_allcfg_service(self):
data={}
try:
meta=self.zk.get_children("/metadata")
for st in meta:
path="/metadata/%s"%(st)
data[path],stat=self.zk.get(path)
cons = self.zk.get_children("/metadata/consumer")
for st in cons:
path = "/metadata/consumer/%s" % (st)
data[path], stat = self.zk.get(path)
prvd = self.zk.get_children("/metadata/provider")
for st in prvd:
path = "/metadata/provider/%s" % (st)
data[path], stat = self.zk.get(path)
return data
except Exception as e:
log.exception("zk数据读取错误:")
raise e
def test_export_hierarchy(self):
auth = ('digest', 'aa', 'pw_aa')
hosts = '127.0.0.1:21811'
users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'}
hierarchy = {
'node0': {
'__val__': '',
'__acl__': {
'aa': 'cdrwa',
'bb': 'rw'
}
},
'node1': {
'__val__': 'node1_val',
'__acl__': {
'aa': 'cdrwa',
'bb': 'rw'
},
'node11': {
'__val__': 'node11_val',
'__acl__': {
'aa': 'cdrwa',
'cc': 'r'
},
},
'node12': {
'__val__': 'node12_val',
'node121': {
'__val__': 'node121_val'
}
},
'node13': {
'__acl__': {
'aa': 'cdrwa'
}
}
},
'node2': {
'__val__': 'node2_val',
'node21': {
'__val__': 'node21_val'
},
'node22': {
'__acl__': {
'aa': 'rwa'
}
}
},
'node3': {
'__acl__': {
'aa': 'carw',
'cc': 'r'
},
'node31': {
'node311': {
'node3111': {},
'node3112': {}
}
}
}
}
zkutil.zkutil.init_hierarchy(hosts, hierarchy, users, auth)
zkcli = KazooClient(hosts)
zkcli.start()
zkcli.add_auth('digest', 'aa:pw_aa')
invalid_zkpath_cases = ('a', 'a/', 'a/b')
for zkpath in invalid_zkpath_cases:
with self.assertRaises(zkutil.zkutil.ZkPathError):
zkutil.zkutil.export_hierarchy(zkcli, zkpath)
valid_cases = (('/', {
'__acl__': {
u'anyone': 'cdrwa'
},
'__val__': '',
u'node0': {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '""'
},
u'node1': {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '"node1_val"',
u'node11': {
'__acl__': {
u'aa': 'cdrwa',
u'cc': 'r'
},
'__val__': '"node11_val"'
},
u'node12': {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '"node12_val"',
u'node121': {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '"node121_val"'
}
},
u'node13': {
'__acl__': {
u'aa': 'cdrwa'
},
'__val__': '{}'
}
},
u'node2': {
'__acl__': {
u'anyone': 'cdrwa'
},
'__val__': '"node2_val"',
u'node21': {
'__acl__': {
u'anyone': 'cdrwa'
},
'__val__': '"node21_val"'
},
u'node22': {
'__acl__': {
u'aa': 'rwa'
},
'__val__': '{}'
}
},
u'node3': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}',
u'node31': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}',
u'node311': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}',
u'node3111': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}'
},
u'node3112': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}'
}
}
}
},
u'zookeeper': {
'__acl__': {
u'anyone': 'cdrwa'
},
'__val__': '',
u'quota': {
'__acl__': {
u'anyone': 'cdrwa'
},
'__val__': ''
}
}
}), ('/node0', {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '""'
}), ('/node1', {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '"node1_val"',
u'node11': {
'__acl__': {
u'aa': 'cdrwa',
u'cc': 'r'
},
'__val__': '"node11_val"'
},
u'node12': {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '"node12_val"',
u'node121': {
'__acl__': {
u'aa': 'cdrwa',
u'bb': 'rw'
},
'__val__': '"node121_val"'
}
},
u'node13': {
'__acl__': {
u'aa': 'cdrwa'
},
'__val__': '{}'
}
}), ('/node1/node11', {
'__acl__': {
u'aa': 'cdrwa',
u'cc': 'r'
},
'__val__': '"node11_val"'
}), ('/node2', {
'__acl__': {
u'anyone': 'cdrwa'
},
'__val__': '"node2_val"',
u'node21': {
'__acl__': {
u'anyone': 'cdrwa'
},
'__val__': '"node21_val"'
},
u'node22': {
'__acl__': {
u'aa': 'rwa'
},
'__val__': '{}'
}
}), ('/node3', {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}',
u'node31': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}',
u'node311': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}',
u'node3111': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}'
},
u'node3112': {
'__acl__': {
u'aa': 'rwca',
u'cc': 'r'
},
'__val__': '{}'
}
}
}
}))
for path, expected_rst in valid_cases:
rst = zkutil.export_hierarchy(zkcli, path)
self.assertEqual(rst, expected_rst)
zkcli.stop()
class WebWindow(object):
def setupUi(self, Window):
Window.setObjectName("Window")
self.centralwidget = QtWidgets.QWidget(Window)
self.centralwidget.setObjectName("centralwidget")
self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget)
self.verticalLayout.setContentsMargins(0, 0, 0, 0)
self.verticalLayout.setObjectName("verticalLayout")
self.zk = None
self.webView = QtWebKitWidgets.QWebView(self.centralwidget)
self.webView.setObjectName("webView")
self.webView.setRenderHint(QPainter.Antialiasing, True)
self.webView.setRenderHint(QPainter.TextAntialiasing, True)
self.webView.setRenderHint(QPainter.SmoothPixmapTransform, True)
self.webView.setRenderHint(QPainter.HighQualityAntialiasing, True)
self.webView.setPage(WebPage())
frame = self.webView.page().mainFrame()
frame.javaScriptWindowObjectCleared.connect(self.initJsComm)
self.verticalLayout.addWidget(self.webView)
Window.setCentralWidget(self.centralwidget)
self.retranslateUi(Window)
QtCore.QMetaObject.connectSlotsByName(Window)
self.loadDefaultAcl()
def retranslateUi(self, Window):
_translate = QtCore.QCoreApplication.translate
Window.setWindowTitle(_translate("Window", "Zookeeper GUI"))
def loadLocalFile(self, filename):
localdir = os.path.abspath(sys.path[0])
if os.path.isfile(localdir):
localdir = os.path.dirname(localdir)
self.webView.setUrl( QtCore.QUrl.fromLocalFile(localdir+'/'+filename) )
def getCfgVar(self, name):
localdir = os.path.abspath(sys.path[0])
if os.path.isfile(localdir):
localdir = os.path.dirname(localdir)
cfg = {}
obj = None
try:
obj = open(localdir+'/cfg.json','r')
cfg = json.loads(obj.read())
except Exception as e:
logging.info(str(e))
finally:
if obj is not None:
obj.close()
if name in cfg:
return str(cfg[name])
return ''
def setCfgVar(self, name, value):
localdir = os.path.abspath(sys.path[0])
if os.path.isfile(localdir):
localdir = os.path.dirname(localdir)
cfg = {}
obj = None
try:
obj = open(localdir+'/cfg.json','r')
cfg = json.loads(obj.read())
except Exception as e:
pass
finally:
if obj is not None:
obj.close()
cfg[name] = value
obj = None
try:
obj = open(localdir+'/cfg.json','w')
obj.truncate()
obj.write(json.dumps(cfg))
except Exception as e:
logging.info(str(e))
finally:
if obj is not None:
obj.close()
def makeDigestCred(self, user, plain_pass):
m = hashlib.sha1( bytes(user,'utf8') + b':' + bytes(plain_pass,'utf8') ).digest()
return user+':'+base64.b64encode(m).strip().decode('utf8')
def initJsComm(self):
frame = self.webView.page().mainFrame()
frame.addToJavaScriptWindowObject('py',self)
@pyqtSlot(str)
def jsSetWinTitle(self, title):
_translate = QtCore.QCoreApplication.translate
self.setWindowTitle(_translate("Window", title))
@pyqtSlot(str, result=str)
def jsCheckYaml(self, s):
try:
a = yaml.load(s)
except Exception as e:
return str(e)
if a is None:
return 'Failed'
return ''
@pyqtSlot(str,result=str)
def jsGetCfg(self, name):
return self.getCfgVar(name)
@pyqtSlot(str,str)
def jsSetCfg(self, name, value):
self.setCfgVar(name, value)
def loadDefaultAcl(self):
self.updateDefaultAclCache( self.getCfgVar('defaultacl') )
def updateDefaultAclCache(self, list_str):
if list_str is not None and len(list_str)>0:
cache = json.loads(list_str)
self.default_acl_plain = []
self.default_acl = []
for one in cache:
if(one['scheme']=='world'):
self.default_acl_plain.append(one)
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') )
self.default_acl.append(acl)
elif(one['scheme']=='digest'):
self.default_acl_plain.append(one)
if 'id' in one:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) )
else:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) )
self.default_acl.append(acl)
elif(one['scheme']=='ip'):
self.default_acl_plain.append(one)
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) )
self.default_acl.append(acl)
else:
self.default_acl_plain = []
self.default_acl = None
@pyqtSlot(str,result=str)
def jsSetDefaultAcl(self, list_str):
self.updateDefaultAclCache(list_str)
self.setCfgVar('defaultacl', json.dumps(self.default_acl_plain))
@pyqtSlot(result=str)
def jsGetDefaultAcl(self):
return json.dumps(self.default_acl_plain)
@pyqtSlot()
def jsGetZk(self):
return self.zk
@pyqtSlot(result=int)
def jsZkIsConnected(self):
if self.zk is not None:
return int(self.zk.state=='CONNECTED')
return 0
@pyqtSlot(str, str, result=str)
def jsZkConnect(self,host, auth_list_str):
try:
if self.zk is not None:
#self.zk.remove_listener(self.onZkStateChange)
self.zk.stop()
self.zk.close()
self.zk = KazooClient(hosts=host)
#self.zk.add_listener(self.onZkStateChange)
self.zk.start(15)
auth_list = json.loads(auth_list_str)
for one in auth_list:
cred = self.makeDigestCred(one['user'], one['pass'])
self.zk.add_auth('digest', one['user']+':'+one['pass'])
except Exception as e:
logging.error("jsZkConnect, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
#def onZkStateChange(self,state):
# frame = self.webView.page().mainFrame()
# frame.evaluateJavaScript("onPyZkStateChange('"+state+"')")
@pyqtSlot(str, result=QVariant)
def jsZkGetChildren(self, path):
try:
logging.info("jsZkGetChildren, path="+path)
children = self.zk.get_children(path)
except NoNodeError:
logging.error("jsZkGetChildren, NoNodeError")
return QVariant({"err":"node not exists"})
except Exception as e:
logging.error("jsZkGetChildren, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return QVariant({"err":str(e)+', traceback='+str(strlist)})
return QVariant({"err":"", "children":children})
@pyqtSlot(str, result=QVariant)
def jsZkGet(self, path):
try:
logging.info("jsZkGet, path="+path)
ret = self.zk.get(path)
except NoNodeError:
logging.error("jsZkGet, NoNodeError")
return QVariant({"err":"node not exists"})
except Exception as e:
logging.error("jsZkGet, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return QVariant({"err":str(e)+', traceback='+str(strlist)})
ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].ctime/1000))
mtime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].mtime/1000))
stat = {'ctime':ctime,'mtime':mtime,'version':ret[1].version}
data = ''
if ret[0] is not None:
data = ret[0].decode('utf8')
else:
logging.info('jsZkGet data None, path='+path)
return QVariant({"err":"", "data":data, "stat":QVariant(stat)})
@pyqtSlot(str, str, int, result=str)
def jsZkSet(self, path, data, ver):
try:
logging.info("jsZkSet, path="+path+',ver='+str(ver))
self.zk.set(path, bytes(data, 'utf8'),ver)
except NoNodeError as e:
logging.error("jsZkSet, NoNodeError")
return "node not exists"
except BadVersionError as e:
logging.error("jsZkSet, BadVersionError")
return "bad version"
except Exception as e:
logging.error("jsZkSet, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, result=QVariant)
def jsZkGetAcl(self, path):
try:
logging.info("jsZkGetAcl, path="+path)
ret = self.zk.get_acls(path)
except NoNodeError as e:
logging.error("jsZkGetAcl, NoNodeError")
return QVariant({"err":"node not exists"})
except Exception as e:
logging.error("jsZkGetAcl, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return QVariant({"err":str(e)+', traceback='+str(strlist)})
lst = []
for acl in ret[0]:
dacl = {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id}
lst.append(QVariant(dacl))
stat = {'ctime':ret[1].ctime,'mtime':ret[1].mtime,'version':ret[1].version}
return QVariant({"err":"", "acl_list":QVariant(lst), "stat":QVariant(stat)})
@pyqtSlot(str, str, result=str)
def jsZkSetAcl(self, path, list_str):
try:
acl_list = None
if list_str is not None and len(list_str)>0:
cache = json.loads(list_str)
acl_list = []
for one in cache:
if(one['scheme']=='world'):
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') )
acl_list.append(acl)
elif(one['scheme']=='digest'):
if 'id' in one:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) )
else:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) )
acl_list.append(acl)
elif(one['scheme']=='ip'):
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) )
acl_list.append(acl)
self.zk.set_acls(path, acl_list)
except NoNodeError as e:
logging.error("jsZkSetAcl, NoNodeError")
return "node not exists"
except InvalidACLError as e:
logging.error("jsZkSetAcl, InvalidACLError")
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return "invalid acl, traceback: "+str(strlist)
except BadVersionError as e:
logging.error("jsZkSetAcl, BadVersionError")
return "bad version error"
except Exception as e:
logging.error("jsZkSetAcl, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str,str,int,int,result=str)
def jsZkCreate(self, path, data, ephem, seq):
try:
logging.info("jsZkCreate, path="+path)
self.zk.create(path=path, value=bytes(data,'utf8'), ephemeral=bool(ephem), sequence=bool(seq))
if self.default_acl is not None and len(self.default_acl)>0:
self.zk.set_acls(path, self.default_acl)
except NoNodeError as e:
logging.error("jsZkCreate, NoNodeError")
return "node not exists"
except NodeExistsError as e:
logging.error("jsZkCreate, NodeExistsError")
return "node already exists"
except NoChildrenForEphemeralsError as e:
logging.error("jsZkCreate, NoChildrenForEphemeralsError")
return "ephemeral node can not have child"
except Exception as e:
logging.error("jsZkCreate, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, int, int, result=str)
def jsZkDelete(self, path, ver, recursive):
try:
logging.info("jsZkDelete, path="+path+',ver='+str(ver)+', recursive='+str(recursive))
self.zk.delete(path, ver, bool(recursive))
except NoNodeError as e:
logging.error("jsZkDelete, NoNodeError")
return "node not exists"
except BadVersionError as e:
logging.error("jsZkDelete, BadVersionError")
return "bad version"
except NotEmptyError as e:
logging.error("jsZkDelete, NotEmptyError")
return "node not empty"
except Exception as e:
logging.error("jsZkDelete, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, str, int, int, result=str)
def jsZkCopy(self, dest_path, ori_path, max_depth, children_only):
logging.info("jsZkCopy, dest_path="+dest_path+", ori_path="+ori_path+", children_only="+str(children_only))
#copy node first
if children_only==0:
try:
ori_data = self.zk.get(ori_path)
if self.zk.exists(dest_path) is None:
self.zk.create(dest_path, ori_data[0], acl=self.default_acl)
else:
self.zk.set(dest_path, ori_data[0])
except NoNodeError as e:
logging.error("jsZkCopy, node, NoNodeError, ori_path="+ori_path+', dest_path='+dest_path)
return "node not exists"
except Exception as e:
logging.error("jsZkCopy, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
#copy children
path = ''
try:
max_depth -= 1
path = ori_path
ori_children = self.zk.get_children(ori_path)
for child in ori_children:
path = ori_path+'/'+child
ret = self.jsZkCopy(dest_path+'/'+child, ori_path+'/'+child, max_depth, 0)
if isinstance(ret, QVariant):
return ret
elif len(ret)>0:
return ret
except NoNodeError as e:
logging.error("jsZkCopy, child, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkCopy, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
'''
@pyqtSlot(str, str, int, result=str)
def jsZkCopyChildren(self, dest_path, ori_path, max_depth):
path = ''
try:
max_depth -= 1;
logging.info("jsZkCopyChildren, dest_path="+dest_path+", ori_path="+ori_path)
path = ori_path
ori_children = self.zk.get_children(ori_path)
path = dest_path
dest_children = self.zk.get_children(dest_path)
for child in ori_children:
if child in dest_children:
return 'child ['+child+'] is found in both path'
for child in ori_children:
path = ori_path+'/'+child
data = self.zk.get(path)[0]
path = dest_path+'/'+child
self.zk.create(path, data, acl=self.default_acl)
if max_depth>0:
ret = self.jsZkCopyChildren(dest_path+'/'+child, ori_path+'/'+child, max_depth)
if len(ret)>0:
return ret
except NoNodeError as e:
logging.info("jsZkCopyChildren, NoNodeError")
return "node not exists, path="+path
except ZookeeperError as e:
logging.info("jsZkCopyChildren, ZookeeperError")
return str(e)+', path='+path
return ''
'''
@pyqtSlot(str, int, result=str)
def jsZkDeleteChildren(self, main_path, max_depth):
path = ''
try:
max_depth -= 1
path = main_path
logging.info("jsZkDeleteChildren, path="+main_path)
children = self.zk.get_children(path)
for child in children:
path = main_path+'/'+child
if max_depth>0:
ret = self.jsZkDeleteChildren(path, max_depth)
if len(ret)>0:
return ret
self.zk.delete(path)
except NoNodeError as e:
logging.error("jsZkDeleteChildren, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkDeleteChildren, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, str, int, int, result=str)
def jsZkExport(self, local_dir, main_path, max_depth, without_acl):
path = ''
try:
max_depth -= 1
path = main_path
logging.info("jsZkExport, path="+main_path+' to local dir '+local_dir)
data = self.zk.get(main_path)
p = pathlib.Path(local_dir)
if not p.exists():
p.mkdir(parents=True)
elif not p.is_dir():
return 'local '+local_dir+' exists, but not a directory'
for child in p.iterdir():
return 'local path '+local_dir+' is not empty, clear it first'
p = pathlib.Path(local_dir+'/____data')
p.touch()
obj = open(str(p),'wb')
try:
if data[0] is not None:
obj.write(data[0])
finally:
obj.close()
if not without_acl:
ret = self.zk.get_acls(path)
lst = []
if ret is not None:
for acl in ret[0]:
lst.append( {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id} )
p = pathlib.Path(local_dir+'/____acl')
p.touch()
obj = open(str(p),'w')
try:
obj.write(json.dumps(lst))
finally:
obj.close()
children = self.zk.get_children(path)
if children is not None:
for child in children:
if child=='zookeeper':
continue
path = main_path+'/'+child
if max_depth>0:
ret = self.jsZkExport(local_dir+'/'+child, path, max_depth, without_acl)
if len(ret)>0:
return ret
except NoNodeError as e:
logging.error("jsZkExport, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkExport, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''
@pyqtSlot(str, str, int, int, result=str)
def jsZkImport(self, local_dir, main_path, max_depth, without_acl):
path = ''
try:
max_depth -= 1
path = main_path
logging.info("jsZkImport, path="+main_path+' from local dir '+local_dir)
obj = open(local_dir+'/____data', 'rb')
if self.zk.exists(path) is None:
self.zk.create(path, obj.read(), acl=self.default_acl)
else:
self.zk.set(path, obj.read())
if not without_acl:
obj = open(local_dir+'/____acl', 'r')
acl_list = None
list_str = obj.read()
if list_str is not None and len(list_str)>0:
cache = json.loads(list_str)
acl_list = []
for one in cache:
acl = kazoo.security.ACL( one['perm'], kazoo.security.Id(one['scheme'], one['id']) )
acl_list.append(acl)
self.zk.set_acls(path, acl_list)
p = pathlib.Path(local_dir)
for child in p.iterdir():
if not child.is_dir():
continue
if child.name=='zookeeper':
continue
ret = self.jsZkImport(str(child), path+'/'+child.name, max_depth, without_acl)
if len(ret)>0:
return ret
except NoNodeError as e:
logging.error("jsZkImport, NoNodeError")
return "node not exists, path="+path
except Exception as e:
logging.error("jsZkImport, "+str(e))
t,v,tb = sys.exc_info()
strlist = traceback.format_tb(tb)
return str(e)+', traceback='+str(strlist)
return ''