class ZookeeperHostSource(HostSource): def __init__(self, config): self.zk = KazooClient(config.hosts.zookeeper.connection_string) self.zk.start() credentials = ":".join( (config.hosts.zookeeper.username, config.hosts.zookeeper.password)) self.zk.add_auth("digest", credentials) self.retry = KazooRetry(max_tries=3) def get_all_hosts(self): try: return self.retry(self.zk.get_children, "/server") except KazooException as e: raise HostLookupError("zk host enumeration failed: %r", e) def should_host_be_alive(self, host_name): try: host_root = "/server/" + host_name state = self.retry(self.zk.get, host_root + "/state")[0] if state in ("kicking", "unhealthy"): return False is_autoscaled = self.retry(self.zk.exists, host_root + "/asg") is_running = self.retry(self.zk.exists, host_root + "/running") return not is_autoscaled or is_running except NoNodeException: return False except KazooException as e: raise HostLookupError("zk host aliveness check failed: %r", e) def shut_down(self): self.zk.stop()
class ZookeeperHostSource(HostSource): def __init__(self, config): self.zk = KazooClient(config.hosts.zookeeper.connection_string) self.zk.start() credentials = ":".join((config.hosts.zookeeper.username, config.hosts.zookeeper.password)) self.zk.add_auth("digest", credentials) self.retry = KazooRetry(max_tries=3) def get_all_hosts(self): try: return self.retry(self.zk.get_children, "/server") except KazooException as e: raise HostLookupError("zk host enumeration failed: %r", e) def should_host_be_alive(self, host_name): try: host_root = "/server/" + host_name state = self.retry(self.zk.get, host_root + "/state")[0] if state in ("kicking", "unhealthy"): return False is_autoscaled = self.retry(self.zk.exists, host_root + "/asg") is_running = self.retry(self.zk.exists, host_root + "/running") return not is_autoscaled or is_running except NoNodeException: return False except KazooException as e: raise HostLookupError("zk host aliveness check failed: %r", e) def shut_down(self): self.zk.stop()
def init_hierarchy(hosts, hierarchy, users, auth): zkcli = KazooClient(hosts) zkcli.start() scheme, name, passw = auth zkcli.add_auth(scheme, name + ':' + passw) def _init_hierarchy(hierarchy, parent_path): if len(hierarchy) == 0: return for node, attr_children in hierarchy.items(): val = attr_children.get('__val__', {}) val = utfjson.dump(val) acl = attr_children.get('__acl__') path = _init_node(zkcli, parent_path, node, val, acl, users) children = { k: v for k, v in attr_children.items() if k not in ('__val__', '__acl__') } _init_hierarchy(children, path) _init_hierarchy(hierarchy, '/') close_zk(zkcli)
def _loop_acquire(self, n, ident): zk = KazooClient(hosts='127.0.0.1:21811') zk.start() scheme, name, passw = zk_test_auth zk.add_auth(scheme, name + ':' + passw) for ii in range(n): l = zkutil.ZKLock('foo_name', zkclient=zk) with l: self.total += 1 self.counter += 1 self.assertTrue(self.counter == 1) time.sleep(0.01) self.counter -= 1 dd("id={ident:0>2} n={ii:0>2} got and released lock: {holder}".format( ident=ident, ii=ii, holder=l.lock_holder)) zk.stop()
def get_zk(): # create : acl=CREATOR_ALL_ACL # zk = KazooClient(hosts=zk_host, auth_data=[(scheme, credential)]) zk = KazooClient(hosts=zk_host) zk.start() zk.add_auth(scheme, credential) return zk
def init_hierarchy(hosts, hierarchy, users, auth): zkcli = KazooClient(hosts) zkcli.start() scheme, name, passw = auth zkcli.add_auth(scheme, name + ':' + passw) def _init_hierarchy(hierarchy, parent_path): if len(hierarchy) == 0: return for node, attr_children in hierarchy.items(): val = attr_children.get('__val__', {}) val = utfjson.dump(val) acl = attr_children.get('__acl__') path = _init_node(zkcli, parent_path, node, val, acl, users) children = {k: v for k, v in attr_children.items() if k not in ('__val__', '__acl__') } _init_hierarchy(children, path) _init_hierarchy(hierarchy, '/') close_zk(zkcli)
class ShellTestCase(unittest.TestCase): """ base class for all tests """ def setUp(self): """ make sure that the prefix dir is empty """ self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests") self.zk_host = os.getenv("ZKSHELL_ZK_HOST", "localhost:2181") self.username = os.getenv("ZKSHELL_USER", "user") self.password = os.getenv("ZKSHELL_PASSWD", "user") self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=") self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "secret") self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest") self.client = KazooClient(self.zk_host, 5) self.client.start() self.client.add_auth(self.scheme, self.auth_id) if self.client.exists(self.tests_path): self.client.delete(self.tests_path, recursive=True) self.client.create(self.tests_path, str.encode("")) self.output = StringIO() self.shell = Shell([self.zk_host], 5, self.output, setup_readline=False, async=False) # Create an empty test dir (needed for some tests) self.temp_dir = tempfile.mkdtemp() @property def auth_id(self): return "%s:%s" % (self.username, self.password) @property def auth_digest(self): return "%s:%s" % (self.username, self.digested_password) def tearDown(self): self.output = None self.shell = None if os.path.isdir(self.temp_dir): shutil.rmtree(self.temp_dir) if self.client.exists(self.tests_path): self.client.delete(self.tests_path, recursive=True) self.client.stop() ### # Helpers. ## def create_compressed(self, path, value): """ ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client to create a znode with zlib compressed content. """ compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value) self.client.create(path, compressed, makepath=True)
def auth_kazoo(username, password): auth_client = KazooClient(hosts=ZOO_HOSTS) try: auth_client.start(timeout=5) auth_client.add_auth('digest', '{username}:{password}'.format(username=username, password=password)) yield auth_client finally: auth_client.stop()
def make_owning_zkclient(hosts, auth): zkclient = KazooClient(hosts=hosts) zkclient.start() if auth is not None: scheme, name, passw = auth zkclient.add_auth(scheme, name + ':' + passw) return zkclient
def make_owning_zkclient(hosts, auth): zkclient = KazooClient(hosts=hosts) zkclient.start() if auth is not None: scheme, name, passw = auth zkclient.add_auth(scheme, name + ':' + passw) return zkclient
def zk_client(host, scheme, credential): """ returns a connected (and possibly authenticated) ZK client """ if not re.match(r".*:\d+$", host): host = "%s:%d" % (host, DEFAULT_ZK_PORT) client = KazooClient(hosts=host) client.start() if scheme != "": client.add_auth(scheme, credential) return client
def get_client(args): client = KazooClient(connection_retry=3, command_retry=3, timeout=1, hosts=args.hosts, use_ssl=args.secure, certfile=args.cert, keyfile=args.key, ca=args.ca) client.start() if (args.user and args.password): client.add_auth('digest', f'{args.user}:{args.password}') return client
def zk_client(host, scheme, credential): """ returns a connected (and possibly authenticated) ZK client """ if not re.match(r".*:\d+$", host): host = "%s:%d" % (host, DEFAULT_ZK_PORT) client = KazooClient(hosts=host) client.start() if scheme != "": client.add_auth(scheme, credential) return client
class ZKWrap(object): def __init__(self): self.zk_hosts = settings.ZK_HOSTS self.zk_root = settings.ZK_ROOT self.zk_timeout = settings.ZK_TIMEOUT self.zk_servers = settings.ZK_SERVERS self.zk_clients = settings.ZK_CLIENTS self.zk_configs = settings.ZK_CONFIGS self.zk_statistics = settings.ZK_STATISTICS self.zk_username = settings.ZK_USERNAME self.zk_password = settings.ZK_PASSWORD self.zk_flush = time.time() self.lock = threading.Lock() self.resource = {} self.zk_harpc_acl = make_digest_acl(username=self.zk_username, password=self.zk_password, all=True) self.zk_anyone_acl = make_acl(scheme='world', credential='anyone', read=True) def children_callback(self, children): print '****', children def get_children(self, path): return self.zk.get_children(path) def get(self, path): return self.zk.get(path) def get_acls(self, path): return self.zk.get_acls(path) def load_zk(self): self.init_connect() def get_resource(self): return self.resource def init_connect(self): try: self.zk = KazooClient(hosts=self.zk_hosts, timeout=self.zk_timeout) self.zk.start(timeout=self.zk_timeout) self.zk.add_auth(scheme='digest', credential=self.zk_username + ':' + self.zk_password) logger.info("connect to zk servers") except Exception, e: logger.error('manage.ZKWrap.init_connect:' + getExceptionDesc(e)) return False
def get_zk(): global _zk if _zk is None: _zk = KazooClient( app.config['ZK_CONNECTION_STRING'], default_acl=( # grants read permissions to anyone. make_acl('world', 'anyone', read=True), # grants all permissions to the creator of the node. make_acl('auth', '', all=True) ) ) _zk.start() _zk.add_auth('digest', jones_credential) _zk.DataWatch('/services', func=ensure_root) return _zk
def connect_to_zookeeper(hostlist, credentials): """Create a connection to the ZooKeeper ensemble. If authentication credentials are provided (as a two-tuple: username, password), we will ensure that they are provided to the server whenever we establish a connection. """ client = KazooClient(hostlist, timeout=5, max_retries=3) # convenient helper function for making credentials client.make_acl = functools.partial(make_digest_acl, *credentials) client.start() client.add_auth("digest", ":".join(credentials)) return client
def connect_to_zookeeper(hostlist, credentials): """Create a connection to the ZooKeeper ensemble. If authentication credentials are provided (as a two-tuple: username, password), we will ensure that they are provided to the server whenever we establish a connection. """ client = KazooClient(hostlist, timeout=5, max_retries=3) # convenient helper function for making credentials client.make_acl = functools.partial(make_digest_acl, *credentials) client.connect() client.add_auth("digest", ":".join(credentials)) return client
class ShellTestCase(unittest.TestCase): """ base class for all tests """ @classmethod def setUpClass(cls): get_global_cluster().start() def setUp(self): """ make sure that the prefix dir is empty """ self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests") self.zk_hosts = ",".join(server.address for server in get_global_cluster()) self.username = os.getenv("ZKSHELL_USER", "user") self.password = os.getenv("ZKSHELL_PASSWD", "user") self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=") self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "test") self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest") self.client = KazooClient(self.zk_hosts, 5) self.client.start() self.client.add_auth(self.scheme, self.auth_id) if self.client.exists(self.tests_path): self.client.delete(self.tests_path, recursive=True) self.client.create(self.tests_path, str.encode("")) self.output = XStringIO() self.shell = Shell([self.zk_hosts], 5, self.output, setup_readline=False, asynchronous=False) # Create an empty test dir (needed for some tests) self.temp_dir = tempfile.mkdtemp() @property def auth_id(self): return "%s:%s" % (self.username, self.password) @property def auth_digest(self): return "%s:%s" % (self.username, self.digested_password) def tearDown(self): if self.output is not None: self.output.close() self.output = None if self.shell is not None: self.shell._disconnect() self.shell = None if os.path.isdir(self.temp_dir): shutil.rmtree(self.temp_dir) if self.client is not None: if self.client.exists(self.tests_path): self.client.delete(self.tests_path, recursive=True) self.client.stop() self.client.close() self.client = None ### # Helpers. ## def create_compressed(self, path, value): """ ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client to create a znode with zlib compressed content. """ compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value) self.client.create(path, compressed, makepath=True)
def test_init_hierarchy(self): auth = ('digest', 'aa', 'pw_aa') hosts = '127.0.0.1:21811' users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'} hierarchy = { 'node1': { '__val__': 'node1_val', '__acl__': {'aa': 'cdrwa', 'bb': 'rw'}, 'node11': { '__val__': 'node11_val', '__acl__': {'aa': 'cdrwa', 'cc': 'r'}, }, 'node12': { '__val__': 'node12_val', 'node121': {'__val__': 'node121_val'} }, 'node13': { '__acl__': {'aa': 'cdrwa'} } }, 'node2': { '__val__': 'node2_val', 'node21': {'__val__': 'node21_val'}, 'node22': {'__acl__': {'aa': 'rwa'}} }, 'node3': { '__acl__': {'aa': 'carw', 'cc': 'r'}, 'node31': {'node311': {'node3111': {}, 'node3112': {}}} } } zkutil.init_hierarchy(hosts, hierarchy, users, auth) zkcli = KazooClient(hosts) zkcli.start() zkcli.add_auth('digest', 'aa:pw_aa') expected_nodes = ( ('/node1', '"node1_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set(['node11', 'node12', 'node13'])), ('/node1/node11', '"node11_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'cc', 'r')], set([])), ('/node1/node12', '"node12_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set(['node121'])), ('/node1/node12/node121', '"node121_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set([])), ('/node1/node13', '{}', [('digest', 'aa', 'cdrwa')], set([])), ('/node2', '"node2_val"', [('world', 'anyone', 'cdrwa')], set(['node21', 'node22'])), ('/node2/node21', '"node21_val"', [('world', 'anyone', 'cdrwa')], set([])), ('/node2/node22', '{}', [('digest', 'aa', 'rwa')], set([])), ('/node3', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set(['node31'])), ('/node3/node31', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set(['node311'])), ('/node3/node31/node311', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set(['node3111', 'node3112'])), ('/node3/node31/node311/node3111', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set([])), ('/node3/node31/node311/node3112', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set([])), ) for node, val, acl, children in expected_nodes: actual_acl = zkutil.parse_kazoo_acl(zkcli.get_acls(node)[0]) self.assertEqual(val, zkcli.get(node)[0]) self.assertEqual(acl, actual_acl) self.assertEqual(children, set(zkcli.get_children(node))) zkcli.stop()
sentry = Sentry(app) jones_credential = make_digest_acl_credential( 'Jones', app.config['ZK_DIGEST_PASSWORD'] ) zk = KazooClient( app.config['ZK_CONNECTION_STRING'], default_acl=( # grants read permissions to anyone. make_acl('world', 'anyone', read=True), # grants all permissions to the creator of the node. make_acl('auth', '', all=True) ) ) zk.connect() zk.add_auth('digest', jones_credential) zk.ensure_path('/services') def request_wants(t): types = ['text/plain', 'text/html', 'application/json'] assert t in types best = request.accept_mimetypes \ .best_match(types) return best == t @app.template_filter() def as_json(d, indent=None): return Markup(json.dumps(d, indent=indent))
def main(): module = AnsibleModule(argument_spec=MODULE_ARGUMENTS) host_a = module.params.get('host', None) authmethod_a = module.params.get('authmethod', None) type_a = module.params.get('type', None) dir_a = module.params.get('dir', None) acl_a = module.params.get('acl', None) value_a = module.params.get('value', '') # return error if library is not installed if not KAZOO_IMPORTED: module.fail_json(msg='Kazoo python library is not installed. Use "pip install kazoo" before running script') # try connect to zookeeper server try: client = KazooClient(host_a) client.start() except: module.fail_json(msg='Could not connect to the zookeeper server') # add auth method to client if authmethod_a is not None: try: method, username, password = [param.replace(':','') for param in authmethod_a.split(':')] if not client.add_auth(method, username + ':' + password): module.fail_json(msg='Auth failed') except AuthFailedError: module.fail_json(msg='Could not auth as specified user') except: module.fail_json(msg='Param authmethod is not correct') # make sure one of setting acls is also used in authmethod and have admin access or is set to world:anyone # access can be empty so this acl will be deleted if acl_a is not None: acl_list = generate_acl_list(module, acl_a) else: acl_list = None # failing if path if match('^(/[^ /]+)+$', dir_a) is None: module.fail_json(msg='Specified directory does not match zookeeper format') changed = False # create and update acl for znode if type_a == 'create': # check if znode parent exists if not client.exists(get_parent(dir_a)): module.fail_json(msg='Znode {0} parent doesnt exists'.format(dir_a)) changed = create_dir(module, client, dir_a, acl_list, value_a if value_a is not None else '') elif type_a == 'create_r': path = [a.replace('/', '') for a in dir_a.split('/') if a is not '' and a is not None] dir = '' for id in range(0, len(path)): dir += '/{0}'.format(path[id]) changed = create_dir(module, client, dir, acl_list) or changed # selected type does not match options else: module.fail_json(msg='Unknown option: {0}'.format(type_a)) client.stop() client.close() module.exit_json(changed=changed, msg='Znode and acls match')
def test_export_hierarchy(self): auth = ('digest', 'aa', 'pw_aa') hosts = '127.0.0.1:21811' users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'} hierarchy = { 'node0': { '__val__': '', '__acl__': {'aa': 'cdrwa', 'bb': 'rw'}}, 'node1': { '__val__': 'node1_val', '__acl__': {'aa': 'cdrwa', 'bb': 'rw'}, 'node11': { '__val__': 'node11_val', '__acl__': {'aa': 'cdrwa', 'cc': 'r'}, }, 'node12': { '__val__': 'node12_val', 'node121': {'__val__': 'node121_val'} }, 'node13': { '__acl__': {'aa': 'cdrwa'} } }, 'node2': { '__val__': 'node2_val', 'node21': {'__val__': 'node21_val'}, 'node22': {'__acl__': {'aa': 'rwa'}} }, 'node3': { '__acl__': {'aa': 'carw', 'cc': 'r'}, 'node31': { 'node311': { 'node3111': {}, 'node3112': {} } } } } zkutil.zkutil.init_hierarchy(hosts, hierarchy, users, auth) zkcli = KazooClient(hosts) zkcli.start() zkcli.add_auth('digest', 'aa:pw_aa') invalid_zkpath_cases = ('a', 'a/', 'a/b') for zkpath in invalid_zkpath_cases: with self.assertRaises(zkutil.zkutil.ZkPathError): zkutil.zkutil.export_hierarchy(zkcli, zkpath) valid_cases = ( ('/', {'__acl__': {u'anyone': 'cdrwa'}, '__val__': '', u'node0': { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '""'}, u'node1': { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '"node1_val"', u'node11': { '__acl__': {u'aa': 'cdrwa', u'cc': 'r'}, '__val__': '"node11_val"'}, u'node12': { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '"node12_val"', u'node121': { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '"node121_val"'}}, u'node13': { '__acl__': {u'aa': 'cdrwa'}, '__val__': '{}'}}, u'node2': { '__acl__': {u'anyone': 'cdrwa'}, '__val__': '"node2_val"', u'node21': { '__acl__': {u'anyone': 'cdrwa'}, '__val__': '"node21_val"'}, u'node22': {'__acl__': {u'aa': 'rwa'}, '__val__': '{}'}}, u'node3': { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}', u'node31': { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}', u'node311': { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}', u'node3111': { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}'}, u'node3112': { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}'}}}}, u'zookeeper': { '__acl__': {u'anyone': 'cdrwa'}, '__val__': '', u'quota': { '__acl__': {u'anyone': 'cdrwa'}, '__val__': ''}}}), ('/node0', { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '""'}), ('/node1', { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '"node1_val"', u'node11': { '__acl__': {u'aa': 'cdrwa', u'cc': 'r'}, '__val__': '"node11_val"'}, u'node12': { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '"node12_val"', u'node121': { '__acl__': {u'aa': 'cdrwa', u'bb': 'rw'}, '__val__': '"node121_val"'}}, u'node13': { '__acl__': {u'aa': 'cdrwa'}, '__val__': '{}'}}), ('/node1/node11', { '__acl__': {u'aa': 'cdrwa', u'cc': 'r'}, '__val__': '"node11_val"'}), ('/node2', { '__acl__': {u'anyone': 'cdrwa'}, '__val__': '"node2_val"', u'node21': { '__acl__': {u'anyone': 'cdrwa'}, '__val__': '"node21_val"'}, u'node22': { '__acl__': {u'aa': 'rwa'}, '__val__': '{}'}}), ('/node3', { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}', u'node31': { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}', u'node311': { '__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}', u'node3111': {'__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}'}, u'node3112': {'__acl__': {u'aa': 'rwca', u'cc': 'r'}, '__val__': '{}'}}}})) for path, expected_rst in valid_cases: rst = zkutil.export_hierarchy(zkcli, path) self.assertEqual(rst, expected_rst) zkcli.stop()
app.config.from_envvar('JONES_SETTINGS', silent=True) if 'SENTRY_DSN' in app.config: sentry = Sentry(app) jones_credential = make_digest_acl_credential('Jones', app.config['ZK_DIGEST_PASSWORD']) zk = KazooClient( app.config['ZK_CONNECTION_STRING'], default_acl=( # grants read permissions to anyone. make_acl('world', 'anyone', read=True), # grants all permissions to the creator of the node. make_acl('auth', '', all=True))) zk.start() zk.add_auth('digest', jones_credential) @zk.DataWatch('/services') def ensure_root(data, stat): if not data: zk.ensure_path('/services') def request_wants(t): types = ['text/plain', 'text/html', 'application/json'] assert t in types best = request.accept_mimetypes \ .best_match(types) return best == t
class ZkOperation(object): def __init__(self, zk_hosts, zk_root): self.zk = KazooClient(zk_hosts) self.root = zk_root self.tasks = set() self.event = threading.Event() def start(self): if self.zk.exists: self.zk.start() self.zk.add_auth('digest', 'publish:publish') if self.zk.connected: self.zk.ensure_path(self.root) def is_job_exist(self, job_id): if job_id == '': raise Exception('job_id is ""') node = self.root + '/jobs/' + job_id return self.zk.exists(node) def check_task_status(self, path): if path == '': raise Exception('path is ""') node = self.root + path data, _ = self.zk.get(node) return data['Status'] def _is_exist(self, node): if self.zk.connected and self.zk.exists(node): if self.zk.exists(node): return True else: return False else: logger.error('zk not connected or node is exists') return False def _create_node(self, node, value=None): if value is None: value = '' value = json.dumps(value) if self.zk.connected and not self.zk.exists(node): self.zk.create(node, makepath=True, value=value.encode()) return True else: logger.error('zk not connected or node is exists') return False def _update_node(self, node, value): if self.zk.connected and self.zk.exists(node): tx = self.zk.transaction() tx.set_data(node, json.dumps(value).encode()) tx.commit() return True else: logger.error( 'update node failed: zk not connected or node is not exists') return False def _get_node(self, node): if self.zk.connected and self.zk.exists(node): node_value, _ = self.zk.get(node) if node_value: return json.loads(node_value.decode()) else: return {} else: logger.error('zk not connected or node is exists') return False def _delete_node(self, node): if self.zk.connected: if not self.zk.exists(node): return True else: self.zk.delete(node, recursive=True) return True else: logger.error('zk not connected') return False # is exist def is_exist_signal(self, job_id): node = '/{}/signal/{}'.format(self.root, job_id) return self._is_exist(node) # CREATE def create_new_job(self, job_id, job_value=None): if job_value is None: job_value = '' if job_id != '': node = self.root + '/jobs/' + job_id ret = self._create_node(node, job_value) return ret else: logger.error('job_id is null') return False def create_new_target(self, job_id, target, target_value): node = '/{}/jobs/{}/targets/{}'.format(self.root, job_id, target) ret = self._create_node(node, target_value) return ret def create_new_task(self, job_id, target, task): node = '/{}/jobs/{}/targets/{}/tasks/{}'.format( self.root, job_id, target, task['task_id']) ret = self._create_node(node, task) return ret def create_job_signal(self, job_id): node = '/{}/signal/{}'.format(self.root, job_id) ret = self._create_node(node, uuid.uuid4().hex) return ret # GET def get_job_info(self, job_id): job_node = '{}/jobs/{}'.format(self.root, job_id) job_value, _ = self.zk.get(job_node) job_info = json.loads(job_value.decode()) return job_info def get_target_info(self, job_id, target): target_node = '{}/jobs/{}/targets/{}'.format(self.root, job_id, target) target_value, _ = self.zk.get(target_node) target_info = json.loads(target_value.decode()) return target_info def get_task_info(self, job_id, target, task_id): task_node = '{}/jobs/{}/targets/{}/tasks/{}'.format( self.root, job_id, target, task_id) task_value, _ = self.zk.get(task_node) task_info = json.loads(task_value.decode()) return task_info # UPDATE def update_job_status(self, job_id, task): if job_id != '' and task is not None: node = self.root + '/signal/' + job_id else: raise Exception('job_id is ""') if self.zk.connected and self.is_job_exist(job_id): tx = self.zk.transaction() tx.set_data(node, task.encode()) tx.commit() def handler_task(self, job_id, task_id, task_name, task_message, status): # 为不必传回target, 遍历任务节点 if not self.is_job_exist(job_id): logger.error("can not find this jobid: {}".format(job_id)) return False job_node = "{}/jobs/{}/targets".format(self.root, job_id) for target in self.zk.get_children(job_node): target_node = "{}/{}/tasks".format(job_node, target) for task in self.zk.get_children(target_node): if task == task_id: task_node = "{}/{}".format(target_node, task) task_value, _ = self.zk.get(task_node) new_task_value = json.loads(task_value.decode()) new_task_value['status'] = status tx = self.zk.transaction() tx.set_data(task_node, json.dumps(new_task_value).encode()) tx.commit() task_value, _ = self.zk.get(task_node) return True logger.error("can not find this taskid: {} in {}".format( task_id, job_id)) return False def get_target_by_taskid(self, job_id, task_id): if self.is_job_exist(job_id): node = "{}/jobs/{}/targets".format(self.root, job_id) for target in self.zk.get_children(node): path = '{}/{}/tasks'.format(node, target) for taskid in self.zk.get_children(path): if taskid == task_id: return target return False else: logger.error("job is not exist: job_id={}".format(job_id)) def send_signal(self, job_id): node = '{}/signal/{}'.format(self.root, job_id) logger.info("send singal: {}".format(job_id)) tx = self.zk.transaction() tx.set_data(node, uuid.uuid4().bytes) tx.commit() # DELETE def delete_job(self, job_id): node = "{}/jobs/{}".format(self.root, job_id) logger.info("delete job: job_id={}".format(job_id)) self._delete_node(node) def delete_signal(self, job_id): node = '{}/signal/{}'.format(self.root, job_id) logger.info("delete singal: {}".format(job_id)) self._delete_node(node) def delete_target(self, job_id, target): target_node = '{}/jobs/{}/targets/{}'.format(self.root, job_id, target) logger.info("delete target: job_id={}, target={}".format( job_id, target)) self._delete_node(target_node) def delete_task(self, job_id, target, task_id): task_node = '{}/jobs/{}/targets/{}/tasks/{}'.format( self.root, job_id, target, task_id) logger.info("delete task: job_id ={}, target={}, task_id={}".format( job_id, target, task_id)) self._delete_node(task_node) ################################# # CALLBACK ## exsit CALLBACK def is_exist_callback(self, callback_node): node = "{}/callback/{}".format(self.root, callback_node) if self.zk.exists(node): return True else: return False ## INIT CALLBACK def init_callback_by_jobid(self, job_id): node = "{}/callback/{}".format(self.root, job_id) job_callback_value = { "job_id": job_id, "status": JobStatus.init.value, "messages": "" } callback = { "callback_level": CallbackLevel.job.value, "callback_info": job_callback_value } ret = self._create_node(node, callback) return ret def init_callback_by_target(self, job_id, target): node = "{}/callback/{}".format(self.root, job_id + "_" + target) target_callback_value = { "job_id": job_id, "target": target, "status": TargetStatus.init.value, "messages": "" } callback = { "callback_level": CallbackLevel.target.value, "callback_info": target_callback_value } ret = self._create_node(node, callback) return ret def init_callback_by_taskid(self, job_id, target, task_id, task_name): node = "{}/callback/{}".format(self.root, task_id) taskid_callback_value = { "job_id": job_id, "target": target, "task_name": task_name, "status": JobStatus.init.value, "messages": "", } callback = { "callback_level": CallbackLevel.task.value, "callback_info": taskid_callback_value } ret = self._create_node(node, callback) return ret ## GET CALLBACK def get_callback_info(self, callback): node = "{}/callback/{}".format(self.root, callback) if self.zk.exists(node): node_value = self._get_node(node) return node_value else: return False ## UPDATE CALLBACK def update_callback_by_jobid(self, job_id, status, messages=None): node = "{}/callback/{}".format(self.root, job_id) if not self.zk.exists(node): return False node_value = self._get_node(node) node_value["callback_info"]["status"] = status if messages is not None: node_value["callback_info"]["messages"] = messages ret = self._update_node(node, node_value) return ret def update_callback_by_target(self, job_id, target, status, messages=None): node = "{}/callback/{}".format(self.root, job_id + "_" + target) if not self.zk.exists(node): return False node_value = self._get_node(node) node_value["callback_info"]["status"] = status if messages is not None: node_value["callback_info"]["messages"] = messages ret = self._update_node(node, node_value) return ret def update_callback_by_taskid(self, job_id, taskid, status, messages=None): node = "{}/callback/{}".format(self.root, taskid) if not self.zk.exists(node): return False node_value = self._get_node(node) node_value["callback_info"]["status"] = status if messages is not None: node_value["callback_info"]["messages"] = messages ret = self._update_node(node, node_value) return ret ## DELETE CALLBACK def delete_callback_node(self, callback): node = "{}/callback/{}".format(self.root, callback) ret = self._delete_node(node) if ret: logger.info( "delete callback node success: callback={}".format(node)) else: logger.error("delete callback node fail: callback={}".format(node)) return ret
class TestZKLock(unittest.TestCase): @classmethod def setUpClass(cls): utdocker.pull_image(zk_test_tag) def setUp(self): config.zk_acl = zk_test_acl config.zk_auth = zk_test_auth self.counter = 0 self.running = True utdocker.create_network() utdocker.start_container( zk_test_name, zk_test_tag, env={ "ZOO_MY_ID": 1, "ZOO_SERVERS": "server.1=0.0.0.0:2888:3888", }, port_bindings={ 2181: 21811, } ) self.zk = KazooClient(hosts='127.0.0.1:21811') self.zk.start() scheme, name, passw = zk_test_auth self.zk.add_auth(scheme, name + ':' + passw) # create lock base dir acl = zkutil.make_kazoo_digest_acl(zk_test_acl) self.zk.create('lock/', acl=acl) self.lck = zkutil.ZKLock('foo_name', zkclient=self.zk) def tearDown(self): self.zk.stop() utdocker.remove_container(zk_test_name) def _on_conn_change(self, state): self.lsn_count += 1 def test_bounded_listener(self): # ensure that adding a bounded listener(self.on_xxx) is ok self.lsn_count = 0 self.zk.add_listener(self._on_conn_change) self.zk.add_listener(self._on_conn_change) self.zk.stop() self.assertEqual(1, self.lsn_count) def _loop_acquire(self, n, ident): zk = KazooClient(hosts='127.0.0.1:21811') zk.start() scheme, name, passw = zk_test_auth zk.add_auth(scheme, name + ':' + passw) for ii in range(n): l = zkutil.ZKLock('foo_name', zkclient=zk) with l: self.total += 1 self.counter += 1 self.assertTrue(self.counter == 1) time.sleep(0.01) self.counter -= 1 dd("id={ident:0>2} n={ii:0>2} got and released lock: {holder}".format( ident=ident, ii=ii, holder=l.lock_holder)) zk.stop() def test_concurrent(self): self.running = True self.total = 0 n_repeat = 40 n_thread = 5 ths = [] for ii in range(n_thread): t = threadutil.start_daemon_thread(self._loop_acquire, args=(n_repeat, ii,)) ths.append(t) for th in ths: th.join() self.running = False self.assertEqual(n_repeat * n_thread, self.total) def test_persistent(self): l = zkutil.ZKLock('foo_name', ephemeral=False, on_lost=lambda: True) try: with l: l.zkclient.stop() except ConnectionClosedError: pass self.assertRaises(zkutil.LockTimeout, self.lck.acquire, timeout=0.2) def test_timeout(self): l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True) l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True) with l1: with ututil.Timer() as t: self.assertRaises(zkutil.LockTimeout, l2.acquire, timeout=0.2) self.assertAlmostEqual(0.2, t.spent(), places=1) with ututil.Timer() as t: self.assertRaises(zkutil.LockTimeout, l2.acquire, timeout=-1) self.assertAlmostEqual(0.0, t.spent(), delta=0.01) try: l2.acquire(timeout=-1) except zkutil.LockTimeout: self.fail('timeout<0 should could acquire') def test_lock_holder(self): a = zkutil.ZKLock('foo_name', on_lost=lambda: True) b = zkutil.ZKLock('foo_name', on_lost=lambda: True) with a: self.assertEqual((a.identifier, 0), a.lock_holder) val, zstate = self.zk.get(a.lock_path) self.assertEqual((val, zstate.version), a.lock_holder) locked, holder, ver = b.try_acquire() self.assertFalse(locked) self.assertEqual((a.identifier, 0), (holder, ver)) self.assertEqual((val, zstate.version), (holder, ver)) def test_watch_acquire(self): a = zkutil.ZKLock('foo', on_lost=lambda: True) b = zkutil.ZKLock('foo', on_lost=lambda: True) # no one locked n = 0 for holder, ver in a.acquire_loop(): n += 1 self.assertEqual(0, n, 'acquired directly') # watch node change it = b.acquire_loop() holder, ver = it.next() self.assertEqual((a.identifier, 0), (holder, ver)) self.zk.set(a.lock_path, 'xx') holder, ver = it.next() self.assertEqual(('xx', 1), (holder, ver), 'watched node change') a.release() try: holder, ver = it.next() self.fail('should not have next yield') except StopIteration: pass self.assertTrue(b.is_locked()) def test_try_lock(self): l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True) l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True) with l1: with ututil.Timer() as t: locked, holder, ver = l2.try_acquire() self.assertFalse(locked) self.assertEqual(l1.identifier, holder) self.assertGreaterEqual(ver, 0) self.assertAlmostEqual(0.0, t.spent(), delta=0.05) with ututil.Timer() as t: locked, holder, ver = l2.try_acquire() self.assertTrue(locked) self.assertEqual(l2.identifier, holder) self.assertEqual(ver, 0) self.assertAlmostEqual(0.0, t.spent(), delta=0.05) def test_try_release(self): l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True) l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True) released, holder, ver = l1.try_release() self.assertEqual((True, l1.identifier, -1), (released, holder, ver)) with l2: released, holder, ver = l1.try_release() self.assertEqual((False, l2.identifier, 0), (released, holder, ver)) released, holder, ver = l2.try_release() self.assertEqual((True, l2.identifier, 0), (released, holder, ver)) def test_zk_lost(self): sess = {'acquired': True} def watch(state): dd('zk node state changed to: ', state) sess['acquired'] = False self.zk.add_listener(watch) # test zk close l = zkutil.ZKLock('foo_name', zkclient=self.zk) l.acquire() self.zk.stop() time.sleep(0.1) self.assertFalse(sess['acquired']) # test node delete sess['acquired'] = True self.zk.start() self.zk.add_listener(watch) l = zkutil.ZKLock('foo_name', zkclient=self.zk) with l: time.sleep(0.1) self.zk.delete(l.zkconf.lock('foo_name')) time.sleep(0.1) self.assertFalse(sess['acquired']) def test_node_change_after_acquired(self): sess = {'acquired': True} def on_lost(): sess['acquired'] = False l = zkutil.ZKLock('foo_name', zkclient=self.zk, on_lost=on_lost) with l: sess['acquired'] = True self.zk.delete(l.zkconf.lock('foo_name')) time.sleep(0.1) self.assertFalse(sess['acquired']) l = zkutil.ZKLock('foo_name', zkclient=self.zk, on_lost=on_lost) with l: sess['acquired'] = True self.zk.set(l.zkconf.lock('foo_name'), 'xxx') time.sleep(0.1) self.assertFalse(sess['acquired']) def test_node_change_after_released(self): sess = {'acquired': True} def on_lost(): sess['acquired'] = False l = zkutil.ZKLock('foo_name', zkclient=self.zk, on_lost=on_lost) with l: sess['acquired'] = True time.sleep(0.1) self.assertTrue(sess['acquired']) def test_is_locked(self): l = zkutil.ZKLock('foo_name', zkclient=self.zk) with l: pass self.assertFalse(l.is_locked()) l = zkutil.ZKLock('foo_name', zkclient=self.zk) l.acquire() self.assertTrue(l.is_locked()) l.try_release() self.assertFalse(l.is_locked()) def test_conn_lost_when_blocking_acquiring(self): l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True) th = threadutil.start_daemon(target=self.zk.stop, after=0.5) with l2: try: self.lck.acquire(timeout=1) self.fail('expected connection error') except ConnectionClosedError: pass th.join() def test_internal_zkclient(self): sess = {'acquired': True} def on_lost(): sess['acquired'] = False # There must be a listener specified to watch connection issue self.assertRaises(ValueError, zkutil.ZKLock, 'foo_name') l = zkutil.ZKLock('foo_name', on_lost=on_lost) with l: self.zk.delete(l.zkconf.lock('foo_name')) time.sleep(0.1) self.assertFalse(sess['acquired']) def test_acl(self): with self.lck: acls, zstat = self.zk.get_acls(self.lck.lock_path) dd(acls) self.assertEqual(1, len(acls)) acl = acls[0] expected = zkutil.perm_to_long(zk_test_acl[0][2], lower=False) self.assertEqual(set(expected), set(acl.acl_list)) self.assertEqual('digest', acl.id.scheme) self.assertEqual(zk_test_acl[0][0], acl.id.id.split(':')[0]) def test_config(self): old = (config.zk_acl, config.zk_auth, config.zk_node_id) config.zk_acl = (('foo', 'bar', 'cd'), ('xp', '123', 'cdrwa')) config.zk_auth = ('digest', 'xp', '123') config.zk_node_id = 'abc' l = zkutil.ZKLock('foo_name', on_lost=lambda: True) dd(l.zkconf.acl()) def _check_ac(ac): self.assertEqual('digest', ac.id.scheme) self.assertEqual('foo', ac.id.id.split(':')[0]) self.assertEqual(set(['CREATE', 'DELETE']), set(ac.acl_list)) _check_ac(l.zkconf.kazoo_digest_acl()[0]) with l: # should have created lock node data, zstate = self.zk.get(l.lock_path) dd(data) self.assertEqual('abc', data.split('-')[0]) acls, zstate = self.zk.get_acls(l.lock_path) dd(acls) _check_ac(acls[0]) (config.zk_acl, config.zk_auth, config.zk_node_id) = old def test_hosts(self): l = zkutil.ZKLock('foo_name', zkconf=dict( hosts='127.0.0.1:21811', ), on_lost=lambda: True) with l: self.assertEqual('127.0.0.1:21811', l._hosts) def test_specify_identifier(self): a = zkutil.ZKLock('foo_name', zkconf=dict( hosts='127.0.0.1:21811', ), identifier='faked', on_lost=lambda: True) b = zkutil.ZKLock('foo_name', zkconf=dict( hosts='127.0.0.1:21811', ), identifier='faked', on_lost=lambda: True) a.acquire() b.acquire() dd('a and b has the same identifier thus they both can acquire the lock') def test_release_listener_removed(self): self.lck.release() self.assertNotIn(self.lck.on_connection_change, self.zk.state_listeners) def test_release_owning_client_stopped(self): l = zkutil.ZKLock('foo_name', zkconf=dict( hosts='127.0.0.1:21811', ), on_lost=lambda: True) l.release() self.assertTrue(l.zkclient._stopped.is_set())
class ZooKeeper(object): def __init__(self, hosts, user=None, password=None): self._zookeeper = KazooClient(hosts=hosts) self._zookeeper.start() if user and password: self._zookeeper.add_auth('digest', '{}:{}'.format(user, password)) def stop(self): if self._zookeeper: self._zookeeper.stop() self._zookeeper.close() self._zookeeper = None def list(self, path): try: return self._zookeeper.get_children(path) except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) except NoAuthError: raise ZooKeeperException("No access to list node: {}".format(path)) def get(self, path): try: value, _ = self._zookeeper.get(path) if value: value = value.decode('utf-8') else: value = "" return value except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) except NoAuthError: raise ZooKeeperException("No access to get node: {}".format(path)) def set(self, path, data): try: self._zookeeper.set(path, data.encode()) except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) except NoAuthError: raise ZooKeeperException("No access to set data on node: {}".format(path)) def create(self, path, data=None, ephemeral=False, sequence=False, makepath=False): if data: data = data.encode() else: data = b"" try: self._zookeeper.create(path, value=data, ephemeral=ephemeral, sequence=sequence, makepath=makepath) except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) except NodeExistsError: raise ZooKeeperException("Node already exists: {}".format(path)) except NoAuthError: raise ZooKeeperException("No access to create node: {}".format(path)) def delete(self, path, recursive=False): try: self._zookeeper.delete(path, recursive=recursive) except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) except NotEmptyError: raise ZooKeeperException("Node contains sub-nodes") except NoAuthError: raise ZooKeeperException("No access to delete node: {}".format(path)) def stat(self, path): try: _, stat = self._zookeeper.get(path) return stat except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) def get_acl(self, path): try: acl, _ = self._zookeeper.get_acls(path) return acl except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) def add_acl(self, path, permissions, scheme, id): perms = get_permissions(permissions) if scheme == "digest": username, password = id.split(":") acl = make_digest_acl(username, password, **perms) else: acl = make_acl(scheme, id, **perms) current_acls = self.get_acl(path) current_acls.append(acl) try: self._zookeeper.set_acls(path, current_acls) except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) except InvalidACLError as exc: raise ZooKeeperException("Invalid ACL format: {}".format(str(exc))) except NoAuthError: raise ZooKeeperException("No access to add acl on node: {}".format(path)) def delete_acl(self, path, index): current_acls = self.get_acl(path) deleted = current_acls.pop(index) try: self._zookeeper.set_acls(path, current_acls) except NoNodeError: raise ZooKeeperException("No such node: {}".format(path)) except NoAuthError: raise ZooKeeperException("No access to delete acl from node: {}".format(path)) return deleted
class Zooconf: __zkcon = None __serviceConfig = None __fsList = None def __init__(self): sys.stderr.write("Fileservicepython start initialization at " + str(int(round(time.time() * 1000)))) sys.stderr.flush() self.__initConfProperties() self.__zooConnect() self.__publishService() self.__initFsWatches() def __initConfProperties(self): sys.stderr.write(dirname(dirname(abspath(__file__)))) sys.stderr.flush() self.__serviceConfig = dict() with open( dirname(dirname(abspath(__file__))) + "/config.properties", 'r') as f: for line in f: line = line.rstrip() if "=" not in line: continue #skips blanks and comments w/o = if line.startswith("#"): continue #skips comments which contain = k, v = line.split("=", 1) self.__serviceConfig[k] = v f.close() def __zooConnect(self): sys.stderr.write("Start zooconnect") sys.stderr.flush() config = self.getServiceConfig() self.__zkcon = KazooClient(hosts=config["ZOOKEEPER_HOST"]) self.__zkcon.start() digest_auth = "%s:%s" % (config["ZOOKEEPER_USER"], config["ZOOKEEPER_PASSWORD"]) self.__zkcon.add_auth("digest", digest_auth) def __initFsWatches(self): # lists are supposed to be thread safe in python self.__fsList = [] @self.__zkcon.ChildrenWatch("/plh414python/fileservices") def watch_children(children): self.__fsList = [] sys.stderr.write("Children are now: %s" % children) sys.stderr.flush() for child in children: self.__fsList.append(child) # Above function called immediately, and from then on def __publishService(self): from kazoo.security import make_digest_acl config = self.getServiceConfig() acl = make_digest_acl(config["ZOOKEEPER_USER"], config["ZOOKEEPER_PASSWORD"], all=True) dataJsonDict = {} dataJsonDict['SERVERHOSTNAME'] = config["SERVERHOSTNAME"] dataJsonDict['SERVER_PORT'] = config["SERVER_PORT"] dataJsonDict['SERVER_SCHEME'] = config["SERVER_SCHEME"] dataJsonDict['CONTEXT'] = config["CONTEXT"] self.__zkcon.set("/plh414python/directoryservice/", json.JSONEncoder().encode(dataJsonDict).encode(), -1) def getAvailableFs(self): return self.__fsList def getZooConnection(self): return self.__zkcon def getServiceConfig(self): return self.__serviceConfig
from kazoo.client import KazooClient zk = KazooClient(hosts='127.0.0.1:2181') zk.start() print zk.get_children("/") try: zk.add_auth("digest", "jamesbond:password123") print zk.get("/area51") except: print "Auth error" zk.stop()
class Zooconf: connection = None webService = None applicationService = None storageServicesList = None authenticationServiceList = None status = None def initialize(self): self.__zooConnect() self.__publishService() self.__initAuthenticationServiceWatches() self.__initStorageServiceWatches() self.heartbeat() def __zooConnect(self): print("Connecting to ZooKeeper") self.connection = KazooClient(hosts=settings.ZOOKEEPER_HOST) self.connection.start() digest_auth = "%s:%s" % (settings.ZOOKEEPER_USER, settings.ZOOKEEPER_PASSWORD) self.connection.add_auth("digest", digest_auth) def __publishService(self): acl = make_digest_acl(settings.ZOOKEEPER_USER, settings.ZOOKEEPER_PASSWORD, all=True) dataJsonDict = { 'SERVER_HOSTNAME': settings.SERVER_HOSTNAME, 'SERVER_PORT': settings.SERVER_PORT, 'SHARED_KEY_BASE_64': settings.SHARED_KEY_BASE_64, 'CHILDREN': [] } if settings.ZOOKEEPER_PATH_TO_NODE == 'Auth/': dataJsonDict['AUTH_SYSTEM'] = settings.AUTH_SYSTEM if self.connection.exists(path=settings.ZOOKEEPER_ROOT + settings.ZOOKEEPER_PATH_TO_NODE + settings.ZOOKEEPER_NODE_ID): self.connection.set( path=settings.ZOOKEEPER_ROOT + settings.ZOOKEEPER_PATH_TO_NODE + settings.ZOOKEEPER_NODE_ID, value=json.JSONEncoder().encode(dataJsonDict).encode()) else: self.connection.create_async( path=settings.ZOOKEEPER_ROOT + settings.ZOOKEEPER_PATH_TO_NODE + settings.ZOOKEEPER_NODE_ID, value=json.JSONEncoder().encode(dataJsonDict).encode(), ephemeral=settings.ZOOKEEPER_NODE_EPHIMERAL) if settings.ZOOKEEPER_PATH_TO_NODE != '': data, stat = self.connection.get(settings.ZOOKEEPER_ROOT + settings.ZOOKEEPER_PATH_TO_NODE) dataJsonDict = json.loads(data.decode("utf-8")) if settings.ZOOKEEPER_NODE_ID not in dataJsonDict['CHILDREN']: dataJsonDict['CHILDREN'].append(settings.ZOOKEEPER_NODE_ID) self.connection.set( path=settings.ZOOKEEPER_ROOT + settings.ZOOKEEPER_PATH_TO_NODE, value=json.JSONEncoder().encode(dataJsonDict).encode()) def __initStorageServiceWatches(self): self.storageServicesList = [] @self.connection.ChildrenWatch(settings.ZOOKEEPER_ROOT + "StorageServices") def watch_children(children): print(self.readTree()) if settings.ZOOKEEPER_PATH_TO_NODE == "StorageServices/": node = { 'SERVER_HOSTNAME': settings.SERVER_HOSTNAME, 'SERVER_PORT': settings.SERVER_PORT, 'CHILDREN': [] } if node not in self.storageServicesList: self.__publishService() def __initAuthenticationServiceWatches(self): self.authenticationServiceList = [] @self.connection.ChildrenWatch(settings.ZOOKEEPER_ROOT + "Auth") def watch_children(children): print(self.readTree()) if settings.ZOOKEEPER_PATH_TO_NODE == "Auth/": node = { 'SERVER_HOSTNAME': settings.SERVER_HOSTNAME, 'SERVER_PORT': settings.SERVER_PORT, 'SHARED_KEY_BASE_64': settings.SHARED_KEY_BASE_64, 'CHILDREN': [], 'AUTH_SYSTEM': settings.AUTH_SYSTEM } if node not in self.authenticationServiceList: self.__publishService() def getAvailableFs(self): return self.storageServicesList def getZooConnection(self): return self.connection def readTree(self): result = "{" rootChildren = self.connection.get_children(settings.ZOOKEEPER_ROOT) for child in rootChildren: data, stat = self.connection.get(settings.ZOOKEEPER_ROOT + child) result += '"' + child + '": ' + data.decode("utf-8") + "," try: grandchildren = self.connection.get_children( settings.ZOOKEEPER_ROOT + child) for grandchild in grandchildren: data, stat = self.connection.get(settings.ZOOKEEPER_ROOT + child + '/' + grandchild) result += '"' + grandchild + '": ' + data.decode( "utf-8") + "," data, stat = self.connection.get(settings.ZOOKEEPER_ROOT + child + '/') dataJsonDict = json.loads(data.decode("utf-8")) dataJsonDict['CHILDREN'] = grandchildren self.connection.set( path=settings.ZOOKEEPER_ROOT + child, value=json.JSONEncoder().encode(dataJsonDict).encode()) except Exception: pass result = result[:-1] + '}' self.status = json.loads(result) self.initZkTree() return self.status def getStatus(self): try: return self.readTree() except Exception: self.__zooConnect() return self.getStatus() def getNodeData(self, node): try: return self.status[node] except Exception: return {} def initZkTree(self): serviceData = self.getNodeData('WebService') self.webService = serviceData['SERVER_HOSTNAME'] + ':' + serviceData[ 'SERVER_PORT'] + '/' serviceData = self.getNodeData('ApplicationService') self.applicationService = serviceData[ 'SERVER_HOSTNAME'] + ':' + serviceData['SERVER_PORT'] + '/' serviceData = self.getNodeData('Auth') self.authenticationServiceList = [] for authService in serviceData['CHILDREN']: authServiceData = self.getNodeData(authService) if authServiceData != {}: baseUrl = authServiceData['SERVER_HOSTNAME'] + ( (':' + authServiceData['SERVER_PORT']) if authServiceData['SERVER_PORT'] != '' else '') + '/' loginUrl = baseUrl + 'login?system=' + authServiceData[ 'AUTH_SYSTEM'] + '&callback=' registerUrl = baseUrl + 'register?system=' + authServiceData[ 'AUTH_SYSTEM'] + '&callback=' authServiceDict = { 'name': authService, 'url': baseUrl, 'loginUrl': loginUrl, 'registerUrl': registerUrl, 'sharedKey': authServiceData['SHARED_KEY_BASE_64'], 'system': authServiceData['AUTH_SYSTEM'] } self.authenticationServiceList.append(authServiceDict) serviceData = self.getNodeData('StorageServices') self.storageServicesList = [] for storageService in serviceData['CHILDREN']: storageServiceData = self.getNodeData(storageService) if storageServiceData != {}: storageServiceDict = { 'name': storageService, 'url': storageServiceData['SERVER_HOSTNAME'] + ':' + storageServiceData['SERVER_PORT'] + '/' } self.storageServicesList.append(storageServiceDict) def heartbeat(self): threading.Timer(300.0, self.heartbeat).start() print("Heartbeat") print(str(self.getStatus()))
def test_init_hierarchy(self): auth = ('digest', 'aa', 'pw_aa') hosts = '127.0.0.1:21811' users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'} hierarchy = { 'node1': { '__val__': 'node1_val', '__acl__': { 'aa': 'cdrwa', 'bb': 'rw' }, 'node11': { '__val__': 'node11_val', '__acl__': { 'aa': 'cdrwa', 'cc': 'r' }, }, 'node12': { '__val__': 'node12_val', 'node121': { '__val__': 'node121_val' } }, 'node13': { '__acl__': { 'aa': 'cdrwa' } } }, 'node2': { '__val__': 'node2_val', 'node21': { '__val__': 'node21_val' }, 'node22': { '__acl__': { 'aa': 'rwa' } } }, 'node3': { '__acl__': { 'aa': 'carw', 'cc': 'r' }, 'node31': { 'node311': { 'node3111': {}, 'node3112': {} } } } } zkutil.init_hierarchy(hosts, hierarchy, users, auth) zkcli = KazooClient(hosts) zkcli.start() zkcli.add_auth('digest', 'aa:pw_aa') expected_nodes = ( ('/node1', '"node1_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set(['node11', 'node12', 'node13'])), ('/node1/node11', '"node11_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'cc', 'r')], set([])), ('/node1/node12', '"node12_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set(['node121'])), ('/node1/node12/node121', '"node121_val"', [('digest', 'aa', 'cdrwa'), ('digest', 'bb', 'rw')], set([])), ('/node1/node13', '{}', [('digest', 'aa', 'cdrwa')], set([])), ('/node2', '"node2_val"', [('world', 'anyone', 'cdrwa')], set(['node21', 'node22'])), ('/node2/node21', '"node21_val"', [('world', 'anyone', 'cdrwa')], set([])), ('/node2/node22', '{}', [('digest', 'aa', 'rwa')], set([])), ('/node3', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set(['node31'])), ('/node3/node31', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set(['node311'])), ('/node3/node31/node311', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set(['node3111', 'node3112'])), ('/node3/node31/node311/node3111', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set([])), ('/node3/node31/node311/node3112', '{}', [('digest', 'aa', 'rwca'), ('digest', 'cc', 'r')], set([])), ) for node, val, acl, children in expected_nodes: actual_acl = zkutil.parse_kazoo_acl(zkcli.get_acls(node)[0]) self.assertEqual(val, zkcli.get(node)[0]) self.assertEqual(acl, actual_acl) self.assertEqual(children, set(zkcli.get_children(node))) zkcli.stop()
# READ ZK Data print("Reading unprotected ZK Data back in.") zk = KazooClient(hosts=zk_hosts) zk.start() for zk_path in all_host_paths: data, stat = zk.get(zk_path) print("{} {}{}{}".format("ZK Node version: is ", stat.version, ". ZK Data sample ", data.decode("utf-8").split(' ')[:3])) zk.stop() # Get Bouncer Data print("Reading digest protected /bouncer/data/data.json data from zk") zk = KazooClient(hosts=zk_hosts) zk.start() zk.add_auth('digest', 'super:secret') data, stat = zk.get('/bouncer/datastore/data.json') # print("{} {}{}{}".format("ZK Version: is ",stat.version, ".\nBouncer ZK Data is \n", data.decode("utf-8"))) print("{} {} {}".format("Size of bouncer config is ~ ", len(data), "bytes.\n")) zk.stop() bouncer_json = json.loads(data.decode("utf-8")) # Iterate through the bouncer_json dict user_count = 0 for user in bouncer_json['users']: print('Bouncer User = ' + user) user_count = user_count + 1 group_count = 0
class Zooconf: __zkcon = None __serviceConfig = None def __init__(self): sys.stderr.write("Fileservicepython start initialization at " + str(int(round(time.time() * 1000)))) sys.stderr.flush() self.__initConfProperties() self.__zooConnect() self.__publishService() def __initConfProperties(self): sys.stderr.write(dirname(dirname(abspath(__file__)))) sys.stderr.flush() self.__serviceConfig = dict() with open( dirname(dirname(abspath(__file__))) + "/config.properties", 'r') as f: for line in f: line = line.rstrip() if "=" not in line: continue #skips blanks and comments w/o = if line.startswith("#"): continue #skips comments which contain = k, v = line.split("=", 1) self.__serviceConfig[k] = v f.close() def __zooConnect(self): sys.stderr.write("Start zooconnect") sys.stderr.flush() config = self.getServiceConfig() self.__zkcon = KazooClient(hosts=config["ZOOKEEPER_HOST"]) self.__zkcon.start() digest_auth = "%s:%s" % (config["ZOOKEEPER_USER"], config["ZOOKEEPER_PASSWORD"]) self.__zkcon.add_auth("digest", digest_auth) def __publishService(self): from kazoo.security import make_digest_acl config = self.getServiceConfig() acl = make_digest_acl(config["ZOOKEEPER_USER"], config["ZOOKEEPER_PASSWORD"], all=True) dataJsonDict = {} dataJsonDict['SERVERHOSTNAME'] = config["SERVERHOSTNAME"] dataJsonDict['SERVER_PORT'] = config["SERVER_PORT"] dataJsonDict['SERVER_SCHEME'] = config["SERVER_SCHEME"] dataJsonDict['HMACKEY'] = config["HMACKEY"] dataJsonDict['CONTEXT'] = config["CONTEXT"] self.__zkcon.create("/plh414python/fileservices/" + config["ID"], json.JSONEncoder().encode(dataJsonDict).encode(), [acl], ephemeral=True) def getZooConnection(self): return self.__zkcon def getServiceConfig(self): return self.__serviceConfig
class ZkClient: def __init__(self, url): self.upCont = 0 self.addCont = 0 self.url = url try: connection_retry = KazooRetry(max_tries=3, delay=0.1) command_retry = KazooRetry(max_tries=3, delay=0.1) self.zk = KazooClient(hosts=url, timeout=10.0, connection_retry=connection_retry, command_retry=command_retry, logger=None) if self.zk: self.zk.start(timeout=60) log.info("连接zk服务器%s成功", url) # 设置ACL self.zk.add_auth("digest", "SmartESC:startesc123") except Exception as e: raise e def close(self): if self.zk: self.zk.stop() self.zk.close() log.info("断开zk连接...") def writeData(self, path, data): try: self.zk.set(path, data) except Exception as e: log.exception("数据写入错误") raise e def set_zookeeper_data(self, data={}): """ 父节点的数据不存在时,则创建父节点 data数据中需要包含路径 :param data: :return: """ for path in data.keys(): try: if self.zk.exists(path): self.zk.set(path, data[path]) self.upCont += 1 else: self.zk.create(path, data[path], makepath=True) self.addCont += 1 # log.info("路径:%s",path) except Exception as e: log.exception("数据%s写入错误", path) raise e def set_service_data(self, data={}): """ 父节点的数据不存在时,不创建父节点 data中包含zookeeper的路径, :param data: :return: """ for path in data.keys(): try: if self.zk.exists(path): self.zk.set(path, data[path]) self.upCont += 1 else: self.zk.create(path, data[path]) self.addCont += 1 # log.info("路径:%s",path) except Exception as e: log.exception("数据%s写入错误", path) raise e def get_onecfg_service(self,sCode): data={} try: meta=self.zk.get_children("/metadata") for st in meta: if sCode in st: path="/metadata/%s"%(st) data[path],stat=self.zk.get(path) cons = self.zk.get_children("/metadata/consumer") for st in cons: if sCode in st: path = "/metadata/consumer/%s" % (st) data[path], stat = self.zk.get(path) prvd = self.zk.get_children("/metadata/provider") for st in prvd: if sCode in st: path = "/metadata/provider/%s" % (st) data[path], stat = self.zk.get(path) return data except Exception as e: log.exception("zk数据读取错误:") raise e def get_cfg_service(self,sCodeList=[]): """ 在zookeeper中查找服务,获取zookeeper中服务的数据 :param sCodeList: :return: """ data = {} for sCode in sCodeList: try: meta = self.zk.get_children("/metadata") for st in meta: if sCode in st: path = "/metadata/%s" % (st) data[path], stat = self.zk.get(path) cons = self.zk.get_children("/metadata/consumer") for st in cons: if sCode in st: path = "/metadata/consumer/%s" % (st) data[path], stat = self.zk.get(path) prvd = self.zk.get_children("/metadata/provider") for st in prvd: if sCode in st: path = "/metadata/provider/%s" % (st) data[path], stat = self.zk.get(path) except Exception as e: log.exception("zk数据读取错误:") raise e return data def get_cfg_service_by_service_code(self, service_code): """ 在zookeeper中查找服务,获取zookeeper中服务的数据 :param service_code: :return: """ data = {} try: meta = self.zk.get_children("/metadata") for st in meta: if service_code in st: path = "/metadata/%s" % (st) data[path], stat = self.zk.get(path) cons = self.zk.get_children("/metadata/consumer") for st in cons: if service_code in st: path = "/metadata/consumer/%s" % (st) data[path], stat = self.zk.get(path) prvd = self.zk.get_children("/metadata/provider") for st in prvd: if service_code in st: path = "/metadata/provider/%s" % (st) data[path], stat = self.zk.get(path) except Exception as e: log.exception("zk数据读取错误:") raise e return data def get_cfg_service_auth_by_system_name(self, system_name): """ 在zookeeper中查找服务的授权信息 :param system_name: 系统名称 :return: """ data = {} new_path = '' try: meta = self.zk.get_children("/configs") for file_name in meta: if system_name in file_name: path = "/configs/%s" % (file_name) new_path = path+"/consumer/authCtrl/"+system_name if self.zk.exists(new_path): data[new_path], stat = self.zk.get(new_path) except Exception as e: log.exception("zk数据读取授权数据错误:%s", new_path) raise e return data def get_allcfg_service(self): data={} try: meta=self.zk.get_children("/metadata") for st in meta: path="/metadata/%s"%(st) data[path],stat=self.zk.get(path) cons = self.zk.get_children("/metadata/consumer") for st in cons: path = "/metadata/consumer/%s" % (st) data[path], stat = self.zk.get(path) prvd = self.zk.get_children("/metadata/provider") for st in prvd: path = "/metadata/provider/%s" % (st) data[path], stat = self.zk.get(path) return data except Exception as e: log.exception("zk数据读取错误:") raise e
def test_export_hierarchy(self): auth = ('digest', 'aa', 'pw_aa') hosts = '127.0.0.1:21811' users = {'aa': 'pw_aa', 'bb': 'pw_bb', 'cc': 'pw_cc'} hierarchy = { 'node0': { '__val__': '', '__acl__': { 'aa': 'cdrwa', 'bb': 'rw' } }, 'node1': { '__val__': 'node1_val', '__acl__': { 'aa': 'cdrwa', 'bb': 'rw' }, 'node11': { '__val__': 'node11_val', '__acl__': { 'aa': 'cdrwa', 'cc': 'r' }, }, 'node12': { '__val__': 'node12_val', 'node121': { '__val__': 'node121_val' } }, 'node13': { '__acl__': { 'aa': 'cdrwa' } } }, 'node2': { '__val__': 'node2_val', 'node21': { '__val__': 'node21_val' }, 'node22': { '__acl__': { 'aa': 'rwa' } } }, 'node3': { '__acl__': { 'aa': 'carw', 'cc': 'r' }, 'node31': { 'node311': { 'node3111': {}, 'node3112': {} } } } } zkutil.zkutil.init_hierarchy(hosts, hierarchy, users, auth) zkcli = KazooClient(hosts) zkcli.start() zkcli.add_auth('digest', 'aa:pw_aa') invalid_zkpath_cases = ('a', 'a/', 'a/b') for zkpath in invalid_zkpath_cases: with self.assertRaises(zkutil.zkutil.ZkPathError): zkutil.zkutil.export_hierarchy(zkcli, zkpath) valid_cases = (('/', { '__acl__': { u'anyone': 'cdrwa' }, '__val__': '', u'node0': { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '""' }, u'node1': { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '"node1_val"', u'node11': { '__acl__': { u'aa': 'cdrwa', u'cc': 'r' }, '__val__': '"node11_val"' }, u'node12': { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '"node12_val"', u'node121': { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '"node121_val"' } }, u'node13': { '__acl__': { u'aa': 'cdrwa' }, '__val__': '{}' } }, u'node2': { '__acl__': { u'anyone': 'cdrwa' }, '__val__': '"node2_val"', u'node21': { '__acl__': { u'anyone': 'cdrwa' }, '__val__': '"node21_val"' }, u'node22': { '__acl__': { u'aa': 'rwa' }, '__val__': '{}' } }, u'node3': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}', u'node31': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}', u'node311': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}', u'node3111': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}' }, u'node3112': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}' } } } }, u'zookeeper': { '__acl__': { u'anyone': 'cdrwa' }, '__val__': '', u'quota': { '__acl__': { u'anyone': 'cdrwa' }, '__val__': '' } } }), ('/node0', { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '""' }), ('/node1', { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '"node1_val"', u'node11': { '__acl__': { u'aa': 'cdrwa', u'cc': 'r' }, '__val__': '"node11_val"' }, u'node12': { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '"node12_val"', u'node121': { '__acl__': { u'aa': 'cdrwa', u'bb': 'rw' }, '__val__': '"node121_val"' } }, u'node13': { '__acl__': { u'aa': 'cdrwa' }, '__val__': '{}' } }), ('/node1/node11', { '__acl__': { u'aa': 'cdrwa', u'cc': 'r' }, '__val__': '"node11_val"' }), ('/node2', { '__acl__': { u'anyone': 'cdrwa' }, '__val__': '"node2_val"', u'node21': { '__acl__': { u'anyone': 'cdrwa' }, '__val__': '"node21_val"' }, u'node22': { '__acl__': { u'aa': 'rwa' }, '__val__': '{}' } }), ('/node3', { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}', u'node31': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}', u'node311': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}', u'node3111': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}' }, u'node3112': { '__acl__': { u'aa': 'rwca', u'cc': 'r' }, '__val__': '{}' } } } })) for path, expected_rst in valid_cases: rst = zkutil.export_hierarchy(zkcli, path) self.assertEqual(rst, expected_rst) zkcli.stop()
class WebWindow(object): def setupUi(self, Window): Window.setObjectName("Window") self.centralwidget = QtWidgets.QWidget(Window) self.centralwidget.setObjectName("centralwidget") self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget) self.verticalLayout.setContentsMargins(0, 0, 0, 0) self.verticalLayout.setObjectName("verticalLayout") self.zk = None self.webView = QtWebKitWidgets.QWebView(self.centralwidget) self.webView.setObjectName("webView") self.webView.setRenderHint(QPainter.Antialiasing, True) self.webView.setRenderHint(QPainter.TextAntialiasing, True) self.webView.setRenderHint(QPainter.SmoothPixmapTransform, True) self.webView.setRenderHint(QPainter.HighQualityAntialiasing, True) self.webView.setPage(WebPage()) frame = self.webView.page().mainFrame() frame.javaScriptWindowObjectCleared.connect(self.initJsComm) self.verticalLayout.addWidget(self.webView) Window.setCentralWidget(self.centralwidget) self.retranslateUi(Window) QtCore.QMetaObject.connectSlotsByName(Window) self.loadDefaultAcl() def retranslateUi(self, Window): _translate = QtCore.QCoreApplication.translate Window.setWindowTitle(_translate("Window", "Zookeeper GUI")) def loadLocalFile(self, filename): localdir = os.path.abspath(sys.path[0]) if os.path.isfile(localdir): localdir = os.path.dirname(localdir) self.webView.setUrl( QtCore.QUrl.fromLocalFile(localdir+'/'+filename) ) def getCfgVar(self, name): localdir = os.path.abspath(sys.path[0]) if os.path.isfile(localdir): localdir = os.path.dirname(localdir) cfg = {} obj = None try: obj = open(localdir+'/cfg.json','r') cfg = json.loads(obj.read()) except Exception as e: logging.info(str(e)) finally: if obj is not None: obj.close() if name in cfg: return str(cfg[name]) return '' def setCfgVar(self, name, value): localdir = os.path.abspath(sys.path[0]) if os.path.isfile(localdir): localdir = os.path.dirname(localdir) cfg = {} obj = None try: obj = open(localdir+'/cfg.json','r') cfg = json.loads(obj.read()) except Exception as e: pass finally: if obj is not None: obj.close() cfg[name] = value obj = None try: obj = open(localdir+'/cfg.json','w') obj.truncate() obj.write(json.dumps(cfg)) except Exception as e: logging.info(str(e)) finally: if obj is not None: obj.close() def makeDigestCred(self, user, plain_pass): m = hashlib.sha1( bytes(user,'utf8') + b':' + bytes(plain_pass,'utf8') ).digest() return user+':'+base64.b64encode(m).strip().decode('utf8') def initJsComm(self): frame = self.webView.page().mainFrame() frame.addToJavaScriptWindowObject('py',self) @pyqtSlot(str) def jsSetWinTitle(self, title): _translate = QtCore.QCoreApplication.translate self.setWindowTitle(_translate("Window", title)) @pyqtSlot(str, result=str) def jsCheckYaml(self, s): try: a = yaml.load(s) except Exception as e: return str(e) if a is None: return 'Failed' return '' @pyqtSlot(str,result=str) def jsGetCfg(self, name): return self.getCfgVar(name) @pyqtSlot(str,str) def jsSetCfg(self, name, value): self.setCfgVar(name, value) def loadDefaultAcl(self): self.updateDefaultAclCache( self.getCfgVar('defaultacl') ) def updateDefaultAclCache(self, list_str): if list_str is not None and len(list_str)>0: cache = json.loads(list_str) self.default_acl_plain = [] self.default_acl = [] for one in cache: if(one['scheme']=='world'): self.default_acl_plain.append(one) acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') ) self.default_acl.append(acl) elif(one['scheme']=='digest'): self.default_acl_plain.append(one) if 'id' in one: acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) ) else: acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) ) self.default_acl.append(acl) elif(one['scheme']=='ip'): self.default_acl_plain.append(one) acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) ) self.default_acl.append(acl) else: self.default_acl_plain = [] self.default_acl = None @pyqtSlot(str,result=str) def jsSetDefaultAcl(self, list_str): self.updateDefaultAclCache(list_str) self.setCfgVar('defaultacl', json.dumps(self.default_acl_plain)) @pyqtSlot(result=str) def jsGetDefaultAcl(self): return json.dumps(self.default_acl_plain) @pyqtSlot() def jsGetZk(self): return self.zk @pyqtSlot(result=int) def jsZkIsConnected(self): if self.zk is not None: return int(self.zk.state=='CONNECTED') return 0 @pyqtSlot(str, str, result=str) def jsZkConnect(self,host, auth_list_str): try: if self.zk is not None: #self.zk.remove_listener(self.onZkStateChange) self.zk.stop() self.zk.close() self.zk = KazooClient(hosts=host) #self.zk.add_listener(self.onZkStateChange) self.zk.start(15) auth_list = json.loads(auth_list_str) for one in auth_list: cred = self.makeDigestCred(one['user'], one['pass']) self.zk.add_auth('digest', one['user']+':'+one['pass']) except Exception as e: logging.error("jsZkConnect, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' #def onZkStateChange(self,state): # frame = self.webView.page().mainFrame() # frame.evaluateJavaScript("onPyZkStateChange('"+state+"')") @pyqtSlot(str, result=QVariant) def jsZkGetChildren(self, path): try: logging.info("jsZkGetChildren, path="+path) children = self.zk.get_children(path) except NoNodeError: logging.error("jsZkGetChildren, NoNodeError") return QVariant({"err":"node not exists"}) except Exception as e: logging.error("jsZkGetChildren, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return QVariant({"err":str(e)+', traceback='+str(strlist)}) return QVariant({"err":"", "children":children}) @pyqtSlot(str, result=QVariant) def jsZkGet(self, path): try: logging.info("jsZkGet, path="+path) ret = self.zk.get(path) except NoNodeError: logging.error("jsZkGet, NoNodeError") return QVariant({"err":"node not exists"}) except Exception as e: logging.error("jsZkGet, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return QVariant({"err":str(e)+', traceback='+str(strlist)}) ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].ctime/1000)) mtime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].mtime/1000)) stat = {'ctime':ctime,'mtime':mtime,'version':ret[1].version} data = '' if ret[0] is not None: data = ret[0].decode('utf8') else: logging.info('jsZkGet data None, path='+path) return QVariant({"err":"", "data":data, "stat":QVariant(stat)}) @pyqtSlot(str, str, int, result=str) def jsZkSet(self, path, data, ver): try: logging.info("jsZkSet, path="+path+',ver='+str(ver)) self.zk.set(path, bytes(data, 'utf8'),ver) except NoNodeError as e: logging.error("jsZkSet, NoNodeError") return "node not exists" except BadVersionError as e: logging.error("jsZkSet, BadVersionError") return "bad version" except Exception as e: logging.error("jsZkSet, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' @pyqtSlot(str, result=QVariant) def jsZkGetAcl(self, path): try: logging.info("jsZkGetAcl, path="+path) ret = self.zk.get_acls(path) except NoNodeError as e: logging.error("jsZkGetAcl, NoNodeError") return QVariant({"err":"node not exists"}) except Exception as e: logging.error("jsZkGetAcl, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return QVariant({"err":str(e)+', traceback='+str(strlist)}) lst = [] for acl in ret[0]: dacl = {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id} lst.append(QVariant(dacl)) stat = {'ctime':ret[1].ctime,'mtime':ret[1].mtime,'version':ret[1].version} return QVariant({"err":"", "acl_list":QVariant(lst), "stat":QVariant(stat)}) @pyqtSlot(str, str, result=str) def jsZkSetAcl(self, path, list_str): try: acl_list = None if list_str is not None and len(list_str)>0: cache = json.loads(list_str) acl_list = [] for one in cache: if(one['scheme']=='world'): acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') ) acl_list.append(acl) elif(one['scheme']=='digest'): if 'id' in one: acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) ) else: acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) ) acl_list.append(acl) elif(one['scheme']=='ip'): acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) ) acl_list.append(acl) self.zk.set_acls(path, acl_list) except NoNodeError as e: logging.error("jsZkSetAcl, NoNodeError") return "node not exists" except InvalidACLError as e: logging.error("jsZkSetAcl, InvalidACLError") t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return "invalid acl, traceback: "+str(strlist) except BadVersionError as e: logging.error("jsZkSetAcl, BadVersionError") return "bad version error" except Exception as e: logging.error("jsZkSetAcl, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' @pyqtSlot(str,str,int,int,result=str) def jsZkCreate(self, path, data, ephem, seq): try: logging.info("jsZkCreate, path="+path) self.zk.create(path=path, value=bytes(data,'utf8'), ephemeral=bool(ephem), sequence=bool(seq)) if self.default_acl is not None and len(self.default_acl)>0: self.zk.set_acls(path, self.default_acl) except NoNodeError as e: logging.error("jsZkCreate, NoNodeError") return "node not exists" except NodeExistsError as e: logging.error("jsZkCreate, NodeExistsError") return "node already exists" except NoChildrenForEphemeralsError as e: logging.error("jsZkCreate, NoChildrenForEphemeralsError") return "ephemeral node can not have child" except Exception as e: logging.error("jsZkCreate, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' @pyqtSlot(str, int, int, result=str) def jsZkDelete(self, path, ver, recursive): try: logging.info("jsZkDelete, path="+path+',ver='+str(ver)+', recursive='+str(recursive)) self.zk.delete(path, ver, bool(recursive)) except NoNodeError as e: logging.error("jsZkDelete, NoNodeError") return "node not exists" except BadVersionError as e: logging.error("jsZkDelete, BadVersionError") return "bad version" except NotEmptyError as e: logging.error("jsZkDelete, NotEmptyError") return "node not empty" except Exception as e: logging.error("jsZkDelete, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' @pyqtSlot(str, str, int, int, result=str) def jsZkCopy(self, dest_path, ori_path, max_depth, children_only): logging.info("jsZkCopy, dest_path="+dest_path+", ori_path="+ori_path+", children_only="+str(children_only)) #copy node first if children_only==0: try: ori_data = self.zk.get(ori_path) if self.zk.exists(dest_path) is None: self.zk.create(dest_path, ori_data[0], acl=self.default_acl) else: self.zk.set(dest_path, ori_data[0]) except NoNodeError as e: logging.error("jsZkCopy, node, NoNodeError, ori_path="+ori_path+', dest_path='+dest_path) return "node not exists" except Exception as e: logging.error("jsZkCopy, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) #copy children path = '' try: max_depth -= 1 path = ori_path ori_children = self.zk.get_children(ori_path) for child in ori_children: path = ori_path+'/'+child ret = self.jsZkCopy(dest_path+'/'+child, ori_path+'/'+child, max_depth, 0) if isinstance(ret, QVariant): return ret elif len(ret)>0: return ret except NoNodeError as e: logging.error("jsZkCopy, child, NoNodeError") return "node not exists, path="+path except Exception as e: logging.error("jsZkCopy, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' ''' @pyqtSlot(str, str, int, result=str) def jsZkCopyChildren(self, dest_path, ori_path, max_depth): path = '' try: max_depth -= 1; logging.info("jsZkCopyChildren, dest_path="+dest_path+", ori_path="+ori_path) path = ori_path ori_children = self.zk.get_children(ori_path) path = dest_path dest_children = self.zk.get_children(dest_path) for child in ori_children: if child in dest_children: return 'child ['+child+'] is found in both path' for child in ori_children: path = ori_path+'/'+child data = self.zk.get(path)[0] path = dest_path+'/'+child self.zk.create(path, data, acl=self.default_acl) if max_depth>0: ret = self.jsZkCopyChildren(dest_path+'/'+child, ori_path+'/'+child, max_depth) if len(ret)>0: return ret except NoNodeError as e: logging.info("jsZkCopyChildren, NoNodeError") return "node not exists, path="+path except ZookeeperError as e: logging.info("jsZkCopyChildren, ZookeeperError") return str(e)+', path='+path return '' ''' @pyqtSlot(str, int, result=str) def jsZkDeleteChildren(self, main_path, max_depth): path = '' try: max_depth -= 1 path = main_path logging.info("jsZkDeleteChildren, path="+main_path) children = self.zk.get_children(path) for child in children: path = main_path+'/'+child if max_depth>0: ret = self.jsZkDeleteChildren(path, max_depth) if len(ret)>0: return ret self.zk.delete(path) except NoNodeError as e: logging.error("jsZkDeleteChildren, NoNodeError") return "node not exists, path="+path except Exception as e: logging.error("jsZkDeleteChildren, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' @pyqtSlot(str, str, int, int, result=str) def jsZkExport(self, local_dir, main_path, max_depth, without_acl): path = '' try: max_depth -= 1 path = main_path logging.info("jsZkExport, path="+main_path+' to local dir '+local_dir) data = self.zk.get(main_path) p = pathlib.Path(local_dir) if not p.exists(): p.mkdir(parents=True) elif not p.is_dir(): return 'local '+local_dir+' exists, but not a directory' for child in p.iterdir(): return 'local path '+local_dir+' is not empty, clear it first' p = pathlib.Path(local_dir+'/____data') p.touch() obj = open(str(p),'wb') try: if data[0] is not None: obj.write(data[0]) finally: obj.close() if not without_acl: ret = self.zk.get_acls(path) lst = [] if ret is not None: for acl in ret[0]: lst.append( {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id} ) p = pathlib.Path(local_dir+'/____acl') p.touch() obj = open(str(p),'w') try: obj.write(json.dumps(lst)) finally: obj.close() children = self.zk.get_children(path) if children is not None: for child in children: if child=='zookeeper': continue path = main_path+'/'+child if max_depth>0: ret = self.jsZkExport(local_dir+'/'+child, path, max_depth, without_acl) if len(ret)>0: return ret except NoNodeError as e: logging.error("jsZkExport, NoNodeError") return "node not exists, path="+path except Exception as e: logging.error("jsZkExport, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return '' @pyqtSlot(str, str, int, int, result=str) def jsZkImport(self, local_dir, main_path, max_depth, without_acl): path = '' try: max_depth -= 1 path = main_path logging.info("jsZkImport, path="+main_path+' from local dir '+local_dir) obj = open(local_dir+'/____data', 'rb') if self.zk.exists(path) is None: self.zk.create(path, obj.read(), acl=self.default_acl) else: self.zk.set(path, obj.read()) if not without_acl: obj = open(local_dir+'/____acl', 'r') acl_list = None list_str = obj.read() if list_str is not None and len(list_str)>0: cache = json.loads(list_str) acl_list = [] for one in cache: acl = kazoo.security.ACL( one['perm'], kazoo.security.Id(one['scheme'], one['id']) ) acl_list.append(acl) self.zk.set_acls(path, acl_list) p = pathlib.Path(local_dir) for child in p.iterdir(): if not child.is_dir(): continue if child.name=='zookeeper': continue ret = self.jsZkImport(str(child), path+'/'+child.name, max_depth, without_acl) if len(ret)>0: return ret except NoNodeError as e: logging.error("jsZkImport, NoNodeError") return "node not exists, path="+path except Exception as e: logging.error("jsZkImport, "+str(e)) t,v,tb = sys.exc_info() strlist = traceback.format_tb(tb) return str(e)+', traceback='+str(strlist) return ''