def test_parse_kazoo_acl(self):
inp = (
security.make_acl('world', 'anyone', all=True),
security.make_digest_acl('foo', 'bar', create=True, read=True),
security.make_digest_acl('xp', '123', all=True),
)
expected = (('world', 'anyone', 'cdrwa'), ('digest', 'foo', 'rc'),
('digest', 'xp', 'cdrwa'))
dd(inp)
rst = zkutil.parse_kazoo_acl(inp)
dd(rst)
self.assertEqual(expected, tuple(rst))
def get_zoo_client(cluster_name="qconf"):
"""get zoo client by cluster_name
"""
global ZOO_CLIENTS
if cluster_name not in ZOO_CLIENTS:
# get zookeeper hosts info
zookeeper = ZdZookeeper.one(cluster_name=cluster_name, deleted="0")
if not zookeeper:
raise ZookeeperConfError("Zookeeper not configured for cluster: {}!".format(cluster_name))
# connect to zookeeper
try:
username = ZK_AUTH.get('user')
pwd = ZK_AUTH.get('pwd')
client = KazooClient(hosts=zookeeper.hosts,
connection_retry={"max_tries": 3, "backoff": 2},
default_acl=make_digest_acl(username, pwd,
read=True, write=True, create=True, delete=True, admin=True),
auth_data=[("digest", '%s:%s' % (username, pwd))]
)
client.start(timeout=3)
ZOO_CLIENTS[cluster_name] = client
except KazooTimeoutError as exc:
log.error('Failed to connnect zookeeper, %s', str(exc))
return
# check connection's state, if not connected, reconect
zoo_client = ZOO_CLIENTS[cluster_name]
if not zoo_client.connected:
zoo_client.start()
return zoo_client
def __publishService(self):
acl = make_digest_acl(settings.ZOOKEEPER_USER,
settings.ZOOKEEPER_PASSWORD,
all=True)
dataJsonDict = {
'SERVER_HOSTNAME': settings.SERVER_HOSTNAME,
'SERVER_PORT': settings.SERVER_PORT,
'SHARED_KEY_BASE_64': settings.SHARED_KEY_BASE_64,
'CHILDREN': []
}
if settings.ZOOKEEPER_PATH_TO_NODE == 'Auth/':
dataJsonDict['AUTH_SYSTEM'] = settings.AUTH_SYSTEM
if self.connection.exists(path=settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE +
settings.ZOOKEEPER_NODE_ID):
self.connection.set(
path=settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE + settings.ZOOKEEPER_NODE_ID,
value=json.JSONEncoder().encode(dataJsonDict).encode())
else:
self.connection.create_async(
path=settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE + settings.ZOOKEEPER_NODE_ID,
value=json.JSONEncoder().encode(dataJsonDict).encode(),
ephemeral=settings.ZOOKEEPER_NODE_EPHIMERAL)
if settings.ZOOKEEPER_PATH_TO_NODE != '':
data, stat = self.connection.get(settings.ZOOKEEPER_ROOT +
settings.ZOOKEEPER_PATH_TO_NODE)
dataJsonDict = json.loads(data.decode("utf-8"))
if settings.ZOOKEEPER_NODE_ID not in dataJsonDict['CHILDREN']:
dataJsonDict['CHILDREN'].append(settings.ZOOKEEPER_NODE_ID)
self.connection.set(
path=settings.ZOOKEEPER_ROOT + settings.ZOOKEEPER_PATH_TO_NODE,
value=json.JSONEncoder().encode(dataJsonDict).encode())
def main():
""" Builds a zookeeper tree with ACLS on from a config file"""
options = {
'servers':('list of zk servers', 'strlist', 'store', None)
}
go = simple_option(options)
rpasswd, rpath = get_rootinfo(go.configfile_remainder)
znodes, users = parse_zkconfig(go.configfile_remainder)
logger.debug("znodes: %s" % znodes)
logger.debug("users: %s" % users)
# Connect to zookeeper
# initial authentication credentials and acl for admin on root level
acreds = [('digest', 'root:' + rpasswd)]
root_acl = make_digest_acl('root', rpasswd, all=True)
# Create kazoo/zookeeper connection with root credentials
servers = go.options.servers
zkclient = VscKazooClient(servers, auth_data=acreds)
# Iterate paths
for path, attrs in znodes.iteritems():
logger.debug("path %s attribs %s" % (path, attrs))
acls = dict((arg, attrs[arg]) for arg in attrs if arg not in ('value', 'ephemeral', 'sequence', 'makepath'))
acl_list = parse_acls(acls, users, root_acl)
kwargs = dict((arg, attrs[arg]) for arg in attrs if arg in ('ephemeral', 'sequence', 'makepath'))
if not zkclient.exists_znode(path):
zkclient.make_znode(path, value=attrs.get('value', ''), acl=acl_list, **kwargs)
else:
logger.warning('node %s already exists' % path)
zkclient.znode_acls(path, acl_list)
zkclient.exit()
def extract_acl(cls, acl):
""" parse an individual ACL (i.e.: world:anyone:cdrwa) """
try:
scheme, rest = acl.split(":", 1)
credential = ":".join(rest.split(":")[0:-1])
cdrwa = rest.split(":")[-1]
except ValueError:
raise cls.BadACL("Bad ACL: %s. Format is scheme:id:perms" % (acl))
if scheme not in cls.valid_schemes:
raise cls.BadACL("Invalid scheme: %s" % (acl))
create = True if "c" in cdrwa else False
read = True if "r" in cdrwa else False
write = True if "w" in cdrwa else False
delete = True if "d" in cdrwa else False
admin = True if "a" in cdrwa else False
if scheme == "username_password":
try:
username, password = credential.split(":", 1)
except ValueError:
raise cls.BadACL("Bad ACL: %s. Format is scheme:id:perms" %
(acl))
return make_digest_acl(username, password, read, write, create,
delete, admin)
else:
return make_acl(scheme, credential, read, write, create, delete,
admin)
def test_parse_kazoo_acl(self):
inp = (security.make_acl('world', 'anyone', all=True),
security.make_digest_acl('foo', 'bar', create=True, read=True),
security.make_digest_acl('xp', '123', all=True),
)
expected = (('world', 'anyone', 'cdrwa'),
('digest', 'foo', 'rc'),
('digest', 'xp', 'cdrwa'))
dd(inp)
rst = zkutil.parse_kazoo_acl(inp)
dd(rst)
self.assertEqual(expected, tuple(rst))
def zkrsync_parse(options):
"""Takes options of simple_option and returns all the parameters after checks"""
if not options.servers:
logger.error("No servers given!")
sys.exit(1)
if not options.rsyncpath:
logger.error("Path is mandatory!")
sys.exit(1)
if options.depth <= 0:
logger.error("Invalid depth!")
sys.exit(1)
rootcreds = [('digest', options.user + ':' + options.passwd)]
admin_acl = make_digest_acl(options.user, options.passwd, all=True)
if options.source and options.destination:
logger.error("Client can not be the source AND the destination")
sys.exit(1)
elif options.source:
rstype = CL_SOURCE
elif options.destination:
rstype = CL_DEST
else:
rstype = None
return rootcreds, admin_acl, rstype
def make_digest_acl(cls, username, password, read=False, write=False,
create=False, delete=False, admin=False, all=False):
"""Create a digest ACL for Zookeeper with the given permissions
for use with ZKClient's ACL methods
:param username: Username to use for the ACL.
:param password: A plain-text password to hash.
:param write: Write permission.
:type write: bool
:param create: Create permission.
:type create: bool
:param delete: Delete permission.
:type delete: bool
:param admin: Admin permission.
:type admin: bool
:param all: All permissions.
:type all: bool
:rtype: :class:`kazoo.security.ACL`
"""
from kazoo.security import make_digest_acl
return make_digest_acl(username, password,
read=read, write=write, create=create,
delete=delete, admin=admin, all=all)
def zk_connect(zk_addr: str,
zk_user: Optional[str] = None,
zk_secret: Optional[str] = None) -> KazooClient:
"""Connect to ZooKeeper.
On connection failure, the function attempts to reconnect indefinitely with
exponential backoff up to 3 seconds. If a command fails, that command is
retried every 300ms for 3 attempts before failing.
These values are chosen to suit a human-interactive time.
Args:
zk_addr: The address to connect to
zk_user: The username to use when connecting to ZooKeeper or `None`
if no authentication is necessary.
zk_secret: The secret to use when connecting to ZooKeeper or `None`
if no authentication is necessary.
Returns:
A ZooKeeper client connection in the form of a `kazoo.client.KazooClient`.
"""
# Try to reconnect indefinitely, with time between updates going
# exponentially to ~3s. Then every retry occurs every ~3 seconds.
conn_retry_policy = KazooRetry(
max_tries=-1,
delay=0.3,
backoff=1.3,
max_delay=3,
ignore_expire=True,
)
# Retry commands every 0.3 seconds, for a total of <1s (usually 0.9)
cmd_retry_policy = KazooRetry(
max_tries=3,
delay=0.3,
backoff=1,
max_delay=1,
ignore_expire=False,
)
default_acl = None
auth_data = None
if zk_user and zk_secret:
default_acl = [make_digest_acl(zk_user, zk_secret, all=True)]
scheme = 'digest'
credential = "{}:{}".format(zk_user, zk_secret)
auth_data = [(scheme, credential)]
zk = KazooClient(
hosts=zk_addr,
timeout=30,
connection_retry=conn_retry_policy,
command_retry=cmd_retry_policy,
default_acl=default_acl,
auth_data=auth_data,
)
zk.start()
return zk
def get_zk(hosts):
if CenterConfig.USE_ACL:
default_acl = make_digest_acl(CenterConfig.ZK_USERNAME, CenterConfig.ZK_PASSWORD, all=True)
zk = KazooClient(hosts=hosts, default_acl=[default_acl], auth_data=[("digest", "{}:{}".format(
CenterConfig.ZK_USERNAME, CenterConfig.ZK_PASSWORD))])
else:
zk = KazooClient(hosts=hosts)
return zk
def test_get_acls(self):
from kazoo.security import make_digest_acl
acl = make_digest_acl('user', 'pass', all=True)
client = self.client
try:
client.create('/a', acl=[acl])
self.assertTrue(acl in client.get_acls('/a')[0])
finally:
client.delete('/a')
def test_get_acls(self):
from kazoo.security import make_digest_acl
acl = make_digest_acl('user', 'pass', all=True)
client = self.client
try:
client.create('/a', acl=[acl])
self.assertTrue(acl in client.get_acls('/a')[0])
finally:
client.delete('/a')
def add_default_acl(self, username, password, read=False, write=False,
create=False, delete=False, admin=False, all=False):
warnings.warn(DeprecationWarning(
'This method will be removed in next release. Please DO NOT USE.'
))
if username and password:
acl = make_digest_acl(username, password, read, write, create,
delete, admin, all)
self.default_acl.append(acl)
self.client.default_acl = self.default_acl
def test_get_acls(self):
from kazoo.security import make_digest_acl
acl = make_digest_acl("user", "pass", all=True)
client = self.client
try:
client.create("/a", acl=[acl])
self.assertTrue(acl in client.get_acls("/a")[0])
finally:
client.delete("/a")
def setUp(self):
""" Uses a test output of simple_option configparser """
self.cfgremainder = {
'user4': {
'passwd': 'bla'
},
'user5': {
'passwd': 'bla'
},
'user2': {
'passwd': 'bar'
},
'user3': {
'passwd': 'bla'
},
'user1': {
'passwd': 'xyz'
},
'rsync': {
'passwd': 'w00f'
},
'sys': {
'passwd': 'sis'
},
'kwaegema': {
'passwd': 'w00f'
},
'/admin/tools': {
'all': 'sys'
},
'/admin/rsync': {
'rwcd': 'rsync,kwaegema',
'all': ''
},
'/admin': {
'rwcd': 'user1',
'c': 'user3',
'rw': 'user4,user3',
'all': '',
'cd': 'user2',
'r': 'user5'
},
'/admin/user1': {
'all': 'user1',
'r': 'user2,user3',
'rw': 'user4'
},
'root': {
'passwd': 'admin',
'path': '/'
}
}
self.rootacl = make_digest_acl('root', 'admin', all=True)
super(ConfigParserTest, self).setUp()
def test_get_acls(self):
from kazoo.security import make_digest_acl
user = '******'
passw = 'pass'
acl = make_digest_acl(user, passw, all=True)
client = self.client
try:
client.create('/a', acl=[acl])
client.add_auth('digest', '{}:{}'.format(user, passw))
self.assertTrue(acl in client.get_acls('/a')[0])
finally:
client.delete('/a')
def zk_connect(zk_user: Optional[str] = None, zk_secret: Optional[str] = None) -> KazooClient:
"""Connect to ZooKeeper.
On connection failure, the function attempts to reconnect indefinitely with exponential backoff
up to 3 seconds. If a command fails, that command is retried every 300ms for 3 attempts before failing.
These values are chosen to suit a human-interactive time.
Args:
zk_user:
The username to use when connecting to ZooKeeper or `None` if no authentication is necessary.
zk_secret:
The secret to use when connecting to ZooKeeper or `None` if no authentication is necessary.
Returns:
A ZooKeeper client connection in the form of a `kazoo.client.KazooClient`.
"""
# Try to reconnect indefinitely, with time between updates going
# exponentially to ~3s. Then every retry occurs every ~3 seconds.
conn_retry_policy = KazooRetry(
max_tries=-1,
delay=0.3,
backoff=1.3,
max_jitter=1,
max_delay=3,
ignore_expire=True,
)
# Retry commands every 0.3 seconds, for a total of <1s (usually 0.9)
cmd_retry_policy = KazooRetry(
max_tries=3,
delay=0.3,
backoff=1,
max_jitter=0.1,
max_delay=1,
ignore_expire=False,
)
default_acl = None
auth_data = None
if zk_user and zk_secret:
default_acl = [make_digest_acl(zk_user, zk_secret, all=True)]
scheme = 'digest'
credential = "{}:{}".format(zk_user, zk_secret)
auth_data = [(scheme, credential)]
zk = KazooClient(
hosts="127.0.0.1:2181",
timeout=30,
connection_retry=conn_retry_policy,
command_retry=cmd_retry_policy,
default_acl=default_acl,
auth_data=auth_data,
)
zk.start()
return zk
def get_auth_data_and_acl(username, password):
if username and password:
auth_scheme = "digest"
auth_credential = "%s:%s" % (username, password)
auth_data = [(auth_scheme, auth_credential)]
default_acl = [make_digest_acl(username, password, all=True)]
elif username or password:
raise ValueError("both username and password must be specified, if any")
else:
auth_data = None
default_acl = None
return auth_data, default_acl
def get_zk(cls, ):
zk_config = get_base_config("zookeeper", {})
if zk_config.get("use_acl", False):
default_acl = make_digest_acl(zk_config.get("user", ""),
zk_config.get("password", ""),
all=True)
zk = KazooClient(hosts=zk_config.get("hosts", []),
default_acl=[default_acl],
auth_data=[("digest", "{}:{}".format(
zk_config.get("user", ""),
zk_config.get("password", "")))])
else:
zk = KazooClient(hosts=zk_config.get("hosts", []))
return zk
def connect(self):
"""Initialize a connection to the Zookeeper quorum.
:return: Kazoo client object as connection.
"""
client_kwargs = dict(
hosts=self.app.config['KAZOO_HOSTS'],
timeout=self.app.config['KAZOO_SESSION_TIMEOUT'],
connection_retry=self.app.config['KAZOO_RETRY'],
command_retry=self.app.config['KAZOO_RETRY']
)
# is ACL ?
username = self.app.config.get('KAZOO_ACL_USERNAME', None)
password = self.app.config.get('KAZOO_ACL_PASSWORD', None)
if username and password:
client_kwargs.update(dict(
default_acl=[
make_digest_acl(
username=username,
password=password,
read=self.app.config.get(
'KAZOO_ACL_READ', False
),
write=self.app.config.get(
'KAZOO_ACL_WRITE', False
),
create=self.app.config.get(
'KAZOO_ACL_CREATE', False
),
delete=self.app.config.get(
'KAZOO_ACL_DELETE', False
),
admin=self.app.config.get(
'KAZOO_ACL_ADMIN', False
),
all=self.app.config.get(
'KAZOO_ACL_ALL', False
)
)
],
auth_data=[("digest", ":".join((username, password)))],
))
client = KazooClient(**client_kwargs)
client.start(timeout=self.app.config['KAZOO_START_TIMEOUT'])
client.add_listener(self.connection_state_listener)
return client
def __init__(self, processor, handler, config):
self._logger = logging.getLogger(__name__)
self._section_name = utils.get_module(__name__)
self._server = None
self._service_name = config.get(self._section_name, "service", required=True)
self._port = config.getint(self._section_name, "port", required=True)
self._zk_connect_str = config.get(self._section_name, "zk_connect_str", required=True)
self._auth_user = config.get(self._section_name, "auth_user", required=True)
self._auth_password = config.get(self._section_name, "auth_password", required=True)
self._monitor = config.getboolean(self._section_name, "monitor", default=settings.SERVICE_MONITOR)
self._c_module_serialize = config.getboolean(self._section_name, "c_module_serialize",
default=settings.USE_C_MODULE_SERIALIZE)
self._server_name = config.get(self._section_name, "name")
self._owner = config.get(self._section_name, "owner")
hosts = "%s/%s" % (self._zk_connect_str, settings.DEFAULT_ZK_NAMESPACE_ROOT)
self._server_path = "%s/%s" % (self._service_name, settings.DEFAULT_ZK_NAMESPACE_SERVERS)
self._statistic_path = "%s/%s/%s" % (self._service_name, settings.DEFAULT_ZK_NAMESPACE_STATISTICS,
settings.DEFAULT_ZK_NAMESPACE_SERVERS)
# create zk acl
self._acls = []
self._acls.append(make_digest_acl(self._auth_user, self._auth_password, all=True))
self._acls.append(make_acl("world", "anyone", read=True))
# create zk_client
self._zkclient = HARpcZKClientManager.make(hosts, config, "server")
self._zkclient.add_auth("digest", "%s:%s" % (self._auth_user, self._auth_password))
# create zkpath
if not self._zkclient.exists(self._service_name):
self._zkclient.create(self._service_name, makepath=True)
if not self._zkclient.exists(self._server_path):
self._zkclient.create(self._server_path, acl=self._acls)
if not self._zkclient.exists(self._statistic_path):
self._zkclient.create(self._statistic_path, makepath=True)
self.transport = TSocket.TServerSocket(port=self._port)
self.tfactory = TTransport.TBufferedTransportFactory()
if self._monitor:
self._collector = StatisticsCollector(self._zkclient, config, is_server=True)
self._processor = self._statistics_wrapper(processor)(handler)
else:
self._processor = processor(handler)
if self._c_module_serialize:
self.pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
else:
self.pfactory = TBinaryProtocol.TBinaryProtocolFactory()
def test_default_acl(self):
from kazoo.security import make_digest_acl
username = uuid.uuid4().hex
password = uuid.uuid4().hex
digest_auth = "%s:%s" % (username, password)
acl = make_digest_acl(username, password, all=True)
self.client.add_auth("digest", digest_auth)
self.client.default_acl = (acl,)
t = self.client.transaction()
t.create('/freddy')
results = t.commit()
eq_(results[0], '/freddy')
def test_default_acl(self):
from kazoo.security import make_digest_acl
username = uuid.uuid4().hex
password = uuid.uuid4().hex
digest_auth = "%s:%s" % (username, password)
acl = make_digest_acl(username, password, all=True)
self.client.add_auth("digest", digest_auth)
self.client.default_acl = (acl, )
t = self.client.transaction()
t.create('/freddy')
results = t.commit()
eq_(results[0], '/freddy')
def __publishService(self):
from kazoo.security import make_digest_acl
config = self.getServiceConfig()
acl = make_digest_acl(config["ZOOKEEPER_USER"],
config["ZOOKEEPER_PASSWORD"],
all=True)
dataJsonDict = {}
dataJsonDict['SERVERHOSTNAME'] = config["SERVERHOSTNAME"]
dataJsonDict['SERVER_PORT'] = config["SERVER_PORT"]
dataJsonDict['SERVER_SCHEME'] = config["SERVER_SCHEME"]
dataJsonDict['CONTEXT'] = config["CONTEXT"]
self.__zkcon.set("/plh414python/directoryservice/",
json.JSONEncoder().encode(dataJsonDict).encode(), -1)
def make_kazoo_digest_acl(acl):
# acl = (('xp', '123', 'cdrwa'),
# ('foo', 'passw', 'rw'),
# )
if acl is None:
return None
rst = []
for name, passw, perms in acl:
perm_dict = {p: True for p in perm_to_long(perms)}
acl_entry = security.make_digest_acl(name, passw, **perm_dict)
rst.append(acl_entry)
return rst
def make_kazoo_digest_acl(acl):
# acl = (('xp', '123', 'cdrwa'),
# ('foo', 'passw', 'rw'),
# )
if acl is None:
return None
rst = []
for name, passw, perms in acl:
perm_dict = {p: True
for p in perm_to_long(perms)}
acl_entry = security.make_digest_acl(name, passw, **perm_dict)
rst.append(acl_entry)
return rst
def __init__(self):
self.zk_hosts = settings.ZK_HOSTS
self.zk_root = settings.ZK_ROOT
self.zk_timeout = settings.ZK_TIMEOUT
self.zk_servers = settings.ZK_SERVERS
self.zk_clients = settings.ZK_CLIENTS
self.zk_configs = settings.ZK_CONFIGS
self.zk_statistics = settings.ZK_STATISTICS
self.zk_username = settings.ZK_USERNAME
self.zk_password = settings.ZK_PASSWORD
self.zk_flush = time.time()
self.lock = threading.Lock()
self.resource = {}
self.zk_harpc_acl = make_digest_acl(username=self.zk_username, password=self.zk_password, all=True)
self.zk_anyone_acl = make_acl(scheme='world', credential='anyone', read=True)
def __publishService(self):
from kazoo.security import make_digest_acl
config = self.getServiceConfig()
acl = make_digest_acl(config["ZOOKEEPER_USER"],
config["ZOOKEEPER_PASSWORD"],
all=True)
dataJsonDict = {}
dataJsonDict['SERVERHOSTNAME'] = config["SERVERHOSTNAME"]
dataJsonDict['SERVER_PORT'] = config["SERVER_PORT"]
dataJsonDict['SERVER_SCHEME'] = config["SERVER_SCHEME"]
dataJsonDict['HMACKEY'] = config["HMACKEY"]
dataJsonDict['CONTEXT'] = config["CONTEXT"]
self.__zkcon.create("/plh414python/fileservices/" + config["ID"],
json.JSONEncoder().encode(dataJsonDict).encode(),
[acl],
ephemeral=True)
def setUp(self):
""" Uses a test output of simple_option configparser """
self.cfgremainder = {
"user4": {"passwd": "bla"},
"user5": {"passwd": "bla"},
"user2": {"passwd": "bar"},
"user3": {"passwd": "bla"},
"user1": {"passwd": "xyz"},
"rsync": {"passwd": "w00f"},
"sys": {"passwd": "sis"},
"kwaegema": {"passwd": "w00f"},
"/admin/tools": {"all": "sys"},
"/admin/rsync": {"rwcd": "rsync,kwaegema", "all": ""},
"/admin": {"rwcd": "user1", "c": "user3", "rw": "user4,user3", "all": "", "cd": "user2", "r": "user5"},
"/admin/user1": {"all": "user1", "r": "user2,user3", "rw": "user4"},
"root": {"passwd": "admin", "path": "/"},
}
self.rootacl = make_digest_acl("root", "admin", all=True)
def add_acl(self, path, permissions, scheme, id):
perms = get_permissions(permissions)
if scheme == "digest":
username, password = id.split(":")
acl = make_digest_acl(username, password, **perms)
else:
acl = make_acl(scheme, id, **perms)
current_acls = self.get_acl(path)
current_acls.append(acl)
try:
self._zookeeper.set_acls(path, current_acls)
except NoNodeError:
raise ZooKeeperException("No such node: {}".format(path))
except InvalidACLError as exc:
raise ZooKeeperException("Invalid ACL format: {}".format(str(exc)))
except NoAuthError:
raise ZooKeeperException("No access to add acl on node: {}".format(path))
def _make_client(self, parsed_url, options):
# Creates a kazoo client,
# See: https://github.com/python-zk/kazoo/blob/2.2.1/kazoo/client.py
# for what options a client takes...
if parsed_url.username and parsed_url.password:
username = parsed_url.username
password = parsed_url.password
digest_auth = "%s:%s" % (username, password)
digest_acl = security.make_digest_acl(username, password, all=True)
default_acl = (digest_acl,)
auth_data = [('digest', digest_auth)]
else:
default_acl = None
auth_data = None
maybe_hosts = [parsed_url.netloc] + list(options.get('hosts', []))
hosts = list(compat_filter(None, maybe_hosts))
if not hosts:
hosts = ['localhost:2181']
randomize_hosts = options.get('randomize_hosts', True)
client_kwargs = {
'hosts': ",".join(hosts),
'timeout': float(options.get('timeout', self.timeout)),
'connection_retry': options.get('connection_retry'),
'command_retry': options.get('command_retry'),
'randomize_hosts': strutils.bool_from_string(randomize_hosts),
'auth_data': auth_data,
'default_acl': default_acl,
}
handler_kind = options.get('handler')
if handler_kind:
try:
handler_cls = self.HANDLERS[handler_kind]
except KeyError:
raise ValueError("Unknown handler '%s' requested"
" valid handlers are %s"
% (handler_kind,
sorted(self.HANDLERS.keys())))
client_kwargs['handler'] = handler_cls()
return client.KazooClient(**client_kwargs)
def _make_client(self, parsed_url, options):
# Creates a kazoo client,
# See: https://github.com/python-zk/kazoo/blob/2.2.1/kazoo/client.py
# for what options a client takes...
if parsed_url.username and parsed_url.password:
username = parsed_url.username
password = parsed_url.password
digest_auth = "%s:%s" % (username, password)
digest_acl = security.make_digest_acl(username, password, all=True)
default_acl = (digest_acl,)
auth_data = [('digest', digest_auth)]
else:
default_acl = None
auth_data = None
maybe_hosts = [parsed_url.netloc] + list(options.get('hosts', []))
hosts = list(compat_filter(None, maybe_hosts))
if not hosts:
hosts = ['localhost:2181']
randomize_hosts = options.get('randomize_hosts', True)
client_kwargs = {
'hosts': ",".join(hosts),
'timeout': float(options.get('timeout', self.timeout)),
'connection_retry': options.get('connection_retry'),
'command_retry': options.get('command_retry'),
'randomize_hosts': strutils.bool_from_string(randomize_hosts),
'auth_data': auth_data,
'default_acl': default_acl,
}
handler_kind = options.get('handler')
if handler_kind:
try:
handler_cls = self.HANDLERS[handler_kind]
except KeyError:
raise ValueError("Unknown handler '%s' requested"
" valid handlers are %s"
% (handler_kind,
sorted(self.HANDLERS.keys())))
client_kwargs['handler'] = handler_cls()
return client.KazooClient(**client_kwargs)
def create():
dirname = ROOT + request.json['path']
name = request.json['name']
value = request.json['value']
try:
options = request.json['options']
options_params = {}
new_path = dirname + '/' + name
if options['create']:
username, password = get_credential()
with auth_kazoo(username, password) as auth_client:
admin_acl = make_digest_acl(username, password, **options)
acl_list = [anonymous_acl, admin_acl]
options_params['acl'] = acl_list
auth_client.create(new_path, value=value.encode('utf8'), **options_params)
else:
client.create(new_path, value=value.encode('utf8'), **options_params)
return make_response(success=True, msg='创建成功')
except Exception as e:
logging.exception('error when create node: %s', str(e))
return make_response(success=False, msg='创建节点出错: %s' % str(e))
def test_start_acl_behavior(self):
# Set up a username/password so an ACL is created
ACL = security.make_digest_acl('user', 'password', all=True)
self.zk.add_auth('digest', 'user:password')
self.zk.default_acl = [ACL]
# Test creating a nested path, and validate that the ACLs for
# that neseted path are setup properly
path = '%s/unit/test/host:22' % self.sandbox
data = {}
reg1 = RegistrationBase(zk=self.zk, path=path, data=data)
# Now registre the path and wait for it to finish
reg1.start()
waituntil(reg1.data, None, 5)
# Now, lets check the ACL for each path that would have
# been created.
self.assertEquals(security.OPEN_ACL_UNSAFE,
self.zk.get_acls('%s/unit' % self.sandbox)[0])
self.assertEquals([ACL],
self.zk.get_acls('%s/unit/test' % self.sandbox)[0])
def parse_acls(acls, users, root_acl):
"""takes a dictionary of acls : users and returns list of ACLs"""
acl_list = [root_acl]
for acl, acl_users in acls.iteritems():
if acl == 'all':
acl_r, acl_w, acl_c, acl_d, acl_a = True, True, True, True, True
else:
acl_r = 'r' in acl
acl_w = 'w' in acl
acl_c = 'c' in acl
acl_d = 'd' in acl
acl_a = 'a' in acl
for acl_user in acl_users:
if acl_user not in users:
logger.raiseException('User %s not configured!' % acl_user)
else:
tacl = make_digest_acl(acl_user, str(users[acl_user].get('passwd')), \
read=acl_r, write=acl_w, create=acl_c, delete=acl_d, admin=acl_a)
acl_list.append(tacl)
logger.debug("acl list %s" % acl_list)
return acl_list
def parse_acls(acls, users, root_acl):
"""takes a dictionary of acls : users and returns list of ACLs"""
acl_list = [root_acl]
for acl, acl_users in acls.iteritems():
if acl == 'all':
acl_r, acl_w, acl_c, acl_d, acl_a = True, True, True, True, True
else:
acl_r = 'r' in acl
acl_w = 'w' in acl
acl_c = 'c' in acl
acl_d = 'd' in acl
acl_a = 'a' in acl
for acl_user in acl_users:
if acl_user not in users:
logger.raiseException('User %s not configured!' % acl_user)
else:
tacl = make_digest_acl(acl_user, str(users[acl_user].get('passwd')), \
read=acl_r, write=acl_w, create=acl_c, delete=acl_d, admin=acl_a)
acl_list.append(tacl)
logger.debug("acl list %s" % acl_list)
return acl_list
def main():
""" Builds a zookeeper tree with ACLS on from a config file"""
options = {'servers': ('list of zk servers', 'strlist', 'store', None)}
go = simple_option(options)
rpasswd, _ = get_rootinfo(go.configfile_remainder)
znodes, users = parse_zkconfig(go.configfile_remainder)
logger.debug("znodes: %s", znodes)
logger.debug("users: %s", users)
# Connect to zookeeper
# initial authentication credentials and acl for admin on root level
acreds = [('digest', 'root:' + rpasswd)]
root_acl = make_digest_acl('root', rpasswd, all=True)
# Create kazoo/zookeeper connection with root credentials
servers = go.options.servers
zkclient = VscKazooClient(servers, auth_data=acreds)
# Iterate paths
for path, attrs in znodes.iteritems():
logger.debug("path %s attribs %s", path, attrs)
acls = dict(
(arg, attrs[arg]) for arg in attrs
if arg not in ('value', 'ephemeral', 'sequence', 'makepath'))
acl_list = parse_acls(acls, users, root_acl)
kwargs = dict((arg, attrs[arg]) for arg in attrs
if arg in ('ephemeral', 'sequence', 'makepath'))
if not zkclient.exists_znode(path):
zkclient.make_znode(path,
value=attrs.get('value', ''),
acl=acl_list,
**kwargs)
else:
logger.warning('node %s already exists', path)
zkclient.znode_acls(path, acl_list)
zkclient.exit()
def extract_acl(cls, acl):
""" parse an individual ACL (i.e.: world:anyone:cdrwa) """
try:
scheme, rest = acl.split(":", 1)
credential = ":".join(rest.split(":")[0:-1])
cdrwa = rest.split(":")[-1]
except ValueError:
raise cls.BadACL("Bad ACL: %s. Format is scheme:id:perms" % (acl))
if scheme not in cls.valid_schemes:
raise cls.BadACL("Invalid scheme: %s" % (acl))
create = True if "c" in cdrwa else False
read = True if "r" in cdrwa else False
write = True if "w" in cdrwa else False
delete = True if "d" in cdrwa else False
admin = True if "a" in cdrwa else False
if scheme == "username_password":
try:
username, password = credential.split(":", 1)
except ValueError:
raise cls.BadACL("Bad ACL: %s. Format is scheme:id:perms" % (acl))
return make_digest_acl(username,
password,
read,
write,
create,
delete,
admin)
else:
return make_acl(scheme,
credential,
read,
write,
create,
delete,
admin)
def connect(self):
"""Initialize a connection to the Zookeeper quorum.
:return: Kazoo client object as connection.
"""
client_kwargs = dict(hosts=self.app.config['KAZOO_HOSTS'],
timeout=self.app.config['KAZOO_SESSION_TIMEOUT'],
connection_retry=self.app.config['KAZOO_RETRY'],
command_retry=self.app.config['KAZOO_RETRY'])
# is ACL ?
username = self.app.config.get('KAZOO_ACL_USERNAME', None)
password = self.app.config.get('KAZOO_ACL_PASSWORD', None)
if username and password:
client_kwargs.update(
dict(
default_acl=[
make_digest_acl(
username=username,
password=password,
read=self.app.config.get('KAZOO_ACL_READ', False),
write=self.app.config.get('KAZOO_ACL_WRITE',
False),
create=self.app.config.get('KAZOO_ACL_CREATE',
False),
delete=self.app.config.get('KAZOO_ACL_DELETE',
False),
admin=self.app.config.get('KAZOO_ACL_ADMIN',
False),
all=self.app.config.get('KAZOO_ACL_ALL', False))
],
auth_data=[("digest", ":".join((username, password)))],
))
client = KazooClient(**client_kwargs)
client.start(timeout=self.app.config['KAZOO_START_TIMEOUT'])
client.add_listener(self.connection_state_listener)
return client
def zk_connect():
print("Connecting to ZooKeeper")
zk_user = os.environ.get('CALICO_ZK_USER')
zk_secret = os.environ.get('CALICO_ZK_SECRET')
conn_retry_policy = KazooRetry(
max_tries=-1,
delay=0.3,
backoff=1.3,
max_jitter=1,
max_delay=3,
ignore_expire=True,
)
# Retry commands every 0.3 seconds, for a total of <1s (usually 0.9)
cmd_retry_policy = KazooRetry(
max_tries=3,
delay=0.3,
backoff=1,
max_jitter=0.1,
max_delay=1,
ignore_expire=False,
)
default_acl = None
auth_data = None
if zk_user and zk_secret:
default_acl = [make_digest_acl(zk_user, zk_secret, all=True)]
scheme = 'digest'
credential = "{}:{}".format(zk_user, zk_secret)
auth_data = [(scheme, credential)]
zk = KazooClient(
hosts=ZK_SERVER,
timeout=30,
connection_retry=conn_retry_policy,
command_retry=cmd_retry_policy,
default_acl=default_acl,
auth_data=auth_data,
)
zk.start()
return zk
def test_start_acl_behavior(self):
# Set up a username/password so an ACL is created
ACL = security.make_digest_acl('user', 'password', all=True)
self.zk.add_auth('digest', 'user:password')
self.zk.default_acl = [ACL]
# Test creating a nested path, and validate that the ACLs for
# that neseted path are setup properly
path = '%s/unit/test/host:22' % self.sandbox
data = {}
reg1 = RegistrationBase(zk=self.zk, path=path, data=data)
# Now registre the path and wait for it to finish
reg1.start()
waituntil(reg1.data, None, 5)
# Now, lets check the ACL for each path that would have
# been created.
self.assertEquals(
security.OPEN_ACL_UNSAFE,
self.zk.get_acls('%s/unit' % self.sandbox)[0])
self.assertEquals(
[ACL], self.zk.get_acls('%s/unit/test' % self.sandbox)[0])
def zk_connect(uri):
"""
Connects to zookeeper
:return: zookeeper connection, or None if unsuccessful
"""
logger = logging.getLogger('gconfiglib')
zk_uri = urlparse.urlparse(uri)
if zk_uri.scheme != 'zookeeper':
logger.log(logging.ERROR, 'Expecting uri in zookeeper://user:passwd@host/prefix format')
return False
host = zk_uri.hostname
credentials = zk_uri.username + ':' + zk_uri.password
prefix = zk_uri.path
try:
zk_conn = KazooClient(hosts=host,
default_acl=[make_digest_acl(*credentials.split(':'), all=True)],
auth_data=[('digest', credentials)])
zk_conn.start()
zk_conn.ensure_path(prefix)
logger.log(logging.DEBUG, 'Connected to Zookeeper at ' + host)
except kazoo.handlers.threading.KazooTimeoutError as exception:
logger.log(logging.ERROR, 'Could not connect to ZooKeeper server, ' + str(exception))
return None
return zk_conn
except Exception, ex:
self.init_ret = False
self.err_str = "Create KazooClient failed! Exception: %s" % str(ex)
try:
print self.lock_path
self.lock_handle = Lock(self.zk_client, self.lock_path)
self.zk_client.set(self.lock_path, self.lock_value)
except Exception, ex:
self.init_ret = False
self.err_str = "Create lock failed! Exception: %s" % str(ex)
from kazoo.security import make_digest_acl
client = self.zk_client
acl = make_digest_acl('pcm', 'pcm', all=True)
try:
client.set_acls(self.lock_path, [acl])
finally:
print("set KazooClient acl SUCCESS!")
def destroy_lock(self):
# self.release()
if self.zk_client != None:
self.zk_client.stop()
self.zk_client = None
def acquire(self, blocking=True, timeout=None):
if self.lock_handle == None:
return None
def _makeAuth(self, *args, **kwargs):
from kazoo.security import make_digest_acl
return make_digest_acl(*args, **kwargs)
def setUp(argv, remaining_args=None):
global zk, myAcl, adminAcls, hiddenAcls
f = None
config_file = os.getenv("ZOOSYNC_CONF", "/etc/zoosyncrc")
try:
f = open(config_file, "r")
except:
pass
if f:
for line in f:
line = line.rstrip("\r\n")
if re.match(r"^\s*#.*", line):
continue
keyvalue = re.split(r"\s*=\s*", line, 1)
key = keyvalue[0]
if len(keyvalue) > 1:
value = keyvalue[1]
else:
value = None
parse_option(key=key, arg=value)
f.close()
try:
opts, args = getopt.getopt(
argv,
"ha:b:H:m:np:s:u:vw:z:",
[
"help",
"acl=",
"base=",
"hostname=",
"dry",
"multi=",
"user="******"password="******"services=",
"version",
"wait=",
"zookeeper=",
],
)
except getopt.GetoptError:
print "Error parsing arguments"
usage()
sys.exit(2)
for opt, arg in opts:
if opt in ["-h", "--help"]:
usage()
sys.exit(0)
elif opt in ["-v", "--version"]:
print version.__version__
sys.exit(0)
parse_option(opt=opt, arg=arg)
logging.basicConfig()
myAcl = make_digest_acl(user, password, all=True)
if admin_acl:
admin_acl_list = re.split(",", admin_acl)
for s in admin_acl_list:
acl = str2acl(s)
if acl:
adminAcls += [acl]
acl = str2acl(s, hidden=True)
if acl:
hiddenAcls += [acl]
# print '# Admin ACL: %s' % adminAcls
# print '# Secret ACL: %s' % hiddenAcls
zk = KazooClient(zookeeper_hosts, auth_data=[("digest", "%s:%s" % (user, password))])
zk.start()
if remaining_args != None:
remaining_args.extend(args)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2018/10/10
# @Author : Wenhao Shan
import log
from zktool.enum.log_enum import KazooStatus
from kazoo.client import KazooClient, KazooState
from kazoo.security import make_digest_acl
Zk_Address = '192.168.1.89:2181'
ACL_MY = make_digest_acl(username="******",
password="******",
read=True,
write=True,
create=True,
delete=True,
admin=True,
all=True)
class ZKClientAsync(KazooClient):
"""
该类继承自KazooClient
"""
def __init__(self,
hosts='127.0.0.1:2181',
timeout=10.0,
client_id=None,
handler=None,
default_acl=None,
def _makeAuth(self, *args, **kwargs):
from kazoo.security import make_digest_acl
return make_digest_acl(*args, **kwargs)
logging.basicConfig()
app = Flask(__name__)
def my_listener(state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
print "Session lost!"
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
print "Disconnected from zookeeper!"
else:
# Handle being connected/reconnected to Zookeeper
print "Connected to zookeeper!"
acls = [make_digest_acl('dcosadmin','zjdcos01',all=True),make_digest_acl('scrm','scrm1121',read=True),]
zk = KazooClient(hosts='10.78.180.44:2181,10.78.180.45:2181,10.78.180.46:2181',auth_data=[('digest','dcosadmin:zjdcos01')])
zk.start()
kr = KazooRetry(max_tries=5, ignore_expire=False)
result = kr(zk.get, "/dcos/chk")
zk.add_listener(my_listener)
crmweb_path = "/ywzc/crm/crm2014/webs"
crmapp_path = "/ywzc/crm/crm2014/apps/cluster"
db_conn = 'dcos/v1g2m60id2499yz@pdbdcos'
marathon_list = ['10.78.182.12','10.78.217.11']
marathon_url = '10.78.182.12:8080'
#Test marathon
def testMarathon():
global marathon_url
for i in marathon_list:
def __init__(self, processor, handler, config):
self._logger = logging.getLogger(__name__)
self._section_name = utils.get_module(__name__)
self._server = None
self._service_name = config.get(self._section_name,
"service",
required=True)
self._port = config.getint(self._section_name, "port", required=True)
self._zk_connect_str = config.get(self._section_name,
"zk_connect_str",
required=True)
self._auth_user = config.get(self._section_name,
"auth_user",
required=True)
self._auth_password = config.get(self._section_name,
"auth_password",
required=True)
self._monitor = config.getboolean(self._section_name,
"monitor",
default=settings.SERVICE_MONITOR)
self._c_module_serialize = config.getboolean(
self._section_name,
"c_module_serialize",
default=settings.USE_C_MODULE_SERIALIZE)
self._server_name = config.get(self._section_name, "name")
self._owner = config.get(self._section_name, "owner")
hosts = "%s/%s" % (self._zk_connect_str,
settings.DEFAULT_ZK_NAMESPACE_ROOT)
self._server_path = "%s/%s" % (self._service_name,
settings.DEFAULT_ZK_NAMESPACE_SERVERS)
self._statistic_path = "%s/%s/%s" % (
self._service_name, settings.DEFAULT_ZK_NAMESPACE_STATISTICS,
settings.DEFAULT_ZK_NAMESPACE_SERVERS)
# create zk acl
self._acls = []
self._acls.append(
make_digest_acl(self._auth_user, self._auth_password, all=True))
self._acls.append(make_acl("world", "anyone", read=True))
# create zk_client
self._zkclient = HARpcZKClientManager.make(hosts, config, "server")
self._zkclient.add_auth(
"digest", "%s:%s" % (self._auth_user, self._auth_password))
# create zkpath
if not self._zkclient.exists(self._service_name):
self._zkclient.create(self._service_name, makepath=True)
if not self._zkclient.exists(self._server_path):
self._zkclient.create(self._server_path, acl=self._acls)
if not self._zkclient.exists(self._statistic_path):
self._zkclient.create(self._statistic_path, makepath=True)
self.transport = TSocket.TServerSocket(port=self._port)
self.tfactory = TTransport.TBufferedTransportFactory()
if self._monitor:
self._collector = StatisticsCollector(self._zkclient,
config,
is_server=True)
self._processor = self._statistics_wrapper(processor)(handler)
else:
self._processor = processor(handler)
if self._c_module_serialize:
self.pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
else:
self.pfactory = TBinaryProtocol.TBinaryProtocolFactory()