def __init__(self, barriername, number_of_workers):
self.cv = threading.Condition()
self.connected = False
self.barrier = "/" + barriername
self.workers = number_of_workers
zookeeper.set_log_stream(open('/dev/null'))
def watcher(handle, type, state, path):
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle, self.barrier, '\x00', [ZOO_OPEN_ACL_UNSAFE], 0)
except zookeeper.NodeExistsException:
pass
except Exception, ex:
print ex
raise ex
def test_sync_get_large_datanode(self):
"""
Test that we can retrieve datanode sizes up to
1Mb with default parameters (depends on ZooKeeper server).
"""
data = ''.join(["A" for x in range(1024 * 1023)])
self.ensureDeleted("/zk-python-test-large-datanode")
zookeeper.create(self.handle, "/zk-python-test-large-datanode", data,
[{
"perms": 0x1f,
"scheme": "world",
"id": "anyone"
}])
(ret, stat) = zookeeper.get(self.handle,
"/zk-python-test-large-datanode")
self.assertEqual(
len(ret), 1024 * 1023,
"Should have got 1Mb returned, instead got %s" % len(ret))
(ret, stat) = zookeeper.get(self.handle,
"/zk-python-test-large-datanode", None,
500)
self.assertEqual(
len(ret), 500,
"Should have got 500 bytes returned, instead got %s" % len(ret))
def __init__(self, service_name, connect_string, timeout=DEFAULT_TIMEOUT, default_port=6664):
self.SERVICE_NODE = "/" + service_name
self.AVAILABILITY_NODE = self.SERVICE_NODE + "/available"
self.MEMBERSHIP_NODE = self.SERVICE_NODE + "/members"
self.connected = False
self.timeout = timeout
self.default_port = default_port
self.conn_cv = threading.Condition()
self.conn_cv.acquire()
self.handle = zookeeper.init(connect_string, self.connection_watcher, timeout)
self.conn_cv.wait(timeout / 1000)
self.conn_cv.release()
self.watcher_lock = threading.Lock()
self.logger = logging.getLogger("sincc")
if not self.connected:
raise SinClusterClientError("Unable to connect to %s" % connect_string)
for path in [self.SERVICE_NODE, self.AVAILABILITY_NODE, self.MEMBERSHIP_NODE]:
if not zookeeper.exists(self.handle, path):
zookeeper.create(self.handle, path, "", [ZOO_OPEN_ACL_UNSAFE], 0)
self.listeners = []
# Start to watch both /members and /available
zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
self.available_nodes = {}
for node_id in available:
self.available_nodes[int(node_id)] = Node(int(node_id),
zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
def connect(host=None):
global conn_cv, connected, handle
global reconnect_host
if host is None:
host = "localhost:2181"
reconnect_host = host
conn_cv.acquire()
handle = zookeeper.init(host, my_connection_watcher, 10000)
while not connected:
conn_cv.wait()
conn_cv.release()
connected = False
while not connected:
try:
zookeeper.create(handle, ROOT, "1", [ZOO_OPEN_ACL_UNSAFE], 0)
connected = True
except zookeeper.NodeExistsException as e:
# No worries
connected = True
except zookeeper.ConnectionLossException:
continue
except:
raise
def register_consumer(self, consumer_group, consumer_id, topic):
"""Creates the following permanent node, if it does not exist already:
/consumers/{consumer_group}/ids
The data written at this node is just the consumer_id so that we can
later track who created what.
We then create the following emphemeral node:
/consumers/{consumer_group}/ids/{consumer_id}
The data written in this node is a dictionary of topic names (in
unicode) to the number of threads that this consumer has registered
for this topic (in our case, always one).
"""
self._create_path_if_needed(self.path_for_consumer_ids(consumer_group),
consumer_id)
# Create an emphermal node for this consumer
consumer_id_path = self.path_for_consumer_id(consumer_group, consumer_id)
log.info("Registering Consumer {0}, trying to create {1}"
.format(consumer_id, consumer_id_path))
zookeeper.create(self._zk.handle,
consumer_id_path,
json.dumps({topic : 1}), # topic : # of threads
ZKUtil.ACL,
zookeeper.EPHEMERAL)
def __init__(self, queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle, type, state, path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle, self.queuename, "queue top level",
[ZOO_OPEN_ACL_UNSAFE], 0)
except zookeeper.NodeExistsException:
print "Queue already exists"
def __tryGC(self, app_id, app_path):
try:
# print "try to obtrain app %s lock" % app_id
val = zookeeper.get(self.handle, PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), None)[0]
lasttime = float(val)
except zookeeper.NoNodeException:
lasttime = 0
if lasttime + GC_INTERVAL < time.time():
# try to gc this app.
gc_path = PATH_SEPARATOR.join([app_path, GC_LOCK_PATH])
try:
now = str(time.time())
zookeeper.create(self.handle, gc_path, now, ZOO_ACL_OPEN, zookeeper.EPHEMERAL)
# succeed to obtain lock.
# TODO: should we handle the timeout of gc also?
try:
self.__executeGC(app_id, app_path)
# update lasttime when gc was succeeded.
now = str(time.time())
self.__updateNode(PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), now)
except Exception, e:
print "warning: gc error %s" % e
traceback.print_exc()
zookeeper.delete(self.handle, gc_path, -1)
except zookeeper.NodeExistsException:
# fail to obtain lock. try next time.
pass
def __init__(self,queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000, 0)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
except IOError, e:
if e.message == zookeeper.zerror(zookeeper.NODEEXISTS):
print "Queue already exists"
else:
raise e
def create_node(zh, options, path):
try:
zookeeper.create(zh, path, options.VALUE, acl_openbar,
zookeeper.EPHEMERAL)
logging.info("OK create %s", path)
except Exception as e:
logging.info("ERROR create %s: %s", path, e)
def set(self, path, data='', previous_data=None):
path = '%s%s' % (self.prefix, path)
try:
current_data, current_meta = zookeeper.get(self.handle, path, None)
except zookeeper.NoNodeException:
if not previous_data:
try:
zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE])
return True
except:
self.log.exception('Failed to create a missing key %s', path)
return False
else:
return False
except:
self.log.exception('Failed to set key %s', path)
return False
version = None
if previous_data:
if current_data != previous_data:
self.log.error('Previous data constraint failed')
return False
version = current_meta['version']
try:
if version is None:
zookeeper.set(self.handle, path, data)
else:
zookeeper.set(self.handle, path, data, version)
except:
self.log.exception('Set failed')
return False
return True
def connect(host=None):
global conn_cv, connected, handle
global reconnect_host
if host is None:
host = "localhost:2181"
reconnect_host = host
conn_cv.acquire()
handle = zookeeper.init(host, my_connection_watcher, 10000)
while not connected:
conn_cv.wait()
conn_cv.release()
connected = False
while not connected:
try:
zookeeper.create(handle, ROOT, "1", [ZOO_OPEN_ACL_UNSAFE], 0)
connected = True
except zookeeper.NodeExistsException as e:
# No worries
connected = True
except zookeeper.ConnectionLossException:
continue
except:
raise
def setUp( self ):
zktestbase.TestBase.setUp(self)
try:
zookeeper.create(self.handle, "/zk-python-existstest","existstest", [ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL)
zookeeper.create(self.handle, "/zk-python-aexiststest","existstest",[ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL)
except:
pass
def watch_session(handle, event_type, state, path):
assert event_type == zookeeper.SESSION_EVENT
assert not path
if state == zookeeper.CONNECTED_STATE:
if self.handle is None:
self.handle = handle
for watch in self.watches.clear():
self._watch(watch)
for path, data in list(self.ephemeral.items()):
zookeeper.create(self.handle, path, data['data'],
data['acl'], data['flags'])
else:
assert handle == self.handle
connected.set()
logger.info('connected %s', handle)
elif state == zookeeper.CONNECTING_STATE:
connected.clear()
elif state == zookeeper.EXPIRED_SESSION_STATE:
connected.clear()
if self.handle is not None:
zookeeper.close(self.handle)
self.handle = None
init()
else:
logger.critical('unexpected session event %s %s', handle, state)
def zookeeper_node_set(nodepath, nodevalue, perms, **option):
ephemeral = 0
if option.has_key('ephemeral') and option['ephemeral']:
ephemeral = zookeeper.EPHEMERAL
parent_perms = perms
if option.has_key('parent_perms'):
parent_perms = option['parent_perms']
p = FilePath('/')
for v in nodepath.split('/'):
p = p / v
if not zookeeper.exists(G.zookeeper, p):
if not option.has_key('nocreate') or not option['nocreate']:
if p == nodepath:
print zookeeper.create(G.zookeeper, p, nodevalue,
[{
"perms": perms,
"scheme": "world",
"id": "anyone"
}], ephemeral)
return True
else:
zookeeper.create(G.zookeeper, p, '', [{
"perms": parent_perms,
"scheme": "world",
"id": "anyone"
}], 0)
elif p == nodepath:
print zookeeper.set(G.zookeeper, p, nodevalue)
return True
return False
def create(self, path, data="", flags=0, makepath=False, force=False):
"""
创建节点
"""
acl2 = {"perms": 0x1f, "scheme": "world", "id": "anyone"}
if self.acl["scheme"] == "digest":
user, _ = self.acl["id"].split(':')
id2 = "%s:%s" % (user,
base64.b64encode(
hashlib.new("sha1", self.acl["id"]).digest()))
acl2 = {"perms": self.acl["perms"], "scheme": "digest", "id": id2}
log.info("zk create [%s]" % path)
if makepath:
if not self.exists(os.path.dirname(path)):
self.create(path=os.path.dirname(path),
data="",
flags=flags,
makepath=True)
log.info(path)
if force and self.exists(path):
self.delete(path)
zookeeper.create(self.handle, path, data, [acl2], flags)
if self.exists(path):
return 1
else:
return 0
def register_consumer(self, consumer_group, consumer_id, topic):
"""Creates the following permanent node, if it does not exist already:
/consumers/{consumer_group}/ids
The data written at this node is just the consumer_id so that we can
later track who created what.
We then create the following emphemeral node:
/consumers/{consumer_group}/ids/{consumer_id}
The data written in this node is a dictionary of topic names (in
unicode) to the number of threads that this consumer has registered
for this topic (in our case, always one).
"""
self._create_path_if_needed(self.path_for_consumer_ids(consumer_group),
consumer_id)
# Create an emphermal node for this consumer
consumer_id_path = self.path_for_consumer_id(consumer_group,
consumer_id)
log.info("Registering Consumer {0}, trying to create {1}".format(
consumer_id, consumer_id_path))
zookeeper.create(
self._zk.handle,
consumer_id_path,
json.dumps({topic: 1}), # topic : # of threads
ZKUtil.ACL,
zookeeper.EPHEMERAL)
def _create_path_if_needed(self, path, data=None):
"""Creates permanent nodes for all elements in the path if they don't
already exist. Places data for each node created. (You'll probably want
to use the consumer_id for this, just for accounting purposes -- it's
not used as part of the balancing algorithm).
Our zc.zk.ZooKeeper object doesn't know how to create nodes, so we
have to dig into the zookeeper base library. Note that we can't create
all of it in one go -- ZooKeeper only allows atomic operations, and
so we're creating these one at a time.
"""
if not path.startswith("/"):
raise ValueError("Paths must be fully qualified (begin with '/').")
def _build_path(nodes):
return "/" + "/".join(nodes)
nodes_to_create = path[1:].split("/") # remove beginning "/"
created_so_far = []
for node in nodes_to_create:
created_path = _build_path(created_so_far)
if node and node not in self._zk.children(created_path).data:
node_to_create = _build_path(created_so_far + [node])
# If data is a string, we'll initialize the node with it...
if isinstance(data, basestring):
init_data = data
else:
init_data = json.dumps(data)
zookeeper.create(self._zk.handle, node_to_create, init_data,
ZKUtil.ACL)
created_so_far.append(node)
def _real_register(self):
for znode, data in self._register_entry_set.iter_entry():
if self._base_znode == "/":
znode = "/%s" % znode
else:
znode = "%s/%s" % (self._base_znode, znode)
while True:
try:
zookeeper.create(
self._register_handler,
znode,
data,
# 权限
[{
"perms": 0x1f,
"scheme": "world",
"id": "anyone"
}],
# 节点类型
zookeeper.EPHEMERAL)
LOGGER.info("create znode: %s successfully" % znode)
break
except zookeeper.NodeExistsException:
if self._delete_if_exists:
LOGGER.info("%s already exists, delete it anyway" %
znode)
try:
zookeeper.delete(self._register_handler, znode)
except zookeeper.NoNodeException:
pass
continue
raise
def __tryGC(self, app_id, app_path):
try:
# print "try to obtrain app %s lock" % app_id
val = zookeeper.get(self.handle, PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), None)[0]
lasttime = float(val)
except zookeeper.NoNodeException:
lasttime = 0
if lasttime + GC_INTERVAL < time.time():
# try to gc this app.
gc_path = PATH_SEPARATOR.join([app_path, GC_LOCK_PATH])
try:
now = str(time.time())
zookeeper.create(self.handle, gc_path, now, ZOO_ACL_OPEN, zookeeper.EPHEMERAL)
# succeed to obtain lock.
# TODO: should we handle the timeout of gc also?
try:
self.__executeGC(app_id, app_path)
# update lasttime when gc was succeeded.
now = str(time.time())
self.__updateNode(PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), now)
except Exception, e:
print "warning: gc error %s" % e
traceback.print_exc()
zookeeper.delete(self.handle, gc_path, -1)
except zookeeper.NodeExistsException:
# fail to obtain lock. try next time.
pass
def __init__(self,queuename, port, is_producer=False):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
sys.exit()
self.cv.release()
if is_producer:
while True:
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
print "Fila criada, OK"
return
except zookeeper.NodeExistsException:
print "Tratorando filas existentes"
while True:
children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
if len(children) == 0:
(data,stat) = zookeeper.get(self.handle, self.queuename, None)
zookeeper.delete(self.handle, self.queuename, stat["version"])
break
for child in children:
data = self.get_and_delete(self.queuename + "/" + child)
def _create_path_if_needed(self, path, data=None):
"""Creates permanent nodes for all elements in the path if they don't
already exist. Places data for each node created. (You'll probably want
to use the consumer_id for this, just for accounting purposes -- it's
not used as part of the balancing algorithm).
Our zc.zk.ZooKeeper object doesn't know how to create nodes, so we
have to dig into the zookeeper base library. Note that we can't create
all of it in one go -- ZooKeeper only allows atomic operations, and
so we're creating these one at a time.
"""
if not path.startswith("/"):
raise ValueError("Paths must be fully qualified (begin with '/').")
def _build_path(nodes):
return "/" + "/".join(nodes)
nodes_to_create = path[1:].split("/") # remove beginning "/"
created_so_far = []
for node in nodes_to_create:
created_path = _build_path(created_so_far)
if node and node not in self._zk.children(created_path).data:
node_to_create = _build_path(created_so_far + [node])
# If data is a string, we'll initialize the node with it...
if isinstance(data, basestring):
init_data = data
else:
init_data = json.dumps(data)
zookeeper.create(self._zk.handle, node_to_create, init_data, ZKUtil.ACL)
created_so_far.append(node)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v', '--verbose', action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_config().get(ns, 'zookeeper')
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
create_tree(zh, boot_tree())
init_namespace(zh, ns)
zookeeper.close(zh)
def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'):
"""zkhosts: a string or a list. list will be ','.join-ed into a string.
root: root node path (any parents must exist, if any)
"""
if not isinstance(zkhosts, basestring):
zkhosts = ','.join(zkhosts)
self.zkhosts = zkhosts
self.ROOT = root
self.alivenode = alivenode
self.nodename = os.uname()[1]
self.NODE_SERVERS = self.ROOT + '/servers'
self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename
self.zh = zk.init(self.zkhosts, self.__watcher)
self.jobs = {}
self.initialize_node_structure()
if not zk.exists(self.zh, self.NODE_ME):
zk.create(self.zh, self.NODE_ME, '', PERM_WORLD)
# setup notification
zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher)
#zk.get_children(self.zh, self.NODE_ME, self.__watcher)
self.NODE_JOBS = self.NODE_ME+'/jobs'
zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
def try_lock(self):
while True:
try:
zookeeper.create(self.handle, self.lockname, self.uuid,
[ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL)
except:
self.log.debug('Failed to acquire lock (%r)' % self.lockname)
return False
try:
(data, _) = zookeeper.get(self.handle, self.lockname, None)
break
except:
self.log.exception(
'try_lock: create succeeded but get failed? (%r)' %
self.lockname)
continue
if data == self.uuid:
self.log.debug('Lock acquired (%r)' % self.lockname)
return True
else:
self.log.error(
'try_lock: create succeeded but data is wrong? (%r)' %
self.lockname)
print "failed to acquire lock"
return False
def register_machine(self, machine, data):
if not isinstance(data, odin_pb2.Machine):
raise TypeError('data must be a protobuffer Machine')
zookeeper.create(self._z,
machine,
data.SerializeToString(),
[ZOO_OPEN_ACL_UNSAFE],
zookeeper.EPHEMERAL)
def create(self, path="", value=""):
try:
zookeeper.create(self.zk, path, value, self.acl, self.flags)
except zookeeper.NodeExistsException:
pass
except zookeeper.NoNodeException:
self.create(path=path.rsplit("/", 1)[0], refactor=False)
self.create(path=path, value=value, refactor=False)
def mark_node_available(self, node_id, data=""):
"""Mark a node available."""
path = self.AVAILABILITY_NODE + "/" + str(node_id)
try:
zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL)
self.logger.info("Node %d: %s is now available" % (node_id, data))
except zookeeper.NodeExistsException:
self.logger.warn("%s already exists" % path)
def set(i):
nodeinfo = '/node%d' % i
datainfo = 'datajfajdfkjaslkjdfajsfiwerkjkjelkjkljafd%d' % i
zookeeper.create(zk, nodeinfo, datainfo, [{
"perms": 0x1f,
"scheme": "world",
"id": "anyone"
}], 0)
print 'insert number %d is ok! ' % i
def write(self, path, contents, ephemeral=False, exclusive=False):
"""
Writes the contents to the path in zookeeper. It will create the path in
zookeeper if it does not already exist.
This method will return True if the value is written, False otherwise.
(The value will not be written if the exclusive is True and the node
already exists.)
"""
partial_path = ''
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not(zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
except zookeeper.NodeExistsException:
pass
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return True
else:
flags = (ephemeral and zookeeper.EPHEMERAL or 0)
try:
zookeeper.create(self.handle, path, contents, [self.acl], flags)
return True
except zookeeper.NodeExistsException:
if not(exclusive):
# Woah, something happened between the top and here.
# We just restart and retry the whole routine.
self.write(path, contents, ephemeral=ephemeral)
return True
else:
return False
def __init__(self, semaphorename, hostname, port, initial_value = 0):
ZooKeeperBase.__init__(self, hostname, port)
self.semaphorename = "/" + semaphorename
try:
zookeeper.create(self.handle,self.semaphorename,"semaphore top level", [ZOO_OPEN_ACL_UNSAFE],0)
self.signal(initial_value)
print "Created new semaphore, OK"
except zookeeper.NodeExistsException:
print "Semaphore Already Exists"
def __init__(self,queuename, hostname, port, is_producer=False):
ZooKeeperBase.__init__(self, hostname, port)
self.queuename = "/" + queuename
if is_producer:
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
print "Created new Queue, OK"
except zookeeper.NodeExistsException:
print "Queue Already Exists"
def register_node(self, node_id, host, port=0):
"""Register a node to the cluster."""
path = self.MEMBERSHIP_NODE + "/" + str(node_id)
if port <= 0:
port = self.default_port
try:
data = host + ":" + str(port)
zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE], 0)
self.logger.info("Node %d: %s is registered" % (node_id, data))
except zookeeper.NodeExistsException:
self.logger.warn("%s already exists" % path)
def createRootNode(self, root):
#Create root node recursively from back to front to back
if root == '/': return
while True:
try:
root = root[:-1]
zookeeper.create(self.zh, root, '', [ZOO_OPEN_ACL_UNSAFE], 0)
return
except zookeeper.NodeExistsException:
return
except zookeeper.NoNodeException:
self.createRootNode(root.rsplit('/', 1)[0] + '/')
def create_zookeeper_without_delete_children(zk, path, data):
try:
if zookeeper.exists(zk, path):
return
zookeeper.create(zk, path, data, [{
"perms": 15,
"scheme": "world",
"id": "anyone"
}], 0)
except zookeeper.NodeExistsException, e:
err_str = "path %s exist\n" % path
handle_output(False, err_str)
def createRootNode(self, root):
#Create root node recursively from back to front to back
if root == '/': return
while True:
try:
root = root[:-1]
zookeeper.create(self.zh, root, '', [ZOO_OPEN_ACL_UNSAFE], 0)
return
except zookeeper.NodeExistsException:
return
except zookeeper.NoNodeException:
self.createRootNode(root.rsplit('/', 1)[0] + '/')
def _action_generator(self, parsed_args):
for zh, group in self.iterate_groups(parsed_args):
for key, first, last in self._list_elections(zh, group):
tail = str(1+int(last)).rjust(10, '0')
suffix = key + '-' + tail
path = group + '/' + suffix
try:
zookeeper.create(zh, path, parsed_args.value,
_acl_openbar, 0)
yield group, suffix, "OK"
except Exception as ex:
yield group, suffix, str(ex)
def test_empty_node(self):
"""
Test for a bug when instead of empty string we can get
random data from buffer malloc'ed to hold node contents.
See ZOOKEEPER-1906 for details
"""
NODE_PATH = "/zk-python-test-empty-node"
self.ensureDeleted(NODE_PATH)
zookeeper.create(self.handle, NODE_PATH, "",
[{"perms":0x1f, "scheme":"world", "id" :"anyone"}])
(data,stat) = zookeeper.get(self.handle, NODE_PATH, None)
self.assertEqual(data, "", "Data is not empty as expected: " + data)
def node_add():
basepath = request.args.get('path', '')
if request.method == 'POST':
path = request.form.get('path')
data = request.form.get('data')
print request.form
print request.values
verify = True
if len(path) == 0:
verify = False
flash('"Node Path" not allow empty')
elif path[0] == '/':
verify = False
flash('"Node Path" can not start with "/"')
else:
path = CONFIG_NODE / basepath / path
if zookeeper.exists(zk, path):
verify = False
flash('Path already exists:"%s"' % (path))
if verify:
nodepath = FilePath('/')
try:
for node in path.split('/'):
nodepath = nodepath / node
if not zookeeper.exists(zk, nodepath):
if nodepath != path:
zookeeper.create(zk, nodepath, '',
[{
"perms": zookeeper.PERM_ALL,
"scheme": 'world',
"id": 'anyone'
}], 0)
else:
zookeeper.create(zk, nodepath, data,
[{
"perms": zookeeper.PERM_ALL,
"scheme": 'world',
"id": 'anyone'
}], 0)
return redirect('/node')
except BaseException, e:
flash(e.message)
formdata = {
'path': request.form.get('path') if path != '' else '',
'data': data
}
def _write(self, path, contents, ephemeral, exclusive, sequential,
mustexist):
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
partial_path = ''
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not (zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl],
0)
except zookeeper.NodeExistsException:
pass
if sequential:
exists = False
else:
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# Check if we require the node to exist.
if not exists and mustexist:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return path
else:
flags = 0
if ephemeral:
flags = flags | zookeeper.EPHEMERAL
if sequential:
flags = flags | zookeeper.SEQUENCE
# NOTE: We return the final path created.
return zookeeper.create(self.handle, path, contents, [self.acl],
flags)
def before_request():
if not zookeeper.exists(zk, ROOT_NODE):
zookeeper.create(zk, ROOT_NODE, '', [{
"perms": zookeeper.PERM_ALL,
"scheme": "world",
"id": "anyone"
}], 0)
elif not zookeeper.exists(zk, CONFIG_NODE):
zookeeper.create(zk, CONFIG_NODE, '', [{
"perms": zookeeper.PERM_ALL,
"scheme": "world",
"id": "anyone"
}], 0)
def post(self):
request_dict = self.request.arguments
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
new_node = (request_dict['New_post_node'])[0]
new_value = (request_dict['new_node_value'])[0]
if zookeeper.exists(zk,new_node):
zookeeper.close(zk)
self.write("此节点存在")
else:
zookeeper.create(zk,new_node,new_value,[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)
zookeeper.close(zk)
self.write("增加成功")
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option(
'-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
help='Only play with small batches to avoid timeouts on slow hosts.')
parser.add_option(
'--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Do not populate entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
def _write(self, path, contents, ephemeral, exclusive, sequential, mustexist):
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
partial_path = ''
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not(zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
except zookeeper.NodeExistsException:
pass
if sequential:
exists = False
else:
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# Check if we require the node to exist.
if not exists and mustexist:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return path
else:
flags = 0
if ephemeral:
flags = flags | zookeeper.EPHEMERAL
if sequential:
flags = flags | zookeeper.SEQUENCE
# NOTE: We return the final path created.
return zookeeper.create(self.handle, path, contents, [self.acl], flags)
def post(self):
request_dict = self.request.arguments
node_json_list = (eval(request_dict['node_json'][0])).values()
cluster_name = (request_dict['cluster_name'])[0]
current_node = (request_dict['current_node'])[0]
if current_node == "/" :
current_node = ""
zk=zookeeper.init(self.zk_connect(cluster_name))
for item in node_json_list:
key = current_node + "/" + item[0]
value = item[1]
if not zookeeper.exists(zk, key):
zookeeper.create(zk, key , value , [{"perms":0x1f,"scheme":"world","id":"anyone"}],0)
self.write("写入成功")
zookeeper.close(zk)
def electLeader(self, handle, nodename, candidateName):
'''
This method elects the new leader from the list of candidates.
The candidate with the lowest sequence Id gets to be the leader.
'''
candidates = zookeeper.get_children(handle, nodename)
myName = candidateName.rsplit('/',1)[1]
leader = sorted(candidates)[0]
if ( myName == leader):
try:
zookeeper.create(handle, nodename+"/leader",myName,[ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL)
log.info(" I won the elections, I am leader now :)")
self.isLeader = True
except zookeeper.NodeExistsException,e:
log.info(e)
def _recursivelyCreatePath(self, path, intermediate_data=''):
'''
Should do a check to see if this dir already exists.
'''
current_path = ''
for part in path.split('/'):
if len(part) > 0:
try:
current_path += '/' + part
zookeeper.create(self._z,
current_path,
intermediate_data,
[ZOO_OPEN_ACL_UNSAFE])
except zookeeper.NodeExistsException:
pass
def __init__(self, name):
self.name = ROOT + '/' + name
self.cv = threading.Condition()
created = False
while not created:
try:
zookeeper.create(handle, self.name, "0", [ZOO_OPEN_ACL_UNSAFE], 0)
created=True
except zookeeper.NodeExistsException:
# S'ok.
created = True
except zookeeper.ConnectionLossException:
continue
self.znode = None
def test_sync_get_large_datanode(self):
"""
Test that we can retrieve datanode sizes up to
1Mb with default parameters (depends on ZooKeeper server).
"""
data = "".join(["A" for x in xrange(1024 * 1023)])
self.ensureDeleted("/zk-python-test-large-datanode")
zookeeper.create(
self.handle, "/zk-python-test-large-datanode", data, [{"perms": 0x1F, "scheme": "world", "id": "anyone"}]
)
(ret, stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode")
self.assertEqual(len(ret), 1024 * 1023, "Should have got 1Mb returned, instead got %s" % len(ret))
(ret, stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode", None, 500)
self.assertEqual(len(ret), 500, "Should have got 500 bytes returned, instead got %s" % len(ret))
def test_empty_node(self):
"""
Test for a bug when instead of empty string we can get
random data from buffer malloc'ed to hold node contents.
See ZOOKEEPER-1906 for details
"""
NODE_PATH = "/zk-python-test-empty-node"
self.ensureDeleted(NODE_PATH)
zookeeper.create(self.handle, NODE_PATH, "", [{
"perms": 0x1f,
"scheme": "world",
"id": "anyone"
}])
(data, stat) = zookeeper.get(self.handle, NODE_PATH, None)
self.assertEqual(data, "", "Data is not empty as expected: " + data)
def create_namespace_tree(zh, ns, logger, batch_size=2048, precheck=False):
if precheck and _probe(zh, ns, logger):
return 0, 0
# Synchronous creation of the root, helps detecting a lot of
# problems with the connection
try:
zookeeper.create(zh, _PREFIX, '', _acl_openbar, 0)
logger.info("Created %s", _PREFIX)
except zookeeper.NodeExistsException:
logger.info("Already %s", _PREFIX)
pass
nodes = generate_namespace_tree(ns, [t for t, _, _ in _srvtypes])
return _create_tree(zh, nodes, logger, int(batch_size))
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
help='Only play with small batches to avoid timeouts on slow hosts.')
parser.add_option(
'--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Do not populate entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
def testmanyhandles(self):
"""
Test the ability of the module to support many handles.
"""
# We'd like to do more, but currently the C client doesn't
# work with > 83 handles (fails to create a pipe) on MacOS 10.5.8
handles = [zookeeper.init(self.host) for i in xrange(63)]
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True,
"Connection timed out to " + self.host)
for i, h in enumerate(handles):
path = "/zkpython-test-handles-%s" % str(i)
self.assertEqual(
path,
zookeeper.create(h, path, "", [ZOO_OPEN_ACL_UNSAFE],
zookeeper.EPHEMERAL))
self.assertEqual(
True, all(zookeeper.close(h) == zookeeper.OK for h in handles))
def create(self, path, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]):
start = time.time()
result = zookeeper.create(self.handle, path, data, acl, flags)
if VERBOSE:
print("Node %s created in %d ms"
% (path, int((time.time() - start) * 1000)))
return result
def create(self, path, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]):
start = time.time()
result = zookeeper.create(self.handle, path, data, acl, flags)
if verbose:
print("Node %s created in %d ms" %
(path, int((time.time() - start) * 1000)))
return result
def test_async_delete(self):
ZOO_OPEN_ACL_UNSAFE = {
"perms": 0x1f,
"scheme": "world",
"id": "anyone"
}
self.assertEqual(self.connected, True)
ret = zookeeper.create(self.handle, "/zk-python-adeletetest",
"nodecontents", [ZOO_OPEN_ACL_UNSAFE],
zookeeper.EPHEMERAL)
self.assertEqual(ret, "/zk-python-adeletetest")
self.cv = threading.Condition()
self.callback_flag = False
self.rc = -1
def callback(handle, rc):
self.cv.acquire()
self.callback_flag = True
self.cv.notify()
self.rc = rc # don't assert this here, as if the assertion fails, the test will block
self.cv.release()
self.cv.acquire()
ret = zookeeper.adelete(self.handle, "/zk-python-adeletetest", -1,
callback)
self.assertEqual(ret, zookeeper.OK, "adelete failed")
while not self.callback_flag:
self.cv.wait(15)
self.cv.release()
self.assertEqual(self.callback_flag, True, "adelete timed out")
self.assertEqual(self.rc, zookeeper.OK)
