class ZooWrap():
def __init__(self):
self.zk = KazooClient(hosts="%s,%s" % (
str(cc.conf['zookeeper']['host']),
str(cc.conf['zookeeper']['port'])))
self.zk.start()
self.root = cc.conf['zookeeper']['rootpath']
self.log = logging.getLogger('L.ZOOKEEPER')
self.zk.ensure_path('/%s/sleeping' % (self.root))
self.whoami = cc.conf['whoami']
def get_sleeping(self):
return self.zk.get_children('/%s/sleeping' % (self.root))
def sleep(self):
try:
self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
self.log.info('Sleeping correctly')
except NodeExistsError:
self.log.error('Node already sleeping... seems weird')
def wake(self):
try:
self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
except NoNodeError:
self.log.error('Node was not sleeping... seems weird')
class Zookeeper(KeyManager):
def __init__(self, hosts):
self._hosts = hosts
self.zk = KazooClient(hosts=hosts)
self.zk.start()
def get(self, key):
result = self.zk.get(key)[0]
if result == "":
result = []
children = self.zk.get_children(key)
for i in children:
result.append({'name': i, 'value': self.zk.get(os.path.join(key, i))[0]})
return result
else:
return self.zk.get(key)[0]
def set(self, key, data):
try:
self.zk.set(key, data.encode('utf-8'))
except:
self.zk.create(key, data.encode('utf-8'))
def mkdir(self, key):
self.set(key, "")
def close(self):
self.zk.stop()
self.zk.close()
@property
def hosts(self):
return self._hosts
def add_and_update(key, value):
print(key, value, '#' * 20)
zk = KazooClient(hosts='127.0.0.1:2181')
zk.start()
# children = zk.get_children('/')
# 创建节点：makepath 设置为 True ，父节点不存在则创建，其他参数不填均为默认
# zk.create('/zookeeper/goodboy')
# 操作完后，别忘了关闭zk连接
# zk.stop()
print('#' * 20)
data, stat = zk.get(key)
print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
if zk.exists(key):
zk.set(key, value)
print('@' * 10, 'setting')
else:
zk.create(key, value, makepath=True)
# zk.create(key)
# print(value, type(value),'#'*10)
# if value != b"":
# zk.set(key, value)
zk.stop()
def zk_flag_set(zk: KazooClient, name: str, value: str):
path = "{}/{}".format(ZK_PREFIX, name)
data = value.encode('utf-8')
try:
zk.set(path, data)
except NoNodeError:
zk.create(path, data, makepath=True)
class KazooService():
zk = None
rootPath = None
def __init__(self, rootPath):
qs = Zookeeper.objects.filter().first()
if qs is None:
raise LookupError("Unable to find Zookeeper configuration!")
self.zk = KazooClient(hosts=qs.url)
self.rootPath = rootPath
self.zk.start(timeout=qs.timeout)
self.zk.ensure_path(rootPath)
def createOrUpdate(self, pathAffix, nodeValue):
if isinstance(nodeValue, str):
bValue = bytes(nodeValue, 'utf-8')
else:
bValue = bytes(str(nodeValue), 'utf-8')
path = self.rootPath + pathAffix
if self.zk.exists(path) is None:
self.zk.create(path, bValue)
else:
self.zk.set(path, bValue)
def __del__(self):
if self.zk is not None:
self.zk.stop()
class ServicePublisher:
def __init__(self, hosts, timeout, publish_port):
self._logger = logging.getLogger(self.__class__.__name__)
self._publish_port = publish_port
self._zk = KazooClient(hosts=hosts)
event = self._zk.start_async()
event.wait(timeout=timeout)
if self._zk.connected:
self._logger.info('Kazoo client successfully connected')
self._publish_status()
else:
self._zk.stop()
self._logger.error('Kazoo client failed to connect')
def _publish_status(self):
full_path = '%s/%s' % (PARENT_NODE, socket.gethostname())
data = {
'started': str(datetime.datetime.now())[:19],
'port': self._publish_port
}
json_data = json.dumps(data).encode(encoding='utf-8')
self._logger.info('Publishing status %s to path %s' %
(data, full_path))
self._zk.create(full_path, json_data, ephemeral=True, makepath=True)
class DistributedLock:
def __init__(self, hosts, logger=None):
self.client = KazooClient(hosts=hosts, logger=logger)
def ensure_path(self, path):
self.client.start()
self.client.ensure_path(path)
self.client.stop()
def try_lock(self, path):
try:
self.client.start()
self.client.create(path, ephemeral=True)
return True
except:
logging.exception("try lock exception. path: {}".format(path))
return False
def try_unlock(self, path):
try:
self.client.delete(path)
self.client.stop()
return True
except:
logging.exception("try unlock exception. path: {}".format(path))
return False
class Store(object):
def __init__(self,**kwargs):
self.config = kwargs
self.client = None
def get_client(self):
return self.client
def open(self):
self.client = KazooClient(**self.config)
self.client.add_listener
self.client.start()
def close(self):
self.client.stop()
def read(self,path):
return self.client.get(path)
def write(self,path,value):
base_path = os.path.dirname(path)
self.client.ensure_path(base_path)
self.client.create(path,value)
def overwrite(self,path,value):
self.client.set(path,value)
def exists(self,path):
return self.client.exists(path)
class ES_zk_interface:
def __init__(self, my_ip, es_interface, es_callback):
self.ip = my_ip
self.es_callback = es_callback
self.es_interface = es_interface
self.zk = KazooClient(hosts=HOST_IP)
self.zk.start()
self.zk.ensure_path("/ass3")
def start_election_process(self):
print "Starting Election process"
#print("Previous Leader: %s" % (self.es_callback.get_leader()))
election = self.zk.Election("/electionpath", self.ip)
election.run(self.leader_function)
def leader_function(self):
print "Won the election: " + str(self.ip)
if self.zk.exists(LEADER_PATH):
self.zk.set(LEADER_PATH, self.ip)
else:
self.zk.create(LEADER_PATH, self.ip)
#print "\nStarting Leader thread.... \n"
self.es_interface.enter_message_loop(self.es_callback, True)
def get_zookeeper_client(self):
return self.zk
def get_leader_path(self):
return LEADER_PATH
def test_pub_response(pub_node, server_list, host='127.0.0.1', port=2181):
zookeeper = KazooClient('%s:%s' % (
host,
port,
))
zookeeper.start()
for s in server_list:
if isinstance(s, int) is True:
continue
try:
node = '/test/to_pub_result/%s/%s' % (
pub_node,
s,
)
if zookeeper.exists(node) is None:
zookeeper.create(node,
json.dumps({
'update_time': time.time(),
'status': 'ok'
}),
makepath=True)
except kazoo.exceptions.NodeExistsError:
pass
def connect(hosts="127.0.0.1:2181"):
retval = KazooClient(hosts=hosts)
def my_listener(state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
logging.warn("Lost connection to Zookeeper")
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
logging.warn("Zookeeper connection suspended")
elif state == KazooState.CONNECTED:
pass
else:
# Handle being connected/reconnected to Zookeeper
logging.info("Other state: %s" % state)
retval.add_listener(my_listener)
retval.start()
if not retval.exists(key):
logging.info("Created '%s'" % key)
retval.create(key)
return (retval)
class ZKBase(object):
"""
scheduler需要定义以下函数：
- callback(KazooState.*)： 处理CONN: LOST,SUSPENDED
-
"""
def __init__(self,conf,scheduler):
self.conf = conf
self.state = "STOPPED"
self.zk = KazooClient(self.conf.processor.zk_url, timeout=self.conf.processor.timeout)
try:
self.zk.start()
except Exception:
LOG.error("Start zk error: %s" % traceback.format_exc(limit=2))
self.zk.add_listener(self.listen)
self.scheduler = scheduler
self.state = "RUNNING" # STOPPED
self.state_changes = False
def listen(self, state):
if state == KazooState.LOST or state == KazooState.SUSPENDED:
LOG.error("Session state change: %s" % state)
if self.state == "STOPPED":
return
self.state_changes = True
self.scheduler.callback(state)
elif state == KazooState.CONNECTED:
LOG.info("Connectied to ZK.")
else:
LOG.error("Session timeout. Cannot conect to ZK.")
def reinit(self,state):
while self.zk.state != KazooState.CONNECTED:
LOG.info("Restart zk connection until connected.")
try:
self.zk.restart()
except Exception:
LOG.error("Reinit: %s" % traceback.format_exc(limit=1))
self.state_changes = False
def create(self,path, value='', acl=None, ephemeral=False, sequence=False, makepath=False):
LOG.info("Create: path %s" % path)
self.zk.create(path=path,
value=value.encode(),
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
def get(self,path,watch=None):
self.zk.get(path,watch)
def get_children(self,path,watch=None):
return self.zk.get_children(path,watch=watch)
def terminate(self):
LOG.info("Terminate ZK connection.")
self.state = "STOPPED"
self.zk.stop()
self.zk.close()
class ZKState():
def __init__(self, path, timeout=30):
super(ZKState, self).__init__()
self._zk = KazooClient(hosts=ZK_HOSTS, timeout=timeout)
self._zk.start(timeout=timeout)
self._path = path
self._zk.ensure_path(path)
def processed(self):
return self._zk.exists(self._path + "/complete")
def process_start(self):
if self.processed():
return False
if self._zk.exists(self._path + "/processing"):
return False
try:
self._zk.create(self._path + "/processing", ephemeral=True)
return True
except NodeExistsError: # another process wins
return False
def process_end(self):
self._zk.create(self._path + "/complete")
self._zk.delete(self._path + "/processing")
def process_abort(self):
try:
self._zk.delete(self._path + "/processing")
except NoNodeError:
pass
def close(self):
self._zk.stop()
self._zk.close()
def main():
logging.basicConfig()
zk = KazooClient(hosts=zookeeper_hostname)
zk.start()
zk.ensure_path("/worker")
node_name = "/worker/" + os.environ["NODE_NAME"]
if not zk.exists(node_name):
msg = "Creating node: " + node_name
print(msg, file=sys.stdout)
db_name = os.environ["DB_HOSTNAME"]
zk.create(node_name, db_name.encode(), ephemeral=True)
if os.environ["WORKER_TYPE"] == "master":
rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True)
rpc_server.start()
else:
try:
if node_name != "/worker/slave1":
master_db = zk.get("/worker/master")[0].decode()
print("[*] Cloning database from master db: " + master_db, file=sys.stdout)
subprocess.call(
"mongodump --host " + master_db + " --port 27017 --db rideshare && mongorestore --host " + os.environ[
'DB_HOSTNAME'] + " --port 27017",
stdout=sys.stdout,
stderr=sys.stdout,
shell=True
)
except Exception as e:
print(e, file=sys.stdout)
old_name = os.environ["NODE_NAME"]
p1 = multiprocessing.Process(target=slave_rpc_server)
p2 = multiprocessing.Process(target=become_master, args=(p1, old_name,))
p1.start()
p2.start()
def registry(self, port, service, methods, zks):
if zks is None:
raise Exception("zk 地址为空")
providers_path = "/dubbo/%s/providers" % (service)
ip = self.local_host_ip()
provider_url = self.generate_provider_url(host=ip,
service=service,
port=port,
methods=methods)
self.service_path[service] = "%s/%s" % (providers_path,
quote_plus(provider_url))
value = str(ip).encode("utf8")
zk = KazooClient(hosts=zks)
self.registry_zk[service] = zk
zk.start()
#zk.ensure_path(providers_path+"/")
try:
zk.delete(self.service_path[service])
except Exception as e:
pass
finally:
zk.create(self.service_path[service],
value,
ephemeral=True,
sequence=False,
makepath=True)
def zk_init(server_address, zk_node, zk_address = ''):
#初始化zk节点
zk_cli = KazooClient(hosts = zk_address)
try:
zk_cli.start()
except:
logging.error('zk Init error, can not connect %s' %(str(zk_address)))
return -1
try:
zk_cli.get(zk_node)
except:
logging.warn('can not find zk path %s, creat it' %(str(zk_node)))
zk_cli.ensure_path(zk_node)
if zk_node[-1] != '/':
zk_node += '/'
try:
zk_cli.create(zk_node + server_address, '1', ephemeral=True)
except:
if zk_cli.get(zk_node + server_address):
return 0
else:
logging.error('create zk_node error, can not create node %s' %(str(zk_node) + str(zk_address)))
return -1
return 0
class zkClient():
def __init__(self):
self.zk = KazooClient(hosts='10.0.0.2:2181')
self.zk.start()
def get_hostname_ip(self):
host_name = socket.getfqdn(socket.gethostname())
host_ip = socket.gethostbyname(host_name)
return "/" + host_name + "_gateway", host_ip
def register(self):
name, ip = self.get_hostname_ip()
if not self.zk.exists(name):
self.zk.create(name, bytes(ip, 'utf-8'))
else:
self.zk.set(name, bytes(ip, 'utf-8'))
def get_kvs_ips(self):
device_ips = []
for device_name in self.zk.get_children('/'):
if device_name.endswith('_kvs'):
data, stat = self.zk.get(device_name)
device_ips.append(data.decode("utf-8"))
return device_ips
def get_nodes_ips(self, device_list):
device_ips = []
for device_name in self.zk.get_children('/'):
if device_name in device_list:
data, stat = self.zk.get(device_name)
device_ips.append(data.decode("utf-8"))
return device_ips
class ZookeeperClient(object):
def __init__(self, server_list):
self._retry = KazooRetry(max_tries=None,
max_delay=300,
sleep_func=gevent.sleep)
self._zk_client = KazooClient(hosts=','.join(server_list),
timeout=400,
handler=SequentialGeventHandler(),
logger=logger,
connection_retry=self._retry,
command_retry=self._retry)
def connect(self):
self._zk_client.start()
def disconnect(self):
self._zk_client.stop()
self._zk_client.close()
def create_node(self, path, value=None):
if value is None:
value = uuid.uuid4()
try:
self._zk_client.create(path, str(value), makepath=True)
except NodeExistsError:
self._zk_client.set(path, str(value))
class ZooWrap():
def __init__(self):
self.zk = KazooClient(hosts="%s,%s" %
(str(cc.conf['zookeeper']['host']),
str(cc.conf['zookeeper']['port'])))
self.zk.start()
self.root = cc.conf['zookeeper']['rootpath']
self.log = logging.getLogger('L.ZOOKEEPER')
self.zk.ensure_path('/%s/sleeping' % (self.root))
self.whoami = cc.conf['whoami']
def get_sleeping(self):
return self.zk.get_children('/%s/sleeping' % (self.root))
def sleep(self):
try:
self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
self.log.info('Sleeping correctly')
except NodeExistsError:
self.log.error('Node already sleeping... seems weird')
def wake(self):
try:
self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
except NoNodeError:
self.log.error('Node was not sleeping... seems weird')
class ZKMData(object):
def __init__(self):
super(ZKMData, self).__init__()
self._zk = KazooClient(hosts=ZK_HOSTS)
self._zk.start()
def set(self, path, value):
value = json.dumps(value).encode('utf-8')
if self._zk.exists(path):
try:
self._zk.set(path, value)
except NoNodeError:
pass
try:
self._zk.create(path, value, makepath=True)
except NodeExistsError:
pass
def get(self, path):
try:
value, stat = self._zk.get(path)
if not value: return {}
return json.loads(value.decode('utf-8'))
except Exception as e:
return {}
def close(self):
self._zk.stop()
self._zk.close()
class Register(object):
def __init__(self,
value,
path='/scrapyd-cluster/worker',
hosts='127.0.0.1:2181'):
self.path = path
self.value = value
self.expire = False
self.zk = KazooClient(hosts)
self.zk.start()
self.zk.add_listener(self.listener)
self.zk.ensure_path(os.path.dirname(self.path))
self.path = self.zk.create(self.path,
self.value,
ephemeral=True,
sequence=True)
logger.info('Connected To ZooKeeper(%s) Succeed! Path : %s' %
(hosts, self.path))
def register(self):
if self.expire:
logger.info('Renew Ephemeral Node. Path : %s Value: %s' %
(self.path, self.value))
self.zk.create(self.path, self.value, ephemeral=True)
self.expire = False
def listener(self, state):
if state == KazooState.LOST:
logger.info('ZooKeeper Connection Lost')
self.expire = True
elif state == KazooState.CONNECTED:
self.zk.handler.spawn(self.register)
def test_zookeeper(zookeeper_server):
test_value = str(uuid.uuid4()).encode("utf-8")
zk_client1 = KazooClient(
hosts="127.0.0.1:{}".format(zookeeper_server.port))
zk_client2 = KazooClient(
hosts="127.0.0.1:{}".format(zookeeper_server.port))
zk_client1.start()
zk_client2.start()
zk_client1.ensure_path("/my")
assert zk_client1.exists("/you") is None
assert zk_client2.exists("/you") is None
assert zk_client1.exists("/my") is not None
assert zk_client2.exists("/my") is not None
zk_client1.create("/my/testplan", test_value)
data, _ = zk_client1.get("/my/testplan")
assert data == test_value
data, _ = zk_client2.get("/my/testplan")
assert data == test_value
zk_client2.set("/my/testplan", str(uuid.uuid4()).encode("utf-8"))
data, _ = zk_client1.get("/my/testplan")
assert data != test_value
data, _ = zk_client2.get("/my/testplan")
assert data != test_value
class ZKStore:
def __init__(self, hosts):
self.zk = KazooClient(hosts=hosts)
self.zk.add_listener(listener)
self.zk.start()
def isConnected(self):
if __state__ == 1:
return True
return False
def write(self, path, node, value):
self.zk.ensure_path(path)
if self.zk.exists(path+"/"+node):
self.zk.set(path+"/"+node, value)
else:
self.zk.create(path + "/" + node, value)
def read(self, path):
if self.zk.exists(path):
data, stat = self.zk.get(path)
return data
return None
def add_node_to_zookeeper():
zk = KazooClient(hosts=zookeeper_hosts)
@zk.add_listener
def zookeeper_listener(state):
if state == KazooState.LOST:
logging.info("zookeeper connect status:LOST")
# Register somewhere that the session was lost
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
logging.info("zookeeper connect status:SUSPENDED")
elif state == KazooState.CONNECTED:
logging.info("zookeeper connect status:CONNECTED")
else:
logging.info(str(state))
zk.start()
# 创建“/spiderServer”永久节点，后面每个有效的redis节点，或动态添加redis节点后，都会在
# “/spiderServer”节点下创建对应的redis服务节点（如：/spiderServer/128.0.0.203），服务节点名称已redis服务所在的机器的ip地址命名
if zk.exists(SPIDER_SERVER_NODE):
pass
else:
try:
zk.create(SPIDER_SERVER_NODE)
except Exception, e:
if isinstance(e, NodeExistsException):
pass
else:
logging.exception(str(e))
def test_init_servers(num, host='127.0.0.1', port=2181):
zookeeper = KazooClient('%s:%s' % (
host,
port,
))
zookeeper.start()
try:
node = '/test/server_list'
if zookeeper.exists(node) is None:
zookeeper.create(node,
json.dumps({'update_time': time.time()}),
makepath=True)
except kazoo.exceptions.NodeExistsError:
pass
v = 1
while v <= num:
try:
node = '/test/server_list/s%s' % v
if zookeeper.exists(node) is None:
zookeeper.create(node,
json.dumps({
'update_time': time.time(),
'server_name': 's%s' % v,
'server_id': v,
}),
makepath=True)
except kazoo.exceptions.NodeExistsError:
pass
v += 1
class ZookClient():
def __init__(self, hosts):
self.kazooclient = KazooClient(hosts=hosts)
self.kazooclient.start()
def foundApp(self, appname): #Ìí¼Óapp
if self.kazooclient.exists('/' + appname):
pass
else:
self.kazooclient.ensure_path('/' + appname)
def foundNode(self, appname, port, host): #Ìí¼Ó½Úµã
if self.kazooclient.exists('/' + appname + '/' + host + ':' +
str(port)):
pass
else:
self.kazooclient.create('/' + appname + '/' + host + ':' +
str(port))
def close(self):
self.kazooclient.close()
def register(self, appname, port, host):
self.foundApp(appname)
self.foundNode(appname, port, host)
# self.close()
def test_subscriber_connect(self):
test_zk = KazooClient(self.zk_sub.zookeeper_connection_url)
try:
test_zk.start(5)
except Exception:
self.assertRaises("Couldn't connect to zookeeper server")
# Ensure a path, create if necessary
test_zk.ensure_path(self.zk_sub.default_node_path)
fake_host_ip_lst = ["0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.3"]
try:
for host_ip in fake_host_ip_lst:
# Create a node with data
node = "node" + str(uuid.uuid4())
path = self.default_node_path + '/' + node
test_zk.create(path, bytes(host_ip, "utf-8"))
self.zk_sub.subscriber_connect("subscriber")
finally:
nodes_to_delete = test_zk.get_children(self.default_node_path)
for node in nodes_to_delete:
path = self.default_node_path + '/' + node
test_zk.delete(path)
children = test_zk.get_children(self.default_node_path)
self.assertEqual(children, [])
class ZookeeperClient(object):
def __init__(self, server_list):
self._retry = KazooRetry(max_tries=None, max_delay=300,
sleep_func=gevent.sleep)
self._zk_client = KazooClient(
hosts=','.join(server_list),
timeout=400,
handler=SequentialGeventHandler(),
logger=logger,
connection_retry=self._retry,
command_retry=self._retry)
def connect(self):
self._zk_client.start()
def disconnect(self):
self._zk_client.stop()
self._zk_client.close()
def create_node(self, path, value=None):
if value is None:
value = uuid.uuid4()
try:
self._zk_client.create(path, str(value), makepath=True)
except NodeExistsError:
self._zk_client.set(path, str(value))
def init_zk(namespace=''):
"""Initialize Zookeeper Instance."""
hosts = os.getenv('ZOOKEEPER_CONN_STRING', 'localhost:2181')
timeout = 600
timeout_treshold = time() + timeout
while True:
try:
# check if namespace exists
zk = KazooClient(hosts,
read_only=False)
zk.start()
if zk.exists(namespace) is None:
zk.create(namespace)
zk.stop
zk.close
zk = KazooClient(hosts + namespace,
read_only=False)
zk.start()
if type(zk) is not KazooClient:
logging.error("can't connect to Zookeeper ...")
exit(1)
return zk
except KazooTimeoutError:
if time() > timeout_treshold:
logging.error("can't connect to Zookeeper ...")
exit(1)
else:
logging.warning("can't connect to Zookeeper, " +
"trying again since timeout " +
"is not reached yet ...")
class ValidatorDetector:
def __init__(self):
self.zk = KazooClient(
hosts="bigdata1:2181,bigdata2:2181,bigdata3:2181")
self.validator_children_watcher = ChildrenWatch(client=self.zk,
path="/servers",
func=self.watcher_func)
self.zk.start()
def watcher_func(self, children):
print("----------------------start-----------------------------")
for c in children:
data, stat = self.zk.get('/servers/' + c)
if stat:
print("Children is:" + c + "\n" + "Data is:",
str(data, encoding='utf-8'))
print("-----------------------end----------------------------")
print("\n\n")
def create_node(self, hostname):
self.zk.create('/servers/server',
bytes(hostname, encoding='utf-8'),
ephemeral=True,
sequence=True,
makepath=True)
def close_client(self, hostname):
self.zk.close()
class ZKData(object):
def __init__(self):
super(ZKData,self).__init__()
options={"max_tries":-1,"max_delay":5,"ignore_expire":True}
self._zk=KazooClient(hosts=ZK_HOSTS,connection_retry=options)
try:
self._zk.start(timeout=3600)
except:
print(traceback.format_exc(), flush=True)
def set(self, path, value):
value=json.dumps(value).encode('utf-8')
try:
self._zk.create(path, value, makepath=True)
except NodeExistsError:
self._zk.set(path,value)
def get(self, path):
try:
value, stat= self._zk.get(path)
if not value: return {}
return json.loads(value.decode('utf-8'))
except Exception as e:
return {}
def close(self):
self._zk.stop()
self._zk.close()
class IDGenerator(object):
'''
ID生成器
'''
# 控制输出信息的锁
print_mutex = threading.Lock()
DELAY_TIME = 3
def __init__(self, zk_server_address, id_base_path):
self.id_base_path = id_base_path
# 创建一个zookeeper客户端
self.zkclient = KazooClient(zk_server_address)
# 添加连接状态监听器
self.zkclient.add_listener(self.zk_connect_listener)
# 与zookeeper开启连接
self.zkclient.start()
# 连接状态监听器
def zk_connect_listener(self, state):
# 获取打印锁
IDGenerator.print_mutex.acquire()
if state == KeeperState.CONNECTED:
print "命名服务器已经开启..."
elif state == KazooState.LOST:
print "命名服务器停止服务..."
else:
raise Exception("命名服务器未正常开启...")
# 获取打印锁
IDGenerator.print_mutex.release()
def start(self):
# 创建父节点
self.create_name_node()
def create_name_node(self):
# 先检查父节点，如果父节点不存在
if not self.zkclient.exists(self.id_base_path):
# 先创建父节点
self.zkclient.create(self.id_base_path)
# 生成id
def generate_id(self):
# 拼凑出服务器子节点的完整路径
node_path = self.id_base_path + "/"
# 创建临时顺序节点
self.node_path = self.zkclient.create(node_path, "",
self.zkclient.default_acl, True,
True)
# 准备删除创建出来的节点
self.zkclient.delete(self.node_path)
return self.node_path.split("/")[-1]
# 停止服务
def stop(self):
self.zkclient.stop()
self.zkclient.close()
class ZKClient:
def __init__(self, zk_url, zk_timeout=5):
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
self.logger = logging.getLogger()
try:
self.client = KazooClient(hosts=zk_url)
self.client.start(timeout=zk_timeout)
except handlers.threading.KazooTimeoutError:
self.logger.error("5 seconds connection timeout")
raise
def store_value(self, value, path):
fmt_value = str(value).encode('utf-8')
self.client.ensure_path(path)
self.client.set(path, fmt_value)
self.client.stop()
def retrieve_value(self, path):
self.client.ensure_path(path)
data, stat = self.client.get(path)
self.client.stop()
if not data:
raise ValueError
else:
return int(data.decode('utf-8'))
def register_server(self, path):
self.client.create(path, ephemeral=True, makepath=True)
def execute(self, request):
zk = KazooClient(hosts=request['zk_url'])
try:
zk.start()
path_ = request['node_path']
value_ = request['node_value'].encode('utf-8')
is_update_timestamp_ = request.get('is_update_timestamp', False)
is_delete_node_ = request.get('is_delete_node', None)
if zk.exists(path_):
if is_delete_node_:
zk.delete(path_, recursive=True)
return self.resp_dict, ''
if is_update_timestamp_:
zk.set(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes())
if value_:
zk.set(path_, value_)
else:
data_, stat_ = zk.get(path_)
if data_:
self.resp_dict['result_value'] = data_.decode()
elif value_:
# 新加节点时也需要添加 '/TIMESTAMP' 节点
zk.create(path_, value_, makepath=True)
zk.create(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes(), makepath=True)
return self.resp_dict, ''
except Exception as e:
logger.exception('zkTestPoint execute error, cause: %s', str(e))
raise Exception('zkTestPoint execute error, cause: ' + str(e))
finally:
zk.stop()
def become_master(slave_process, old_name):
s = socket.socket()
s.bind(("", 23456))
print("[*] Listening for command from orchestrator to become master ...", file=sys.stdout)
s.listen(2)
c, address = s.accept()
print("Received command from orchestrator to become master: " + c.recv(1024).decode(), file=sys.stdout)
slave_process.terminate()
os.environ["WORKER_TYPE"] = "master"
os.environ["NODE_NAME"] = "master"
client = docker.DockerClient(base_url="tcp://172.17.0.1:4444")
cnt = client.containers.get(old_name)
cnt.rename("master")
logging.basicConfig()
zk = KazooClient(hosts=zookeeper_hostname)
zk.start()
node_name = "/worker/" + os.environ["NODE_NAME"]
if not zk.exists(node_name):
msg = "Creating node: " + node_name
print(msg, file=sys.stdout)
db_name = os.environ["DB_HOSTNAME"]
zk.create(node_name, db_name.encode(), ephemeral=True)
time.sleep(3)
zk.delete("/worker/" + old_name)
rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True)
rpc_server.start()
def do_zookeeper_create(addr, path, value): print(path) zk = KazooClient(addr) zk.start() zk.create(path, bytes(value, 'utf-8')) do_zookeeper_read(addr, path)
class ZooKeeper(object):
def __init__(self, hosts="127.0.0.1:2181", prefix="/"):
prefix = prefix.rstrip("/")
if prefix and not prefix.startswith("/"):
raise ValueError("prefix must start with /")
self._prefix = prefix
self._zk = KazooClient(hosts=hosts, handler=_get_zk_handler())
self._zk.start()
def __del__(self):
self.close()
def _path(self, path):
if self._prefix:
return "{0}/{1}".format(self._prefix, path)
return path
def close(self):
if not self._zk:
return
try:
self._zk.stop()
self._zk.close()
except Exception:
pass
finally:
self._zk = None
def create(self, path, value="", makepath=True):
self._zk.create(self._path(path), value, makepath=makepath)
def delete(self, path, recursive=True):
self._zk.delete(self._path(path), recursive=recursive)
def set(self, path, value, makepath=True):
path = self._path(path)
try:
self._zk.set(path, value)
except zkexc.NoNodeError:
if not makepath:
raise
self._zk.create(path, value, makepath=True)
def get(self, path, none=True):
try:
return self._zk.get(self._path(path))
except zkexc.NoNodeError:
if none:
return None
raise
def ls(self, path, none=True):
try:
return self._zk.get_children(self._path(path))
except zkexc.NoNodeError:
if none:
return None
raise
class ShellTestCase(unittest.TestCase):
""" base class for all tests """
def setUp(self):
"""
make sure that the prefix dir is empty
"""
self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests")
self.zk_host = os.getenv("ZKSHELL_ZK_HOST", "localhost:2181")
self.username = os.getenv("ZKSHELL_USER", "user")
self.password = os.getenv("ZKSHELL_PASSWD", "user")
self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=")
self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "secret")
self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest")
self.client = KazooClient(self.zk_host, 5)
self.client.start()
self.client.add_auth(self.scheme, self.auth_id)
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.create(self.tests_path, str.encode(""))
self.output = StringIO()
self.shell = Shell([self.zk_host], 5, self.output, setup_readline=False, async=False)
# Create an empty test dir (needed for some tests)
self.temp_dir = tempfile.mkdtemp()
@property
def auth_id(self):
return "%s:%s" % (self.username, self.password)
@property
def auth_digest(self):
return "%s:%s" % (self.username, self.digested_password)
def tearDown(self):
self.output = None
self.shell = None
if os.path.isdir(self.temp_dir):
shutil.rmtree(self.temp_dir)
if self.client.exists(self.tests_path):
self.client.delete(self.tests_path, recursive=True)
self.client.stop()
###
# Helpers.
##
def create_compressed(self, path, value):
"""
ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client
to create a znode with zlib compressed content.
"""
compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value)
self.client.create(path, compressed, makepath=True)
def zookeeper_ephemeral_node(name):
zk = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2181'))
zk.start()
zk.create("/my/zookeeper_test/node1", b"process1 running", ephemeral=True)
#At 10 validate that ephemeral node exist that is the process is still running
time.sleep(10)
assert zk.exists("/my/zookeeper_test/node1"), "process node is not found at 10 s when it is still running"
time.sleep(20)
zk.stop()
class ZookeeperContext:
def __init__(self, zookeeper_url, test_id, node_id):
self.assertNotNone(zookeeper_url)
self.assertNotNone(test_id)
self.assertNotNone(node_id)
self.zookeeper_url = zookeeper_url
self.test_id = test_id
self.node_id = node_id
self.zk_path = '/bolt/testing/%s/%s' % (self.test_id, self.node_id)
logger.info("Initialized decorator: %s " % self.__str__())
self.__connect_zookeeper()
def assertNotNone(self,thing):
if thing == None:
logger.fatal("Invalid key in initializer")
def __connect_zookeeper(self):
try:
logger.info('Attempting to connect to: %s' % self.zookeeper_url)
self.zk = KazooClient(hosts=self.zookeeper_url)
self.zk.start(timeout=10)
logger.info('initialized zookeeper')
except Exception as exception:
logger.error('Failed to connect to zk')
logger.fatal(exception)
def __str__(self):
return json.dumps(self, default=lambda o: o.__dict__, indent=4)
def __del__(self):
self.zk.stop()
def publish(self, key, data):
self.assertNotNone(key)
self.assertNotNone(data)
path = self.zk_path + "/" + key
logger.info("Creating data in: %s" % path)
try:
bytes = pickle.dumps(data);
if not self.zk.exists(path):
logger.info("Path does not exist in zk, creating... %s" % path)
self.zk.create(path, value=bytes, makepath=True)
else:
self.zk.set(path, value=bytes)
except Exception as exception:
logger.error('Error setting data in ' % path)
logger.fatal(exception)
def main():
'''
A manual configuration file pusher for the crawlers. This will update
Zookeeper with the contents of the file specified in the args.
'''
import argparse
from kazoo.client import KazooClient
parser = argparse.ArgumentParser(
description="Crawler config file pusher to Zookeeper")
parser.add_argument('-f', '--file', action='store', required=True,
help="The yaml file to use")
parser.add_argument('-i', '--id', action='store', default="all",
help="The crawler id to use in zookeeper")
parser.add_argument('-p', '--path', action='store',
default="/scrapy-cluster/crawler/",
help="The zookeeper path to use")
parser.add_argument('-w', '--wipe', action='store_const', const=True,
help="Remove the current config")
parser.add_argument('-z', '--zoo-keeper', action='store', required=True,
help="The Zookeeper connection <host>:<port>")
args = vars(parser.parse_args())
filename = args['file']
id = args['id']
wipe = args['wipe']
zoo = args['zoo_keeper']
path = args['path']
zk = KazooClient(hosts=zoo)
zk.start()
# ensure path exists
zk.ensure_path(path)
bytes = open(filename, 'rb').read()
if zk.exists(path):
# push the conf file
if not zk.exists(path + id) and not wipe:
print "creaing conf node"
zk.create(path + id, bytes)
elif not wipe:
print "updating conf file"
zk.set(path + id, bytes)
if wipe:
zk.set(path + id, None)
zk.stop()
def put_topology(topology, config=config):
zk = KazooClient(**config)
zk.start()
for service, links in topology.iteritems():
if zk.exists("/sdn/services/{name}".format(name=service)):
zk.set("/sdn/services/{name}".format(name=service), json.dumps(links))
else:
zk.ensure_path("/sdn/services")
zk.create("/sdn/services/{name}".format(name=service), json.dumps(links))
ret_check = zk.get_children("/sdn/services")
zk.stop()
return ret_check
def test_create_makepath_incompatible_acls(self):
from kazoo.client import KazooClient
from kazoo.security import make_digest_acl_credential, CREATOR_ALL_ACL
credential = make_digest_acl_credential("username", "password")
alt_client = KazooClient(self.cluster[0].address + self.client.chroot,
max_retries=5, auth_data=[("digest", credential)])
alt_client.start()
alt_client.create("/1/2", b"val2", makepath=True, acl=CREATOR_ALL_ACL)
try:
self.assertRaises(NoAuthError, self.client.create, "/1/2/3/4/5",
b"val2", makepath=True)
finally:
alt_client.delete('/', recursive=True)
alt_client.stop()
class HHFrontierWorker(FrontierWorker):
def __init__(self, settings, no_batches, no_scoring, no_incoming):
super(HHFrontierWorker, self).__init__(settings, no_batches, no_scoring, no_incoming)
self.init_zookeeper()
def init_zookeeper(self):
self._zk = KazooClient(hosts=settings.get('ZOOKEEPER_LOCATION'))
self._zk.add_listener(self.zookeeper_listener)
self._zk.start()
self.znode_path = self._zk.create("/frontera/hh-f-worker", ephemeral=True, sequence=True, makepath=True)
def zookeeper_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
pass
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
pass
else:
# Handle being connected/reconnected to Zookeeper
pass
def set_process_info(self, process_info):
self.process_info = process_info
self._zk.set(self.znode_path, self.process_info)
def set_job_id(self, job_id):
self._backend.set_job_id(job_id)
self.job_id = job_id
class ZookeeperSession(object):
def __init__(self, locations, name_prefix, root_prefix='/frontera'):
self._zk = KazooClient(hosts=locations)
self._zk.add_listener(self.zookeeper_listener)
self._zk.start()
self.root_prefix = root_prefix
self.znode_path = self._zk.create("%s/%s" % (self.root_prefix, name_prefix),
ephemeral=True,
sequence=True,
makepath=True)
def zookeeper_listener(self, state):
if state == KazooState.LOST:
# Register somewhere that the session was lost
pass
elif state == KazooState.SUSPENDED:
# Handle being disconnected from Zookeeper
pass
else:
# Handle being connected/reconnected to Zookeeper
pass
def set(self, value):
self._zk.set(self.znode_path, value)
def get_workers(self, prefix='', exclude_prefix=''):
for znode_name in self._zk.get_children(self.root_prefix):
if prefix and not znode_name.startswith(prefix):
continue
if exclude_prefix and znode_name.startswith(exclude_prefix):
continue
location, _ = self._zk.get(self.root_prefix+"/"+znode_name)
yield location
def _receive(self):
server, port, buf_que = self._createThriftServer()
kazoo_client = KazooClient(self.zk_address)
kazoo_client.start()
path = '%s/%s:%d' % (self.zk_path, socket.gethostname(), port)
kazoo_client.create(path, ephemeral=True, makepath=True)
while not server._stop:
try:
message = buf_que.pop()
yield message
except:
time.sleep(0.1)
server.close()
kazoo_client.close()
def model_environments(ip_address, file_path):
zk = KazooClient(hosts=ip_address)
zk.start()
for root, dirs, files in os.walk(file_path):
node_path = root.replace(os.path.dirname(file_path), "") # trim root from path, except parent
try:
zk.create(node_path, makepath=True) # make parent directories as needed
except NodeExistsException:
print "%s already exists" % node_path
if len(files) > 0:
file = os.path.join(root, files[0])
__load_file_into_znode(zk, file, node_path)
zk.stop()
def test_pub_response(pub_node, server_list, host = '127.0.0.1', port = 2181):
zookeeper = KazooClient('%s:%s' % (host, port,))
zookeeper.start()
for s in server_list:
if isinstance(s, int) is True:
continue
try:
node = '/test/to_pub_result/%s/%s' % (pub_node, s, )
if zookeeper.exists(node) is None:
zookeeper.create(node, json.dumps({
'update_time' : time.time(),
'status' : 'ok'
}), makepath = True)
except kazoo.exceptions.NodeExistsError:
pass
def join(server, environment, name, on_parent_data_change = None):
zk = KazooClient(hosts=server)
zk.start() # connect to zookeeper
# create an ephemeral znode under the given environment,
# if the given name ends in '-' make it sequential.
zk.create( path = '{0}/{1}'.format(environment, name),
ephemeral = True,
sequence = (name[-1:] == '-'))
# if a callback was provided trigger it whenever parents data changes...
# for example to update variables on config changes.
if on_parent_data_change:
zk.DataWatch(environment, func = on_parent_data_change)
#hand zookeeper client back to caller
return zk
def zkset():
file = "/tmp/redis_config.properties"
zk = KazooClient(hosts='172.29.22.15:2181')
zk.start()
f=open(file,'rb')
m=f.read()
#Ensure a path , Create if necessary.
zk.ensure_path("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes")
#Create a node with data.
try:
zk.create("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
except kazoo.exceptions.NodeExistsError:
zk.set("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
zk.stop()
class ZookeeperClient(object):
def __init__(self, zk_hosts):
self.zk = KazooClient(hosts=zk_hosts)
self.zk.start()
def put(self, entity, id, data):
path = '/{}/{}'.format(entity, id)
binary_value = json.dumps(data).encode('utf-8')
try:
self.zk.create(path, binary_value, makepath=True)
return True
except NodeExistsError:
self.zk.set(path, binary_value)
return False
def get(self, entity, id):
path = '/{}/{}'.format(entity, id)
try:
binary_data, _ = self.zk.get(path)
except NoNodeError:
return None
return json.loads(binary_data.decode('utf-8'))
def delete(self, entity, id):
path = '/{}/{}'.format(entity, id)
try:
self.zk.delete(path)
return True
except NoNodeError:
return False
def list(self, entity):
path = '/{}'.format(entity)
try:
children = self.zk.get_children(path)
for child in children:
value = self.get(entity, child)
yield (child, value)
except NoNodeError:
yield []
class MasterElection(object):
def __init__(self):
self.zk = KazooClient(hosts=state.ARGS.zookeeper)
self.master_notified = False
self.my_node = None
self.zk.start() # Stop never called
self.zk.ensure_path(ELECTION_PATH)
def start_election(self, master_callback):
"""
Start the master election.
If this node is the master, the callback will be called once.
:params master_callback: Called if this node is the master
"""
self.callback = master_callback
self.my_node = self.zk.create(ELECTION_PATH + '/n_',
ephemeral=True, sequence=True)
self.zk.get_children(ELECTION_PATH, watch=self._election_update)
self._election_update()
def _election_update(self, data=None):
def worker():
try:
self.master_notified = True
self.callback()
except Exception as e:
self.master_notified = False
log.info("Failed to activate master, redoing election: %r", e)
self.zk.delete(self.my_node)
self.my_node = self.zk.create(ELECTION_PATH + '/n_',
ephemeral=True, sequence=True)
self._election_update()
if not self.master_notified and \
sorted(self.zk.get_children(ELECTION_PATH))[0] == \
os.path.basename(self.my_node):
t = threading.Thread(target=worker)
t.daemon = True
t.start()
def test_init_servers(server_id, host = '127.0.0.1', port = 2181):
zookeeper = KazooClient('%s:%s' % (host, port,))
zookeeper.start()
try:
node = '/test/server_list'
if zookeeper.exists(node) is None:
zookeeper.create(node, json.dumps({'update_time' : time.time()}), makepath = True)
except kazoo.exceptions.NodeExistsError:
pass
try:
node = '/test/server_list/s%s' % server_id
zookeeper.delete(node)
zookeeper.create(node, json.dumps({
'update_time' : time.time(),
'server_name' : 's%s' % server_id,
'server_id' : server_id,
}), makepath = True)
except kazoo.exceptions.NodeExistsError:
pass
def main():
(options, args) = parse_options(sys.argv[1:])
data = options.znode_data_size * b"D"
s = KazooClient(options.server)
s.start()
if s.exists(options.root_znode):
children = s.get_children(options.root_znode)
print("delete old entries: %d" % len(children))
for child in children:
s.delete("%s/%s" % (options.root_znode, child))
else:
s.create(options.root_znode, "kazoo root znode")
evaluation(s, options.root_znode, data, options)
s.stop()
print("Performance test complete")
class ZookClient(object):
def __init__(self, api_client=None):
logging.basicConfig()
# Create a client and start it
self.zk = KazooClient()
self.zk.start()
def create_accounts_path(self, name, **kwargs):
path = "/dso/" + name
self.zk.ensure_path(path)
self.zk.set(path, b"id: 7b4235ca-00fb-4dca-ad3e-8b6e3662631a\ngroupname: hr\ndescription: 人力资源")
def create_accountinfo_path(self, account_id, **kwargs):
self.zk.ensure_path("/app/someservice")
def create_path(self, path, **kwargs):
self.zk.ensure_path(path)
def get_data(self, path):
return self.zk.get(path)
def test_tran(self):
self.zk.delete("/app22")
self.zk.create("/app22", b"" + '{"12": "12"}')
tran = self.zk.transaction()
tran.delete("/app22")
tran.create("/app22", b"" + '{"22": "22"}')
tran.commit()
print "commit"
def stop(self):
# In the end, stop it
self.zk.stop()
def job_submitter_main():
try:
zk = init()
cpool = ConnectionPool(host='localhost', port=6379, db=0)
r = Redis(connection_pool=cpool)
added = 0
tried = 0
max_add_try = 5000
jobname = uuid.uuid4().hex
added_nums = set()
while tried < max_add_try:
value = randint(5000, 90000000)
tried += 1
if value not in added_nums:
added_nums.add(value)
else:
continue
while True:
try:
r.lpush(jobname, value)
added += 1
break
except Exception as e:
sleep(1)
print "Lpush ", jobname, e
zk = KazooClient(hosts='127.0.0.1:2181')
zk.add_listener(state_listener)
zk.start()
value = SUBMITTED + "=" + str(added)
zk.create('/jobs/' + jobname, value=value)
zk.stop()
except Exception as e:
print 'Big problem in submitting job ', e
sys.exit(1)
print 'Job submitted ' + jobname
def join_peacock_network():
"""
주키퍼에 watch를 등록하고 해당 노드를 가입시킨다.
"""
from config import ZOOKEEPER_HOST
zk = KazooClient(hosts=ZOOKEEPER_HOST)
zk.start()
zk.ChildrenWatch("/peacock/job/nodes", job_node_watch)
my_node = zk.create("/peacock/job/nodes/node", ephemeral=True, sequence=True, makepath=True)
global my_node_name
my_node_name = my_node[my_node.rfind('/') + 1:]
job_node_watch(zk.get_children("/peacock/job/nodes"))
def main():
zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0)
zk.add_listener(my_listener)
zk.start()
if zk.exists("/ELECTION") == None:
zk.ensure_path("/ELECTION")
c = 1
node_pathes = []
while c < 10:
c += 1
node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True)
node_pathes.append(node_path)
my_path = random.choice(node_pathes)
my_path = my_path.replace("/ELECTION/", "")
# print "print my_path", my_path
children = zk.get_children("/ELECTION/", watch=election_child_watcher)
get_next_path = False
prev_path = None
for child_path in sorted(children):
if child_path == my_path:
break
prev_path = child_path
# I'm the leader
if prev_path == None:
print "OK I'm leader don't have to watch"
return
# fires twice, once on creation ignore
@zk.DataWatch("/ELECTION/" + prev_path)
def watch_node(data, stat):
# only watch for first change
if stat.version == 1:
print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
print "setting watch on " + prev_path
print "my", my_path
zk.set("/ELECTION/" + prev_path, b"some data")
print "boom. watch triggered?"
# time.sleep(10)
print "bye"
zk.stop()
class TestSessions(unittest.TestCase):
def setUp(self):
from kazoo.client import KazooClient
from kazoo.protocol.states import KazooState
from kazoo.testing.common import ZookeeperCluster
ZK_HOME = os.environ.get("ZOOKEEPER_PATH")
ZK_CLASSPATH = os.environ.get("ZOOKEEPER_CLASSPATH")
self.cluster = ZookeeperCluster(ZK_HOME, size=1, port_offset=21000, classpath=ZK_CLASSPATH)
self.cluster.start()
atexit.register(lambda cluster: self.cluster.terminate(), self.cluster)
self.client = KazooClient(self.cluster[0].address, max_retries=5)
self.ev = threading.Event()
def back(state):
if state == KazooState.CONNECTED:
self.ev.set()
self.client.start()
self.path = self.client.create(uuid.uuid4().hex)
self.client.add_listener(back)
def test_restarted_server(self):
raise SkipTest('Patch missing')
self.cluster.stop()
self.cluster.start()
self.ev.wait(5)
eq_(self.ev.is_set(), True)
self.assertTrue(self.client.retry(self.client.exists, self.path))
def test_terminated_server(self):
raise SkipTest('Patch missing')
self.cluster.reset()
self.cluster.start()
self.ev.wait(5)
eq_(self.ev.is_set(), True)
self.assertFalse(self.client.retry(self.client.exists, self.path))
def tearDown(self):
self.ev.clear()
self.client.stop()
self.cluster.stop()