class ZooWrap(): def __init__(self): self.zk = KazooClient(hosts="%s,%s" % ( str(cc.conf['zookeeper']['host']), str(cc.conf['zookeeper']['port']))) self.zk.start() self.root = cc.conf['zookeeper']['rootpath'] self.log = logging.getLogger('L.ZOOKEEPER') self.zk.ensure_path('/%s/sleeping' % (self.root)) self.whoami = cc.conf['whoami'] def get_sleeping(self): return self.zk.get_children('/%s/sleeping' % (self.root)) def sleep(self): try: self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami)) self.log.info('Sleeping correctly') except NodeExistsError: self.log.error('Node already sleeping... seems weird') def wake(self): try: self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami)) except NoNodeError: self.log.error('Node was not sleeping... seems weird')
class Zookeeper(KeyManager): def __init__(self, hosts): self._hosts = hosts self.zk = KazooClient(hosts=hosts) self.zk.start() def get(self, key): result = self.zk.get(key)[0] if result == "": result = [] children = self.zk.get_children(key) for i in children: result.append({'name': i, 'value': self.zk.get(os.path.join(key, i))[0]}) return result else: return self.zk.get(key)[0] def set(self, key, data): try: self.zk.set(key, data.encode('utf-8')) except: self.zk.create(key, data.encode('utf-8')) def mkdir(self, key): self.set(key, "") def close(self): self.zk.stop() self.zk.close() @property def hosts(self): return self._hosts
def add_and_update(key, value): print(key, value, '#' * 20) zk = KazooClient(hosts='127.0.0.1:2181') zk.start() # children = zk.get_children('/') # 创建节点:makepath 设置为 True ,父节点不存在则创建,其他参数不填均为默认 # zk.create('/zookeeper/goodboy') # 操作完后,别忘了关闭zk连接 # zk.stop() print('#' * 20) data, stat = zk.get(key) print("Version: %s, data: %s" % (stat.version, data.decode("utf-8"))) if zk.exists(key): zk.set(key, value) print('@' * 10, 'setting') else: zk.create(key, value, makepath=True) # zk.create(key) # print(value, type(value),'#'*10) # if value != b"": # zk.set(key, value) zk.stop()
def zk_flag_set(zk: KazooClient, name: str, value: str): path = "{}/{}".format(ZK_PREFIX, name) data = value.encode('utf-8') try: zk.set(path, data) except NoNodeError: zk.create(path, data, makepath=True)
class KazooService(): zk = None rootPath = None def __init__(self, rootPath): qs = Zookeeper.objects.filter().first() if qs is None: raise LookupError("Unable to find Zookeeper configuration!") self.zk = KazooClient(hosts=qs.url) self.rootPath = rootPath self.zk.start(timeout=qs.timeout) self.zk.ensure_path(rootPath) def createOrUpdate(self, pathAffix, nodeValue): if isinstance(nodeValue, str): bValue = bytes(nodeValue, 'utf-8') else: bValue = bytes(str(nodeValue), 'utf-8') path = self.rootPath + pathAffix if self.zk.exists(path) is None: self.zk.create(path, bValue) else: self.zk.set(path, bValue) def __del__(self): if self.zk is not None: self.zk.stop()
class ServicePublisher: def __init__(self, hosts, timeout, publish_port): self._logger = logging.getLogger(self.__class__.__name__) self._publish_port = publish_port self._zk = KazooClient(hosts=hosts) event = self._zk.start_async() event.wait(timeout=timeout) if self._zk.connected: self._logger.info('Kazoo client successfully connected') self._publish_status() else: self._zk.stop() self._logger.error('Kazoo client failed to connect') def _publish_status(self): full_path = '%s/%s' % (PARENT_NODE, socket.gethostname()) data = { 'started': str(datetime.datetime.now())[:19], 'port': self._publish_port } json_data = json.dumps(data).encode(encoding='utf-8') self._logger.info('Publishing status %s to path %s' % (data, full_path)) self._zk.create(full_path, json_data, ephemeral=True, makepath=True)
class DistributedLock: def __init__(self, hosts, logger=None): self.client = KazooClient(hosts=hosts, logger=logger) def ensure_path(self, path): self.client.start() self.client.ensure_path(path) self.client.stop() def try_lock(self, path): try: self.client.start() self.client.create(path, ephemeral=True) return True except: logging.exception("try lock exception. path: {}".format(path)) return False def try_unlock(self, path): try: self.client.delete(path) self.client.stop() return True except: logging.exception("try unlock exception. path: {}".format(path)) return False
class Store(object): def __init__(self,**kwargs): self.config = kwargs self.client = None def get_client(self): return self.client def open(self): self.client = KazooClient(**self.config) self.client.add_listener self.client.start() def close(self): self.client.stop() def read(self,path): return self.client.get(path) def write(self,path,value): base_path = os.path.dirname(path) self.client.ensure_path(base_path) self.client.create(path,value) def overwrite(self,path,value): self.client.set(path,value) def exists(self,path): return self.client.exists(path)
class ES_zk_interface: def __init__(self, my_ip, es_interface, es_callback): self.ip = my_ip self.es_callback = es_callback self.es_interface = es_interface self.zk = KazooClient(hosts=HOST_IP) self.zk.start() self.zk.ensure_path("/ass3") def start_election_process(self): print "Starting Election process" #print("Previous Leader: %s" % (self.es_callback.get_leader())) election = self.zk.Election("/electionpath", self.ip) election.run(self.leader_function) def leader_function(self): print "Won the election: " + str(self.ip) if self.zk.exists(LEADER_PATH): self.zk.set(LEADER_PATH, self.ip) else: self.zk.create(LEADER_PATH, self.ip) #print "\nStarting Leader thread.... \n" self.es_interface.enter_message_loop(self.es_callback, True) def get_zookeeper_client(self): return self.zk def get_leader_path(self): return LEADER_PATH
def test_pub_response(pub_node, server_list, host='127.0.0.1', port=2181): zookeeper = KazooClient('%s:%s' % ( host, port, )) zookeeper.start() for s in server_list: if isinstance(s, int) is True: continue try: node = '/test/to_pub_result/%s/%s' % ( pub_node, s, ) if zookeeper.exists(node) is None: zookeeper.create(node, json.dumps({ 'update_time': time.time(), 'status': 'ok' }), makepath=True) except kazoo.exceptions.NodeExistsError: pass
def connect(hosts="127.0.0.1:2181"): retval = KazooClient(hosts=hosts) def my_listener(state): if state == KazooState.LOST: # Register somewhere that the session was lost logging.warn("Lost connection to Zookeeper") elif state == KazooState.SUSPENDED: # Handle being disconnected from Zookeeper logging.warn("Zookeeper connection suspended") elif state == KazooState.CONNECTED: pass else: # Handle being connected/reconnected to Zookeeper logging.info("Other state: %s" % state) retval.add_listener(my_listener) retval.start() if not retval.exists(key): logging.info("Created '%s'" % key) retval.create(key) return (retval)
class ZKBase(object): """ scheduler需要定义以下函数: - callback(KazooState.*): 处理CONN: LOST,SUSPENDED - """ def __init__(self,conf,scheduler): self.conf = conf self.state = "STOPPED" self.zk = KazooClient(self.conf.processor.zk_url, timeout=self.conf.processor.timeout) try: self.zk.start() except Exception: LOG.error("Start zk error: %s" % traceback.format_exc(limit=2)) self.zk.add_listener(self.listen) self.scheduler = scheduler self.state = "RUNNING" # STOPPED self.state_changes = False def listen(self, state): if state == KazooState.LOST or state == KazooState.SUSPENDED: LOG.error("Session state change: %s" % state) if self.state == "STOPPED": return self.state_changes = True self.scheduler.callback(state) elif state == KazooState.CONNECTED: LOG.info("Connectied to ZK.") else: LOG.error("Session timeout. Cannot conect to ZK.") def reinit(self,state): while self.zk.state != KazooState.CONNECTED: LOG.info("Restart zk connection until connected.") try: self.zk.restart() except Exception: LOG.error("Reinit: %s" % traceback.format_exc(limit=1)) self.state_changes = False def create(self,path, value='', acl=None, ephemeral=False, sequence=False, makepath=False): LOG.info("Create: path %s" % path) self.zk.create(path=path, value=value.encode(), ephemeral=ephemeral, sequence=sequence, makepath=makepath) def get(self,path,watch=None): self.zk.get(path,watch) def get_children(self,path,watch=None): return self.zk.get_children(path,watch=watch) def terminate(self): LOG.info("Terminate ZK connection.") self.state = "STOPPED" self.zk.stop() self.zk.close()
class ZKState(): def __init__(self, path, timeout=30): super(ZKState, self).__init__() self._zk = KazooClient(hosts=ZK_HOSTS, timeout=timeout) self._zk.start(timeout=timeout) self._path = path self._zk.ensure_path(path) def processed(self): return self._zk.exists(self._path + "/complete") def process_start(self): if self.processed(): return False if self._zk.exists(self._path + "/processing"): return False try: self._zk.create(self._path + "/processing", ephemeral=True) return True except NodeExistsError: # another process wins return False def process_end(self): self._zk.create(self._path + "/complete") self._zk.delete(self._path + "/processing") def process_abort(self): try: self._zk.delete(self._path + "/processing") except NoNodeError: pass def close(self): self._zk.stop() self._zk.close()
def main(): logging.basicConfig() zk = KazooClient(hosts=zookeeper_hostname) zk.start() zk.ensure_path("/worker") node_name = "/worker/" + os.environ["NODE_NAME"] if not zk.exists(node_name): msg = "Creating node: " + node_name print(msg, file=sys.stdout) db_name = os.environ["DB_HOSTNAME"] zk.create(node_name, db_name.encode(), ephemeral=True) if os.environ["WORKER_TYPE"] == "master": rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True) rpc_server.start() else: try: if node_name != "/worker/slave1": master_db = zk.get("/worker/master")[0].decode() print("[*] Cloning database from master db: " + master_db, file=sys.stdout) subprocess.call( "mongodump --host " + master_db + " --port 27017 --db rideshare && mongorestore --host " + os.environ[ 'DB_HOSTNAME'] + " --port 27017", stdout=sys.stdout, stderr=sys.stdout, shell=True ) except Exception as e: print(e, file=sys.stdout) old_name = os.environ["NODE_NAME"] p1 = multiprocessing.Process(target=slave_rpc_server) p2 = multiprocessing.Process(target=become_master, args=(p1, old_name,)) p1.start() p2.start()
def registry(self, port, service, methods, zks): if zks is None: raise Exception("zk 地址为空") providers_path = "/dubbo/%s/providers" % (service) ip = self.local_host_ip() provider_url = self.generate_provider_url(host=ip, service=service, port=port, methods=methods) self.service_path[service] = "%s/%s" % (providers_path, quote_plus(provider_url)) value = str(ip).encode("utf8") zk = KazooClient(hosts=zks) self.registry_zk[service] = zk zk.start() #zk.ensure_path(providers_path+"/") try: zk.delete(self.service_path[service]) except Exception as e: pass finally: zk.create(self.service_path[service], value, ephemeral=True, sequence=False, makepath=True)
def zk_init(server_address, zk_node, zk_address = ''): #初始化zk节点 zk_cli = KazooClient(hosts = zk_address) try: zk_cli.start() except: logging.error('zk Init error, can not connect %s' %(str(zk_address))) return -1 try: zk_cli.get(zk_node) except: logging.warn('can not find zk path %s, creat it' %(str(zk_node))) zk_cli.ensure_path(zk_node) if zk_node[-1] != '/': zk_node += '/' try: zk_cli.create(zk_node + server_address, '1', ephemeral=True) except: if zk_cli.get(zk_node + server_address): return 0 else: logging.error('create zk_node error, can not create node %s' %(str(zk_node) + str(zk_address))) return -1 return 0
class zkClient(): def __init__(self): self.zk = KazooClient(hosts='10.0.0.2:2181') self.zk.start() def get_hostname_ip(self): host_name = socket.getfqdn(socket.gethostname()) host_ip = socket.gethostbyname(host_name) return "/" + host_name + "_gateway", host_ip def register(self): name, ip = self.get_hostname_ip() if not self.zk.exists(name): self.zk.create(name, bytes(ip, 'utf-8')) else: self.zk.set(name, bytes(ip, 'utf-8')) def get_kvs_ips(self): device_ips = [] for device_name in self.zk.get_children('/'): if device_name.endswith('_kvs'): data, stat = self.zk.get(device_name) device_ips.append(data.decode("utf-8")) return device_ips def get_nodes_ips(self, device_list): device_ips = [] for device_name in self.zk.get_children('/'): if device_name in device_list: data, stat = self.zk.get(device_name) device_ips.append(data.decode("utf-8")) return device_ips
class ZookeeperClient(object): def __init__(self, server_list): self._retry = KazooRetry(max_tries=None, max_delay=300, sleep_func=gevent.sleep) self._zk_client = KazooClient(hosts=','.join(server_list), timeout=400, handler=SequentialGeventHandler(), logger=logger, connection_retry=self._retry, command_retry=self._retry) def connect(self): self._zk_client.start() def disconnect(self): self._zk_client.stop() self._zk_client.close() def create_node(self, path, value=None): if value is None: value = uuid.uuid4() try: self._zk_client.create(path, str(value), makepath=True) except NodeExistsError: self._zk_client.set(path, str(value))
class ZooWrap(): def __init__(self): self.zk = KazooClient(hosts="%s,%s" % (str(cc.conf['zookeeper']['host']), str(cc.conf['zookeeper']['port']))) self.zk.start() self.root = cc.conf['zookeeper']['rootpath'] self.log = logging.getLogger('L.ZOOKEEPER') self.zk.ensure_path('/%s/sleeping' % (self.root)) self.whoami = cc.conf['whoami'] def get_sleeping(self): return self.zk.get_children('/%s/sleeping' % (self.root)) def sleep(self): try: self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami)) self.log.info('Sleeping correctly') except NodeExistsError: self.log.error('Node already sleeping... seems weird') def wake(self): try: self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami)) except NoNodeError: self.log.error('Node was not sleeping... seems weird')
class ZKMData(object): def __init__(self): super(ZKMData, self).__init__() self._zk = KazooClient(hosts=ZK_HOSTS) self._zk.start() def set(self, path, value): value = json.dumps(value).encode('utf-8') if self._zk.exists(path): try: self._zk.set(path, value) except NoNodeError: pass try: self._zk.create(path, value, makepath=True) except NodeExistsError: pass def get(self, path): try: value, stat = self._zk.get(path) if not value: return {} return json.loads(value.decode('utf-8')) except Exception as e: return {} def close(self): self._zk.stop() self._zk.close()
class Register(object): def __init__(self, value, path='/scrapyd-cluster/worker', hosts='127.0.0.1:2181'): self.path = path self.value = value self.expire = False self.zk = KazooClient(hosts) self.zk.start() self.zk.add_listener(self.listener) self.zk.ensure_path(os.path.dirname(self.path)) self.path = self.zk.create(self.path, self.value, ephemeral=True, sequence=True) logger.info('Connected To ZooKeeper(%s) Succeed! Path : %s' % (hosts, self.path)) def register(self): if self.expire: logger.info('Renew Ephemeral Node. Path : %s Value: %s' % (self.path, self.value)) self.zk.create(self.path, self.value, ephemeral=True) self.expire = False def listener(self, state): if state == KazooState.LOST: logger.info('ZooKeeper Connection Lost') self.expire = True elif state == KazooState.CONNECTED: self.zk.handler.spawn(self.register)
def test_zookeeper(zookeeper_server): test_value = str(uuid.uuid4()).encode("utf-8") zk_client1 = KazooClient( hosts="127.0.0.1:{}".format(zookeeper_server.port)) zk_client2 = KazooClient( hosts="127.0.0.1:{}".format(zookeeper_server.port)) zk_client1.start() zk_client2.start() zk_client1.ensure_path("/my") assert zk_client1.exists("/you") is None assert zk_client2.exists("/you") is None assert zk_client1.exists("/my") is not None assert zk_client2.exists("/my") is not None zk_client1.create("/my/testplan", test_value) data, _ = zk_client1.get("/my/testplan") assert data == test_value data, _ = zk_client2.get("/my/testplan") assert data == test_value zk_client2.set("/my/testplan", str(uuid.uuid4()).encode("utf-8")) data, _ = zk_client1.get("/my/testplan") assert data != test_value data, _ = zk_client2.get("/my/testplan") assert data != test_value
class ZKStore: def __init__(self, hosts): self.zk = KazooClient(hosts=hosts) self.zk.add_listener(listener) self.zk.start() def isConnected(self): if __state__ == 1: return True return False def write(self, path, node, value): self.zk.ensure_path(path) if self.zk.exists(path+"/"+node): self.zk.set(path+"/"+node, value) else: self.zk.create(path + "/" + node, value) def read(self, path): if self.zk.exists(path): data, stat = self.zk.get(path) return data return None
def add_node_to_zookeeper(): zk = KazooClient(hosts=zookeeper_hosts) @zk.add_listener def zookeeper_listener(state): if state == KazooState.LOST: logging.info("zookeeper connect status:LOST") # Register somewhere that the session was lost elif state == KazooState.SUSPENDED: # Handle being disconnected from Zookeeper logging.info("zookeeper connect status:SUSPENDED") elif state == KazooState.CONNECTED: logging.info("zookeeper connect status:CONNECTED") else: logging.info(str(state)) zk.start() # 创建“/spiderServer”永久节点,后面每个有效的redis节点,或动态添加redis节点后,都会在 # “/spiderServer”节点下创建对应的redis服务节点(如:/spiderServer/128.0.0.203),服务节点名称已redis服务所在的机器的ip地址命名 if zk.exists(SPIDER_SERVER_NODE): pass else: try: zk.create(SPIDER_SERVER_NODE) except Exception, e: if isinstance(e, NodeExistsException): pass else: logging.exception(str(e))
def test_init_servers(num, host='127.0.0.1', port=2181): zookeeper = KazooClient('%s:%s' % ( host, port, )) zookeeper.start() try: node = '/test/server_list' if zookeeper.exists(node) is None: zookeeper.create(node, json.dumps({'update_time': time.time()}), makepath=True) except kazoo.exceptions.NodeExistsError: pass v = 1 while v <= num: try: node = '/test/server_list/s%s' % v if zookeeper.exists(node) is None: zookeeper.create(node, json.dumps({ 'update_time': time.time(), 'server_name': 's%s' % v, 'server_id': v, }), makepath=True) except kazoo.exceptions.NodeExistsError: pass v += 1
class ZookClient(): def __init__(self, hosts): self.kazooclient = KazooClient(hosts=hosts) self.kazooclient.start() def foundApp(self, appname): #Ìí¼Óapp if self.kazooclient.exists('/' + appname): pass else: self.kazooclient.ensure_path('/' + appname) def foundNode(self, appname, port, host): #Ìí¼Ó½Úµã if self.kazooclient.exists('/' + appname + '/' + host + ':' + str(port)): pass else: self.kazooclient.create('/' + appname + '/' + host + ':' + str(port)) def close(self): self.kazooclient.close() def register(self, appname, port, host): self.foundApp(appname) self.foundNode(appname, port, host) # self.close()
def test_subscriber_connect(self): test_zk = KazooClient(self.zk_sub.zookeeper_connection_url) try: test_zk.start(5) except Exception: self.assertRaises("Couldn't connect to zookeeper server") # Ensure a path, create if necessary test_zk.ensure_path(self.zk_sub.default_node_path) fake_host_ip_lst = ["0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.3"] try: for host_ip in fake_host_ip_lst: # Create a node with data node = "node" + str(uuid.uuid4()) path = self.default_node_path + '/' + node test_zk.create(path, bytes(host_ip, "utf-8")) self.zk_sub.subscriber_connect("subscriber") finally: nodes_to_delete = test_zk.get_children(self.default_node_path) for node in nodes_to_delete: path = self.default_node_path + '/' + node test_zk.delete(path) children = test_zk.get_children(self.default_node_path) self.assertEqual(children, [])
class ZookeeperClient(object): def __init__(self, server_list): self._retry = KazooRetry(max_tries=None, max_delay=300, sleep_func=gevent.sleep) self._zk_client = KazooClient( hosts=','.join(server_list), timeout=400, handler=SequentialGeventHandler(), logger=logger, connection_retry=self._retry, command_retry=self._retry) def connect(self): self._zk_client.start() def disconnect(self): self._zk_client.stop() self._zk_client.close() def create_node(self, path, value=None): if value is None: value = uuid.uuid4() try: self._zk_client.create(path, str(value), makepath=True) except NodeExistsError: self._zk_client.set(path, str(value))
def init_zk(namespace=''): """Initialize Zookeeper Instance.""" hosts = os.getenv('ZOOKEEPER_CONN_STRING', 'localhost:2181') timeout = 600 timeout_treshold = time() + timeout while True: try: # check if namespace exists zk = KazooClient(hosts, read_only=False) zk.start() if zk.exists(namespace) is None: zk.create(namespace) zk.stop zk.close zk = KazooClient(hosts + namespace, read_only=False) zk.start() if type(zk) is not KazooClient: logging.error("can't connect to Zookeeper ...") exit(1) return zk except KazooTimeoutError: if time() > timeout_treshold: logging.error("can't connect to Zookeeper ...") exit(1) else: logging.warning("can't connect to Zookeeper, " + "trying again since timeout " + "is not reached yet ...")
class ValidatorDetector: def __init__(self): self.zk = KazooClient( hosts="bigdata1:2181,bigdata2:2181,bigdata3:2181") self.validator_children_watcher = ChildrenWatch(client=self.zk, path="/servers", func=self.watcher_func) self.zk.start() def watcher_func(self, children): print("----------------------start-----------------------------") for c in children: data, stat = self.zk.get('/servers/' + c) if stat: print("Children is:" + c + "\n" + "Data is:", str(data, encoding='utf-8')) print("-----------------------end----------------------------") print("\n\n") def create_node(self, hostname): self.zk.create('/servers/server', bytes(hostname, encoding='utf-8'), ephemeral=True, sequence=True, makepath=True) def close_client(self, hostname): self.zk.close()
class ZKData(object): def __init__(self): super(ZKData,self).__init__() options={"max_tries":-1,"max_delay":5,"ignore_expire":True} self._zk=KazooClient(hosts=ZK_HOSTS,connection_retry=options) try: self._zk.start(timeout=3600) except: print(traceback.format_exc(), flush=True) def set(self, path, value): value=json.dumps(value).encode('utf-8') try: self._zk.create(path, value, makepath=True) except NodeExistsError: self._zk.set(path,value) def get(self, path): try: value, stat= self._zk.get(path) if not value: return {} return json.loads(value.decode('utf-8')) except Exception as e: return {} def close(self): self._zk.stop() self._zk.close()
class IDGenerator(object): ''' ID生成器 ''' # 控制输出信息的锁 print_mutex = threading.Lock() DELAY_TIME = 3 def __init__(self, zk_server_address, id_base_path): self.id_base_path = id_base_path # 创建一个zookeeper客户端 self.zkclient = KazooClient(zk_server_address) # 添加连接状态监听器 self.zkclient.add_listener(self.zk_connect_listener) # 与zookeeper开启连接 self.zkclient.start() # 连接状态监听器 def zk_connect_listener(self, state): # 获取打印锁 IDGenerator.print_mutex.acquire() if state == KeeperState.CONNECTED: print "命名服务器已经开启..." elif state == KazooState.LOST: print "命名服务器停止服务..." else: raise Exception("命名服务器未正常开启...") # 获取打印锁 IDGenerator.print_mutex.release() def start(self): # 创建父节点 self.create_name_node() def create_name_node(self): # 先检查父节点,如果父节点不存在 if not self.zkclient.exists(self.id_base_path): # 先创建父节点 self.zkclient.create(self.id_base_path) # 生成id def generate_id(self): # 拼凑出服务器子节点的完整路径 node_path = self.id_base_path + "/" # 创建临时顺序节点 self.node_path = self.zkclient.create(node_path, "", self.zkclient.default_acl, True, True) # 准备删除创建出来的节点 self.zkclient.delete(self.node_path) return self.node_path.split("/")[-1] # 停止服务 def stop(self): self.zkclient.stop() self.zkclient.close()
class ZKClient: def __init__(self, zk_url, zk_timeout=5): logging.basicConfig(stream=sys.stdout, level=logging.INFO) self.logger = logging.getLogger() try: self.client = KazooClient(hosts=zk_url) self.client.start(timeout=zk_timeout) except handlers.threading.KazooTimeoutError: self.logger.error("5 seconds connection timeout") raise def store_value(self, value, path): fmt_value = str(value).encode('utf-8') self.client.ensure_path(path) self.client.set(path, fmt_value) self.client.stop() def retrieve_value(self, path): self.client.ensure_path(path) data, stat = self.client.get(path) self.client.stop() if not data: raise ValueError else: return int(data.decode('utf-8')) def register_server(self, path): self.client.create(path, ephemeral=True, makepath=True)
def execute(self, request): zk = KazooClient(hosts=request['zk_url']) try: zk.start() path_ = request['node_path'] value_ = request['node_value'].encode('utf-8') is_update_timestamp_ = request.get('is_update_timestamp', False) is_delete_node_ = request.get('is_delete_node', None) if zk.exists(path_): if is_delete_node_: zk.delete(path_, recursive=True) return self.resp_dict, '' if is_update_timestamp_: zk.set(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes()) if value_: zk.set(path_, value_) else: data_, stat_ = zk.get(path_) if data_: self.resp_dict['result_value'] = data_.decode() elif value_: # 新加节点时也需要添加 '/TIMESTAMP' 节点 zk.create(path_, value_, makepath=True) zk.create(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes(), makepath=True) return self.resp_dict, '' except Exception as e: logger.exception('zkTestPoint execute error, cause: %s', str(e)) raise Exception('zkTestPoint execute error, cause: ' + str(e)) finally: zk.stop()
def become_master(slave_process, old_name): s = socket.socket() s.bind(("", 23456)) print("[*] Listening for command from orchestrator to become master ...", file=sys.stdout) s.listen(2) c, address = s.accept() print("Received command from orchestrator to become master: " + c.recv(1024).decode(), file=sys.stdout) slave_process.terminate() os.environ["WORKER_TYPE"] = "master" os.environ["NODE_NAME"] = "master" client = docker.DockerClient(base_url="tcp://172.17.0.1:4444") cnt = client.containers.get(old_name) cnt.rename("master") logging.basicConfig() zk = KazooClient(hosts=zookeeper_hostname) zk.start() node_name = "/worker/" + os.environ["NODE_NAME"] if not zk.exists(node_name): msg = "Creating node: " + node_name print(msg, file=sys.stdout) db_name = os.environ["DB_HOSTNAME"] zk.create(node_name, db_name.encode(), ephemeral=True) time.sleep(3) zk.delete("/worker/" + old_name) rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True) rpc_server.start()
def do_zookeeper_create(addr, path, value): print(path) zk = KazooClient(addr) zk.start() zk.create(path, bytes(value, 'utf-8')) do_zookeeper_read(addr, path)
class ZooKeeper(object): def __init__(self, hosts="127.0.0.1:2181", prefix="/"): prefix = prefix.rstrip("/") if prefix and not prefix.startswith("/"): raise ValueError("prefix must start with /") self._prefix = prefix self._zk = KazooClient(hosts=hosts, handler=_get_zk_handler()) self._zk.start() def __del__(self): self.close() def _path(self, path): if self._prefix: return "{0}/{1}".format(self._prefix, path) return path def close(self): if not self._zk: return try: self._zk.stop() self._zk.close() except Exception: pass finally: self._zk = None def create(self, path, value="", makepath=True): self._zk.create(self._path(path), value, makepath=makepath) def delete(self, path, recursive=True): self._zk.delete(self._path(path), recursive=recursive) def set(self, path, value, makepath=True): path = self._path(path) try: self._zk.set(path, value) except zkexc.NoNodeError: if not makepath: raise self._zk.create(path, value, makepath=True) def get(self, path, none=True): try: return self._zk.get(self._path(path)) except zkexc.NoNodeError: if none: return None raise def ls(self, path, none=True): try: return self._zk.get_children(self._path(path)) except zkexc.NoNodeError: if none: return None raise
class ShellTestCase(unittest.TestCase): """ base class for all tests """ def setUp(self): """ make sure that the prefix dir is empty """ self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests") self.zk_host = os.getenv("ZKSHELL_ZK_HOST", "localhost:2181") self.username = os.getenv("ZKSHELL_USER", "user") self.password = os.getenv("ZKSHELL_PASSWD", "user") self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=") self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "secret") self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest") self.client = KazooClient(self.zk_host, 5) self.client.start() self.client.add_auth(self.scheme, self.auth_id) if self.client.exists(self.tests_path): self.client.delete(self.tests_path, recursive=True) self.client.create(self.tests_path, str.encode("")) self.output = StringIO() self.shell = Shell([self.zk_host], 5, self.output, setup_readline=False, async=False) # Create an empty test dir (needed for some tests) self.temp_dir = tempfile.mkdtemp() @property def auth_id(self): return "%s:%s" % (self.username, self.password) @property def auth_digest(self): return "%s:%s" % (self.username, self.digested_password) def tearDown(self): self.output = None self.shell = None if os.path.isdir(self.temp_dir): shutil.rmtree(self.temp_dir) if self.client.exists(self.tests_path): self.client.delete(self.tests_path, recursive=True) self.client.stop() ### # Helpers. ## def create_compressed(self, path, value): """ ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client to create a znode with zlib compressed content. """ compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value) self.client.create(path, compressed, makepath=True)
def zookeeper_ephemeral_node(name): zk = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2181')) zk.start() zk.create("/my/zookeeper_test/node1", b"process1 running", ephemeral=True) #At 10 validate that ephemeral node exist that is the process is still running time.sleep(10) assert zk.exists("/my/zookeeper_test/node1"), "process node is not found at 10 s when it is still running" time.sleep(20) zk.stop()
class ZookeeperContext: def __init__(self, zookeeper_url, test_id, node_id): self.assertNotNone(zookeeper_url) self.assertNotNone(test_id) self.assertNotNone(node_id) self.zookeeper_url = zookeeper_url self.test_id = test_id self.node_id = node_id self.zk_path = '/bolt/testing/%s/%s' % (self.test_id, self.node_id) logger.info("Initialized decorator: %s " % self.__str__()) self.__connect_zookeeper() def assertNotNone(self,thing): if thing == None: logger.fatal("Invalid key in initializer") def __connect_zookeeper(self): try: logger.info('Attempting to connect to: %s' % self.zookeeper_url) self.zk = KazooClient(hosts=self.zookeeper_url) self.zk.start(timeout=10) logger.info('initialized zookeeper') except Exception as exception: logger.error('Failed to connect to zk') logger.fatal(exception) def __str__(self): return json.dumps(self, default=lambda o: o.__dict__, indent=4) def __del__(self): self.zk.stop() def publish(self, key, data): self.assertNotNone(key) self.assertNotNone(data) path = self.zk_path + "/" + key logger.info("Creating data in: %s" % path) try: bytes = pickle.dumps(data); if not self.zk.exists(path): logger.info("Path does not exist in zk, creating... %s" % path) self.zk.create(path, value=bytes, makepath=True) else: self.zk.set(path, value=bytes) except Exception as exception: logger.error('Error setting data in ' % path) logger.fatal(exception)
def main(): ''' A manual configuration file pusher for the crawlers. This will update Zookeeper with the contents of the file specified in the args. ''' import argparse from kazoo.client import KazooClient parser = argparse.ArgumentParser( description="Crawler config file pusher to Zookeeper") parser.add_argument('-f', '--file', action='store', required=True, help="The yaml file to use") parser.add_argument('-i', '--id', action='store', default="all", help="The crawler id to use in zookeeper") parser.add_argument('-p', '--path', action='store', default="/scrapy-cluster/crawler/", help="The zookeeper path to use") parser.add_argument('-w', '--wipe', action='store_const', const=True, help="Remove the current config") parser.add_argument('-z', '--zoo-keeper', action='store', required=True, help="The Zookeeper connection <host>:<port>") args = vars(parser.parse_args()) filename = args['file'] id = args['id'] wipe = args['wipe'] zoo = args['zoo_keeper'] path = args['path'] zk = KazooClient(hosts=zoo) zk.start() # ensure path exists zk.ensure_path(path) bytes = open(filename, 'rb').read() if zk.exists(path): # push the conf file if not zk.exists(path + id) and not wipe: print "creaing conf node" zk.create(path + id, bytes) elif not wipe: print "updating conf file" zk.set(path + id, bytes) if wipe: zk.set(path + id, None) zk.stop()
def put_topology(topology, config=config): zk = KazooClient(**config) zk.start() for service, links in topology.iteritems(): if zk.exists("/sdn/services/{name}".format(name=service)): zk.set("/sdn/services/{name}".format(name=service), json.dumps(links)) else: zk.ensure_path("/sdn/services") zk.create("/sdn/services/{name}".format(name=service), json.dumps(links)) ret_check = zk.get_children("/sdn/services") zk.stop() return ret_check
def test_create_makepath_incompatible_acls(self): from kazoo.client import KazooClient from kazoo.security import make_digest_acl_credential, CREATOR_ALL_ACL credential = make_digest_acl_credential("username", "password") alt_client = KazooClient(self.cluster[0].address + self.client.chroot, max_retries=5, auth_data=[("digest", credential)]) alt_client.start() alt_client.create("/1/2", b"val2", makepath=True, acl=CREATOR_ALL_ACL) try: self.assertRaises(NoAuthError, self.client.create, "/1/2/3/4/5", b"val2", makepath=True) finally: alt_client.delete('/', recursive=True) alt_client.stop()
class HHFrontierWorker(FrontierWorker): def __init__(self, settings, no_batches, no_scoring, no_incoming): super(HHFrontierWorker, self).__init__(settings, no_batches, no_scoring, no_incoming) self.init_zookeeper() def init_zookeeper(self): self._zk = KazooClient(hosts=settings.get('ZOOKEEPER_LOCATION')) self._zk.add_listener(self.zookeeper_listener) self._zk.start() self.znode_path = self._zk.create("/frontera/hh-f-worker", ephemeral=True, sequence=True, makepath=True) def zookeeper_listener(self, state): if state == KazooState.LOST: # Register somewhere that the session was lost pass elif state == KazooState.SUSPENDED: # Handle being disconnected from Zookeeper pass else: # Handle being connected/reconnected to Zookeeper pass def set_process_info(self, process_info): self.process_info = process_info self._zk.set(self.znode_path, self.process_info) def set_job_id(self, job_id): self._backend.set_job_id(job_id) self.job_id = job_id
class ZookeeperSession(object): def __init__(self, locations, name_prefix, root_prefix='/frontera'): self._zk = KazooClient(hosts=locations) self._zk.add_listener(self.zookeeper_listener) self._zk.start() self.root_prefix = root_prefix self.znode_path = self._zk.create("%s/%s" % (self.root_prefix, name_prefix), ephemeral=True, sequence=True, makepath=True) def zookeeper_listener(self, state): if state == KazooState.LOST: # Register somewhere that the session was lost pass elif state == KazooState.SUSPENDED: # Handle being disconnected from Zookeeper pass else: # Handle being connected/reconnected to Zookeeper pass def set(self, value): self._zk.set(self.znode_path, value) def get_workers(self, prefix='', exclude_prefix=''): for znode_name in self._zk.get_children(self.root_prefix): if prefix and not znode_name.startswith(prefix): continue if exclude_prefix and znode_name.startswith(exclude_prefix): continue location, _ = self._zk.get(self.root_prefix+"/"+znode_name) yield location
def _receive(self): server, port, buf_que = self._createThriftServer() kazoo_client = KazooClient(self.zk_address) kazoo_client.start() path = '%s/%s:%d' % (self.zk_path, socket.gethostname(), port) kazoo_client.create(path, ephemeral=True, makepath=True) while not server._stop: try: message = buf_que.pop() yield message except: time.sleep(0.1) server.close() kazoo_client.close()
def model_environments(ip_address, file_path): zk = KazooClient(hosts=ip_address) zk.start() for root, dirs, files in os.walk(file_path): node_path = root.replace(os.path.dirname(file_path), "") # trim root from path, except parent try: zk.create(node_path, makepath=True) # make parent directories as needed except NodeExistsException: print "%s already exists" % node_path if len(files) > 0: file = os.path.join(root, files[0]) __load_file_into_znode(zk, file, node_path) zk.stop()
def test_pub_response(pub_node, server_list, host = '127.0.0.1', port = 2181): zookeeper = KazooClient('%s:%s' % (host, port,)) zookeeper.start() for s in server_list: if isinstance(s, int) is True: continue try: node = '/test/to_pub_result/%s/%s' % (pub_node, s, ) if zookeeper.exists(node) is None: zookeeper.create(node, json.dumps({ 'update_time' : time.time(), 'status' : 'ok' }), makepath = True) except kazoo.exceptions.NodeExistsError: pass
def join(server, environment, name, on_parent_data_change = None): zk = KazooClient(hosts=server) zk.start() # connect to zookeeper # create an ephemeral znode under the given environment, # if the given name ends in '-' make it sequential. zk.create( path = '{0}/{1}'.format(environment, name), ephemeral = True, sequence = (name[-1:] == '-')) # if a callback was provided trigger it whenever parents data changes... # for example to update variables on config changes. if on_parent_data_change: zk.DataWatch(environment, func = on_parent_data_change) #hand zookeeper client back to caller return zk
def zkset(): file = "/tmp/redis_config.properties" zk = KazooClient(hosts='172.29.22.15:2181') zk.start() f=open(file,'rb') m=f.read() #Ensure a path , Create if necessary. zk.ensure_path("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes") #Create a node with data. try: zk.create("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m) except kazoo.exceptions.NodeExistsError: zk.set("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m) zk.stop()
class ZookeeperClient(object): def __init__(self, zk_hosts): self.zk = KazooClient(hosts=zk_hosts) self.zk.start() def put(self, entity, id, data): path = '/{}/{}'.format(entity, id) binary_value = json.dumps(data).encode('utf-8') try: self.zk.create(path, binary_value, makepath=True) return True except NodeExistsError: self.zk.set(path, binary_value) return False def get(self, entity, id): path = '/{}/{}'.format(entity, id) try: binary_data, _ = self.zk.get(path) except NoNodeError: return None return json.loads(binary_data.decode('utf-8')) def delete(self, entity, id): path = '/{}/{}'.format(entity, id) try: self.zk.delete(path) return True except NoNodeError: return False def list(self, entity): path = '/{}'.format(entity) try: children = self.zk.get_children(path) for child in children: value = self.get(entity, child) yield (child, value) except NoNodeError: yield []
class MasterElection(object): def __init__(self): self.zk = KazooClient(hosts=state.ARGS.zookeeper) self.master_notified = False self.my_node = None self.zk.start() # Stop never called self.zk.ensure_path(ELECTION_PATH) def start_election(self, master_callback): """ Start the master election. If this node is the master, the callback will be called once. :params master_callback: Called if this node is the master """ self.callback = master_callback self.my_node = self.zk.create(ELECTION_PATH + '/n_', ephemeral=True, sequence=True) self.zk.get_children(ELECTION_PATH, watch=self._election_update) self._election_update() def _election_update(self, data=None): def worker(): try: self.master_notified = True self.callback() except Exception as e: self.master_notified = False log.info("Failed to activate master, redoing election: %r", e) self.zk.delete(self.my_node) self.my_node = self.zk.create(ELECTION_PATH + '/n_', ephemeral=True, sequence=True) self._election_update() if not self.master_notified and \ sorted(self.zk.get_children(ELECTION_PATH))[0] == \ os.path.basename(self.my_node): t = threading.Thread(target=worker) t.daemon = True t.start()
def test_init_servers(server_id, host = '127.0.0.1', port = 2181): zookeeper = KazooClient('%s:%s' % (host, port,)) zookeeper.start() try: node = '/test/server_list' if zookeeper.exists(node) is None: zookeeper.create(node, json.dumps({'update_time' : time.time()}), makepath = True) except kazoo.exceptions.NodeExistsError: pass try: node = '/test/server_list/s%s' % server_id zookeeper.delete(node) zookeeper.create(node, json.dumps({ 'update_time' : time.time(), 'server_name' : 's%s' % server_id, 'server_id' : server_id, }), makepath = True) except kazoo.exceptions.NodeExistsError: pass
def main(): (options, args) = parse_options(sys.argv[1:]) data = options.znode_data_size * b"D" s = KazooClient(options.server) s.start() if s.exists(options.root_znode): children = s.get_children(options.root_znode) print("delete old entries: %d" % len(children)) for child in children: s.delete("%s/%s" % (options.root_znode, child)) else: s.create(options.root_znode, "kazoo root znode") evaluation(s, options.root_znode, data, options) s.stop() print("Performance test complete")
class ZookClient(object): def __init__(self, api_client=None): logging.basicConfig() # Create a client and start it self.zk = KazooClient() self.zk.start() def create_accounts_path(self, name, **kwargs): path = "/dso/" + name self.zk.ensure_path(path) self.zk.set(path, b"id: 7b4235ca-00fb-4dca-ad3e-8b6e3662631a\ngroupname: hr\ndescription: 人力资源") def create_accountinfo_path(self, account_id, **kwargs): self.zk.ensure_path("/app/someservice") def create_path(self, path, **kwargs): self.zk.ensure_path(path) def get_data(self, path): return self.zk.get(path) def test_tran(self): self.zk.delete("/app22") self.zk.create("/app22", b"" + '{"12": "12"}') tran = self.zk.transaction() tran.delete("/app22") tran.create("/app22", b"" + '{"22": "22"}') tran.commit() print "commit" def stop(self): # In the end, stop it self.zk.stop()
def job_submitter_main(): try: zk = init() cpool = ConnectionPool(host='localhost', port=6379, db=0) r = Redis(connection_pool=cpool) added = 0 tried = 0 max_add_try = 5000 jobname = uuid.uuid4().hex added_nums = set() while tried < max_add_try: value = randint(5000, 90000000) tried += 1 if value not in added_nums: added_nums.add(value) else: continue while True: try: r.lpush(jobname, value) added += 1 break except Exception as e: sleep(1) print "Lpush ", jobname, e zk = KazooClient(hosts='127.0.0.1:2181') zk.add_listener(state_listener) zk.start() value = SUBMITTED + "=" + str(added) zk.create('/jobs/' + jobname, value=value) zk.stop() except Exception as e: print 'Big problem in submitting job ', e sys.exit(1) print 'Job submitted ' + jobname
def join_peacock_network(): """ 주키퍼에 watch를 등록하고 해당 노드를 가입시킨다. """ from config import ZOOKEEPER_HOST zk = KazooClient(hosts=ZOOKEEPER_HOST) zk.start() zk.ChildrenWatch("/peacock/job/nodes", job_node_watch) my_node = zk.create("/peacock/job/nodes/node", ephemeral=True, sequence=True, makepath=True) global my_node_name my_node_name = my_node[my_node.rfind('/') + 1:] job_node_watch(zk.get_children("/peacock/job/nodes"))
def main(): zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0) zk.add_listener(my_listener) zk.start() if zk.exists("/ELECTION") == None: zk.ensure_path("/ELECTION") c = 1 node_pathes = [] while c < 10: c += 1 node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True) node_pathes.append(node_path) my_path = random.choice(node_pathes) my_path = my_path.replace("/ELECTION/", "") # print "print my_path", my_path children = zk.get_children("/ELECTION/", watch=election_child_watcher) get_next_path = False prev_path = None for child_path in sorted(children): if child_path == my_path: break prev_path = child_path # I'm the leader if prev_path == None: print "OK I'm leader don't have to watch" return # fires twice, once on creation ignore @zk.DataWatch("/ELECTION/" + prev_path) def watch_node(data, stat): # only watch for first change if stat.version == 1: print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8"))) print "setting watch on " + prev_path print "my", my_path zk.set("/ELECTION/" + prev_path, b"some data") print "boom. watch triggered?" # time.sleep(10) print "bye" zk.stop()
class TestSessions(unittest.TestCase): def setUp(self): from kazoo.client import KazooClient from kazoo.protocol.states import KazooState from kazoo.testing.common import ZookeeperCluster ZK_HOME = os.environ.get("ZOOKEEPER_PATH") ZK_CLASSPATH = os.environ.get("ZOOKEEPER_CLASSPATH") self.cluster = ZookeeperCluster(ZK_HOME, size=1, port_offset=21000, classpath=ZK_CLASSPATH) self.cluster.start() atexit.register(lambda cluster: self.cluster.terminate(), self.cluster) self.client = KazooClient(self.cluster[0].address, max_retries=5) self.ev = threading.Event() def back(state): if state == KazooState.CONNECTED: self.ev.set() self.client.start() self.path = self.client.create(uuid.uuid4().hex) self.client.add_listener(back) def test_restarted_server(self): raise SkipTest('Patch missing') self.cluster.stop() self.cluster.start() self.ev.wait(5) eq_(self.ev.is_set(), True) self.assertTrue(self.client.retry(self.client.exists, self.path)) def test_terminated_server(self): raise SkipTest('Patch missing') self.cluster.reset() self.cluster.start() self.ev.wait(5) eq_(self.ev.is_set(), True) self.assertFalse(self.client.retry(self.client.exists, self.path)) def tearDown(self): self.ev.clear() self.client.stop() self.cluster.stop()