def __init__(self, inception, zk_storage=False): # zk_storage: Decide whether to use zookeeper (True) or not (False) self.zk_storage = zk_storage self.inception = inception if self.zk_storage: # Flag indicating master/slave role self.master_ctl = False self.pos_path = "/pos" self.arp_path = "/arp" self.queue_path = "/queue" self.leader_path = "/election" self.pktin_path = "/log/packet_in" self.rpc_path = "/log/rpc" self.digest_to_pktin = {} zk_logger = logging.getLogger('kazoo') zk_log_level = log.LOG_LEVELS[CONF.zk_log_level] zk_logger.setLevel(zk_log_level) console_handler = logging.StreamHandler() console_handler.setLevel(zk_log_level) console_handler.setFormatter(logging.Formatter(CONF.log_formatter)) zk_logger.addHandler(console_handler) self.zk = client.KazooClient(hosts=CONF.zk_servers, logger=zk_logger) self.zk.start() self.zk.ensure_path(self.pos_path) self.zk.ensure_path(self.arp_path) self.zk.ensure_path(self.pktin_path) self.zk.ensure_path(self.rpc_path) self.pkt_queue = self.zk.LockingQueue(self.queue_path) self.thread_pkt = hub.spawn(self.handle_pkt_queue) hub.spawn(self.run_for_leader)
def _make_client(self, parsed_url, options): # Creates a kazoo client, # See: https://github.com/python-zk/kazoo/blob/2.2.1/kazoo/client.py # for what options a client takes... maybe_hosts = [parsed_url.netloc] + list(options.get('hosts', [])) hosts = list(compat_filter(None, maybe_hosts)) if not hosts: hosts = ['localhost:2181'] randomize_hosts = options.get('randomize_hosts', True) client_kwargs = { 'hosts': ",".join(hosts), 'timeout': float(options.get('timeout', self.timeout)), 'connection_retry': options.get('connection_retry'), 'command_retry': options.get('command_retry'), 'randomize_hosts': strutils.bool_from_string(randomize_hosts), } handler_kind = options.get('handler') if handler_kind: try: handler_cls = self.HANDLERS[handler_kind] except KeyError: raise ValueError("Unknown handler '%s' requested" " valid handlers are %s" % (handler_kind, sorted(self.HANDLERS.keys()))) client_kwargs['handler'] = handler_cls() return client.KazooClient(**client_kwargs)
def make_client(conf): """Creates a kazoo client given a configuration dictionary.""" # See: http://kazoo.readthedocs.org/en/latest/api/client.html client_kwargs = { 'read_only': bool(conf.get('read_only')), 'randomize_hosts': bool(conf.get('randomize_hosts')), } # See: http://kazoo.readthedocs.org/en/latest/api/retry.html if 'command_retry' in conf: client_kwargs['command_retry'] = conf['command_retry'] if 'connection_retry' in conf: client_kwargs['connection_retry'] = conf['connection_retry'] hosts = _parse_hosts(conf.get("hosts", "localhost:2181")) if not hosts or not isinstance(hosts, six.string_types): raise TypeError("Invalid hosts format, expected " "non-empty string/list, not '%s' (%s)" % (hosts, type(hosts))) client_kwargs['hosts'] = hosts if 'timeout' in conf: client_kwargs['timeout'] = float(conf['timeout']) # Kazoo supports various handlers, gevent, threading, eventlet... # allow the user of this client object to optionally specify one to be # used. if 'handler' in conf: client_kwargs['handler'] = conf['handler'] return client.KazooClient(**client_kwargs)
def __init__(self): self._my_client = kz_client.KazooClient(hosts='127.0.0.1:2181', timeout=5) self._my_client.add_listener(CentralAgent.my_listener) self._my_resources = [] self._my_id = str(uuid.uuid4()) print("Agent id: %s" % self._my_id)
def zk_connection(zk_hosts): zk = zk_client.KazooClient(hosts=zk_hosts) try: zk.start() yield zk finally: zk.stop()
def __init__(self, zk_hosts, group_name, consumer_name, allocator_fn, on_rebalance=None): self.zk_hosts = zk_hosts self.group_name = group_name self.consumer_name = consumer_name self.allocator_fn = allocator_fn self.on_rebalance = on_rebalance self.conn = client.KazooClient(hosts=",".join(self.zk_hosts)) self.connected = threading.Event() self.members = set() self.members_collected = threading.Event() self.party = Party(self.conn, self.consumer_name, self.members_path, on_change=self.on_group_members_change) self.partitions = set() self.partitions_collected = threading.Event() self.shared_set = SharedSet(self.conn, self.partition_path, on_change=self.on_partition_change) self.mapping = {}
def connection(): zk = client.KazooClient(hosts=CONF.zookeeper.host) try: zk.start() yield zk finally: zk.stop()
def zoo_client(nodes_ips): hosts_list = [] contrail_controllers_fqdns = settings.CONTRAIL_ROLES_DISTRIBUTION[ settings.ROLE_CONTRAIL_CONTROLLER] for name in nodes_ips: if name in contrail_controllers_fqdns: hosts_list.append("{}:{}".format(nodes_ips[name][0], settings.ZOOKEEPER_PORT)) return client.KazooClient(hosts=','.join(hosts_list))
def connect_to_zookeeper_queue_backend(conf): """Connect to a zookeeper cluster""" storage_backend_hosts = ','.join([ '%s:%s' % (host, conf.queue_backend_port) for host in conf.queue_backend_host ]) zk_client = client.KazooClient(storage_backend_hosts) zk_client.start() return zk_client
def get_zookeeper_client(zookeeper_hosts): # kazoo requires a comma separated string. if isinstance(zookeeper_hosts, list): host_string = u",".join(zookeeper_hosts) elif isinstance(zookeeper_hosts, string_types): host_string = zookeeper_hosts else: raise LockException("zookeeper_hosts arg must be a string or list of strings") logger.info(u"zookeeper hosts={0}".format(host_string)) return kazoo_client.KazooClient(hosts=host_string)
def get_zkclient(self): """ 创建kazoo.client.KazooClient实例 :return: kazoo.client.KazooClient实例 :rtype: kazoo.client.KazooClient :raises: None """ hosts = config.GuardianConfig.get(STATE_SERVICE_HOSTS) zkc = client.KazooClient(hosts=hosts) return zkc
def main(): zk = client.KazooClient(os.environ["ZK_CONNECT"]) zk.retry(zk.start) try: fqdn = socket.getfqdn() for broker_id in zk.retry(zk.get_children, "/brokers/ids"): data = zk.retry(zk.get, "/brokers/ids/" + broker_id)[0] if fqdn == json.loads(data.decode("UTF-8"))["host"]: sys.exit(0) sys.exit(1) finally: zk.retry(zk.stop)
def _lazy_initialize(self): if not self.client: hosts = _parse_hosts(self.config.remote_db_hosts) _handler = eventlet.SequentialEventletHandler() _retry = retry.KazooRetry(max_tries=CLIENT_CONNECTION_RETRIES, delay=0.5, backoff=2, sleep_func=_handler.sleep_func) self.client = client.KazooClient(hosts=hosts, handler=_handler, connection_retry=_retry) self.client.start() self.client.ensure_path(ROOT_NS)
def check_for_zookeeper(timeout=1): sys.stderr.write("Testing for the existence of a zookeeper server...\n") sys.stderr.write("Please wait....\n") with contextlib.closing(client.KazooClient()) as test_client: try: test_client.start(timeout=timeout) except test_client.handler.timeout_exception: sys.stderr.write("Zookeeper is needed for running this example!\n") traceback.print_exc() return False else: test_client.stop() return True
def zk_connection(url): # support an environment and url # if url, it should be like this: # zk://<address>:<port>/<path> zk_hosts = os.environ.get("KOLLA_ZK_HOSTS") if zk_hosts is None: components = parse.urlparse(url) zk_hosts = components.netloc zk = kz_client.KazooClient(hosts=zk_hosts) zk.start() try: yield zk finally: zk.stop()
def _new_session(cls): """ 创建kazoo.client.KazooClient实例 :return: kazoo.client.KazooClient实例 :rtype: kazoo.client.KazooClient :raises: exception.EPConnectTimeout 连接超时异常 """ # 仅在必要的情况下才引入kazoo from kazoo import client hosts = config.GuardianConfig.get(config.STATE_SERVICE_HOSTS_NAME) params = json.loads( config.GuardianConfig.get(cls.PERSIST_PARAMETERS_NAME, '{}')) return ZkPersistence._run_catch( lambda: (client.KazooClient(hosts=hosts, **params)))
def checkConsistencyAndPush(): #Hardcoded . Read from ConfigFile myClusterId = 1 myIp = "172.17.0.2" lockAcquired = False lock = None db_name = "" clientIndex = request.values dbName = clientIndex['databaseName'] #Check if dbRepo already exists in sqlDatabase db = DB.retreiveDbClusterMapping("db1") if (len(db) != 0): create = 1 #check if Db locally present clusterId = db[0][1] if clusterId == myClusterId: serverIP = myIp else: serverIPList = DB.retreiveServerClusterMapping(clusterId) i = random.randint(0, len(serverIPList)) serverIP = serverIPList[i][2] # Index file should be from right server indexFile = "/Source/Repo/db1/.mvc/index.json" #check for consistency and send OK response with IP of right Server who has DB. isConsistent = utilities.checkConsistency(clientIndex, indexFile) if (isConsistent is True): zk_client = C.KazooClient(hosts=serverIP) zk_client.start() lock = zk_client.Lock(dbName) lockAcquired = lock.acquire() #Maintain an entry in the DB for commit #probe all server for completion #if consistent state reached release the lock #return with success response print(serverIP, lockAcquired) if lock is not None: lock.release() #Else return Reject message return "Working Correctly"
def __init__(self, mode, ip_address=None, zk_address=None, strength=0, logfile='log/pub.log', pub_name=None, zk_root=''): self.ip_address = ip_address self.zk_address = zk_address self.strength = strength self.my_client = kz_client.KazooClient(hosts=zk_address) self.my_client.start() self.zk_root = zk_root self.broker_address = None self.pub_mw = None self.mode = mode self.exited = False self.logger = get_logger(logfile) self.pub_name = pub_name
def getBrokerList(zkURL, node_dir='/idmm2/httpbroker'): logging.basicConfig() brokerlist = [] print '%s' % zkURL try: zk = client.KazooClient(hosts='%s' % zkURL) zk.start() addr_list = zk.get_children(node_dir) print 'addr_list=%s' % addr_list #if len(addr_list)>0: # for id in addr_list: # tmp_path = node_dir + '/' + id # data,stat = zk.get('%s' %tmp_path) # d = data.split("//")[1] # addr = d.split("/")[0] # brokerlist.append(addr) zk.stop() return addr_list except Exception, e: print traceback.format_exc()
def _make_client(self, parsed_url, options): # Creates a kazoo client, # See: https://github.com/python-zk/kazoo/blob/2.2.1/kazoo/client.py # for what options a client takes... if parsed_url.username and parsed_url.password: username = parsed_url.username password = parsed_url.password digest_auth = "%s:%s" % (username, password) digest_acl = security.make_digest_acl(username, password, all=True) default_acl = (digest_acl,) auth_data = [('digest', digest_auth)] else: default_acl = None auth_data = None maybe_hosts = [parsed_url.netloc] + list(options.get('hosts', [])) hosts = list(compat_filter(None, maybe_hosts)) if not hosts: hosts = ['localhost:2181'] randomize_hosts = options.get('randomize_hosts', True) client_kwargs = { 'hosts': ",".join(hosts), 'timeout': float(options.get('timeout', self.timeout)), 'connection_retry': options.get('connection_retry'), 'command_retry': options.get('command_retry'), 'randomize_hosts': strutils.bool_from_string(randomize_hosts), 'auth_data': auth_data, 'default_acl': default_acl, } handler_kind = options.get('handler') if handler_kind: try: handler_cls = self.HANDLERS[handler_kind] except KeyError: raise ValueError("Unknown handler '%s' requested" " valid handlers are %s" % (handler_kind, sorted(self.HANDLERS.keys()))) client_kwargs['handler'] = handler_cls() return client.KazooClient(**client_kwargs)
def setUp(self): self.kzclient = kzcl.KazooClient('127.0.0.1:2181')
def _get_zk_client(): c = client.KazooClient(hosts=",".join(ZK_HOST_LIST)) c.start() return c
import os import sys import requests from flask import jsonify, request, make_response, send_from_directory,send_file from kazoo import client as kz_client import logging ROOT_PATH = os.path.dirname(os.path.realpath(__file__)) os.environ.update({'ROOT_PATH': ROOT_PATH}) sys.path.append(os.path.join(ROOT_PATH, 'modules')) from app import app PORT = os.environ.get('PORT') NODE_PATH = "/web" kz = kz_client.KazooClient('ZK') def my_listener(state): if state == kz_client.KazooState.LOST: # Register somewhere that the session was lost print("State: LOST!") elif state == kz_client.KazooState.SUSPENDED: # Handle being disconnected from Zookeeper print("State: SUSPENDED!") else: print("State: CONNECTED!") print("END OF ELSE!") def make_zk_node(): try:
import kazoo.client as kc import socket import datetime as dt zk = kc.KazooClient(hosts='zk1.staging.srv:2181 ') zk.start() def update(): zk.ensure_path('/users/stansun') zk.ensure_path('/users/stansun/ipmapping') zk.ensure_path('/users/stansun/ipmapping/stansun-pc') zk.ensure_path('/users/stansun/ipmapping/stansun-pc-lastupdate') ip = [ l for l in ([ ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.") ][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)] ][0][1]]) if l ][0][0] #print("Set ip to %s"%ip) zk.set('/users/stansun/ipmapping/stansun-pc', ip.encode()) zk.set('/users/stansun/ipmapping/stansun-pc-lastupdate', dt.datetime.now().strftime("%c").encode()) def get_latest(): latestip = zk.get('/users/stansun/ipmapping/stansun-pc')
total = 3 ss = {} cs = {} r1 = {} r2 = {} r3 = {} payload = {"hi": "bye"} y = json.dumps(payload) from kazoo import client as kz_client count = 1 my_client = kz_client.KazooClient(hosts='127.0.0.1:2181') def my_listener(state): if state == kz_client.KazooState.CONNECTED: print("Client connected !") my_client.add_listener(my_listener) my_client.start(timeout=5) a = my_client.create("/parent/" + sys.argv[1], ephemeral=True) print(a) b = my_client.get_children("/parent") print(b) for a in b: print(a)
from kazoo import client from vyper import v client = client.KazooClient() client.start() v.set_config_type("json") v.add_remote_provider("zookeeper", client, "/config.json") v.read_remote_config() print("Hello " + v.get("hello"))
import socket from kazoo import client as kz_client import logging import time import json import uuid logging.basicConfig() import datetime #hosts=ip of the master instance/container followed by the port that he zk is running #change the hosts ip addres. my_client = kz_client.KazooClient(hosts='172.17.0.2:2181') #function to check whether the client has connected to zookeeper or not def my_listener(state): if state == kz_client.KazooState.CONNECTED: print("Client is connected to the SERVER !!!") my_client.add_listener(my_listener) #starting the client my_client.start() t = str(datetime.datetime.today()) #getting the mac address of the instance l = (':'.join([ '{:02x}'.format((uuid.getnode() >> ele) & 0xff) for ele in range(0, 8 * 6, 8)
def __init__(self, member_id, parsed_url, options): super(KazooDriver, self).__init__(member_id, parsed_url, options) self._coord = client.KazooClient(hosts=parsed_url.netloc) self._member_id = member_id
def make_client(conf): """Creates a `kazoo`_ `client`_ given a configuration dictionary. :param conf: configuration dictionary that will be used to configure the created client :type conf: dict The keys that will be extracted are: - ``read_only``: boolean that specifies whether to allow connections to read only servers, defaults to ``False`` - ``randomize_hosts``: boolean that specifies whether to randomize host lists provided, defaults to ``False`` - ``command_retry``: a kazoo `retry`_ object (or dict of options which will be used for creating one) that will be used for retrying commands that are executed - ``connection_retry``: a kazoo `retry`_ object (or dict of options which will be used for creating one) that will be used for retrying connection failures that occur - ``hosts``: a string, list, set (or dict with host keys) that will specify the hosts the kazoo client should be connected to, if none is provided then ``localhost:2181`` will be used by default - ``timeout``: a float value that specifies the default timeout that the kazoo client will use - ``handler``: a kazoo handler object that can be used to provide the client with alternate async strategies (the default is `thread`_ based, but `gevent`_, or `eventlet`_ ones can be provided as needed) .. _client: http://kazoo.readthedocs.org/en/latest/api/client.html .. _kazoo: http://kazoo.readthedocs.org/ .. _retry: http://kazoo.readthedocs.org/en/latest/api/retry.html .. _gevent: http://kazoo.readthedocs.org/en/latest/api/\ handlers/gevent.html .. _eventlet: http://kazoo.readthedocs.org/en/latest/api/\ handlers/eventlet.html .. _thread: http://kazoo.readthedocs.org/en/latest/api/\ handlers/threading.html """ # See: http://kazoo.readthedocs.org/en/latest/api/client.html client_kwargs = { 'read_only': bool(conf.get('read_only')), 'randomize_hosts': bool(conf.get('randomize_hosts')), } # See: http://kazoo.readthedocs.org/en/latest/api/retry.html if 'command_retry' in conf: client_kwargs['command_retry'] = conf['command_retry'] if 'connection_retry' in conf: client_kwargs['connection_retry'] = conf['connection_retry'] hosts = _parse_hosts(conf.get("hosts", "localhost:2181")) if not hosts or not isinstance(hosts, six.string_types): raise TypeError("Invalid hosts format, expected " "non-empty string/list, not '%s' (%s)" % (hosts, type(hosts))) client_kwargs['hosts'] = hosts if 'timeout' in conf: client_kwargs['timeout'] = float(conf['timeout']) # Kazoo supports various handlers, gevent, threading, eventlet... # allow the user of this client object to optionally specify one to be # used. if 'handler' in conf: client_kwargs['handler'] = conf['handler'] return client.KazooClient(**client_kwargs)
from kazoo import client as kz_client from collections import defaultdict zkserver = 'host5:12181' #'ubuntu202:2182' my_client = kz_client.KazooClient(hosts=zkserver) def getValue(instance, parent_path, zk_client): temp_p = parent_path + '/' + str(instance) if zk_client.exists(temp_p): job_instance = zk_client.get(temp_p + '/instance') print "===> ", job_instance[0] return job_instance[0] else: return "" def my_listener(state): if state == kz_client.KazooState.CONNECTED: print("Client connected !") my_client.add_listener(my_listener) my_client.start(timeout=5) #job-v1 (job in yoo) #job (job in yoo-job) job_node = "job-v1" data = my_client.get_children("/" + job_node)