Exemplo n.º 1
0
class ZooWrap():
    def __init__(self):
        self.zk = KazooClient(hosts="%s,%s" % (
            str(cc.conf['zookeeper']['host']),
            str(cc.conf['zookeeper']['port'])))
        self.zk.start()
        self.root = cc.conf['zookeeper']['rootpath']
        self.log = logging.getLogger('L.ZOOKEEPER')
        self.zk.ensure_path('/%s/sleeping' % (self.root))
        self.whoami = cc.conf['whoami']

    def get_sleeping(self):
        return self.zk.get_children('/%s/sleeping' % (self.root))

    def sleep(self):
        try:
            self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
            self.log.info('Sleeping correctly')
        except NodeExistsError:
            self.log.error('Node already sleeping... seems weird')

    def wake(self):
        try:
            self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
        except NoNodeError:
            self.log.error('Node was not sleeping... seems weird')
Exemplo n.º 2
0
class Zookeeper(KeyManager):
    def __init__(self, hosts):
        self._hosts = hosts
        self.zk = KazooClient(hosts=hosts)
        self.zk.start()

    def get(self, key):
        result = self.zk.get(key)[0]
        if result == "":
            result = []
            children = self.zk.get_children(key)
            for i in children:
                result.append({'name': i, 'value': self.zk.get(os.path.join(key, i))[0]})
            return result
        else:
            return self.zk.get(key)[0]

    def set(self, key, data):
        try:
            self.zk.set(key, data.encode('utf-8'))
        except:
            self.zk.create(key, data.encode('utf-8'))

    def mkdir(self, key):
        self.set(key, "")

    def close(self):
        self.zk.stop()
        self.zk.close()

    @property
    def hosts(self):
        return self._hosts
Exemplo n.º 3
0
def add_and_update(key, value):
    print(key, value, '#' * 20)
    zk = KazooClient(hosts='127.0.0.1:2181')
    zk.start()
    # children = zk.get_children('/')

    # 创建节点:makepath 设置为 True ,父节点不存在则创建,其他参数不填均为默认
    # zk.create('/zookeeper/goodboy')
    # 操作完后,别忘了关闭zk连接

    # zk.stop()
    print('#' * 20)
    data, stat = zk.get(key)
    print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
    if zk.exists(key):
        zk.set(key, value)
        print('@' * 10, 'setting')
    else:
        zk.create(key, value, makepath=True)
    # zk.create(key)

    # print(value, type(value),'#'*10)
    # if value != b"":
    #     zk.set(key, value)
    zk.stop()
Exemplo n.º 4
0
def zk_flag_set(zk: KazooClient, name: str, value: str):
    path = "{}/{}".format(ZK_PREFIX, name)
    data = value.encode('utf-8')
    try:
        zk.set(path, data)
    except NoNodeError:
        zk.create(path, data, makepath=True)
Exemplo n.º 5
0
class KazooService():

    zk = None
    rootPath = None

    def __init__(self, rootPath):

        qs = Zookeeper.objects.filter().first()
        if qs is None:
            raise LookupError("Unable to find Zookeeper configuration!")

        self.zk = KazooClient(hosts=qs.url)
        self.rootPath = rootPath

        self.zk.start(timeout=qs.timeout)
        self.zk.ensure_path(rootPath)

    def createOrUpdate(self, pathAffix, nodeValue):

        if isinstance(nodeValue, str):
            bValue = bytes(nodeValue, 'utf-8')
        else:
            bValue = bytes(str(nodeValue), 'utf-8')

        path = self.rootPath + pathAffix

        if self.zk.exists(path) is None:
            self.zk.create(path, bValue)
        else:
            self.zk.set(path, bValue)

    def __del__(self):

        if self.zk is not None:
            self.zk.stop()
Exemplo n.º 6
0
class ServicePublisher:
    def __init__(self, hosts, timeout, publish_port):
        self._logger = logging.getLogger(self.__class__.__name__)
        self._publish_port = publish_port
        self._zk = KazooClient(hosts=hosts)
        event = self._zk.start_async()
        event.wait(timeout=timeout)

        if self._zk.connected:
            self._logger.info('Kazoo client successfully connected')
            self._publish_status()
        else:
            self._zk.stop()
            self._logger.error('Kazoo client failed to connect')

    def _publish_status(self):
        full_path = '%s/%s' % (PARENT_NODE, socket.gethostname())
        data = {
            'started': str(datetime.datetime.now())[:19],
            'port': self._publish_port
        }
        json_data = json.dumps(data).encode(encoding='utf-8')
        self._logger.info('Publishing status %s to path %s' %
                          (data, full_path))
        self._zk.create(full_path, json_data, ephemeral=True, makepath=True)
Exemplo n.º 7
0
class DistributedLock:
    def __init__(self, hosts, logger=None):
        self.client = KazooClient(hosts=hosts, logger=logger)

    def ensure_path(self, path):
        self.client.start()
        self.client.ensure_path(path)
        self.client.stop()

    def try_lock(self, path):
        try:
            self.client.start()
            self.client.create(path, ephemeral=True)
            return True
        except:
            logging.exception("try lock exception. path: {}".format(path))
            return False

    def try_unlock(self, path):
        try:
            self.client.delete(path)
            self.client.stop()
            return True
        except:
            logging.exception("try unlock exception. path: {}".format(path))
            return False
Exemplo n.º 8
0
class Store(object):
    def __init__(self,**kwargs):
        self.config = kwargs
        self.client = None

    def get_client(self):
        return self.client

    def open(self):
        self.client = KazooClient(**self.config)
        self.client.add_listener
        self.client.start()

    def close(self):
        self.client.stop()

    def read(self,path):
        return self.client.get(path)

    def write(self,path,value):
        base_path = os.path.dirname(path)
        self.client.ensure_path(base_path)
        self.client.create(path,value)

    def overwrite(self,path,value):
        self.client.set(path,value)

    def exists(self,path):
        return self.client.exists(path)
Exemplo n.º 9
0
class ES_zk_interface:
    def __init__(self, my_ip, es_interface, es_callback):
        self.ip = my_ip
        self.es_callback = es_callback
        self.es_interface = es_interface
        self.zk = KazooClient(hosts=HOST_IP)
        self.zk.start()

        self.zk.ensure_path("/ass3")

    def start_election_process(self):
        print "Starting Election process"
        #print("Previous Leader: %s" % (self.es_callback.get_leader()))
        election = self.zk.Election("/electionpath", self.ip)
        election.run(self.leader_function)

    def leader_function(self):
        print "Won the election: " + str(self.ip)
        if self.zk.exists(LEADER_PATH):
            self.zk.set(LEADER_PATH, self.ip)
        else:
            self.zk.create(LEADER_PATH, self.ip)

        #print "\nStarting Leader thread.... \n"
        self.es_interface.enter_message_loop(self.es_callback, True)

    def get_zookeeper_client(self):
        return self.zk

    def get_leader_path(self):
        return LEADER_PATH
Exemplo n.º 10
0
def test_pub_response(pub_node, server_list, host='127.0.0.1', port=2181):
    zookeeper = KazooClient('%s:%s' % (
        host,
        port,
    ))
    zookeeper.start()

    for s in server_list:
        if isinstance(s, int) is True:
            continue

        try:
            node = '/test/to_pub_result/%s/%s' % (
                pub_node,
                s,
            )
            if zookeeper.exists(node) is None:
                zookeeper.create(node,
                                 json.dumps({
                                     'update_time': time.time(),
                                     'status': 'ok'
                                 }),
                                 makepath=True)
        except kazoo.exceptions.NodeExistsError:
            pass
Exemplo n.º 11
0
def connect(hosts="127.0.0.1:2181"):

    retval = KazooClient(hosts=hosts)

    def my_listener(state):
        if state == KazooState.LOST:
            # Register somewhere that the session was lost
            logging.warn("Lost connection to Zookeeper")

        elif state == KazooState.SUSPENDED:
            # Handle being disconnected from Zookeeper
            logging.warn("Zookeeper connection suspended")

        elif state == KazooState.CONNECTED:
            pass

        else:
            # Handle being connected/reconnected to Zookeeper
            logging.info("Other state: %s" % state)

    retval.add_listener(my_listener)
    retval.start()

    if not retval.exists(key):
        logging.info("Created '%s'" % key)
        retval.create(key)

    return (retval)
Exemplo n.º 12
0
class ZKBase(object):
    """
    scheduler需要定义以下函数:
    - callback(KazooState.*): 处理CONN: LOST,SUSPENDED
    - 
    """

    def __init__(self,conf,scheduler):
        self.conf = conf
        self.state = "STOPPED" 
        self.zk = KazooClient(self.conf.processor.zk_url, timeout=self.conf.processor.timeout)
        try:
            self.zk.start()
        except Exception:
            LOG.error("Start zk error: %s" % traceback.format_exc(limit=2))
        self.zk.add_listener(self.listen)
        self.scheduler = scheduler
        self.state = "RUNNING" # STOPPED
        self.state_changes = False
        
    def listen(self, state):
        if state == KazooState.LOST or state == KazooState.SUSPENDED:
            LOG.error("Session state change: %s" % state)
            if self.state == "STOPPED":
                return
            self.state_changes = True
            self.scheduler.callback(state)
        elif state == KazooState.CONNECTED:
            LOG.info("Connectied to ZK.")
        else:
            LOG.error("Session timeout. Cannot conect to ZK.")

    def reinit(self,state):
        while self.zk.state != KazooState.CONNECTED:
            LOG.info("Restart zk connection until connected.")
            try:
                self.zk.restart()
            except Exception:
                LOG.error("Reinit: %s" % traceback.format_exc(limit=1))
        self.state_changes = False

    def create(self,path, value='', acl=None, ephemeral=False, sequence=False, makepath=False):
        LOG.info("Create: path %s" % path)
        self.zk.create(path=path,
                        value=value.encode(),
                        ephemeral=ephemeral,
                        sequence=sequence,
                        makepath=makepath)

    def get(self,path,watch=None):
        self.zk.get(path,watch)

    def get_children(self,path,watch=None):
        return self.zk.get_children(path,watch=watch)

    def terminate(self):
        LOG.info("Terminate ZK connection.")
        self.state = "STOPPED"
        self.zk.stop()
        self.zk.close()
Exemplo n.º 13
0
class ZKState():
    def __init__(self, path, timeout=30):
        super(ZKState, self).__init__()
        self._zk = KazooClient(hosts=ZK_HOSTS, timeout=timeout)
        self._zk.start(timeout=timeout)
        self._path = path
        self._zk.ensure_path(path)

    def processed(self):
        return self._zk.exists(self._path + "/complete")

    def process_start(self):
        if self.processed():
            return False
        if self._zk.exists(self._path + "/processing"):
            return False
        try:
            self._zk.create(self._path + "/processing", ephemeral=True)
            return True
        except NodeExistsError:  # another process wins
            return False

    def process_end(self):
        self._zk.create(self._path + "/complete")
        self._zk.delete(self._path + "/processing")

    def process_abort(self):
        try:
            self._zk.delete(self._path + "/processing")
        except NoNodeError:
            pass

    def close(self):
        self._zk.stop()
        self._zk.close()
Exemplo n.º 14
0
def main():
    logging.basicConfig()
    zk = KazooClient(hosts=zookeeper_hostname)
    zk.start()
    zk.ensure_path("/worker")
    node_name = "/worker/" + os.environ["NODE_NAME"]
    if not zk.exists(node_name):
        msg = "Creating node: " + node_name
        print(msg, file=sys.stdout)
        db_name = os.environ["DB_HOSTNAME"]
        zk.create(node_name, db_name.encode(), ephemeral=True)

    if os.environ["WORKER_TYPE"] == "master":
        rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True)
        rpc_server.start()
    else:
        try:
            if node_name != "/worker/slave1":
                master_db = zk.get("/worker/master")[0].decode()
                print("[*] Cloning database from master db: " + master_db, file=sys.stdout)
                subprocess.call(
                    "mongodump --host " + master_db + " --port 27017 --db rideshare && mongorestore --host " + os.environ[
                        'DB_HOSTNAME'] + " --port 27017",
                    stdout=sys.stdout,
                    stderr=sys.stdout,
                    shell=True
                )
        except Exception as e:
            print(e, file=sys.stdout)

        old_name = os.environ["NODE_NAME"]
        p1 = multiprocessing.Process(target=slave_rpc_server)
        p2 = multiprocessing.Process(target=become_master, args=(p1, old_name,))
        p1.start()
        p2.start()
Exemplo n.º 15
0
    def registry(self, port, service, methods, zks):
        if zks is None:
            raise Exception("zk 地址为空")
        providers_path = "/dubbo/%s/providers" % (service)
        ip = self.local_host_ip()
        provider_url = self.generate_provider_url(host=ip,
                                                  service=service,
                                                  port=port,
                                                  methods=methods)
        self.service_path[service] = "%s/%s" % (providers_path,
                                                quote_plus(provider_url))
        value = str(ip).encode("utf8")

        zk = KazooClient(hosts=zks)
        self.registry_zk[service] = zk
        zk.start()
        #zk.ensure_path(providers_path+"/")
        try:
            zk.delete(self.service_path[service])
        except Exception as e:
            pass
        finally:
            zk.create(self.service_path[service],
                      value,
                      ephemeral=True,
                      sequence=False,
                      makepath=True)
Exemplo n.º 16
0
def zk_init(server_address, zk_node, zk_address = ''):
    #初始化zk节点
    zk_cli = KazooClient(hosts = zk_address)
    try:
        zk_cli.start()
    except:
        logging.error('zk Init error, can not connect %s' %(str(zk_address)))
        return -1
    
    try:
        zk_cli.get(zk_node)
    except:
        logging.warn('can not find zk path %s, creat it' %(str(zk_node)))
        zk_cli.ensure_path(zk_node)

    if zk_node[-1] != '/':
        zk_node += '/'
        
    try:
        zk_cli.create(zk_node + server_address, '1', ephemeral=True)
    except:
        if zk_cli.get(zk_node + server_address):
            return 0
        else:
            logging.error('create zk_node error, can not create node %s' %(str(zk_node) + str(zk_address)))
            return -1
    return 0
Exemplo n.º 17
0
class zkClient():
    def __init__(self):
        self.zk = KazooClient(hosts='10.0.0.2:2181')
        self.zk.start()

    def get_hostname_ip(self):
        host_name = socket.getfqdn(socket.gethostname())
        host_ip = socket.gethostbyname(host_name)
        return "/" + host_name + "_gateway", host_ip

    def register(self):
        name, ip = self.get_hostname_ip()
        if not self.zk.exists(name):
            self.zk.create(name, bytes(ip, 'utf-8'))
        else:
            self.zk.set(name, bytes(ip, 'utf-8'))

    def get_kvs_ips(self):
        device_ips = []
        for device_name in self.zk.get_children('/'):
            if device_name.endswith('_kvs'):
                data, stat = self.zk.get(device_name)
                device_ips.append(data.decode("utf-8"))
        return device_ips

    def get_nodes_ips(self, device_list):
        device_ips = []
        for device_name in self.zk.get_children('/'):
            if device_name in device_list:
                data, stat = self.zk.get(device_name)
                device_ips.append(data.decode("utf-8"))
        return device_ips
Exemplo n.º 18
0
class ZookeeperClient(object):
    def __init__(self, server_list):
        self._retry = KazooRetry(max_tries=None,
                                 max_delay=300,
                                 sleep_func=gevent.sleep)
        self._zk_client = KazooClient(hosts=','.join(server_list),
                                      timeout=400,
                                      handler=SequentialGeventHandler(),
                                      logger=logger,
                                      connection_retry=self._retry,
                                      command_retry=self._retry)

    def connect(self):
        self._zk_client.start()

    def disconnect(self):
        self._zk_client.stop()
        self._zk_client.close()

    def create_node(self, path, value=None):
        if value is None:
            value = uuid.uuid4()
        try:
            self._zk_client.create(path, str(value), makepath=True)
        except NodeExistsError:
            self._zk_client.set(path, str(value))
Exemplo n.º 19
0
class ZooWrap():
    def __init__(self):
        self.zk = KazooClient(hosts="%s,%s" %
                              (str(cc.conf['zookeeper']['host']),
                               str(cc.conf['zookeeper']['port'])))
        self.zk.start()
        self.root = cc.conf['zookeeper']['rootpath']
        self.log = logging.getLogger('L.ZOOKEEPER')
        self.zk.ensure_path('/%s/sleeping' % (self.root))
        self.whoami = cc.conf['whoami']

    def get_sleeping(self):
        return self.zk.get_children('/%s/sleeping' % (self.root))

    def sleep(self):
        try:
            self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
            self.log.info('Sleeping correctly')
        except NodeExistsError:
            self.log.error('Node already sleeping... seems weird')

    def wake(self):
        try:
            self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
        except NoNodeError:
            self.log.error('Node was not sleeping... seems weird')
Exemplo n.º 20
0
class ZKMData(object):
    def __init__(self):
        super(ZKMData, self).__init__()
        self._zk = KazooClient(hosts=ZK_HOSTS)
        self._zk.start()

    def set(self, path, value):
        value = json.dumps(value).encode('utf-8')
        if self._zk.exists(path):
            try:
                self._zk.set(path, value)
            except NoNodeError:
                pass
        try:
            self._zk.create(path, value, makepath=True)
        except NodeExistsError:
            pass

    def get(self, path):
        try:
            value, stat = self._zk.get(path)
            if not value: return {}
            return json.loads(value.decode('utf-8'))
        except Exception as e:
            return {}

    def close(self):
        self._zk.stop()
        self._zk.close()
Exemplo n.º 21
0
class Register(object):
    def __init__(self,
                 value,
                 path='/scrapyd-cluster/worker',
                 hosts='127.0.0.1:2181'):
        self.path = path
        self.value = value
        self.expire = False
        self.zk = KazooClient(hosts)
        self.zk.start()
        self.zk.add_listener(self.listener)

        self.zk.ensure_path(os.path.dirname(self.path))
        self.path = self.zk.create(self.path,
                                   self.value,
                                   ephemeral=True,
                                   sequence=True)
        logger.info('Connected To ZooKeeper(%s) Succeed! Path : %s' %
                    (hosts, self.path))

    def register(self):
        if self.expire:
            logger.info('Renew Ephemeral Node. Path : %s  Value: %s' %
                        (self.path, self.value))
            self.zk.create(self.path, self.value, ephemeral=True)
            self.expire = False

    def listener(self, state):
        if state == KazooState.LOST:
            logger.info('ZooKeeper Connection Lost')
            self.expire = True
        elif state == KazooState.CONNECTED:
            self.zk.handler.spawn(self.register)
Exemplo n.º 22
0
def test_zookeeper(zookeeper_server):
    test_value = str(uuid.uuid4()).encode("utf-8")

    zk_client1 = KazooClient(
        hosts="127.0.0.1:{}".format(zookeeper_server.port))
    zk_client2 = KazooClient(
        hosts="127.0.0.1:{}".format(zookeeper_server.port))
    zk_client1.start()
    zk_client2.start()

    zk_client1.ensure_path("/my")
    assert zk_client1.exists("/you") is None
    assert zk_client2.exists("/you") is None
    assert zk_client1.exists("/my") is not None
    assert zk_client2.exists("/my") is not None

    zk_client1.create("/my/testplan", test_value)
    data, _ = zk_client1.get("/my/testplan")
    assert data == test_value
    data, _ = zk_client2.get("/my/testplan")
    assert data == test_value

    zk_client2.set("/my/testplan", str(uuid.uuid4()).encode("utf-8"))
    data, _ = zk_client1.get("/my/testplan")
    assert data != test_value
    data, _ = zk_client2.get("/my/testplan")
    assert data != test_value
Exemplo n.º 23
0
class ZKStore:
    def __init__(self, hosts):
        self.zk = KazooClient(hosts=hosts)
        self.zk.add_listener(listener)
        self.zk.start()


    def isConnected(self):
        if __state__ == 1:
            return True
        return False


    def write(self, path, node, value):
        self.zk.ensure_path(path)
        if self.zk.exists(path+"/"+node):
           self.zk.set(path+"/"+node, value)
        else:
           self.zk.create(path + "/" + node, value)


    def read(self, path):
        if self.zk.exists(path):
            data, stat = self.zk.get(path)
            return data
        return None
Exemplo n.º 24
0
def add_node_to_zookeeper():
    zk = KazooClient(hosts=zookeeper_hosts)

    @zk.add_listener
    def zookeeper_listener(state):
        if state == KazooState.LOST:
            logging.info("zookeeper connect status:LOST")
            # Register somewhere that the session was lost
        elif state == KazooState.SUSPENDED:
            # Handle being disconnected from Zookeeper
            logging.info("zookeeper connect status:SUSPENDED")
        elif state == KazooState.CONNECTED:
            logging.info("zookeeper connect status:CONNECTED")
        else:
            logging.info(str(state))

    zk.start()
    # 创建“/spiderServer”永久节点,后面每个有效的redis节点,或动态添加redis节点后,都会在
    # “/spiderServer”节点下创建对应的redis服务节点(如:/spiderServer/128.0.0.203),服务节点名称已redis服务所在的机器的ip地址命名
    if zk.exists(SPIDER_SERVER_NODE):
        pass
    else:
        try:
            zk.create(SPIDER_SERVER_NODE)
        except Exception, e:
            if isinstance(e, NodeExistsException):
                pass
            else:
                logging.exception(str(e))
Exemplo n.º 25
0
def test_init_servers(num, host='127.0.0.1', port=2181):
    zookeeper = KazooClient('%s:%s' % (
        host,
        port,
    ))
    zookeeper.start()

    try:
        node = '/test/server_list'
        if zookeeper.exists(node) is None:
            zookeeper.create(node,
                             json.dumps({'update_time': time.time()}),
                             makepath=True)
    except kazoo.exceptions.NodeExistsError:
        pass

    v = 1
    while v <= num:
        try:
            node = '/test/server_list/s%s' % v
            if zookeeper.exists(node) is None:
                zookeeper.create(node,
                                 json.dumps({
                                     'update_time': time.time(),
                                     'server_name': 's%s' % v,
                                     'server_id': v,
                                 }),
                                 makepath=True)
        except kazoo.exceptions.NodeExistsError:
            pass
        v += 1
Exemplo n.º 26
0
class ZookClient():
    def __init__(self, hosts):
        self.kazooclient = KazooClient(hosts=hosts)
        self.kazooclient.start()

    def foundApp(self, appname):  #Ìí¼Óapp
        if self.kazooclient.exists('/' + appname):
            pass
        else:
            self.kazooclient.ensure_path('/' + appname)

    def foundNode(self, appname, port, host):  #Ìí¼Ó½Úµã
        if self.kazooclient.exists('/' + appname + '/' + host + ':' +
                                   str(port)):
            pass
        else:
            self.kazooclient.create('/' + appname + '/' + host + ':' +
                                    str(port))

    def close(self):
        self.kazooclient.close()

    def register(self, appname, port, host):
        self.foundApp(appname)
        self.foundNode(appname, port, host)


#         self.close()
    def test_subscriber_connect(self):
        test_zk = KazooClient(self.zk_sub.zookeeper_connection_url)
        try:
            test_zk.start(5)
        except Exception:
            self.assertRaises("Couldn't connect to zookeeper server")

        # Ensure a path, create if necessary
        test_zk.ensure_path(self.zk_sub.default_node_path)

        fake_host_ip_lst = ["0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.3"]

        try:
            for host_ip in fake_host_ip_lst:
                # Create a node with data
                node = "node" + str(uuid.uuid4())
                path = self.default_node_path + '/' + node
                test_zk.create(path, bytes(host_ip, "utf-8"))

            self.zk_sub.subscriber_connect("subscriber")
        finally:
            nodes_to_delete = test_zk.get_children(self.default_node_path)
            for node in nodes_to_delete:
                path = self.default_node_path + '/' + node
                test_zk.delete(path)

        children = test_zk.get_children(self.default_node_path)
        self.assertEqual(children, [])
Exemplo n.º 28
0
class ZookeeperClient(object):
    def __init__(self, server_list):
        self._retry = KazooRetry(max_tries=None, max_delay=300,
                                 sleep_func=gevent.sleep)
        self._zk_client = KazooClient(
            hosts=','.join(server_list),
            timeout=400,
            handler=SequentialGeventHandler(),
            logger=logger,
            connection_retry=self._retry,
            command_retry=self._retry)

    def connect(self):
        self._zk_client.start()

    def disconnect(self):
        self._zk_client.stop()
        self._zk_client.close()

    def create_node(self, path, value=None):
        if value is None:
            value = uuid.uuid4()
        try:
            self._zk_client.create(path, str(value), makepath=True)
        except NodeExistsError:
            self._zk_client.set(path, str(value))
Exemplo n.º 29
0
def init_zk(namespace=''):
    """Initialize Zookeeper Instance."""
    hosts = os.getenv('ZOOKEEPER_CONN_STRING', 'localhost:2181')

    timeout = 600
    timeout_treshold = time() + timeout

    while True:
        try:
            # check if namespace exists
            zk = KazooClient(hosts,
                             read_only=False)
            zk.start()
            if zk.exists(namespace) is None:
                zk.create(namespace)
            zk.stop
            zk.close
            zk = KazooClient(hosts + namespace,
                             read_only=False)
            zk.start()
            if type(zk) is not KazooClient:
                logging.error("can't connect to Zookeeper ...")
                exit(1)
            return zk
        except KazooTimeoutError:
            if time() > timeout_treshold:
                logging.error("can't connect to Zookeeper ...")
                exit(1)
            else:
                logging.warning("can't connect to Zookeeper, " +
                                "trying again since timeout " +
                                "is not reached yet ...")
Exemplo n.º 30
0
class ValidatorDetector:
    def __init__(self):
        self.zk = KazooClient(
            hosts="bigdata1:2181,bigdata2:2181,bigdata3:2181")
        self.validator_children_watcher = ChildrenWatch(client=self.zk,
                                                        path="/servers",
                                                        func=self.watcher_func)
        self.zk.start()

    def watcher_func(self, children):
        print("----------------------start-----------------------------")
        for c in children:
            data, stat = self.zk.get('/servers/' + c)
            if stat:
                print("Children is:" + c + "\n" + "Data is:",
                      str(data, encoding='utf-8'))
        print("-----------------------end----------------------------")
        print("\n\n")

    def create_node(self, hostname):
        self.zk.create('/servers/server',
                       bytes(hostname, encoding='utf-8'),
                       ephemeral=True,
                       sequence=True,
                       makepath=True)

    def close_client(self, hostname):
        self.zk.close()
Exemplo n.º 31
0
class ZKData(object):
    def __init__(self):
        super(ZKData,self).__init__()
        options={"max_tries":-1,"max_delay":5,"ignore_expire":True}
        self._zk=KazooClient(hosts=ZK_HOSTS,connection_retry=options)
        try:
            self._zk.start(timeout=3600)
        except:
            print(traceback.format_exc(), flush=True)

    def set(self, path, value):
        value=json.dumps(value).encode('utf-8')
        try:
            self._zk.create(path, value, makepath=True)
        except NodeExistsError:
            self._zk.set(path,value)

    def get(self, path):
        try:
            value, stat= self._zk.get(path)
            if not value: return {}
            return json.loads(value.decode('utf-8'))
        except Exception as e:
            return {}

    def close(self):
        self._zk.stop()
        self._zk.close()
Exemplo n.º 32
0
class IDGenerator(object):
    '''
    ID生成器
    '''

    # 控制输出信息的锁
    print_mutex = threading.Lock()
    DELAY_TIME = 3

    def __init__(self, zk_server_address, id_base_path):
        self.id_base_path = id_base_path

        # 创建一个zookeeper客户端
        self.zkclient = KazooClient(zk_server_address)
        # 添加连接状态监听器
        self.zkclient.add_listener(self.zk_connect_listener)
        # 与zookeeper开启连接
        self.zkclient.start()

    # 连接状态监听器
    def zk_connect_listener(self, state):
        # 获取打印锁
        IDGenerator.print_mutex.acquire()
        if state == KeeperState.CONNECTED:
            print "命名服务器已经开启..."
        elif state == KazooState.LOST:
            print "命名服务器停止服务..."
        else:
            raise Exception("命名服务器未正常开启...")
        # 获取打印锁
        IDGenerator.print_mutex.release()

    def start(self):
        # 创建父节点
        self.create_name_node()

    def create_name_node(self):
        # 先检查父节点,如果父节点不存在
        if not self.zkclient.exists(self.id_base_path):
            # 先创建父节点
            self.zkclient.create(self.id_base_path)

    # 生成id
    def generate_id(self):
        # 拼凑出服务器子节点的完整路径
        node_path = self.id_base_path + "/"
        # 创建临时顺序节点
        self.node_path = self.zkclient.create(node_path, "",
                                              self.zkclient.default_acl, True,
                                              True)

        # 准备删除创建出来的节点
        self.zkclient.delete(self.node_path)

        return self.node_path.split("/")[-1]

    # 停止服务
    def stop(self):
        self.zkclient.stop()
        self.zkclient.close()
Exemplo n.º 33
0
class ZKClient:
    def __init__(self, zk_url, zk_timeout=5):
        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
        self.logger = logging.getLogger()
        try:
            self.client = KazooClient(hosts=zk_url)
            self.client.start(timeout=zk_timeout)
        except handlers.threading.KazooTimeoutError:
            self.logger.error("5 seconds connection timeout")
            raise

    def store_value(self, value, path):
        fmt_value = str(value).encode('utf-8')
        self.client.ensure_path(path)
        self.client.set(path, fmt_value)
        self.client.stop()

    def retrieve_value(self, path):
        self.client.ensure_path(path)
        data, stat = self.client.get(path)
        self.client.stop()
        if not data:
            raise ValueError
        else:
            return int(data.decode('utf-8'))

    def register_server(self, path):
        self.client.create(path, ephemeral=True, makepath=True)
Exemplo n.º 34
0
    def execute(self, request):
        zk = KazooClient(hosts=request['zk_url'])
        try:
            zk.start()
            path_ = request['node_path']
            value_ = request['node_value'].encode('utf-8')
            is_update_timestamp_ = request.get('is_update_timestamp', False)
            is_delete_node_ = request.get('is_delete_node', None)

            if zk.exists(path_):
                if is_delete_node_:
                    zk.delete(path_, recursive=True)
                    return self.resp_dict, ''
                if is_update_timestamp_:
                    zk.set(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes())
                if value_:
                    zk.set(path_, value_)
                else:
                    data_, stat_ = zk.get(path_)
                    if data_:
                        self.resp_dict['result_value'] = data_.decode()
            elif value_:
                # 新加节点时也需要添加 '/TIMESTAMP' 节点
                zk.create(path_, value_, makepath=True)
                zk.create(path_ + '/TIMESTAMP', ZkTestPoint.get_timestamp_bytes(), makepath=True)

            return self.resp_dict, ''
        except Exception as e:
            logger.exception('zkTestPoint execute error, cause: %s', str(e))
            raise Exception('zkTestPoint execute error, cause: ' + str(e))
        finally:
            zk.stop()
Exemplo n.º 35
0
def become_master(slave_process, old_name):
    s = socket.socket()
    s.bind(("", 23456))
    print("[*] Listening for command from orchestrator to become master ...", file=sys.stdout)
    s.listen(2)
    c, address = s.accept()
    print("Received command from orchestrator to become master: " + c.recv(1024).decode(), file=sys.stdout)
    slave_process.terminate()

    os.environ["WORKER_TYPE"] = "master"
    os.environ["NODE_NAME"] = "master"

    client = docker.DockerClient(base_url="tcp://172.17.0.1:4444")
    cnt = client.containers.get(old_name)
    cnt.rename("master")

    logging.basicConfig()
    zk = KazooClient(hosts=zookeeper_hostname)
    zk.start()

    node_name = "/worker/" + os.environ["NODE_NAME"]
    if not zk.exists(node_name):
        msg = "Creating node: " + node_name
        print(msg, file=sys.stdout)
        db_name = os.environ["DB_HOSTNAME"]
        zk.create(node_name, db_name.encode(), ephemeral=True)

    time.sleep(3)
    zk.delete("/worker/" + old_name)

    rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True)
    rpc_server.start()
Exemplo n.º 36
0
def do_zookeeper_create(addr, path, value):
	print(path)
	zk = KazooClient(addr)
	zk.start()
	zk.create(path, bytes(value, 'utf-8'))

	do_zookeeper_read(addr, path)
Exemplo n.º 37
0
class ZooKeeper(object):
    def __init__(self, hosts="127.0.0.1:2181", prefix="/"):
        prefix = prefix.rstrip("/")
        if prefix and not prefix.startswith("/"):
            raise ValueError("prefix must start with /")
        self._prefix = prefix
        self._zk = KazooClient(hosts=hosts, handler=_get_zk_handler())
        self._zk.start()

    def __del__(self):
        self.close()

    def _path(self, path):
        if self._prefix:
            return "{0}/{1}".format(self._prefix, path)
        return path

    def close(self):
        if not self._zk:
            return

        try:
            self._zk.stop()
            self._zk.close()
        except Exception:
            pass
        finally:
            self._zk = None

    def create(self, path, value="", makepath=True):
        self._zk.create(self._path(path), value, makepath=makepath)

    def delete(self, path, recursive=True):
        self._zk.delete(self._path(path), recursive=recursive)

    def set(self, path, value, makepath=True):
        path = self._path(path)
        try:
            self._zk.set(path, value)
        except zkexc.NoNodeError:
            if not makepath:
                raise
            self._zk.create(path, value, makepath=True)

    def get(self, path, none=True):
        try:
            return self._zk.get(self._path(path))
        except zkexc.NoNodeError:
            if none:
                return None
            raise

    def ls(self, path, none=True):
        try:
            return self._zk.get_children(self._path(path))
        except zkexc.NoNodeError:
            if none:
                return None
            raise
Exemplo n.º 38
0
class ShellTestCase(unittest.TestCase):
    """ base class for all tests """

    def setUp(self):
        """
        make sure that the prefix dir is empty
        """
        self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests")
        self.zk_host = os.getenv("ZKSHELL_ZK_HOST", "localhost:2181")
        self.username = os.getenv("ZKSHELL_USER", "user")
        self.password = os.getenv("ZKSHELL_PASSWD", "user")
        self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=")
        self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "secret")
        self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest")

        self.client = KazooClient(self.zk_host, 5)
        self.client.start()
        self.client.add_auth(self.scheme, self.auth_id)
        if self.client.exists(self.tests_path):
            self.client.delete(self.tests_path, recursive=True)
        self.client.create(self.tests_path, str.encode(""))

        self.output = StringIO()
        self.shell = Shell([self.zk_host], 5, self.output, setup_readline=False, async=False)

        # Create an empty test dir (needed for some tests)
        self.temp_dir = tempfile.mkdtemp()

    @property
    def auth_id(self):
        return "%s:%s" % (self.username, self.password)

    @property
    def auth_digest(self):
        return "%s:%s" % (self.username, self.digested_password)

    def tearDown(self):
        self.output = None
        self.shell = None

        if os.path.isdir(self.temp_dir):
            shutil.rmtree(self.temp_dir)

        if self.client.exists(self.tests_path):
            self.client.delete(self.tests_path, recursive=True)

        self.client.stop()

    ###
    # Helpers.
    ##

    def create_compressed(self, path, value):
        """
        ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client
        to create a znode with zlib compressed content.
        """
        compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value)
        self.client.create(path, compressed, makepath=True)
Exemplo n.º 39
0
def zookeeper_ephemeral_node(name):
  zk = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2181'))
  zk.start()
  zk.create("/my/zookeeper_test/node1", b"process1 running", ephemeral=True)
  #At 10 validate that ephemeral node exist that is the process is still running
  time.sleep(10)
  assert zk.exists("/my/zookeeper_test/node1"), "process node is not found at 10 s when it is still running"

  time.sleep(20)
  zk.stop()
Exemplo n.º 40
0
class ZookeeperContext:

    def __init__(self, zookeeper_url, test_id, node_id):
        self.assertNotNone(zookeeper_url)
        self.assertNotNone(test_id)
        self.assertNotNone(node_id)

        self.zookeeper_url = zookeeper_url
        self.test_id = test_id
        self.node_id = node_id
        self.zk_path = '/bolt/testing/%s/%s' % (self.test_id, self.node_id)
        logger.info("Initialized decorator: %s " % self.__str__())
        self.__connect_zookeeper()

    def assertNotNone(self,thing):
        if thing == None:
            logger.fatal("Invalid key in initializer")

    def __connect_zookeeper(self):
        try:
            logger.info('Attempting to connect to: %s' % self.zookeeper_url)
            self.zk = KazooClient(hosts=self.zookeeper_url)
            self.zk.start(timeout=10)
            logger.info('initialized zookeeper')
        except Exception as exception:
            logger.error('Failed to connect to zk')
            logger.fatal(exception)


    def __str__(self):
        return json.dumps(self, default=lambda o: o.__dict__, indent=4)

    def __del__(self):
        self.zk.stop()

    def publish(self, key, data):
        self.assertNotNone(key)
        self.assertNotNone(data)

        path = self.zk_path + "/" + key
        logger.info("Creating data in: %s" % path)
        try:
            bytes = pickle.dumps(data);
            if not self.zk.exists(path):
                logger.info("Path does not exist in zk, creating... %s" % path)
                self.zk.create(path, value=bytes,  makepath=True)
            else:
                self.zk.set(path, value=bytes)

        except Exception as exception:
            logger.error('Error setting data in ' % path)
            logger.fatal(exception)
Exemplo n.º 41
0
def main():
    '''
    A manual configuration file pusher for the crawlers. This will update
    Zookeeper with the contents of the file specified in the args.
    '''
    import argparse
    from kazoo.client import KazooClient

    parser = argparse.ArgumentParser(
            description="Crawler config file pusher to Zookeeper")
    parser.add_argument('-f', '--file', action='store', required=True,
                        help="The yaml file to use")
    parser.add_argument('-i', '--id', action='store', default="all",
                        help="The crawler id to use in zookeeper")
    parser.add_argument('-p', '--path', action='store',
                        default="/scrapy-cluster/crawler/",
                        help="The zookeeper path to use")
    parser.add_argument('-w', '--wipe', action='store_const', const=True,
                        help="Remove the current config")
    parser.add_argument('-z', '--zoo-keeper', action='store', required=True,
                        help="The Zookeeper connection <host>:<port>")
    args = vars(parser.parse_args())

    filename = args['file']
    id = args['id']
    wipe = args['wipe']
    zoo = args['zoo_keeper']
    path = args['path']

    zk = KazooClient(hosts=zoo)
    zk.start()

    # ensure path exists
    zk.ensure_path(path)

    bytes = open(filename, 'rb').read()

    if zk.exists(path):
        # push the conf file
        if not zk.exists(path + id) and not wipe:
            print "creaing conf node"
            zk.create(path + id, bytes)
        elif not wipe:
            print "updating conf file"
            zk.set(path + id, bytes)

        if wipe:
            zk.set(path + id, None)

    zk.stop()
Exemplo n.º 42
0
def put_topology(topology, config=config):
  zk = KazooClient(**config)
  zk.start()
  
  for service, links in topology.iteritems():
    if zk.exists("/sdn/services/{name}".format(name=service)):
      zk.set("/sdn/services/{name}".format(name=service), json.dumps(links))
    else:
      zk.ensure_path("/sdn/services")
      zk.create("/sdn/services/{name}".format(name=service), json.dumps(links))
      
  ret_check = zk.get_children("/sdn/services")
  zk.stop()
  
  return ret_check
Exemplo n.º 43
0
    def test_create_makepath_incompatible_acls(self):
        from kazoo.client import KazooClient
        from kazoo.security import make_digest_acl_credential, CREATOR_ALL_ACL
        credential = make_digest_acl_credential("username", "password")
        alt_client = KazooClient(self.cluster[0].address + self.client.chroot,
            max_retries=5, auth_data=[("digest", credential)])
        alt_client.start()
        alt_client.create("/1/2", b"val2", makepath=True, acl=CREATOR_ALL_ACL)

        try:
            self.assertRaises(NoAuthError, self.client.create, "/1/2/3/4/5",
                b"val2", makepath=True)
        finally:
            alt_client.delete('/', recursive=True)
            alt_client.stop()
Exemplo n.º 44
0
class HHFrontierWorker(FrontierWorker):
    def __init__(self, settings, no_batches, no_scoring, no_incoming):
        super(HHFrontierWorker, self).__init__(settings, no_batches, no_scoring, no_incoming)
        self.init_zookeeper()

    def init_zookeeper(self):
        self._zk = KazooClient(hosts=settings.get('ZOOKEEPER_LOCATION'))
        self._zk.add_listener(self.zookeeper_listener)
        self._zk.start()
        self.znode_path = self._zk.create("/frontera/hh-f-worker", ephemeral=True, sequence=True, makepath=True)

    def zookeeper_listener(self, state):
        if state == KazooState.LOST:
            # Register somewhere that the session was lost
            pass
        elif state == KazooState.SUSPENDED:
            # Handle being disconnected from Zookeeper
            pass
        else:
            # Handle being connected/reconnected to Zookeeper
            pass

    def set_process_info(self, process_info):
        self.process_info = process_info
        self._zk.set(self.znode_path, self.process_info)

    def set_job_id(self, job_id):
        self._backend.set_job_id(job_id)
        self.job_id = job_id
Exemplo n.º 45
0
class ZookeeperSession(object):

    def __init__(self, locations, name_prefix, root_prefix='/frontera'):
        self._zk = KazooClient(hosts=locations)
        self._zk.add_listener(self.zookeeper_listener)
        self._zk.start()
        self.root_prefix = root_prefix
        self.znode_path = self._zk.create("%s/%s" % (self.root_prefix, name_prefix),
                                          ephemeral=True,
                                          sequence=True,
                                          makepath=True)

    def zookeeper_listener(self, state):
        if state == KazooState.LOST:
            # Register somewhere that the session was lost
            pass
        elif state == KazooState.SUSPENDED:
            # Handle being disconnected from Zookeeper
            pass
        else:
            # Handle being connected/reconnected to Zookeeper
            pass

    def set(self, value):
        self._zk.set(self.znode_path, value)

    def get_workers(self, prefix='', exclude_prefix=''):
        for znode_name in self._zk.get_children(self.root_prefix):
            if prefix and not znode_name.startswith(prefix):
                continue
            if exclude_prefix and znode_name.startswith(exclude_prefix):
                continue
            location, _ = self._zk.get(self.root_prefix+"/"+znode_name)
            yield location
Exemplo n.º 46
0
        def _receive(self):
            server, port, buf_que = self._createThriftServer()

            kazoo_client = KazooClient(self.zk_address)
            kazoo_client.start()
            path = '%s/%s:%d' % (self.zk_path, socket.gethostname(), port)
            kazoo_client.create(path, ephemeral=True, makepath=True)

            while not server._stop:
                try:
                    message = buf_que.pop()
                    yield message
                except:
                    time.sleep(0.1)
            server.close()
            kazoo_client.close()
Exemplo n.º 47
0
def model_environments(ip_address, file_path):
    zk = KazooClient(hosts=ip_address)
    zk.start()
    for root, dirs, files in os.walk(file_path):
        node_path = root.replace(os.path.dirname(file_path), "")  # trim root from path, except parent
        try:
            zk.create(node_path, makepath=True)  # make parent directories as needed

        except NodeExistsException:
            print "%s already exists" % node_path

        if len(files) > 0:
            file = os.path.join(root, files[0])
            __load_file_into_znode(zk, file, node_path)

    zk.stop()
Exemplo n.º 48
0
def test_pub_response(pub_node, server_list, host = '127.0.0.1', port = 2181):
    zookeeper = KazooClient('%s:%s' % (host, port,))
    zookeeper.start()

    for s in server_list:
        if isinstance(s, int) is True:
            continue

        try:
            node = '/test/to_pub_result/%s/%s' % (pub_node, s, )
            if zookeeper.exists(node) is None:
                zookeeper.create(node, json.dumps({
                    'update_time' : time.time(),
                    'status'      : 'ok'
                }), makepath = True)
        except kazoo.exceptions.NodeExistsError:
            pass
Exemplo n.º 49
0
def join(server, environment, name, on_parent_data_change = None):
    zk = KazooClient(hosts=server)
    zk.start() # connect to zookeeper

    # create an ephemeral znode under the given environment,
    # if the given name ends in '-' make it sequential.
    zk.create( path = '{0}/{1}'.format(environment, name),
               ephemeral = True,
               sequence = (name[-1:] == '-'))

    # if a callback was provided trigger it whenever parents data changes...
    # for example to update variables on config changes.
    if on_parent_data_change:
        zk.DataWatch(environment, func = on_parent_data_change)

    #hand zookeeper client back to caller
    return zk
Exemplo n.º 50
0
def zkset():

    file = "/tmp/redis_config.properties"

    zk = KazooClient(hosts='172.29.22.15:2181')
    zk.start()

    f=open(file,'rb')
    m=f.read()

    #Ensure a path , Create if necessary.
    zk.ensure_path("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes")
    #Create a node with data.
    try:
        zk.create("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
    except kazoo.exceptions.NodeExistsError:
        zk.set("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)


    zk.stop()
Exemplo n.º 51
0
class ZookeeperClient(object):

    def __init__(self, zk_hosts):
        self.zk = KazooClient(hosts=zk_hosts)
        self.zk.start()

    def put(self, entity, id, data):
        path = '/{}/{}'.format(entity, id)
        binary_value = json.dumps(data).encode('utf-8')
        try:
            self.zk.create(path, binary_value, makepath=True)
            return True
        except NodeExistsError:
            self.zk.set(path, binary_value)
            return False

    def get(self, entity, id):
        path = '/{}/{}'.format(entity, id)
        try:
            binary_data, _ = self.zk.get(path)
        except NoNodeError:
            return None
        return json.loads(binary_data.decode('utf-8'))

    def delete(self, entity, id):
        path = '/{}/{}'.format(entity, id)
        try:
            self.zk.delete(path)
            return True
        except NoNodeError:
            return False

    def list(self, entity):
        path = '/{}'.format(entity)
        try:
            children = self.zk.get_children(path)
            for child in children:
                value = self.get(entity, child)
                yield (child, value)
        except NoNodeError:
            yield []
Exemplo n.º 52
0
class MasterElection(object):
    def __init__(self):
        self.zk = KazooClient(hosts=state.ARGS.zookeeper)
        self.master_notified = False
        self.my_node = None
        self.zk.start()  # Stop never called
        self.zk.ensure_path(ELECTION_PATH)

    def start_election(self, master_callback):
        """
        Start the master election.

        If this node is the master, the callback will be called once.

        :params master_callback: Called if this node is the master
        """
        self.callback = master_callback
        self.my_node = self.zk.create(ELECTION_PATH + '/n_',
                                      ephemeral=True, sequence=True)
        self.zk.get_children(ELECTION_PATH, watch=self._election_update)
        self._election_update()

    def _election_update(self, data=None):
        def worker():
            try:
                self.master_notified = True
                self.callback()
            except Exception as e:
                self.master_notified = False
                log.info("Failed to activate master, redoing election: %r", e)
                self.zk.delete(self.my_node)
                self.my_node = self.zk.create(ELECTION_PATH + '/n_',
                                              ephemeral=True, sequence=True)
                self._election_update()

        if not self.master_notified and \
                sorted(self.zk.get_children(ELECTION_PATH))[0] == \
                os.path.basename(self.my_node):
            t = threading.Thread(target=worker)
            t.daemon = True
            t.start()
Exemplo n.º 53
0
def test_init_servers(server_id, host = '127.0.0.1', port = 2181):
    zookeeper = KazooClient('%s:%s' % (host, port,))
    zookeeper.start()

    try:
        node = '/test/server_list'
        if zookeeper.exists(node) is None:
            zookeeper.create(node, json.dumps({'update_time' : time.time()}), makepath = True)
    except kazoo.exceptions.NodeExistsError:
        pass

    try:
        node = '/test/server_list/s%s' % server_id
        zookeeper.delete(node)
        zookeeper.create(node, json.dumps({
            'update_time' : time.time(),
            'server_name' : 's%s' % server_id,
            'server_id'   : server_id,
        }), makepath = True)
    except kazoo.exceptions.NodeExistsError:
        pass
Exemplo n.º 54
0
def main():
    (options, args) = parse_options(sys.argv[1:])

    data = options.znode_data_size * b"D"

    s = KazooClient(options.server)
    s.start()

    if s.exists(options.root_znode):
        children = s.get_children(options.root_znode)
        print("delete old entries: %d" % len(children))
        for child in children:
            s.delete("%s/%s" % (options.root_znode, child))
    else:
        s.create(options.root_znode, "kazoo root znode")

    evaluation(s, options.root_znode, data, options)

    s.stop()

    print("Performance test complete")
Exemplo n.º 55
0
class ZookClient(object):

    def __init__(self, api_client=None):
        logging.basicConfig()
        # Create a client and start it
        self.zk = KazooClient()
        self.zk.start()

    def create_accounts_path(self, name, **kwargs):
        path = "/dso/" + name
        self.zk.ensure_path(path)
        self.zk.set(path, b"id: 7b4235ca-00fb-4dca-ad3e-8b6e3662631a\ngroupname: hr\ndescription: 人力资源")


    def create_accountinfo_path(self, account_id, **kwargs):
        self.zk.ensure_path("/app/someservice")

    def create_path(self, path, **kwargs):
        self.zk.ensure_path(path)

    def get_data(self, path):
        return self.zk.get(path)

    def test_tran(self):
        self.zk.delete("/app22")
        self.zk.create("/app22", b"" + '{"12": "12"}')

        tran = self.zk.transaction()
        tran.delete("/app22")
        tran.create("/app22", b"" + '{"22": "22"}')
        tran.commit()
        print "commit"




    def stop(self):
        # In the end, stop it
        self.zk.stop()
def job_submitter_main():
    try:
        zk = init()
        cpool = ConnectionPool(host='localhost', port=6379, db=0)
        r = Redis(connection_pool=cpool)
        added = 0
        tried = 0
        max_add_try = 5000
        jobname = uuid.uuid4().hex
        added_nums = set()

        while tried < max_add_try:
            value = randint(5000, 90000000)
            tried += 1
            if value not in added_nums:
                added_nums.add(value)
            else:
                continue

            while True:
                try:
                    r.lpush(jobname, value)
                    added += 1
                    break
                except Exception as e:
                    sleep(1)
                    print "Lpush ", jobname, e

        zk = KazooClient(hosts='127.0.0.1:2181')
        zk.add_listener(state_listener)
        zk.start()
        value = SUBMITTED + "=" + str(added)
        zk.create('/jobs/' + jobname, value=value)
        zk.stop()

    except Exception as e:
        print 'Big problem in submitting job ', e
        sys.exit(1)
    print 'Job submitted ' + jobname
Exemplo n.º 57
0
def join_peacock_network():
    """
    주키퍼에 watch를 등록하고 해당 노드를 가입시킨다.
    """
    from config import ZOOKEEPER_HOST

    zk = KazooClient(hosts=ZOOKEEPER_HOST)
    zk.start()

    zk.ChildrenWatch("/peacock/job/nodes", job_node_watch)
    my_node = zk.create("/peacock/job/nodes/node", ephemeral=True, sequence=True, makepath=True)
    global my_node_name
    my_node_name = my_node[my_node.rfind('/') + 1:]
    job_node_watch(zk.get_children("/peacock/job/nodes"))
Exemplo n.º 58
0
def main():
    zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0)
    zk.add_listener(my_listener)
    zk.start()

    if zk.exists("/ELECTION") == None:
        zk.ensure_path("/ELECTION")

    c = 1
    node_pathes = []
    while c < 10:
        c += 1
        node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True)
        node_pathes.append(node_path)

    my_path = random.choice(node_pathes)
    my_path = my_path.replace("/ELECTION/", "")
    # print "print my_path", my_path

    children = zk.get_children("/ELECTION/", watch=election_child_watcher)
    get_next_path = False

    prev_path = None
    for child_path in sorted(children):
        if child_path == my_path:
            break
        prev_path = child_path

        # I'm the leader
    if prev_path == None:
        print "OK I'm leader don't have to watch"
        return

        # fires twice, once on creation ignore

    @zk.DataWatch("/ELECTION/" + prev_path)
    def watch_node(data, stat):
        # only watch for first change
        if stat.version == 1:
            print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
            print "setting watch on " + prev_path
            print "my", my_path

    zk.set("/ELECTION/" + prev_path, b"some data")

    print "boom. watch triggered?"
    # time.sleep(10)
    print "bye"

    zk.stop()
Exemplo n.º 59
0
class TestSessions(unittest.TestCase):

    def setUp(self):
        from kazoo.client import KazooClient
        from kazoo.protocol.states import KazooState
        from kazoo.testing.common import ZookeeperCluster
        ZK_HOME = os.environ.get("ZOOKEEPER_PATH")
        ZK_CLASSPATH = os.environ.get("ZOOKEEPER_CLASSPATH")
        self.cluster = ZookeeperCluster(ZK_HOME, size=1, port_offset=21000, classpath=ZK_CLASSPATH)
        self.cluster.start()
        atexit.register(lambda cluster: self.cluster.terminate(), self.cluster)
        self.client = KazooClient(self.cluster[0].address, max_retries=5)
        self.ev = threading.Event()

        def back(state):
            if state == KazooState.CONNECTED:
                self.ev.set()
        self.client.start()
        self.path = self.client.create(uuid.uuid4().hex)
        self.client.add_listener(back)

    def test_restarted_server(self):
        raise SkipTest('Patch missing')
        self.cluster.stop()
        self.cluster.start()
        self.ev.wait(5)
        eq_(self.ev.is_set(), True)
        self.assertTrue(self.client.retry(self.client.exists, self.path))

    def test_terminated_server(self):
        raise SkipTest('Patch missing')
        self.cluster.reset()
        self.cluster.start()
        self.ev.wait(5)
        eq_(self.ev.is_set(), True)
        self.assertFalse(self.client.retry(self.client.exists, self.path))

    def tearDown(self):
        self.ev.clear()
        self.client.stop()
        self.cluster.stop()