Exemple #1
0
class ZKClient:
    def __init__(self, zk_url, zk_timeout=5):
        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
        self.logger = logging.getLogger()
        try:
            self.client = KazooClient(hosts=zk_url)
            self.client.start(timeout=zk_timeout)
        except handlers.threading.KazooTimeoutError:
            self.logger.error("5 seconds connection timeout")
            raise

    def store_value(self, value, path):
        fmt_value = str(value).encode('utf-8')
        self.client.ensure_path(path)
        self.client.set(path, fmt_value)
        self.client.stop()

    def retrieve_value(self, path):
        self.client.ensure_path(path)
        data, stat = self.client.get(path)
        self.client.stop()
        if not data:
            raise ValueError
        else:
            return int(data.decode('utf-8'))

    def register_server(self, path):
        self.client.create(path, ephemeral=True, makepath=True)
Exemple #2
0
class ZK:

    client = None

    def __init__(self, zk_host):
        self.client = KazooClient(zk_host)
        self.client.start()

    def __del__(self):
        self.client.stop()

    def get_node(self, path):
        if not self.client.exists(path):
            return None
        node = ZKNode(path, self)
        return node

    def create_node(self, path):
        self.client.ensure_path(path)
        return self.get_node(path)

    def get_transaction(self):
        return self.client.transaction()

    def get_lock(self, path, id=None):
        return self.client.Lock(path + "/lock", id)

    def has_lock(self, path):
        lock_path = path + "/lock"
        if not self.client.exists(lock_path):
            return False
        if len(self.client.get_children(lock_path)) > 0:
            return True
        else:
            return False
def main(_):
    if MODE.STATUS == FLAGS.mode:
        request = get_model_status_pb2.GetModelStatusRequest()
        request.model_spec.name = 'detection'
        request.model_spec.signature_name = 'serving_default'
    elif MODE.CONFIG == FLAGS.mode:
        request = model_management_pb2.ReloadConfigRequest()
        config = request.config.model_config_list.config.add()
        config.name = 'detection'
        config.base_path = '/models/detection/detection'
        config.model_platform = 'tensorflow'
        config.model_version_policy.specific.versions.append(5)
        config.model_version_policy.specific.versions.append(7)
        config2 = request.config.model_config_list.config.add()
        config2.name = 'pascal'
        config2.base_path = '/models/detection/pascal'
        config2.model_platform = 'tensorflow'
    elif MODE.ZOOKEEPER == FLAGS.mode:
        zk = KazooClient(hosts="10.10.67.225:2181")
        zk.start()
        zk.ensure_path('/serving/cunan')
        zk.set(
            '/serving/cunan',
            get_config('detection', 5, 224, 'serving_default',
                       ','.join(get_classes('model_data/cci.names')),
                       "10.12.102.32:8000"))
        return
    for address in FLAGS.addresses:
        channel = grpc.insecure_channel(address)
        stub = model_service_pb2_grpc.ModelServiceStub(channel)
        if MODE.STATUS == FLAGS.mode:
            result = stub.GetModelStatus(request)
        elif MODE.CONFIG == FLAGS.mode:
            result = stub.HandleReloadConfigRequest(request)
        print(result)
class ZKState():
    def __init__(self, path, timeout=30):
        super(ZKState, self).__init__()
        self._zk = KazooClient(hosts=ZK_HOSTS, timeout=timeout)
        self._zk.start(timeout=timeout)
        self._path = path
        self._zk.ensure_path(path)

    def processed(self):
        return self._zk.exists(self._path + "/complete")

    def process_start(self):
        if self.processed():
            return False
        if self._zk.exists(self._path + "/processing"):
            return False
        try:
            self._zk.create(self._path + "/processing", ephemeral=True)
            return True
        except NodeExistsError:  # another process wins
            return False

    def process_end(self):
        self._zk.create(self._path + "/complete")
        self._zk.delete(self._path + "/processing")

    def process_abort(self):
        try:
            self._zk.delete(self._path + "/processing")
        except NoNodeError:
            pass

    def close(self):
        self._zk.stop()
        self._zk.close()
Exemple #5
0
def main_loop():
    logging.basicConfig()

    zk = KazooClient(hosts=zk_connect_string)
    zk.start()

    # make sure the root folders for the sendgraph and the schedules exist
    zk.ensure_path(metrics_zk_path)
    zk.ensure_path(schedule_zk_path)

    for topology in zk.get_children(metrics_zk_path):
        topology_metrics_zk_path = metrics_zk_path + "/" + topology
        print("registering watcher schedule for " + topology_metrics_zk_path)

        # register a data watch for each
        def watchFunc(data, stat, event):
            #print("watch called")
            if event is not None and event.type == EventType.CHANGED:
                print("new sendgraph data for {0} at {1}".format(topology, byteArrayToInt(data)))
                schedule(zk, topology)
            return True  # returning false will disable the watch

        # install data watch
        #DataWatch(zk, topology_metrics_zk_path, func=watchFunc)

        # if there is some data already, schedule immediately
        if len(zk.get_children(topology_metrics_zk_path)):
            print("existing sendgraph data for {0}".format(topology))
            schedule(zk, topology)
Exemple #6
0
 def zookeeper_crud(self, **kwargs):
     operation = kwargs['operation']
     room = kwargs['room']
     room_cn_name = room.comment[:4]
     try:
         zk = KazooClient(hosts=room.zk_cluster)
         zk.start()
         if operation == 'retrieve':
             path = kwargs['path']
             zk.ensure_path(path)
             data, stat = zk.get(path)
             self.i('%s,获取节点%s数据成功,值为%s' % (room_cn_name, path, data))
             return data
         elif operation == 'update':
             path = kwargs['path']
             data = kwargs['data']
             zk.ensure_path(path)
             zk.set(path, bytes(data))
             print ':'.join([str(zk.hosts), 'set', path, data])
             self.i('%s,更新节点%s成功,值为%s' % (room_cn_name, path, data))
             print ':'.join([str(zk.hosts), 'get', path, zk.get(path)[0]])
             return True
     except KazooException, e:
         self.i('%s,%s操作失败,原因为%s' % (room_cn_name, operation, e.message))
         return False
class ES_zk_interface:
    def __init__(self, my_ip, es_interface, es_callback):
        self.ip = my_ip
        self.es_callback = es_callback
        self.es_interface = es_interface
        self.zk = KazooClient(hosts=HOST_IP)
        self.zk.start()

        self.zk.ensure_path("/ass3")

    def start_election_process(self):
        print "Starting Election process"
        #print("Previous Leader: %s" % (self.es_callback.get_leader()))
        election = self.zk.Election("/electionpath", self.ip)
        election.run(self.leader_function)

    def leader_function(self):
        print "Won the election: " + str(self.ip)
        if self.zk.exists(LEADER_PATH):
            self.zk.set(LEADER_PATH, self.ip)
        else:
            self.zk.create(LEADER_PATH, self.ip)

        #print "\nStarting Leader thread.... \n"
        self.es_interface.enter_message_loop(self.es_callback, True)

    def get_zookeeper_client(self):
        return self.zk

    def get_leader_path(self):
        return LEADER_PATH
Exemple #8
0
class ActorAddressBook(object):
    def __init__(self, zk_hosts, timeout=60.0):
        self.retry = KazooRetry(max_tries=10)
        self.zk = KazooClient(hosts=zk_hosts, timeout=timeout)
        self.zk.start()

    def lookup(self, path):
        return self.retry(self._lookup, path)

    def _lookup(self, path):
        actor_url, stat = self.zk.get(path)
        return RemoteActor(actor_url.decode('utf-8'))

    def register(self, path, actor_url):
        return self.retry(self._register, path, actor_url)

    def _register(self, path, actor_url):
        self.zk.ensure_path(path)
        self.zk.set(path, actor_url.encode('utf-8'))

    def delete(self, path):
        self.zk.delete(path, recursive=True)

    def __del__(self):
        self.zk.stop()
Exemple #9
0
class KazooService():

    zk = None
    rootPath = None

    def __init__(self, rootPath):

        qs = Zookeeper.objects.filter().first()
        if qs is None:
            raise LookupError("Unable to find Zookeeper configuration!")

        self.zk = KazooClient(hosts=qs.url)
        self.rootPath = rootPath

        self.zk.start(timeout=qs.timeout)
        self.zk.ensure_path(rootPath)

    def createOrUpdate(self, pathAffix, nodeValue):

        if isinstance(nodeValue, str):
            bValue = bytes(nodeValue, 'utf-8')
        else:
            bValue = bytes(str(nodeValue), 'utf-8')

        path = self.rootPath + pathAffix

        if self.zk.exists(path) is None:
            self.zk.create(path, bValue)
        else:
            self.zk.set(path, bValue)

    def __del__(self):

        if self.zk is not None:
            self.zk.stop()
Exemple #10
0
class Store(object):
    def __init__(self,**kwargs):
        self.config = kwargs
        self.client = None

    def get_client(self):
        return self.client

    def open(self):
        self.client = KazooClient(**self.config)
        self.client.add_listener
        self.client.start()

    def close(self):
        self.client.stop()

    def read(self,path):
        return self.client.get(path)

    def write(self,path,value):
        base_path = os.path.dirname(path)
        self.client.ensure_path(base_path)
        self.client.create(path,value)

    def overwrite(self,path,value):
        self.client.set(path,value)

    def exists(self,path):
        return self.client.exists(path)
Exemple #11
0
def register_location(host, port):
  """ Register service location with ZooKeeper. """
  global zk_client
  zk_client = KazooClient(hosts=appscale_info.get_zk_locations_string(),
                          connection_retry=ZK_PERSISTENT_RECONNECTS)
  zk_client.start()
  server_node = '{}/{}:{}'.format(UA_SERVERS_NODE, host, port)

  def create_server_node():
    """ Creates a server registration entry in ZooKeeper. """
    try:
      zk_client.retry(zk_client.create, server_node, ephemeral=True)
    except NodeExistsError:
      # If the server gets restarted, the old node may exist for a short time.
      zk_client.retry(zk_client.delete, server_node)
      zk_client.retry(zk_client.create, server_node, ephemeral=True)

    logger.info('UAServer registered at {}'.format(server_node))

  def zk_state_listener(state):
    """ Handles changes to ZooKeeper connection state.

    Args:
      state: A string specifying the new ZooKeeper connection state.
    """
    if state == KazooState.CONNECTED:
      threading.Thread(target=create_server_node).start()

  zk_client.add_listener(zk_state_listener)
  zk_client.ensure_path(UA_SERVERS_NODE)
  # Since the client was started before adding the listener, make sure the
  # server node gets created.
  zk_state_listener(zk_client.state)
Exemple #12
0
class ZookClient():
    def __init__(self, hosts):
        self.kazooclient = KazooClient(hosts=hosts)
        self.kazooclient.start()

    def foundApp(self, appname):  #Ìí¼Óapp
        if self.kazooclient.exists('/' + appname):
            pass
        else:
            self.kazooclient.ensure_path('/' + appname)

    def foundNode(self, appname, port, host):  #Ìí¼Ó½Úµã
        if self.kazooclient.exists('/' + appname + '/' + host + ':' +
                                   str(port)):
            pass
        else:
            self.kazooclient.create('/' + appname + '/' + host + ':' +
                                    str(port))

    def close(self):
        self.kazooclient.close()

    def register(self, appname, port, host):
        self.foundApp(appname)
        self.foundNode(appname, port, host)


#         self.close()
Exemple #13
0
class Register(object):
    def __init__(self,
                 value,
                 path='/scrapyd-cluster/worker',
                 hosts='127.0.0.1:2181'):
        self.path = path
        self.value = value
        self.expire = False
        self.zk = KazooClient(hosts)
        self.zk.start()
        self.zk.add_listener(self.listener)

        self.zk.ensure_path(os.path.dirname(self.path))
        self.path = self.zk.create(self.path,
                                   self.value,
                                   ephemeral=True,
                                   sequence=True)
        logger.info('Connected To ZooKeeper(%s) Succeed! Path : %s' %
                    (hosts, self.path))

    def register(self):
        if self.expire:
            logger.info('Renew Ephemeral Node. Path : %s  Value: %s' %
                        (self.path, self.value))
            self.zk.create(self.path, self.value, ephemeral=True)
            self.expire = False

    def listener(self, state):
        if state == KazooState.LOST:
            logger.info('ZooKeeper Connection Lost')
            self.expire = True
        elif state == KazooState.CONNECTED:
            self.zk.handler.spawn(self.register)
Exemple #14
0
class ZooWrap():
    def __init__(self):
        self.zk = KazooClient(hosts="%s,%s" % (
            str(cc.conf['zookeeper']['host']),
            str(cc.conf['zookeeper']['port'])))
        self.zk.start()
        self.root = cc.conf['zookeeper']['rootpath']
        self.log = logging.getLogger('L.ZOOKEEPER')
        self.zk.ensure_path('/%s/sleeping' % (self.root))
        self.whoami = cc.conf['whoami']

    def get_sleeping(self):
        return self.zk.get_children('/%s/sleeping' % (self.root))

    def sleep(self):
        try:
            self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
            self.log.info('Sleeping correctly')
        except NodeExistsError:
            self.log.error('Node already sleeping... seems weird')

    def wake(self):
        try:
            self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
        except NoNodeError:
            self.log.error('Node was not sleeping... seems weird')
Exemple #15
0
class KazooWrapper:
    """
    Wrapper that ensure that we connect to zookeeper only when need it.
    Prevents connection on error request.
    """
    log = logging.getLogger('KazooWrapper')

    def __init__(self, chroot=None, *args, **kwargs):
        self.zk = KazooClient(*args, **kwargs)
        self._started = False
        self.chroot = chroot

    def start(self):
        self.log.debug('Connect')
        self.zk.start()
        if self.chroot:
            self.zk.ensure_path(self.chroot)
            self.zk.chroot = self.chroot
        self._started = True

    def stop(self):
        if self._started:
            self.zk.stop()

    def __getattr__(self, name):
        if not self._started:
            self.start()
        return getattr(self.zk, name)
Exemple #16
0
def check_active(port):
    zk = KazooClient("localhost:2181")
    zk.start()
    zk.add_listener(my_listener)
    zk.ensure_path("/big_data_assign")
    children = zk.get_children("/big_data_assign")
    path = "/big_data_assign/"
    for each in children:
        path_child = path + each
        #print(path_child)
        c = zk.get(path_child)
        c = c[0]
        c = c.decode()
        c = eval(c, {'OrderedDict': OrderedDict})
        #print("here4",c,type(c))
        #for key, value in c.items():
        key = list(c.items())[0]
        key = key[0]
        #print(key,type(key))
        if (key == port):
            active = True
            break
        else:
            active = False
    #print(active)
    return active
Exemple #17
0
class ZKStore:
    def __init__(self, hosts):
        self.zk = KazooClient(hosts=hosts)
        self.zk.add_listener(listener)
        self.zk.start()


    def isConnected(self):
        if __state__ == 1:
            return True
        return False


    def write(self, path, node, value):
        self.zk.ensure_path(path)
        if self.zk.exists(path+"/"+node):
           self.zk.set(path+"/"+node, value)
        else:
           self.zk.create(path + "/" + node, value)


    def read(self, path):
        if self.zk.exists(path):
            data, stat = self.zk.get(path)
            return data
        return None
Exemple #18
0
class ZooWrap():
    def __init__(self):
        self.zk = KazooClient(hosts="%s,%s" %
                              (str(cc.conf['zookeeper']['host']),
                               str(cc.conf['zookeeper']['port'])))
        self.zk.start()
        self.root = cc.conf['zookeeper']['rootpath']
        self.log = logging.getLogger('L.ZOOKEEPER')
        self.zk.ensure_path('/%s/sleeping' % (self.root))
        self.whoami = cc.conf['whoami']

    def get_sleeping(self):
        return self.zk.get_children('/%s/sleeping' % (self.root))

    def sleep(self):
        try:
            self.zk.create('/%s/sleeping/%s' % (self.root, self.whoami))
            self.log.info('Sleeping correctly')
        except NodeExistsError:
            self.log.error('Node already sleeping... seems weird')

    def wake(self):
        try:
            self.zk.delete('/%s/sleeping/%s' % (self.root, self.whoami))
        except NoNodeError:
            self.log.error('Node was not sleeping... seems weird')
Exemple #19
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-p",
                        "--project",
                        help="project name kafka or schema-registry",
                        default="kafka")
    parser.add_argument(
        "-z",
        "--zookeeper",
        help="zookeeper url with port",
        default='zookeeper.wgdp.io:2181',
    )
    args = parser.parse_args()
    conf_dict = collect_info(args.project)
    print(conf_dict)
    region = define_region_name(conf_dict["hostname"])
    if region == 'edcenter':
        if args.project == 'kafka':
            conf_dict['url'] = 'kafka.wgdp.io'
        else:
            conf_dict['url'] = 'schemas-registry.wgdp.io'
    else:
        conf_dict['url'] = '{}-kafka.wgdp.io'.format(region)
    zk = KazooClient(hosts=args.zookeeper)
    zk.start()
    zk.ensure_path("/app/{}/{}/{}".format(args.project, region,
                                          conf_dict["hostname"]))
    zk.set(
        "/app/{}/{}/{}".format(args.project, region, conf_dict["hostname"]),
        json.dumps(conf_dict),
    )
Exemple #20
0
    def _start(self, err_msg, spawn_monit=False):
        if self._is_destroyed:
            return

        self._client = None
        # Increase the session timeout from 10 to 25 seconds.
        try:
            host_list = self.zk_hosts
            client = KazooClient(
                hosts=",".join(host_list),
                timeout=self._get_session_timeout(),
                max_retries=3,
                handler=SequentialGeventHandler())

            # Increase the start timeout to 20 seconds from 15 seconds.
            # Guard this with explicit gevent timeout to protect us from
            # some corner cases where starting client failed to respect
            # start timeout passed in below.
            with gevent.Timeout(seconds=self._get_start_timeout() + 5):
                client.start(timeout=self._get_start_timeout())
            client.ensure_path("/")
            self._last_success_health_check_ts = time.time()
            log.info("Successfully started kazoo client.")
            self._client = client
        except (Exception, gevent.Timeout):
            self._sc.increment("errors.zk.client.start.failure",
                               tags={'host': hostname},
                               sample_rate=1)
            log.exception(err_msg)
        finally:
            if spawn_monit:
                self._monit_greenlet = gevent.spawn(self._monit)
                gevent.sleep(0)
Exemple #21
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--hosts", required=False, help="zk hosts, example:127.0.0.1:2181")
    parser.add_argument("--module", required=False, help="module name")
    parser.add_argument("--module_dependence", required=False, help="dependence module names, separated by comma")
    parser.add_argument("--redis_names", required=False, help="redis names")
    parser.add_argument("--redis_data", required=False, help="redis data")
    parser.add_argument("--paths", required=False, help="zk absolute path")
    args = parser.parse_args()
    module = args.module
    module_dependence = args.module_dependence

    zk = KazooClient(hosts=args.hosts)
    zk.start()
    if module is not None:
        create_node(zk, module, module + "-test", module_dependence)
    if module_dependence is not None:
        for s in module_dependence.split(","):
            create_node(zk, s, s + "-test")
    if args.redis_names is not None and args.redis_data is not None:
        for s in args.redis_names.split(","):
            create_redis(zk, s, args.redis_data)
    if args.paths is not None:
        for s in args.paths.split(","):
            zk.ensure_path(s)
Exemple #22
0
def test_zookeeper(zookeeper_server):
    test_value = str(uuid.uuid4()).encode("utf-8")

    zk_client1 = KazooClient(
        hosts="127.0.0.1:{}".format(zookeeper_server.port))
    zk_client2 = KazooClient(
        hosts="127.0.0.1:{}".format(zookeeper_server.port))
    zk_client1.start()
    zk_client2.start()

    zk_client1.ensure_path("/my")
    assert zk_client1.exists("/you") is None
    assert zk_client2.exists("/you") is None
    assert zk_client1.exists("/my") is not None
    assert zk_client2.exists("/my") is not None

    zk_client1.create("/my/testplan", test_value)
    data, _ = zk_client1.get("/my/testplan")
    assert data == test_value
    data, _ = zk_client2.get("/my/testplan")
    assert data == test_value

    zk_client2.set("/my/testplan", str(uuid.uuid4()).encode("utf-8"))
    data, _ = zk_client1.get("/my/testplan")
    assert data != test_value
    data, _ = zk_client2.get("/my/testplan")
    assert data != test_value
    def test_subscriber_connect(self):
        test_zk = KazooClient(self.zk_sub.zookeeper_connection_url)
        try:
            test_zk.start(5)
        except Exception:
            self.assertRaises("Couldn't connect to zookeeper server")

        # Ensure a path, create if necessary
        test_zk.ensure_path(self.zk_sub.default_node_path)

        fake_host_ip_lst = ["0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.3"]

        try:
            for host_ip in fake_host_ip_lst:
                # Create a node with data
                node = "node" + str(uuid.uuid4())
                path = self.default_node_path + '/' + node
                test_zk.create(path, bytes(host_ip, "utf-8"))

            self.zk_sub.subscriber_connect("subscriber")
        finally:
            nodes_to_delete = test_zk.get_children(self.default_node_path)
            for node in nodes_to_delete:
                path = self.default_node_path + '/' + node
                test_zk.delete(path)

        children = test_zk.get_children(self.default_node_path)
        self.assertEqual(children, [])
Exemple #24
0
    def zookeeper_watcher(self, q, room):
        service = q.get()
        try:
            zk = KazooClient(hosts=room.zk_cluster)
            zk.start()
            count = 0
            path = '/FlagsCenter/%s/gray_%s/gray_%s.result' % (self.full_app_name, service, service)
            zk.ensure_path(path)

            @zk.DataWatch(path)
            def watch_node(data, stat):
                if count == 0:
                    self.service_dict[room] = self.service_dict.get(room, dict())
                    self.service_dict[room][stat.pzxid] = self.service_dict[room].get(stat.pzxid, dict())
                    self.service_dict[room][stat.pzxid]['name'] = service
                    self.service_dict[room][stat.pzxid]['data'] = False
                else:
                    self.service_dict[room][stat.pzxid]['data'] = data
            while count < (ZOOKEEPER['WATCHER_TIMEOUT'] + ZOOKEEPER['WATCHER_DELAY']) * ZOOKEEPER['WATCHER_TIMES']:
                if all([self.service_dict[room][pzxid]['data'] for pzxid in self.service_dict[room]]):
                    q.task_done()
                    return
                count += 1
                time.sleep(1.0 / ZOOKEEPER['WATCHER_TIMES'])
            q.task_done()
            return
        except KazooException, e:
            self.service_dict[room] = e.message
            self.i(u'%s,watcher操作失败,原因为%s' % (room.comment[:4], self.service_dict))
            q.task_done()
            return
Exemple #25
0
def zk_init(server_address, zk_node, zk_address = ''):
    #初始化zk节点
    zk_cli = KazooClient(hosts = zk_address)
    try:
        zk_cli.start()
    except:
        logging.error('zk Init error, can not connect %s' %(str(zk_address)))
        return -1
    
    try:
        zk_cli.get(zk_node)
    except:
        logging.warn('can not find zk path %s, creat it' %(str(zk_node)))
        zk_cli.ensure_path(zk_node)

    if zk_node[-1] != '/':
        zk_node += '/'
        
    try:
        zk_cli.create(zk_node + server_address, '1', ephemeral=True)
    except:
        if zk_cli.get(zk_node + server_address):
            return 0
        else:
            logging.error('create zk_node error, can not create node %s' %(str(zk_node) + str(zk_address)))
            return -1
    return 0
class DistributedLock:
    def __init__(self, hosts, logger=None):
        self.client = KazooClient(hosts=hosts, logger=logger)

    def ensure_path(self, path):
        self.client.start()
        self.client.ensure_path(path)
        self.client.stop()

    def try_lock(self, path):
        try:
            self.client.start()
            self.client.create(path, ephemeral=True)
            return True
        except:
            logging.exception("try lock exception. path: {}".format(path))
            return False

    def try_unlock(self, path):
        try:
            self.client.delete(path)
            self.client.stop()
            return True
        except:
            logging.exception("try unlock exception. path: {}".format(path))
            return False
Exemple #27
0
class ZooWatch:
    def __init__(self):
        print("Zoo watch init")
        logging.basicConfig()
        self.zk = KazooClient(hosts='zoo:2181')
        self.zk.start()
        self.temp = []
        self.master_db_name = "mongomaster"

    def start(self):
        print("[*] Starting zoo watch", file=sys.stdout)
        self.zk.ensure_path("/worker")

        @self.zk.ChildrenWatch("/worker")
        def callback_worker(workers):
            print("[*] Changes detected", file=sys.stdout)
            print(workers, self.temp)
            if len(workers) < len(self.temp):
                node = listdiff(self.temp, workers)
                print("[-] Node deleted: " + node, file=sys.stdout)
                print("[*] Current workers: " + str(workers), file=sys.stdout)
                if "slv_node" in node:
                    killed_containers = client.containers.list(
                        all=True, filters={"exited": "137"})
                    print("[*] killed killed ++++++++", killed_containers)
                    #slave_cnt = client.containers.get(node)
                    i = node[-1]
                    #name ="slv"+i
                    #slave_cnt = client.containers.get(name)
                    global prune_cont
                    #if(name != prune_cont):
                    #print("[&&] slave cont",slave_cnt)
                    #slave_db_cnt = client.containers.get("mongodb" + i)
                    if prune_cont not in killed_containers and len(
                            killed_containers) > 0:
                        #slave_cnt.remove()
                        #slave_db_cnt.remove()
                        #random_name = "".join(random.choices(string.ascii_lowercase + string.digits, k=7))
                        bring_up_new_worker_container(
                            slave_name="slv" + "newwwwwwwwwwwwwwwww",
                            db_name="mongodb" + "newwwwwwwwwwwwwwwwwwww")
                    else:
                        print("[*] Scaling down - removing " + node)
                        #client.containers.prune()
                        #print("[*] Or newly elected master is deleting its old node")
                else:
                    print("[-] Master failed", file=sys.stdout)

            elif len(workers) > len(self.temp):
                print("[+] Node added: " + listdiff(self.temp, workers),
                      file=sys.stdout)
                print("[*] Current workers: " + str(workers), file=sys.stdout)

            else:
                pass

            self.temp = workers

        while True:
            pass
def main():
    logging.basicConfig()
    zk = KazooClient(hosts=zookeeper_hostname)
    zk.start()
    zk.ensure_path("/worker")
    node_name = "/worker/" + os.environ["NODE_NAME"]
    if not zk.exists(node_name):
        msg = "Creating node: " + node_name
        print(msg, file=sys.stdout)
        db_name = os.environ["DB_HOSTNAME"]
        zk.create(node_name, db_name.encode(), ephemeral=True)

    if os.environ["WORKER_TYPE"] == "master":
        rpc_server = RpcServer(queue_name='writeQ', func=writedb, is_master=True)
        rpc_server.start()
    else:
        try:
            if node_name != "/worker/slave1":
                master_db = zk.get("/worker/master")[0].decode()
                print("[*] Cloning database from master db: " + master_db, file=sys.stdout)
                subprocess.call(
                    "mongodump --host " + master_db + " --port 27017 --db rideshare && mongorestore --host " + os.environ[
                        'DB_HOSTNAME'] + " --port 27017",
                    stdout=sys.stdout,
                    stderr=sys.stdout,
                    shell=True
                )
        except Exception as e:
            print(e, file=sys.stdout)

        old_name = os.environ["NODE_NAME"]
        p1 = multiprocessing.Process(target=slave_rpc_server)
        p2 = multiprocessing.Process(target=become_master, args=(p1, old_name,))
        p1.start()
        p2.start()
def list_worker():
	logging.basicConfig()
	zk = KazooClient(hosts='zoo:2181')
	zk.start()
	zk.ensure_path("/workers")
	children = zk.get_children("/workers")
	return Response(json.dumps(children),status=200,mimetype='application/json')
Exemple #30
0
class ZkHelper(object):
    def __init__(self, address='', port=''):
        assert address and port
        self.zk_address = address
        self.zk_port = port
        self.retry = KazooRetry(max_delay=10000, max_tries=None)
        self.zk = KazooClient(hosts='%s:%s' % (self.zk_address, self.zk_port),
                              connection_retry=self.retry,
                              timeout=20)
        self.zk.add_listener(self._listener)
        self.zk.start()
        logging.info("instance zk client start (%s:%s)" %
                     (self.zk_address, self.zk_port))

    @staticmethod
    def _listener(state):
        if state == KazooState.LOST:
            logging.info(
                "zk connect lost, stop this connection and then start new one!"
            )
        elif state == KazooState.SUSPENDED:
            logging.info(
                "zk connect suspended, stop this connection and then start new one!"
            )

    def write(self, path, data):
        self.zk.ensure_path(path)
        self.retry(self.zk.set, path, data)
        logging.info("write data:%s to path:%s" % (data, path))

    def ensure_path(self, path):
        self.zk.ensure_path(path)

    def get_lock(self, path):
        return self.zk.Lock(path, threading.currentThread())

    def read(self, path):
        if self.zk.exists(path):
            data = self.retry(self.zk.get, path)
            logging.info("read data:%s from path:%s" % (data, path))
            return data[0]
        logging.info("path:%s not exist" % path)

    def get_children_list(self, path):
        if self.zk.exists(path):
            data = self.retry(self.zk.get_children, path)
            logging.info("get children:%s from path:%s" % (data, path))
            return data
        logging.info("path:%s not exist" % path)

    def exists(self, path):
        return self.zk.exists(path)

    def get_lock(self, path):
        lock = self.retry(self.zk.Lock, path, threading.current_thread())
        return lock

    def close(self):
        self.zk.close()
Exemple #31
0
 def setup(self, env, result):
     """
     Setup method that will be executed before all testcases. It is
     used to ensure the path that the testcases require.
     """
     zk = KazooClient(hosts="127.0.0.1:{}".format(env.zk.port))
     zk.start()
     zk.ensure_path("/testplan")
def createZkNode(zk_host, zk_port, host_group, host, serverset_member):
    zk_connection_string = zk_host + ":" + zk_port
    zk = KazooClient(hosts=zk_connection_string)
    zk.start()
    zk.ensure_path("inventory")
    zk_node = "inventory/" + host_group + "/" + host
    zk.ensure_path(zk_node)
    zk.set(zk_node, str(serverset_member).encode())
Exemple #33
0
class ZkHelper:
    def __init__(self,
                 zk_addresses,
                 zk_timeout_sec,
                 zk_root,
                 redis_address,
                 redis_id=None):
        """ create zookeeper client, keep connection and auto re-register when session expired
    :type zk_addresses: str
    :type zk_timeout_sec: int
    :type zk_root: str
    :type redis_address: str
    """
        self.__zk_addresses = zk_addresses  # "zookeeper.n.miliao.com:2181"
        self.__zk_timeout = zk_timeout_sec
        self.__redis_address = bytes(redis_address)  # "192.168.1.1:6379"

        self.__zk_registered_redis_id = redis_id
        self.__zk_redis_path = None
        self.__zk_root = zk_root  # "/xmss/cluster/{clusterid}/filecache/partition_{id}"
        self.__zk_redis_pool = "%s/%s" % (self.__zk_root, "pool")
        self.__zk_lock_path = "%s/%s" % (self.__zk_root, "lock")

        # init zookeeper client
        zk_retry = KazooRetry(max_tries=3, delay=1.0,
                              ignore_expire=False)  # delay in sec
        self.__zk_client = KazooClient(hosts=self.__zk_addresses,
                                       logger=KAZOO_LOGGER,
                                       timeout=self.__zk_timeout,
                                       connection_retry=zk_retry)
        self.__zk_client.add_listener(self.listener)
        self.__zk_lock = lock.Lock(self.__zk_client,
                                   self.__zk_lock_path,
                                   identifier=self.__redis_address)

        # auto re-register worker
        self.__lost_zk_connection = False
        self.__re_register_event = self.__zk_client.handler.event_object()
        self.__re_register_thread = None

        self.__supervisor = None
        self.__stopped = False

    def list_pool(self):
        try:
            if not self.__zk_client.exists(self.__zk_redis_pool):
                LOGGER.warn("zk path[%s] doesn`t exists. create it.",
                            self.__zk_redis_pool)
                self.__zk_client.ensure_path(self.__zk_redis_pool)

            children_str = self.__zk_client.get_children(
                path=self.__zk_redis_pool)
            children = [int(i) for i in children_str]
            LOGGER.info("current pool:[%s]", children)
            return children
        except (ConnectionLoss, ZookeeperError), e:
            LOGGER.error("zk exception when list pool.")
            raise ZkHelperException(e)
Exemple #34
0
def print_children():
    zk = KazooClient("localhost:2181")
    zk.start()
    zk.add_listener(my_listener)
    zk.ensure_path("/big_data_assign")
    #print("All children")
    children = zk.get_children("/big_data_assign")
    print("children: ", children)
    return children
Exemple #35
0
 def register(self, zk, version='1.0.0', revision='1.0.0'):
     client = KazooClient(zk)
     client.start()
     for service, methods in self._services.items():
         logging.info(
             f'register service "{service}", methods "{methods}" to zookeeper "{zk}"'
         )
         url = f'dubbo://{self._host}:{self._port}/{service}?anyhost=true&application={self._app}&dubbo={self._dubbo_version}&interface={service}&methods={",".join(methods)}&pid={next(_pid_gen)}&revision={revision}&side=provider&timestamp={get_timestamp()}&version={version}'
         client.ensure_path(f'/dubbo/{service}/providers/{quote_plus(url)}')
Exemple #36
0
class dzk:
    def __init__(self):
        self.BasePath = "/my/"
        self.zk = KazooClient(hosts='x.24.79.51:2181,x.24.79.53:2181',retry_max_delay=2000)
        self.zk.start()
        self.zk.add_listener(self.listener)

    def listener(state):
        if state == KazooState.LOST:
            self.zk.start()
        elif state == KazooState.SUSPENDED:
            print "*******listener saw KazooState.LOST"
        else:
            print "*******listener saw KazooState.CONNECT"

    def getIpHost(self):
        self.myname  = socket.getfqdn(socket.gethostname())
        myip = socket.gethostbyname(self.myname)
        return  myip

    def register(self):
        ip = self.getIpHost()
        if ip:
            NODE = self.BasePath + ip
            print "register:",NODE
        else:
            print "[ERROR:] %s does not exist " %(NODE)
            sys.exit(2)

        if not self.zk.exists(NODE): 
            self.zk.ensure_path(NODE)

    def getData(self):
        ip = self.getIpHost()
        if ip:
            NODE = self.BasePath + ip
        else:
            print "[ERROR:] %s does not exist " %(NODE) 

        if self.zk.exists(NODE):
            data, stat = self.zk.get(NODE)
            print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))

    def monitor(self):
        pass
    
    def heartbeat(self):
        pass

    def role(self):
        pass
    
    def command(self):
        pass
Exemple #37
0
        def build(self, **connection_kwargs):
            assert self.zk_hosts
            assert self.zk_proxy_dir

            retry = {
                "max_delay": self.zk_max_delay,
                "max_tries": self.zk_max_tries
            }
            zk_client = KazooClient(hosts=self.zk_hosts, connection_retry=retry)
            zk_client.start()
            zk_client.ensure_path(self.zk_proxy_dir)
            return CodisConnectionPool(zk_client, self.zk_proxy_dir, True, **connection_kwargs)
Exemple #38
0
def main():
    '''
    A manual configuration file pusher for the crawlers. This will update
    Zookeeper with the contents of the file specified in the args.
    '''
    import argparse
    from kazoo.client import KazooClient

    parser = argparse.ArgumentParser(
            description="Crawler config file pusher to Zookeeper")
    parser.add_argument('-f', '--file', action='store', required=True,
                        help="The yaml file to use")
    parser.add_argument('-i', '--id', action='store', default="all",
                        help="The crawler id to use in zookeeper")
    parser.add_argument('-p', '--path', action='store',
                        default="/scrapy-cluster/crawler/",
                        help="The zookeeper path to use")
    parser.add_argument('-w', '--wipe', action='store_const', const=True,
                        help="Remove the current config")
    parser.add_argument('-z', '--zoo-keeper', action='store', required=True,
                        help="The Zookeeper connection <host>:<port>")
    args = vars(parser.parse_args())

    filename = args['file']
    id = args['id']
    wipe = args['wipe']
    zoo = args['zoo_keeper']
    path = args['path']

    zk = KazooClient(hosts=zoo)
    zk.start()

    # ensure path exists
    zk.ensure_path(path)

    bytes = open(filename, 'rb').read()

    if zk.exists(path):
        # push the conf file
        if not zk.exists(path + id) and not wipe:
            print "creaing conf node"
            zk.create(path + id, bytes)
        elif not wipe:
            print "updating conf file"
            zk.set(path + id, bytes)

        if wipe:
            zk.set(path + id, None)

    zk.stop()
Exemple #39
0
def main():
    zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0)
    zk.add_listener(my_listener)
    zk.start()

    if zk.exists("/ELECTION") == None:
        zk.ensure_path("/ELECTION")

    c = 1
    node_pathes = []
    while c < 10:
        c += 1
        node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True)
        node_pathes.append(node_path)

    my_path = random.choice(node_pathes)
    my_path = my_path.replace("/ELECTION/", "")
    # print "print my_path", my_path

    children = zk.get_children("/ELECTION/", watch=election_child_watcher)
    get_next_path = False

    prev_path = None
    for child_path in sorted(children):
        if child_path == my_path:
            break
        prev_path = child_path

        # I'm the leader
    if prev_path == None:
        print "OK I'm leader don't have to watch"
        return

        # fires twice, once on creation ignore

    @zk.DataWatch("/ELECTION/" + prev_path)
    def watch_node(data, stat):
        # only watch for first change
        if stat.version == 1:
            print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
            print "setting watch on " + prev_path
            print "my", my_path

    zk.set("/ELECTION/" + prev_path, b"some data")

    print "boom. watch triggered?"
    # time.sleep(10)
    print "bye"

    zk.stop()
Exemple #40
0
def ensure_kafka_zk_path(retries=3):
    while retries >= 0:
        # Connect to the ZooKeeper nodes. Use a pretty large timeout in case they were
        # just started. We should wait for them for a little while.
        zk = KazooClient(hosts=ZOOKEEPER_NODE_LIST, timeout=30000)
        try:
            zk.start()
            zk.ensure_path(KAFKA_ZOOKEEPER_BASE)
            return True
        except:
            retries -= 1
        finally:
            zk.stop()
    return False
Exemple #41
0
def put_topology(topology, config=config):
  zk = KazooClient(**config)
  zk.start()
  
  for service, links in topology.iteritems():
    if zk.exists("/sdn/services/{name}".format(name=service)):
      zk.set("/sdn/services/{name}".format(name=service), json.dumps(links))
    else:
      zk.ensure_path("/sdn/services")
      zk.create("/sdn/services/{name}".format(name=service), json.dumps(links))
      
  ret_check = zk.get_children("/sdn/services")
  zk.stop()
  
  return ret_check
def test_zookeeper_fault_tolerance():
  """
  Kill zookeeper1 and see if other zookeeper instances are in quorum
  """
  zookeper_deployer = runtime.get_deployer("zookeeper")
  kazoo_connection_url = str(runtime.get_active_config('zookeeper_host') + ':2181')
  zkclient = KazooClient(hosts=kazoo_connection_url)

  zkclient.start()

  zkclient.ensure_path("/my/zookeeper_errorinjection")
  # kill the Zookeeper1 instance
  print "killing zoookeeper instance1"
  zookeper_deployer.kill("zookeeper1")
  time.sleep(20)
  zkclient.stop()
Exemple #43
0
    def test_multiclient_rebalance(self, *args):
        """Test rebalancing with many connected clients

        This test is primarily good at ferreting out concurrency bugs
        and therefore doesn't test one specific thing, since such
        bugs are fundamentally hard to trap.

        To test, it simulates 10 consumer connecting over time, rebalancing,
        and eventually disconnecting one by one. In order to accomplish
        this, it creates a separate kazoo client for each consumer. This
        is required because otherwise there is too much thread contention
        over the single kazoo client. Some callbacks won't happen until
        much too late because there aren't enough threads to go around.

        """
        n_partitions = 10
        n_consumers = 10
        t = Topic(self.c, 'testtopic')

        # with self.client, new clients don't see the brokers -- unsure why
        from kazoo.client import KazooClient
        zk_hosts = ','.join(
                ['%s:%s' % (h,p) for h,p in self.client.hosts.hosts])
        zkclient = KazooClient(hosts=zk_hosts)
        zkclient.start()
        self._register_fake_brokers(n_partitions, client=zkclient)
        zkclient.ensure_path('/brokers/topics')

        # bring up consumers
        consumers = []
        for i in xrange(n_consumers):
            newclient = KazooClient(hosts=zk_hosts)
            newclient.start()
            cluster = Cluster(newclient)
            topic = Topic(cluster, 'testtopic')
            consumers.append((newclient, topic.subscribe('group1')))
            time.sleep(1)

        time.sleep(5) # let things settle

        # bring down consumers
        for client,consumer in consumers:
            consumer.stop_partitions()
            client.stop()
            time.sleep(2) # a little more time so we don't kill during rebalance

        newclient.stop()
Exemple #44
0
class NodeMonitor:
    STATIC_NODE_ID = 0
    global t
    def __init__(self):
        self.zk = None
        self.SERVER_IP_AND_PORT = "localhost:2181"
        self.NODE_ID = str(NodeMonitor.STATIC_NODE_ID)
        NodeMonitor.STATIC_NODE_ID += 1 
    
    def start_zk(self):
        self.zk = KazooClient(hosts=self.SERVER_IP_AND_PORT)
        self.zk.add_listener(self._connection_listener)
        self.zk.start();
        
        self.zk.ensure_path("/monitorData/"+ self.NODE_ID)
    
    def start_update_info(self):
        t = threading.Timer(0.0, self._update_info)
        t.start()
        
    
    def _update_info_once(self):
        cmi = CollectMachineInfo()
        async_obj = self.zk.set_async("/monitorData/"+ self.NODE_ID, (cmi.collectInfo()).encode(encoding="utf-8"))
        async_obj.rawlink(self._update_info_callback)
    
        
    def _connection_listener(self, state):
        if state == KazooState.LOST:
            print "connection lost, going to connect again"
            self.start_zk();
        elif state == KazooState.SUSPENDED:
            print "suspended"
        else:
            print "connected ok"
    
    def _update_info_callback(self, async_obj):
        try:
            print "update success"
        except (ConnectionLossException, NoAuthException):
            print "exception!"
    
    def _update_info(self):
        print "begin to update"
        self._update_info_once()
        t = threading.Timer(5.0, self._update_info)
        t.start()
Exemple #45
0
class ZookeeperSession(BaseClient):
  conext_manager = ZookeeperResponseContextManager
  loose_policy = {}
  strict_policy = {}

  def __init__(self,server_list='127.0.0.1:2181',*args,**kwargs):
    super(ZookeeperSession,self).__init__(*args,**kwargs)
    self.session_policy = "loose_policy"
    self._zookeeper_client = None 
    self.server_list = server_list

  def set_session_policy(self,session_policy="loose"):
    '''prototype not currenlty used.
    '''
    self.session_policy = session_policy+"_policy"

  def connect(self,*args,**kwargs):
    '''See http://kazoo.readthedocs.org/en/latest/api/client.html
     for details regarding available options. Any provided client
     start() parameters provided will override defaults.
    '''
    defaults = {
      "hosts" : self.server_list,
      "handler" : SequentialGeventHandler()
    }
    defaults.update(getattr(self,self.session_policy))
    defaults.update(kwargs)
    self._state = KazooState.LOST
    self._zookeeper_client = KazooClient(**defaults)
    self._zookeeper_client.add_listener(self._state_tracker)
    watchable = self._zookeeper_client.start_async()
    watchable.wait(30)
    if not self._zookeeper_client.connected:
      err = "Could not connect to Zookeeper server(s) %(server_list)s" % defaults
      raise ResponseError(err)

  @require_state(KazooState.CONNECTED)
  @record_stats
  def ensure_path(self,path,watcher=None):
    self._zookeeper_client.ensure_path(path,watcher)
    
  def _state_tracker(self,state):
    self._state = state

  def __del__(self):
    if isinstance(self._zookeeper_client, KazooClient):
      self._zookeeper_client.stop() 
def test_zookeeper_process_tracking():
  """
  Tests if process register node correctly with zookeeper and zookeeper deletes it when process terminates
  """
  #Wait for zookeeper to start so that kazoo client can connect correctly
  time.sleep(5)
  #"connecting to esnure /my/zookeeper_test"

  kazoo_connection_url = str(runtime.get_active_config('zookeeper_host') + ':2181')
  zkclient = KazooClient(hosts=kazoo_connection_url)

  zkclient.start()

  zkclient.ensure_path("/my/zookeeper_test")
  #spawn a python multiprocess which creates an ephermeral node
  #once the process ends the node will be deleted.
  p = Process(target=zookeeper_ephemeral_node, args=("process1",))
  p.start()
  zkclient.stop()
class ZkServiceRegister:
    def __init__(self, zk_address, zk_timeout):
        self.__zkClient = KazooClient(hosts=zk_address, timeout=zk_timeout, read_only=False)
        self.__zkListener = ZkServiceRegisterListener(self.__zkClient)
        self.__zkClient.add_listener(self.__zkListener)
        self.__zkClient.start()

    def register(self, path, host, port, weight=DEFAULT_HOST_WEIGHT):
        try:
            if not self.__zkClient.exists(path):
                self.__zkClient.ensure_path(path)
        except Exception, e:
            print e.message

        reg_path = path + '/' + host + ':' + str(port) + ':' + str(weight)
        if self.__zkClient.exists(reg_path):
            self.__zkClient.delete(reg_path)

        self.__zkClient.create(reg_path, value='', ephemeral=True)
Exemple #48
0
def zkset():

    file = "/tmp/redis_config.properties"

    zk = KazooClient(hosts='172.29.22.15:2181')
    zk.start()

    f=open(file,'rb')
    m=f.read()

    #Ensure a path , Create if necessary.
    zk.ensure_path("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes")
    #Create a node with data.
    try:
        zk.create("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)
    except kazoo.exceptions.NodeExistsError:
        zk.set("/instances/dev_conf/webapps/qf-app-bkend/WEB-INF/classes/redis_config.properties",m)


    zk.stop()
Exemple #49
0
class MasterElection(object):
    def __init__(self):
        self.zk = KazooClient(hosts=state.ARGS.zookeeper)
        self.master_notified = False
        self.my_node = None
        self.zk.start()  # Stop never called
        self.zk.ensure_path(ELECTION_PATH)

    def start_election(self, master_callback):
        """
        Start the master election.

        If this node is the master, the callback will be called once.

        :params master_callback: Called if this node is the master
        """
        self.callback = master_callback
        self.my_node = self.zk.create(ELECTION_PATH + '/n_',
                                      ephemeral=True, sequence=True)
        self.zk.get_children(ELECTION_PATH, watch=self._election_update)
        self._election_update()

    def _election_update(self, data=None):
        def worker():
            try:
                self.master_notified = True
                self.callback()
            except Exception as e:
                self.master_notified = False
                log.info("Failed to activate master, redoing election: %r", e)
                self.zk.delete(self.my_node)
                self.my_node = self.zk.create(ELECTION_PATH + '/n_',
                                              ephemeral=True, sequence=True)
                self._election_update()

        if not self.master_notified and \
                sorted(self.zk.get_children(ELECTION_PATH))[0] == \
                os.path.basename(self.my_node):
            t = threading.Thread(target=worker)
            t.daemon = True
            t.start()
Exemple #50
0
class MicroService(object):
	"""docstring for MicroService"""
	def __init__(self, hosts, app):
		from kazoo.client import KazooClient

		self.zk = KazooClient(hosts=hosts)
		self.app = "/" + app

	def zkStart(self):
		self.zk.start()

	def zkStop(self):
		self.zk.stop()
		
	def registerService(self, service, url):
		if self.zk.exists(self.app + "/" + service):
			self.zk.delete(self.app + "/" + service, recursive=True)
			self.zk.delete(self.app, recursive=True)
		self.zk.ensure_path(self.app + "/" + service)
		self.zk.set(self.app + "/" + service, bytes(url, encoding="utf-8"))

		self.__watchServiceChanged(service)

		return 'Service registed'

	def unregisterService(self, service):
		self.zk.delete(self.app + "/" + service, recursive=True)
		return "Service deleted"

	def findService(self, service):
		data, stat = self.zk.get(self.app + "/" + service)
		return data.decode("utf-8")

	def __watchServiceChanged(self, service):

		@self.zk.DataWatch(self.app + "/" + service)
		def changed(data, stat):
			print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
Exemple #51
0
class ZooDict(object):

    def __init__(self, path_root=None):
        self.path_root = path_root or default_path_root

    def __enter__(self):
        self.zk = KazooClient(hosts=zk_hosts)
        self.zk.start(timeout=5)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.zk.stop()

    def start(self):
        return self.__enter__()

    def stop(self):
        return self.__exit__()

    def get(self, key):
        try:
            r = self.zk.get(self.path_root + '/' + key)

            if r and r[0]:
                return json.loads(r[0])
        except NoNodeError as no_node:
            return None

    def set(self, key, data_dict):
        self.zk.ensure_path(self.path_root + '/' + key)
        self.zk.set(self.path_root + '/' + key, json.dumps(data_dict))

    def delete(self, key):
        self.zk.delete(self.path_root + '/' + key)

    def items(self):
        return [ (child, self.get(child)) for child in self.zk.get_children(self.path_root) ]
Exemple #52
0
def doSync(ip, src, dst, deleteFile):
    zk = KazooClient(hosts=ip)
    zk.start()
    
    if not os.path.exists(src):
        print src + ' not exists'
        return
    if not os.path.isdir(src):
        print 'copy file'
        copyFile(zk, src, dst)
        zk.stop()
        return
    for f in os.listdir(src):
        if f.startswith('.svn'):
            continue
        fullPath = os.path.join(src, f)
        zk_path = os.path.join(dst, f)
        if os.path.isdir(fullPath):
            zk.ensure_path(zk_path)
            syncChild(zk, fullPath, zk_path, deleteFile)
            continue
        zk.ensure_path(dst)
        copyFile(zk, fullPath, zk_path)
    zk.stop()
Exemple #53
0
class ZookClient(object):

    def __init__(self, api_client=None):
        logging.basicConfig()
        # Create a client and start it
        self.zk = KazooClient()
        self.zk.start()

    def create_accounts_path(self, name, **kwargs):
        path = "/dso/" + name
        self.zk.ensure_path(path)
        self.zk.set(path, b"id: 7b4235ca-00fb-4dca-ad3e-8b6e3662631a\ngroupname: hr\ndescription: 人力资源")


    def create_accountinfo_path(self, account_id, **kwargs):
        self.zk.ensure_path("/app/someservice")

    def create_path(self, path, **kwargs):
        self.zk.ensure_path(path)

    def get_data(self, path):
        return self.zk.get(path)

    def test_tran(self):
        self.zk.delete("/app22")
        self.zk.create("/app22", b"" + '{"12": "12"}')

        tran = self.zk.transaction()
        tran.delete("/app22")
        tran.create("/app22", b"" + '{"22": "22"}')
        tran.commit()
        print "commit"




    def stop(self):
        # In the end, stop it
        self.zk.stop()
class ZookeeperRegistry(Registry):
    _app_config = ApplicationConfig('default_app')
    _connect_state = 'UNCONNECT'

    def __init__(self, zk_hosts, application_config=None):
        Registry.__init__(self)
        if application_config:
            self._app_config = application_config
        self.__zk = KazooClient(hosts=zk_hosts)
        self.__zk.add_listener(self.__state_listener)
        self.__zk.start()

    def __state_listener(self, state):
        if state == KazooState.LOST:
            # Register somewhere that the session was lost
            self._connect_state = state
        elif state == KazooState.SUSPENDED:
            # Handle being disconnected from Zookeeper
            # print 'disconnect from zookeeper'
            self._connect_state = state
        else:
            # Handle being connected/reconnected to Zookeeper
            # print 'connected'
            self._connect_state = state

    def __unquote(self, origin_nodes):
        return (urllib.unquote(child_node).decode('utf8') for child_node in origin_nodes if child_node)

    def _do_event(self, event):
        # event.path 是类似/dubbo/com.ofpay.demo.api.UserProvider/providers 这样的
        # 如果要删除,必须先把/dubbo/和最后的/providers去掉
        # 将zookeeper中查询到的服务节点列表加入到一个dict中
        # zookeeper中保持的节点url类似如下
        logger.info("receive event is {0}, event state is {1}".format(event, event.state))
        provide_name = event.path[7:event.path.rfind('/')]
        if event.state in ['CONNECTED', 'DELETED']:
            children = self.__zk.get_children(event.path, watch=self.event_listener)
            self._compare_swap_nodes(provide_name, self.__unquote(children))
            configurators_nodes = self._get_provider_configuration(provide_name)
            self._set_provider_configuration(provide_name, configurators_nodes)
        print self._service_providers

    def _do_config_event(self, event):
        """
        zk的目录路径为 /dubbo/com.qianmi.pc.api.es.item.EsGoodsQueryProvider/configurators
        :param event:
        :return:
        """
        logger.info("receive config event is {0}, event state is {1}".format(event, event.state))
        provide_name = event.path[7:event.path.rfind('/')]
        configurators_nodes = self._get_provider_configuration(provide_name)
        self._set_provider_configuration(provide_name, configurators_nodes)

        print self._service_providers

    def register(self, interface, **kwargs):
        ip = self.__zk._connection._socket.getsockname()[0]
        params = {
            'interface': interface,
            'application': self._app_config.name,
            'application.version': self._app_config.version,
            'category': 'consumer',
            'dubbo': 'dubbo-client-py-1.0.0',
            'environment': self._app_config.environment,
            'method': '',
            'owner': self._app_config.owner,
            'side': 'consumer',
            'pid': os.getpid(),
            'version': '1.0'
        }
        url = 'consumer://{0}/{1}?{2}'.format(ip, interface, urllib.urlencode(params))
        # print urllib.quote(url, safe='')

        consumer_path = '{0}/{1}/{2}'.format('dubbo', interface, 'consumers')
        self.__zk.ensure_path(consumer_path)
        self.__zk.create(consumer_path + '/' + urllib.quote(url, safe=''), ephemeral=True)

    def subscribe(self, interface, **kwargs):
        """
        监听注册中心的服务上下线
        :param interface: 类似com.ofpay.demo.api.UserProvider这样的服务名
        :return: 无返回
        """
        version = kwargs.get('version', '')
        group = kwargs.get('group', '')
        providers_children = self.__zk.get_children('{0}/{1}/{2}'.format('dubbo', interface, 'providers'),
                                                    watch=self.event_listener)
        logger.debug("watch node is {0}".format(providers_children))
        self.__zk.get_children('{0}/{1}/{2}'.format('dubbo', interface, 'configurators'),
                               watch=self.configuration_listener)
        # 全部重新添加
        self._compare_swap_nodes(interface, self.__unquote(providers_children))

        configurators_nodes = self._get_provider_configuration(interface)
        self._set_provider_configuration(interface, configurators_nodes)

    def _get_provider_configuration(self, interface):
        """
        获取dubbo自定义配置数据,从"/dubbo/{interface}/configurators" 路径下获取配置
        :param interface:
        :return:
        """
        try:
            configurators_nodes = self.__zk.get_children('{0}/{1}/{2}'.format('dubbo', interface, 'configurators'),
                                                         watch=self.configuration_listener)
            logger.debug("configurators node is {0}".format(configurators_nodes))
            return self.__unquote(configurators_nodes)
        except Exception as e:
            logger.warn("get provider %s configuration error %s", interface, str(e))
Exemple #55
0
class SimpleSwitch13(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SimpleSwitch13, self).__init__(*args, **kwargs)
        self.mac_to_port = {}
        self.zkConf = {'root':'/multicontroller', 'topo':'/topology',
                       'swstat':'/swstat', 'counter': '/counter'}
        self.zk = KazooClient('127.0.0.1:2181')
        self.zk.start()
        self.ip = '202.201.3.51'
        self.sws = {}
        self.gid = random.randint(0, 10000)
        self.dps = {}
        self.links = []
        self.interval = 5
        self.role = OFPCR_ROLE_EQUAL
        self.topoThread = hub.spawn(self._topoThread)
        self.linkThread = hub.spawn(self._linkDiscover)
        self.clearLinkThread = hub.spawn(self._cleanLinks)
        self.clearLinkThread = hub.spawn(self._cleanSwitches)

    def _cleanSwitches(self):
        while  True:
            self.sws = {k:self.sws[k] for k in self.sws if self.sws[k]}
            hub.sleep(self.interval)

    def _topoThread(self):
        linkNode = self.zkConf['root'] + self.zkConf['topo'] + self.ip
        if self.zk.exists(linkNode):
            self.zk.set(linkNode, json.dumps(self.links))
        else:
            self.zk.create(linkNode, json.dumps(self.links))
        hub.sleep(self.interval)

    def _linkDiscover(self):
        while True:
            for dpid in self.dps:
                self.sendSlldp(dpid)
            hub.sleep(self.interval)

    def sendSlldp(self, dpid):
        dp = self.dps.get(dpid)
        if dp is None:
            return
        actions = [dp.ofproto_parser.OFPActionOutput(dp.ofproto.OFPP_FLOOD)]
        pkt = packet.Packet()
        pkt.add_protocol(ethernet.ethernet(ethertype=ETH_TYPE_SLLDP,
                        dst=SLLDP_MAC_DST, src=SLLDP_MAC_SRC))
        pkt.add_protocol(slldp(dp.id))
        pkt.serialize()
        slldpPacket = pkt.data
        out = dp.ofproto_parser.OFPPacketOut(
            datapath=dp, in_port=dp.ofproto.OFPP_CONTROLLER,
            buffer_id=dp.ofproto.OFP_NO_BUFFER, actions=actions,
            data=slldpPacket)
        dp.send_msg(out)

    def getLinks(self):
        topoNode = self.zkConf['root'] + self.zkConf['topo']
        ips = self.zk.get_children(topoNode)
        res = []
        for ip in ips:
            links = self.zk.get(topoNode + ip)[0]
            for link in links:
                res.append(link)
        return res

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):

        msg = ev.msg
        pkt = packet.Packet(msg.data)
        eth = pkt.get_protocols(ethernet.ethernet)[0]
        dst = eth.dst

        # SLLDP packet
        if dst == SLLDP_MAC_DST:
            self.handleSlldp(ev)
            return
        # process packet_in message in subclass
        self.packet_in_process(ev)

    def handleSlldp(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        dpid = datapath.id
        inPort = msg.match['in_port']

        pkt = packet.Packet(msg.data)
        slldpBuff = pkt.get_protocols(ethernet.ethernet)[2]
        dpidSrc, _ = slldp.parser(slldpBuff)
        self.links.append({'srcdpid': dpidSrc,
            'dst': {'dpid': dpid, 'port': inPort},
            'time': time.time()})

    def _cleanLinks(self):
        while True:
            now = time.time()
            self.links = [l for l in self.links if now - l['time'] < self.interval]
            hub.sleep(self.interval)

    @abc.abstractmethod
    def packet_in_process(self, ev):
        pass

    @set_ev_cls(event.EventSwitchEnter)
    def switch_enter(self, ev):
        dpid = ev.datapath.id
        self.sws[dpid] = True
        self.dps[dpid] = ev.datapath
        dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
                + '/' + dpid_to_str(dpid)
        self.zk.ensure_path(dpNode)
        if self.election(dpid):
            self.role = OFPCR_ROLE_MASTER
        else:
            self.role = OFPCR_ROLE_SLAVE
        self.countUp(dpid)
        self.roleRequest(dpid, self.role)
        mflag = dpNode + '/' + 'master'
        DataWatch(self.zk, mflag, self.masterWatcher)

    def masterWatcher(self, data, stat, ev):
        if ev and ev.type == 'DELETED':
            _, _, dpid, _ = ev.path.split('/')
            dpid = str_to_dpid(dpid)
            if self.sws.get(dpid):
                if self.election(dpid):
                    self.role = OFPCR_ROLE_MASTER
                    self.roleRequest(dpid, self.role)
            return self.sws.get(dpid)

    def election(self, dpid):
        dpNode = self.zkConf['root'] + self.zkConf['swstat'] \
                + '/' + dpid_to_str(dpid)
        mflag = dpNode + '/' + 'master'
        while not self.zk.exists(mflag):
            mlock = self.zk.Lock(dpNode + '/' + 'mlock', self.ip)
            with mlock:
                if not self.zk.exists(mflag):
                    self.zk.create(mflag, self.ip, ephemeral=True)
            if self.zk.exists(mflag):
                if self.zk.get(mflag) == self.ip:
                    return True
                else:
                    return False
            else:
                time.sleep(random.randint(0, 100)/500.0)
        return False

    def roleRequest(self, dp, role):
        msg = dp.ofproto_parser.OFPRoleRequest(dp, role, self.gid)
        dp.send_msg(msg)

    def getCount(self, dpid):
        dpNode =  self.zkConf['root'] + self.zkConf['swstat'] \
                + '/' + dpid_to_str(dpid)
        countNode = dpNode + self.zkConf['counter']
        counters = self.zk.get_children(countNode)
        return len(counters)

    def countUp(self, dpid):
        countNode = self.zkConf['root'] + self.zkConf['swstat'] \
                + '/' + dpid_to_str(dpid) + self.zkConf['counter']
        self.zk.ensure_path(countNode)
        self.zk.create(countNode+uuid4().hex, 'alive', ephemeral=True)

    @set_ev_cls(event.EventSwitchLeave)
    def switch_levave(self, ev):
        dpid = ev.datapath
        count = self.getCount(dpid)
        self.sws[dpid] = False
        if count == 0:
            dpNode =  self.zkConf['root'] + self.zkConf['swstat'] \
                    + '/' + dpid_to_str(dpid)
        self.zk.delete(dpNode, recursive=True)
Exemple #56
0
class RPCServer(asyncore.dispatcher):

    zk_root = "/demo"
    zk_rpc = zk_root + "/rpc"

    def __init__(self, host, port):
        asyncore.dispatcher.__init__(self)
        self.host = host
        self.port = port
        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.set_reuse_addr()
        self.bind((host, port))
        self.listen(1)
        self.child_pids = []
        if self.prefork(10):  # 产生子进程
            self.register_zk()  # 注册服务
            self.register_parent_signal()  # 父进程善后处理
        else:
            self.register_child_signal()  # 子进程善后处理

    def prefork(self, n):
        for i in range(n):
            pid = os.fork()
            if pid < 0:  # fork error
                raise
            if pid > 0:  # parent process
                self.child_pids.append(pid)
                continue
            if pid == 0:
                return False  # child process
        return True

    def register_zk(self):
        self.zk = KazooClient(hosts='127.0.0.1:2181')
        self.zk.start()
        self.zk.ensure_path(self.zk_root)  # 创建根节点
        value = json.dumps({"host": self.host, "port": self.port})
        # 创建服务子节点
        self.zk.create(self.zk_rpc, value, ephemeral=True, sequence=True)

    def exit_parent(self, sig, frame):
        self.zk.stop()  # 关闭zk客户端
        self.close()  # 关闭serversocket
        asyncore.close_all()  # 关闭所有clientsocket
        pids = []
        for pid in self.child_pids:
            print 'before kill'
            try:
                os.kill(pid, signal.SIGINT)  # 关闭子进程
                pids.append(pid)
            except OSError, ex:
                if ex.args[0] == errno.ECHILD:  # 目标子进程已经提前挂了
                    continue
                raise ex
            print 'after kill', pid
        for pid in pids:
            while True:
                try:
                    os.waitpid(pid, 0)  # 收割目标子进程
                    break
                except OSError, ex:
                    if ex.args[0] == errno.ECHILD:  # 子进程已经割过了
                        break
                    if ex.args[0] != errno.EINTR:
                        raise ex  # 被其它信号打断了,要重试
            print 'wait over', pid
Exemple #57
0
class Store:
    """
    Uses zoo keeper as a key vaue store. This store is as thread safe as the underlying client.
    """

    def __init__(self, host_list):
        hosts = ",".join(host_list)
        print("zk, connecting to: '%s'" % hosts)
        self.zk = KazooClient(hosts)

    def __enter__(self):
        self.zk.start()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.zk.stop()

    def get(self, uid, ns, key):
        """
        Get the value associated with the key.
        :param uid: The user id to use.
        :param ns: The namespace to use.
        :param key: The key to look up.
        :return: A tuple of (version, value) stored.
        """
        path = "/jwl/sync/%s/%s/%s" % (uid, ns, key)
        print("zk, fetching: '%s'" % path)
        try:
            r = self.zk.get(path)
            print("zk, value at: '%s'='%s'" % (path, r))
            return r[1].version, r[0]
        except NoNodeError:
            print("zk, no value at: '%s'" % path)
            return None, None


    def put(self, uid, ns, expected_version, key, value):
        """
        If the version of the value in the system is what we expect, then update the key with
        the value given. Otherwise return a tuple of (False, cur_serial, cur_value).

        :param uid: The user id to use.
        :param ns: The namespace to use.
        :param expected_version: The version of the record we expect.
        :param key: The key to update.
        :param value: The value to associate with the key.
        :return: A tuple of (True, new_version, new_value) if it works, (False, cur_version, cur_value) on failure.
        """
        path = "/jwl/sync/%s/%s/%s" % (uid, ns, key)
        self.zk.ensure_path(path)

        if expected_version is None:
            self.zk.set(path, value)
            return True, 1, value

        try:
            print("zk, set '%s'='%s'" % (path, value))
            self.zk.set(path, value, version=expected_version)
            return True, expected_version + 1, value
        except BadVersionError:
            print("zk, set failed '%s'='%s'" % (path, value))
            r = self.zk.get(path)
            return False, r[1].version, r[0]