Example #1
0
 def create_service_transaction(self, service, servers):
     """
     创建一个service事务版本
     :param service:
     :param servers:
     :return:
     """
     try:
         self.zk.create(path='%s/%s/%s' % (self.zk_root, service,self.zk_servers), acl=[self.zk_harpc_acl, self.zk_anyone_acl], makepath=True)
         transaction = self.zk.transaction()
         path_list = []
         for server in servers:
             path = path='%s/%s/%s/%s' % (self.zk_root, service, self.zk_servers, server)
             path_list.append(unicode(path))
             transaction.create(path=path,
                        acl=[self.zk_harpc_acl, self.zk_anyone_acl])
         result = transaction.commit()
         if(result == path_list):
             return (True,)
         else:
             logger.error('manage.create_service:' + SafeString(result))
             return (False,"service节点创建成功,但servers节点创建失败!")
     except Exception, e:
         logger.error('manage.create_service:' + getExceptionDesc(e))
         return (False, getExceptionDesc(e))
Example #2
0
 def get_statistics(self, service, key, type, start=0, end=0):
     """
     获取日志
     :param service:
     :param key:
     :param type:
     :param start:
     :param end:
     :return:
     """
     tables = []
     series = []
     statistics_series = settings.ZK_STATISTICS_SERIES
     total = {"avgtime":0,'mintime':0,"maxtime":0,"qps":0.0,"success":0,"failure":0}
     if service and len(service) > 0 and key and len(key) > 0 and type in (self.zk_servers, self.zk_clients):
         try:
             result = self.zk.get("%s/%s/%s/%s/%s/" % (self.zk_root, service, self.zk_statistics, type, key))
             detail_list = {}
             try:
                 result_str = safe_list_get(result,0,"{}")
                 result_dict = json.loads(result_str)
                 detail_list = result_dict['detail']
                 total = result_dict['total']
             except Exception,e:
                 logger.error('manage.harpc.get_statistics getdata:' + getExceptionDesc(e))
                 pass
             for detail in detail_list:
                 date = detail.get("time")
                 if start > 0 and end > 0 and start > int(date) or end < int(date):
                     continue
                 table = [date]
                 for statistics_serie in statistics_series:
                     statistics_serie_name = statistics_serie.get('name', None)
                     value = detail.get(statistics_serie_name, 0)
                     unit = statistics_serie.get('unit', 0)
                     table.append(value)
                     date_struct = time.strptime(SafeString(date), "%Y%m%d%H%M%S")
                     serie_dict = {'name': statistics_serie_name, 'data': [
                                 [date_struct.tm_year, date_struct.tm_mon - 1, date_struct.tm_mday, date_struct.tm_hour,
                                 date_struct.tm_min, date_struct.tm_sec, value]], 'yAxis': unit}
                     for serie in series:
                         if serie.get('name', None) == statistics_serie_name:
                             serie.get('data', []).append(
                                  [date_struct.tm_year, date_struct.tm_mon - 1, date_struct.tm_mday,
                                   date_struct.tm_hour, date_struct.tm_min, date_struct.tm_sec, value])
                             break
                     else:
                          series.append(serie_dict)
                 tables.append(table)
         except Exception,e:
             logger.error('manage.harpc.get_statistics:' + getExceptionDesc(e))
             pass
Example #3
0
    def get_clients_total(self,service):
        """
        获取一个clients的合计信息
        :param service:
        :return:
        """
        statistics_series = settings.ZK_STATISTICS_SERIES
        results = {"avgtime":0,'mintime':0,"maxtime":0,"qps":0.0,"success":0,"failure":0}

        try:
            clients = self.resource[service][self.zk_clients]
            for key in clients:
                try:
                    result = self.zk.get(path="%s/%s/%s/%s/%s/" % (self.zk_root, service, self.zk_statistics, self.zk_clients, key))
                    total = {}
                    try:
                        result_str = safe_list_get(result,0,"{}")
                        result_dict = json.loads(result_str)
                        total = result_dict['total']
                    except Exception,e:
                        logger.error('manage.harpc.get_clients_total getdata:' + getExceptionDesc(e))
                        pass
                    for statistics_serie in statistics_series:
                        serie_name = statistics_serie['name']
                        serie_value = total.get(serie_name,0)
                        results[serie_name] = results.get(serie_name,0)+serie_value
                except Exception,e:
                    continue
Example #4
0
 def create_service(self, service):
     """
     创建一个service
     :param service:
     :param servers:
     :return:
     """
     try:
         self.zk.create(path='%s/%s' % (self.zk_root, service))
         self.zk.create(path='%s/%s/%s' % (self.zk_root, service,self.zk_servers),acl=[self.zk_harpc_acl, self.zk_anyone_acl])
         self.lock.acquire()
         self.resource[service] = {self.zk_servers: {}, self.zk_clients: {}, "acl": True}
         self.lock.release()
         return (True,"创建成功,请刷新页面查看结果")
     except Exception, e:
         logger.error('manage.create_service:' + getExceptionDesc(e))
         return (False, getExceptionDesc(e))
Example #5
0
File: core.py Project: zmyer/harpc
 def init_connect(self):
     try:
         self.zk = KazooClient(hosts=self.zk_hosts, timeout=self.zk_timeout)
         self.zk.start(timeout=self.zk_timeout)
         self.zk.add_auth(scheme='digest', credential=self.zk_username + ':' + self.zk_password)
         logger.info("connect to zk servers")
     except Exception, e:
         logger.error('manage.ZKWrap.init_connect:' + getExceptionDesc(e))
         return False
Example #6
0
    def delete_server(self,service,server):
        """
        删除一个servers
        :param service:
        :param servers:
        :return:
        """
        try:
            path = '%s/%s/%s/%s' % (self.zk_root, service, self.zk_servers, server)
            self.zk.delete(path=path, recursive=True)

            self.lock.acquire()
            self.resource[service][self.zk_servers].pop(server)
            self.lock.release()

            return (True,)
        except Exception, e:
            logger.error('manage.delect_servers:' + getExceptionDesc(e))
            return (False, getExceptionDesc(e))
Example #7
0
    def create_server(self,service,server,server_value):
        """
        创建一个server
        :param service:
        :param servers:
        :return:
        """
        try:
            self.zk.create(path='%s/%s/%s/%s' % (self.zk_root, service, self.zk_servers, server),
                            makepath=True,value=server_value)

            self.lock.acquire()
            self.resource[service][self.zk_servers][server] = ({"data": json.loads(server_value), "acl": True})
            self.lock.release()

            return (True,)
        except Exception, e:
            logger.error('manage.create_servers:' + getExceptionDesc(e))
            return (False, getExceptionDesc(e))
Example #8
0
 def get_service_acl(self, service):
     """
     获取service权限信息
     :param service:
     :return:
     """
     try:
         return self.resource[service]['acl']
     except Exception,e:
         logger.error('manage.get_service_acl:' + getExceptionDesc(e))
         pass
Example #9
0
 def get_service_child_acl(self,service,type,child):
     """
     获取service孩子节点权限
     :param service:
     :param type:
     :param child:
     :return:
     """
     try:
         return self.resource[service][type][child]['acl']
     except Exception,e:
         logger.error('manage.get_service_child_acl:' + getExceptionDesc(e))
Example #10
0
 def get_service_child_info(self,service,type,child):
     """
     获取子节点信息
     :param service:
     :param type:
     :param child:
     :return:
     """
     result = {}
     try:
         result = self.resource[service][type][child]['data']
     except Exception,e:
         logger.error('manage.get_service_child_info:' + getExceptionDesc(e))
         pass
Example #11
0
 def get_clients(self, service):
     """
     获取clients信息
     :param service:
     :return:
     """
     results = []
     try:
         clients_dict = self.resource[service][self.zk_clients]
         for (k,v) in clients_dict.items():
             client_name = k
             client_acl = v.get("acl",False)
             results.append((client_name,client_acl))
     except Exception,e:
         logger.error('manage.get_servers:' + getExceptionDesc(e))
         pass
Example #12
0
 def get_servers(self, service):
     """
     获取指定service信息
     :param service:
     :return:
     """
     results = []
     try:
         servers_dict = self.resource[service][self.zk_servers]
         for (k,v) in servers_dict.items():
             server_name = k
             server_acl = v.get("acl",False)
             results.append((server_name,server_acl))
     except Exception,e:
         logger.error('manage.get_servers:' + getExceptionDesc(e))
         pass
Example #13
0
File: core.py Project: zmyer/harpc
    def watch_clients_exists(self, event):
        logger.info("watch_clients is change:" + str(event))
        type = event.type
        path = event.path
        service = path.split("/")[-2]
        if self.zk.exists(path=path, watch=self.watch_clients_exists):
            if type == 'CREATED':
                clients = {}
                client_list = self.zk.get_children(path="%s/%s/%s/" % (self.zk_root, service, self.zk_clients),
                                                   watch=self.watch_clients)
                for client in client_list:
                    client_data_dict = {}
                    client_data = self.zk.get(path="%s/%s/%s/%s" % (self.zk_root, service, self.zk_clients, client))
                    try:
                        client_data_str = safe_list_get(client_data, 0, "{}")
                        client_data_dict = json.loads(client_data_str)
                    except Exception, e:
                        logger.error(
                            'manage.ZKWrap.watch_clients_exists getdata:' + service + "/" + client + ":" + getExceptionDesc(
                                e))
                        pass

                    client_node_stat = safe_list_get(client_data, 1, None)
                    if client_node_stat:
                        client_data_dict['ctime'] = time.strftime("%Y-%m-%d %H:%M:%S",
                                                                  time.localtime(int(client_node_stat.ctime / 1000)))

                    client_acl = False
                    result = self.get_acls("%s/%s/%s/%s" % (self.zk_root, service, self.zk_clients, client))
                    acls = safe_list_get(result, 0, [])
                    if self.zk_harpc_acl in acls:
                        client_acl = True

                    clients[client] = {"data": client_data_dict, "acl": client_acl}

                self.lock.acquire()
                self.resource[service][self.zk_clients] = clients
                self.lock.release()
Example #14
0
File: core.py Project: zmyer/harpc
    def watch_clients(self, event):
        logger.info("clients is change:" + str(event))
        type = event.type
        path = event.path
        service = path.split("/")[-2]
        new_client_list = self.zk.get_children(path=path, watch=self.watch_clients)
        if type == 'CHILD':
            new_clients = set(new_client_list)
            old_clients = set(self.resource[service][self.zk_clients].keys())

            add_clients = new_clients - old_clients
            del_clients = old_clients - new_clients

            for add_client in add_clients:
                client_data_dict = {}
                client_data = self.zk.get(path="%s/%s/%s/%s" % (self.zk_root, service, self.zk_clients, add_client))
                try:
                    client_data_str = safe_list_get(client_data, 0, "{}")
                    client_data_dict = json.loads(client_data_str)
                except Exception, e:
                    logger.error(
                        'manage.ZKWrap.watch_clients getdata:' + service + "/" + add_client + ":" + getExceptionDesc(e))
                    pass

                client_node_stat = safe_list_get(client_data, 1, None)
                if client_node_stat:
                    client_data_dict['ctime'] = time.strftime("%Y-%m-%d %H:%M:%S",
                                                              time.localtime(int(client_node_stat.ctime / 1000)))

                client_acl = False
                result = self.get_acls("%s/%s/%s/%s" % (self.zk_root, service, self.zk_clients, add_client))
                acls = safe_list_get(result, 0, [])
                if self.zk_harpc_acl in acls:
                    client_acl = True

                self.lock.acquire()
                self.resource[service][self.zk_clients][add_client] = {"data": client_data_dict, "acl": client_acl}
                self.lock.release()

            for del_client in del_clients:
                self.lock.acquire()
                del self.resource[service][self.zk_clients][del_client]
                self.lock.release()
Example #15
0
File: core.py Project: zmyer/harpc
    def watch_services(self, event):
        logger.info("services is change:" + str(event))
        new_service_list = self.zk.get_children(path="/%s" % (self.zk_root), watch=self.watch_services)
        new_service = set(new_service_list)
        old_service = set(self.resource.keys())
        add_services = new_service - old_service
        del_services = old_service - new_service

        for add_service in add_services:
            acl = False
            try:
                result = self.get_acls("%s/%s/%s" % (self.zk_root, add_service, self.zk_servers))
                acls = safe_list_get(result, 0, [])
                if self.zk_harpc_acl in acls:
                    acl = True
            except Exception, e:
                acl = False

            servers = {}
            if self.zk.exists(path="%s/%s/%s" % (self.zk_root, add_service, self.zk_servers),
                              watch=self.watch_servers_exists):
                server_list = self.zk.get_children(path="%s/%s/%s/" % (self.zk_root, add_service, self.zk_servers),
                                                   watch=self.watch_servers)
                for server in server_list:
                    server_data_dict = {}
                    server_data = self.zk.get(path="%s/%s/%s/%s" % (self.zk_root, add_service, self.zk_servers, server))
                    try:
                        server_data_str = safe_list_get(server_data, 0, "{}")
                        server_data_dict = json.loads(server_data_str)
                    except Exception, e:
                        logger.error(
                            'manage.ZKWrap.watch_services getdata:' + add_service + "/" + server + ":" + getExceptionDesc(
                                e))
                        pass

                    server_node_stat = safe_list_get(server_data, 1, None)
                    if server_node_stat:
                        server_data_dict['ctime'] = time.strftime("%Y-%m-%d %H:%M:%S",
                                                                  time.localtime(int(server_node_stat.ctime / 1000)))

                    server_acl = False
                    result = self.get_acls("%s/%s/%s/%s" % (self.zk_root, add_service, self.zk_servers, server))
                    acls = safe_list_get(result, 0, [])
                    if self.zk_harpc_acl in acls:
                        server_acl = True

                    servers[server] = {"data": server_data_dict, "acl": server_acl}
Example #16
0
File: core.py Project: zmyer/harpc
            clients = {}
            # 如果系统节点数过多请注释该条不启用监听器实时更新节点信息,以避免ZK性能受到影响
            # if self.zk.exists(path="%s/%s/%s" % (self.zk_root,service,self.zk_clients),watch=self.watch_clients_exists):
            if self.zk.exists(path="%s/%s/%s" % (self.zk_root, service, self.zk_clients)):
                # 如果系统节点数过多请注释该条不启用监听器实时更新节点信息,以避免ZK性能受到影响
                # client_list = self.zk.get_children(path="%s/%s/%s/" % (self.zk_root, service, self.zk_clients),watch=self.watch_clients)
                client_list = self.zk.get_children(path="%s/%s/%s/" % (self.zk_root, service, self.zk_clients))
                for client in client_list:
                    client_data_dict = {}
                    client_data = self.zk.get(path="%s/%s/%s/%s" % (self.zk_root, service, self.zk_clients, client))
                    try:
                        client_data_str = safe_list_get(client_data, 0, "{}")
                        client_data_dict = json.loads(client_data_str)
                    except Exception, e:
                        logger.error(
                            'manage.ZKWrap.load_zk_resource getdata:' + service + "/" + client + ":" + getExceptionDesc(
                                e))
                        pass
                    client_node_stat = safe_list_get(client_data, 1, None)
                    if client_node_stat:
                        client_data_dict['ctime'] = time.strftime("%Y-%m-%d %H:%M:%S",
                                                                  time.localtime(int(client_node_stat.ctime / 1000)))
                    client_acl = False
                    result = self.get_acls("%s/%s/%s/%s" % (self.zk_root, service, self.zk_clients, client))
                    acls = safe_list_get(result, 0, [])
                    if self.zk_harpc_acl in acls:
                        client_acl = True

                    clients[client] = {"data": client_data_dict, "acl": client_acl}
            services[service] = {self.zk_servers: servers, self.zk_clients: clients, "acl": acl}

        self.lock.acquire()
Example #17
0
File: core.py Project: zmyer/harpc
    def load_zk_resource(self):
        logger.info('get service from zookeeper')
        # 如果系统节点数过多请注释该条不启用监听器实时更新节点信息,以避免ZK性能受到影响
        # service_list = self.zk.get_children(path=self.zk_root, watch=self.watch_services)
        service_list = self.zk.get_children(path=self.zk_root)
        services = {}
        for service in service_list:

            acl = False
            try:
                result = self.get_acls("%s/%s/%s" % (self.zk_root, service, self.zk_servers))
                acls = safe_list_get(result, 0, [])
                if self.zk_harpc_acl in acls:
                    acl = True
            except Exception, e:
                acl = False

            servers = {}
            # 如果系统节点数过多请注释该条不启用监听器实时更新节点信息,以避免ZK性能受到影响
            # if self.zk.exists(path="%s/%s/%s" % (self.zk_root,service,self.zk_servers),watch=self.watch_servers_exists):
            if self.zk.exists(path="%s/%s/%s" % (self.zk_root, service, self.zk_servers)):
                # 如果系统节点数过多请注释该条不启用监听器实时更新节点信息,以避免ZK性能受到影响
                # server_list= self.zk.get_children(path="%s/%s/%s/" % (self.zk_root, service, self.zk_servers),watch=self.watch_servers)
                server_list = self.zk.get_children(path="%s/%s/%s/" % (self.zk_root, service, self.zk_servers))
                for server in server_list:
                    server_data_dict = {}
                    server_data = self.zk.get(path="%s/%s/%s/%s" % (self.zk_root, service, self.zk_servers, server))
                    try:
                        server_data_str = safe_list_get(server_data, 0, "{}")
                        server_data_dict = json.loads(server_data_str)
                    except Exception, e:
                        logger.error(
                            'manage.ZKWrap.load_zk_resource getdata:' + service + "/" + server + "/" + getExceptionDesc(
                                e))
                        pass
                    # if len(server_data_str) > 2:
                    #     server_data_dict = eval(server_data_str)
                    server_node_stat = safe_list_get(server_data, 1, None)
                    if server_node_stat:
                        server_data_dict['ctime'] = time.strftime("%Y-%m-%d %H:%M:%S",
                                                                  time.localtime(int(server_node_stat.ctime / 1000)))
                    server_acl = False
                    result = self.get_acls("%s/%s/%s/%s" % (self.zk_root, service, self.zk_servers, server))
                    acls = safe_list_get(result, 0, [])
                    if self.zk_harpc_acl in acls:
                        server_acl = True

                    servers[server] = {"data": server_data_dict, "acl": server_acl}
Example #18
0
                    for statistics_serie in statistics_series:
                        serie_name = statistics_serie['name']
                        serie_value = total.get(serie_name,0)
                        results[serie_name] = results.get(serie_name,0)+serie_value
                except Exception,e:
                    continue
            for statistics_serie in statistics_series:
                serie_name = statistics_serie['name']
                value = results.get(serie_name,0)
                if serie_name in ['avgtime','mintime','maxtime']:
                    value = value/len(servers)
                if serie_name == 'qps':
                    value = float('%.4f'% value)
                results[serie_name] = value
        except Exception,e:
            logger.error('manage.harpc.get_servers_total:' + getExceptionDesc(e))
            pass
        return results

    def get_clients_total(self,service):
        """
        获取一个clients的合计信息
        :param service:
        :return:
        """
        statistics_series = settings.ZK_STATISTICS_SERIES
        results = {"avgtime":0,'mintime':0,"maxtime":0,"qps":0.0,"success":0,"failure":0}

        try:
            clients = self.resource[service][self.zk_clients]
            for key in clients: