def __init__(self, networkmgr, etcdclient, addr, mode): self.addr = addr logger.info ("begin initialize on %s" % self.addr) self.networkmgr = networkmgr self.etcd = etcdclient self.mode = mode # initialize the network logger.info ("initialize network") # 'docklet-br' not need ip address. Because every user has gateway #[status, result] = self.networkmgr.acquire_sysips_cidr() #self.networkmgr.printpools() #if not status: # logger.info ("initialize network failed, no IP for system bridge") # sys.exit(1) #self.bridgeip = result[0] #logger.info ("initialize bridge wih ip %s" % self.bridgeip) #network.netsetup("init", self.bridgeip) if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") sys.exit(1) # get allnodes self.allnodes = self._nodelist_etcd("allnodes") self.runnodes = self._nodelist_etcd("runnodes") logger.info ("all nodes are: %s" % self.allnodes) logger.info ("run nodes are: %s" % self.runnodes) if len(self.runnodes)>0: logger.error ("init runnodes is not null, need to be clean") sys.exit(1) # init rpc list self.rpcs = [] # start new thread to watch whether a new node joins logger.info ("start thread to watch new nodes ...") self.thread_watchnewnode = threading.Thread(target=self._watchnewnode) self.thread_watchnewnode.start() # wait for all nodes joins while(True): allin = True for node in self.allnodes: if node not in self.runnodes: allin = False break if allin: logger.info("all nodes necessary joins ...") break time.sleep(0.05) logger.info ("run nodes are: %s" % self.runnodes)
def __init__(self, networkmgr, etcdclient, addr, mode): self.addr = addr logger.info("begin initialize on %s" % self.addr) self.networkmgr = networkmgr self.etcd = etcdclient self.mode = mode # initialize the network logger.info("initialize network") # 'docklet-br' not need ip address. Because every user has gateway #[status, result] = self.networkmgr.acquire_sysips_cidr() #self.networkmgr.printpools() #if not status: # logger.info ("initialize network failed, no IP for system bridge") # sys.exit(1) #self.bridgeip = result[0] #logger.info ("initialize bridge wih ip %s" % self.bridgeip) #network.netsetup("init", self.bridgeip) if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") sys.exit(1) # get allnodes self.allnodes = self._nodelist_etcd("allnodes") self.runnodes = self._nodelist_etcd("runnodes") logger.info("all nodes are: %s" % self.allnodes) logger.info("run nodes are: %s" % self.runnodes) if len(self.runnodes) > 0: logger.error("init runnodes is not null, need to be clean") sys.exit(1) # init rpc list self.rpcs = [] # start new thread to watch whether a new node joins logger.info("start thread to watch new nodes ...") self.thread_watchnewnode = threading.Thread(target=self._watchnewnode) self.thread_watchnewnode.start() # wait for all nodes joins while (True): allin = True for node in self.allnodes: if node not in self.runnodes: allin = False break if allin: logger.info("all nodes necessary joins ...") break time.sleep(0.05) logger.info("run nodes are: %s" % self.runnodes)
def del_usrgwbr(self, username, uid, nodemgr): if username not in self.usrgws.keys(): return [False, "user does't have gateway or user doesn't exist."] ip = self.usrgws[username] logger.info("Delete user %s(%s) gateway on %s" %(username, str(uid), ip)) if ip == self.masterip: netcontrol.del_gw('docklet-br-'+str(uid), username) netcontrol.del_bridge('docklet-br-'+str(uid)) else: worker = nodemgr.ip_to_rpc(ip) worker.del_gw('docklet-br-'+str(uid), username) worker.del_bridge('docklet-br-'+str(uid)) del self.usrgws[username] self.etcd.delkey("network/usrgws/"+username) return [True, 'delete user\' gateway success']
def del_usrgwbr(self, username, uid, nodemgr): if username not in self.usrgws.keys(): return [False, "user does't have gateway or user doesn't exist."] ip = self.usrgws[username] logger.info("Delete user %s(%s) gateway on %s" %(username, str(uid), ip)) if ip == self.masterip: netcontrol.del_gw('docklet-br-'+str(uid), username) netcontrol.del_bridge('docklet-br-'+str(uid)) else: worker = nodemgr.ip_to_rpc(ip) worker.del_gw('docklet-br-'+str(uid), username) worker.del_bridge('docklet-br-'+str(uid)) del self.usrgws[username] self.etcd.delkey("network/usrgws/"+username) return [True, 'delete user\' gateway success']
def check_switch(self, switchid): switchid = int(switchid) if switchid not in self.__switches and self.__mode == 'new' and netcontrol.bridge_exists(switchid): [sts, _] = netcontrol.del_bridge(switchid) if not sts: return [False, "del virtual switch failed with mode new"] if not netcontrol.bridge_exists(switchid): [sts, _] = netcontrol.new_bridge(switchid) if not sts: return [False, "add virtual switch failed"] if switchid not in self.__switches: self.__switches.append(switchid) return [True, "check virtual switch okay"]
def check_switch_with_gre(self, switchid): if self.__addr == self.__master: return True switchid = int(switchid) if switchid not in self.__switches and self.__mode == 'new' and netcontrol.bridge_exists(switchid): [sts, _] = netcontrol.del_bridge(switchid) if not sts: return [False, "del virtual switch failed with mode new"] if not netcontrol.bridge_exists(switchid): [sts, _] = netcontrol.new_bridge(switchid) if not sts: return [False, "add virtual switch failed"] if switchid not in self.__switches: self.__switches.append(switchid) if not netcontrol.gre_exists(switchid, self.__master): [sts, _] = netcontrol.setup_gre(switchid, self.__master, switchid) if not sts: return [False, "setup gre to master failed"] return [True, "check virtual switch okay"]
def __init__(self, networkmgr, etcdclient, addr, mode): self.addr = addr logger.info ("begin initialize on %s" % self.addr) self.networkmgr = networkmgr self.etcd = etcdclient self.mode = mode self.workerport = env.getenv('WORKER_PORT') # initialize the network logger.info ("initialize network") # 'docklet-br' not need ip address. Because every user has gateway #[status, result] = self.networkmgr.acquire_sysips_cidr() #self.networkmgr.printpools() #if not status: # logger.info ("initialize network failed, no IP for system bridge") # sys.exit(1) #self.bridgeip = result[0] #logger.info ("initialize bridge wih ip %s" % self.bridgeip) #network.netsetup("init", self.bridgeip) if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") sys.exit(1) # init rpc list self.rpcs = [] # get allnodes # used in recovery mode, find alll the lost running nodes self.allnodes = self._nodelist_etcd("allnodes") self.runnodes = [] [status, runlist] = self.etcd.listdir("machines/runnodes") for node in runlist: nodeip = node['key'].rsplit('/',1)[1] if node['value'] == 'ok': logger.info ("running node %s" % nodeip) self.runnodes.append(nodeip) self.rpcs.append(xmlrpc.client.ServerProxy("http://%s:%s" % (nodeip, self.workerport))) logger.info ("add %s:%s in rpc client list" % (nodeip, self.workerport)) logger.info ("all nodes are: %s" % self.allnodes) logger.info ("run nodes are: %s" % self.runnodes) # start new thread to watch whether a new node joins logger.info ("start thread to watch new nodes ...") self.thread_watchnewnode = threading.Thread(target=self._watchnewnode) self.thread_watchnewnode.start() # wait for all nodes joins while(True): allin = True for node in self.allnodes: if node not in self.runnodes: allin = False break if allin: logger.info("all nodes necessary joins ...") break time.sleep(0.05) logger.info ("run nodes are: %s" % self.runnodes)
def __init__(self, etcdclient, addr, port): self.addr = addr self.port = port logger.info ("begin initialize on %s" % self.addr) self.fspath = env.getenv('FS_PREFIX') self.poolsize = env.getenv('DISKPOOL_SIZE') self.etcd = etcdclient self.master = self.etcd.getkey("service/master")[1] self.mode=None # waiting state is preserved for compatible. self.etcd.setkey("machines/runnodes/"+self.addr, "waiting") # get this node's key to judge how to init. [status, key] = self.etcd.getkey("machines/runnodes/"+self.addr) if status: self.key = generatekey("machines/allnodes/"+self.addr) else: logger.error("get key failed. %s" % 'machines/runnodes/'+self.addr) sys.exit(1) # check token to check global directory [status, token_1] = self.etcd.getkey("token") tokenfile = open(self.fspath+"/global/token", 'r') token_2 = tokenfile.readline().strip() if token_1 != token_2: logger.error("check token failed, global directory is not a shared filesystem") sys.exit(1) logger.info ("worker registered and checked the token") # worker search all run nodes to judge how to init # If the node in all node list, we will recover it. # Otherwise, this node is new added in. value = 'init-new' [status, alllist] = self.etcd.listdir("machines/allnodes") for node in alllist: if node['key'] == self.key: value = 'init-recovery' break logger.info("worker start in "+value+" mode") Containers = container.Container(self.addr, etcdclient) if value == 'init-new': logger.info ("init worker with mode:new") self.mode='new' # check global directory do not have containers on this worker [both, onlylocal, onlyglobal] = Containers.diff_containers() if len(both+onlyglobal) > 0: logger.error ("mode:new will clean containers recorded in global, please check") sys.exit(1) [status, info] = Containers.delete_allcontainers() if not status: logger.error ("delete all containers failed") sys.exit(1) # create new lvm VG at last new_group("docklet-group",self.poolsize,self.fspath+"/local/docklet-storage") #subprocess.call([self.libpath+"/lvmtool.sh", "new", "group", "docklet-group", self.poolsize, self.fspath+"/local/docklet-storage"]) elif value == 'init-recovery': logger.info ("init worker with mode:recovery") self.mode='recovery' # recover lvm VG first recover_group("docklet-group",self.fspath+"/local/docklet-storage") #subprocess.call([self.libpath+"/lvmtool.sh", "recover", "group", "docklet-group", self.fspath+"/local/docklet-storage"]) [status, meg] = Containers.check_allcontainers() if status: logger.info ("all containers check ok") else: logger.info ("not all containers check ok") #sys.exit(1) else: logger.error ("worker init mode:%s not supported" % value) sys.exit(1) # initialize rpc # xmlrpc.server.SimpleXMLRPCServer(addr) -- addr : (ip-addr, port) # if ip-addr is "", it will listen ports of all IPs of this host logger.info ("initialize rpcserver %s:%d" % (self.addr, int(self.port))) # logRequests=False : not print rpc log #self.rpcserver = xmlrpc.server.SimpleXMLRPCServer((self.addr, self.port), logRequests=False) self.rpcserver = ThreadXMLRPCServer((self.addr, int(self.port)), allow_none=True, logRequests=False) self.rpcserver.register_introspection_functions() self.rpcserver.register_instance(Containers) self.rpcserver.register_function(monitor.workerFetchInfo) # register functions or instances to server for rpc #self.rpcserver.register_function(function_name) # init collector to collect monitor infomation self.con_collector = monitor.Container_Collector() self.hosts_collector = monitor.Collector() # initialize the network # if worker and master run on the same node, reuse bridges # don't need to create new bridges if (self.addr == self.master): logger.info ("master also on this node. reuse master's network") else: logger.info ("initialize network") # 'docklet-br' of worker do not need IP Addr. #[status, result] = self.etcd.getkey("network/workbridge") #if not status: # logger.error ("get bridge IP failed, please check whether master set bridge IP for worker") #self.bridgeip = result # create bridges for worker #network.netsetup("init", self.bridgeip) if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") sys.exit(1) logger.info ("setup GRE tunnel to master %s" % self.master) #network.netsetup("gre", self.master) if not netcontrol.gre_exists('docklet-br', self.master): netcontrol.setup_gre('docklet-br', self.master)
def __init__(self, etcdclient, addr, port): self.addr = addr self.port = port logger.info("begin initialize on %s" % self.addr) self.fspath = env.getenv('FS_PREFIX') self.poolsize = env.getenv('DISKPOOL_SIZE') self.etcd = etcdclient self.master = self.etcd.getkey("service/master")[1] self.mode = None # waiting state is preserved for compatible. self.etcd.setkey("machines/runnodes/" + self.addr, "waiting") # get this node's key to judge how to init. [status, key] = self.etcd.getkey("machines/runnodes/" + self.addr) if status: self.key = generatekey("machines/allnodes/" + self.addr) else: logger.error("get key failed. %s" % 'machines/runnodes/' + self.addr) sys.exit(1) # check token to check global directory [status, token_1] = self.etcd.getkey("token") tokenfile = open(self.fspath + "/global/token", 'r') token_2 = tokenfile.readline().strip() if token_1 != token_2: logger.error( "check token failed, global directory is not a shared filesystem" ) sys.exit(1) logger.info("worker registered and checked the token") # worker search all run nodes to judge how to init # If the node in all node list, we will recover it. # Otherwise, this node is new added in. value = 'init-new' [status, alllist] = self.etcd.listdir("machines/allnodes") for node in alllist: if node['key'] == self.key: value = 'init-recovery' break logger.info("worker start in " + value + " mode") Containers = container.Container(self.addr, etcdclient) if value == 'init-new': logger.info("init worker with mode:new") self.mode = 'new' # check global directory do not have containers on this worker [both, onlylocal, onlyglobal] = Containers.diff_containers() if len(both + onlyglobal) > 0: logger.error( "mode:new will clean containers recorded in global, please check" ) sys.exit(1) [status, info] = Containers.delete_allcontainers() if not status: logger.error("delete all containers failed") sys.exit(1) # create new lvm VG at last new_group("docklet-group", self.poolsize, self.fspath + "/local/docklet-storage") #subprocess.call([self.libpath+"/lvmtool.sh", "new", "group", "docklet-group", self.poolsize, self.fspath+"/local/docklet-storage"]) elif value == 'init-recovery': logger.info("init worker with mode:recovery") self.mode = 'recovery' # recover lvm VG first recover_group("docklet-group", self.fspath + "/local/docklet-storage") #subprocess.call([self.libpath+"/lvmtool.sh", "recover", "group", "docklet-group", self.fspath+"/local/docklet-storage"]) [status, meg] = Containers.check_allcontainers() if status: logger.info("all containers check ok") else: logger.info("not all containers check ok") #sys.exit(1) else: logger.error("worker init mode:%s not supported" % value) sys.exit(1) # initialize rpc # xmlrpc.server.SimpleXMLRPCServer(addr) -- addr : (ip-addr, port) # if ip-addr is "", it will listen ports of all IPs of this host logger.info("initialize rpcserver %s:%d" % (self.addr, int(self.port))) # logRequests=False : not print rpc log #self.rpcserver = xmlrpc.server.SimpleXMLRPCServer((self.addr, self.port), logRequests=False) self.rpcserver = ThreadXMLRPCServer((self.addr, int(self.port)), allow_none=True, logRequests=False) self.rpcserver.register_introspection_functions() self.rpcserver.register_instance(Containers) self.rpcserver.register_function(monitor.workerFetchInfo) # register functions or instances to server for rpc #self.rpcserver.register_function(function_name) # initialize the network # if worker and master run on the same node, reuse bridges # don't need to create new bridges if (self.addr == self.master): logger.info("master also on this node. reuse master's network") else: logger.info("initialize network") # 'docklet-br' of worker do not need IP Addr. #[status, result] = self.etcd.getkey("network/workbridge") #if not status: # logger.error ("get bridge IP failed, please check whether master set bridge IP for worker") #self.bridgeip = result # create bridges for worker #network.netsetup("init", self.bridgeip) if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") sys.exit(1) logger.info("setup GRE tunnel to master %s" % self.master) #network.netsetup("gre", self.master) if not netcontrol.gre_exists('docklet-br', self.master): netcontrol.setup_gre('docklet-br', self.master)
def __init__(self, etcdclient, addr, port): self.addr = addr self.port = port logger.info ("begin initialize on %s" % self.addr) self.fspath = env.getenv('FS_PREFIX') self.poolsize = env.getenv('DISKPOOL_SIZE') self.etcd = etcdclient self.master = self.etcd.getkey("service/master")[1] self.mode=None # register self to master self.etcd.setkey("machines/runnodes/"+self.addr, "waiting") for f in range (0, 3): [status, value] = self.etcd.getkey("machines/runnodes/"+self.addr) if not value.startswith("init"): # master wakesup every 0.1s to check register logger.debug("worker % register to master failed %d \ time, sleep %fs" % (self.addr, f+1, 0.1)) time.sleep(0.1) else: break if value.startswith("init"): # check token to check global directory [status, token_1] = self.etcd.getkey("token") tokenfile = open(self.fspath+"/global/token", 'r') token_2 = tokenfile.readline().strip() if token_1 != token_2: logger.error("check token failed, global directory is not a shared filesystem") sys.exit(1) else: logger.error ("worker register in machines/runnodes failed, maybe master not start") sys.exit(1) logger.info ("worker registered in master and checked the token") Containers = container.Container(self.addr, etcdclient) if value == 'init-new': logger.info ("init worker with mode:new") self.mode='new' # check global directory do not have containers on this worker [both, onlylocal, onlyglobal] = Containers.diff_containers() if len(both+onlyglobal) > 0: logger.error ("mode:new will clean containers recorded in global, please check") sys.exit(1) [status, info] = Containers.delete_allcontainers() if not status: logger.error ("delete all containers failed") sys.exit(1) # create new lvm VG at last new_group("docklet-group",self.poolsize,self.fspath+"/local/docklet-storage") #subprocess.call([self.libpath+"/lvmtool.sh", "new", "group", "docklet-group", self.poolsize, self.fspath+"/local/docklet-storage"]) elif value == 'init-recovery': logger.info ("init worker with mode:recovery") self.mode='recovery' # recover lvm VG first recover_group("docklet-group",self.fspath+"/local/docklet-storage") #subprocess.call([self.libpath+"/lvmtool.sh", "recover", "group", "docklet-group", self.fspath+"/local/docklet-storage"]) [status, meg] = Containers.check_allcontainers() if status: logger.info ("all containers check ok") else: logger.info ("not all containers check ok") #sys.exit(1) else: logger.error ("worker init mode:%s not supported" % value) sys.exit(1) # initialize rpc # xmlrpc.server.SimpleXMLRPCServer(addr) -- addr : (ip-addr, port) # if ip-addr is "", it will listen ports of all IPs of this host logger.info ("initialize rpcserver %s:%d" % (self.addr, int(self.port))) # logRequests=False : not print rpc log #self.rpcserver = xmlrpc.server.SimpleXMLRPCServer((self.addr, self.port), logRequests=False) self.rpcserver = ThreadXMLRPCServer((self.addr, int(self.port)), allow_none=True) self.rpcserver.register_introspection_functions() self.rpcserver.register_instance(Containers) # register functions or instances to server for rpc #self.rpcserver.register_function(function_name) # initialize the network # if worker and master run on the same node, reuse bridges # don't need to create new bridges if (self.addr == self.master): logger.info ("master also on this node. reuse master's network") else: logger.info ("initialize network") # 'docklet-br' of worker do not need IP Addr. #[status, result] = self.etcd.getkey("network/workbridge") #if not status: # logger.error ("get bridge IP failed, please check whether master set bridge IP for worker") #self.bridgeip = result # create bridges for worker #network.netsetup("init", self.bridgeip) if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") sys.exit(1) logger.info ("setup GRE tunnel to master %s" % self.master) #network.netsetup("gre", self.master) if not netcontrol.gre_exists('docklet-br', self.master): netcontrol.setup_gre('docklet-br', self.master)
def __init__(self, networkmgr, etcdclient, addr, mode): self.addr = addr logger.info ("begin initialize on %s" % self.addr) self.networkmgr = networkmgr self.etcd = etcdclient self.mode = mode self.workerport = env.getenv('WORKER_PORT') # initialize the network logger.info ("initialize network") # 'docklet-br' not need ip address. Because every user has gateway #[status, result] = self.networkmgr.acquire_sysips_cidr() #self.networkmgr.printpools() #if not status: # logger.info ("initialize network failed, no IP for system bridge") # sys.exit(1) #self.bridgeip = result[0] #logger.info ("initialize bridge wih ip %s" % self.bridgeip) #network.netsetup("init", self.bridgeip) if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") sys.exit(1) # init rpc list self.rpcs = [] # get allnodes self.allnodes = self._nodelist_etcd("allnodes") self.runnodes = [] [status, runlist] = self.etcd.listdir("machines/runnodes") for node in runlist: nodeip = node['key'].rsplit('/',1)[1] if node['value'] == 'ok': logger.info ("running node %s" % nodeip) self.runnodes.append(nodeip) self.rpcs.append(xmlrpc.client.ServerProxy("http://%s:%s" % (nodeip, self.workerport))) logger.info ("add %s:%s in rpc client list" % (nodeip, self.workerport)) logger.info ("all nodes are: %s" % self.allnodes) logger.info ("run nodes are: %s" % self.runnodes) # start new thread to watch whether a new node joins logger.info ("start thread to watch new nodes ...") self.thread_watchnewnode = threading.Thread(target=self._watchnewnode) self.thread_watchnewnode.start() # wait for all nodes joins while(True): allin = True for node in self.allnodes: if node not in self.runnodes: allin = False break if allin: logger.info("all nodes necessary joins ...") break time.sleep(0.05) logger.info ("run nodes are: %s" % self.runnodes)