class ClusterContext(BaseObject): def __init__(self, configFile, platformType): baseDir = dirname(os.path.realpath(__file__)); if not configFile.startswith('/'): configFile = baseDir + "/" + configFile; configData = open(configFile, 'r').read() config = json.loads(configData, object_hook=lambda d: Namespace(**d)) setattr(config.properties, 'base_dir', baseDir) self.config = config PropertyUtil.traverseAndResolvePropertyExpression(config, config.properties) self.images = {} for name in config.images.__dict__.keys(): imgObj = getattr(config.images, name) image = { "name": name, "repository": imgObj.repository } self.images[name] = image self.serverSets = {} for name in config.servers.__dict__.keys(): serverConfig = getattr(config.servers, name) self.serverSets[name] = ServerSet(name, serverConfig) self.services = {} for name in config.services.__dict__.keys(): serviceConfig = getattr(config.services, name) self.services[name] = Service.createService(name, serviceConfig) self.platform = DockerPlatform() def getImages(self): return self.images def getServerSet(self, name): if self.serverSets[name] is not None: return self.serverSets[name] return None def getServerSets(self): return self.serverSets def getServices(self): return self.serverSets def getZookeeperConnect(self): serverSet = self.getServerSet('zookeeper') if serverSet is None: return "localhost:2181" zkConnect = "" first = True for (serverName, server) in serverSet.getServers().iteritems(): if not first: zkConnect += "," zkConnect += server.hostname + ':2181' first = False return zkConnect def getZookeeperClusterConfig(self): serverSet = self.getServerSet('zookeeper') if serverSet is None: return "" config = "\n\n#Zookeeper cluster configuration\n" for (serverName, server) in serverSet.getServers().iteritems(): config += "server." + str(server.serverId) + "=" + server.hostname + ":2888:3888\n" return config def getHadoopMasterConfig(self): serverSet = self.getServerSet('hadoop_master') if serverSet is None: return "" config = "" for (serverName, server) in serverSet.getServers().iteritems(): config += server.hostname + "\n" return config def getHadoopSlaveConfig(self): serverSet = self.getServerSet('hadoop_worker') if serverSet is None: return "" config = "" for (serverName, server) in serverSet.getServers().iteritems(): config += server.hostname + "\n" return config def status(self): self.title("List the servers and the service status") self.serverResolveIp() account = self.config.properties.account_dev headers = ["Host", "IP", "Service", "Process", "PID"] cells = [] for serverSet in sorted(self.serverSets.itervalues(), key=operator.attrgetter("priority")): servers = serverSet.getServers() for (severName, server) in servers.iteritems(): cells.append([server.hostname, server.ip, "", "", ""]) for serviceName in server.services.__dict__.keys(): service = self.services[serviceName] cells.append(["", "", serviceName, "", ""]) processes = service.processes for (processName, process) in processes.iteritems(): pid = server.getProcessPid(account, self.platform, serviceName, processName, process) if pid is not None: cells.append(["", "", "", processName, pid]) print tabulate(cells, headers) def imageBuild(self): return self.platform.imageBuild(self) def imageClean(self): return self.platform.imageClean(self) def serverResolveIp(self): for (serverSetName, serverSet) in self.serverSets.iteritems(): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): if server.ip is None: self.platform.serverResolveIp(server) def server(self, option): self.title("Server " + option) #sortedServerSet = sorted(self.serverSets.items(), key=lambda x : MyDict[x]) #for (serverSetName, serverSet) in self.serverSets.iteritems(): for serverSet in sorted(self.serverSets.itervalues(), key=operator.attrgetter("priority")): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): if option == 'start': self.platform.serverStart(server) if option == 'stop': self.platform.serverStop(server) if option == 'kill': self.platform.serverKill(server) if option == 'destroy': self.platform.serverDestroy(server) if option == 'start': hostIpMap = self.generateHostsMap() for (serverSetName, serverSet) in self.serverSets.iteritems(): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): cmd = "echo '%s' >> /etc/hosts" % (hostIpMap) self.platform.sshExec("root", server, cmd) def serverDestroyAll(self): self.title("Destroy All The Server In The Cluster ") self.platform.serverDestroyAll() def generateHostsMap(self): BEGIN = "##CLUSTER START##" END = "##CLUSTER END##" hostIpMap = BEGIN + '\n' self.serverResolveIp() for (name, serverSet) in self.serverSets.iteritems(): servers = serverSet.getServers() for (hostname, server) in servers.iteritems(): hostIpMap += server.ip + ' ' + hostname + '\n' hostIpMap += END return hostIpMap def updateHostsFile(self): self.info("Generate /etc/hosts content") hostString = self.generateHostsMap() hostFile = open('/etc/hosts', 'r') hostFileContent = hostFile.read() hostFile.close(); regex = re.compile(r"##CLUSTER START##.*##CLUSTER END##", re.DOTALL) hostFileContent = regex.sub('', hostFileContent) hostFileContent += '\n' + hostString; print hostFileContent def service(self, option): self.title("Service " + option) account = self.config.properties.account_dev for serverSet in sorted(self.serverSets.itervalues(), key=operator.attrgetter("priority")): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): for serviceName in server.services.__dict__.keys(): service = self.services[serviceName] context = ServiceContext(account, self, self.platform, server, service) if option == 'install': context.install() elif option == 'configure': context.configure() elif option == 'start': context.start() elif option == 'stop': context.stop() elif option == 'kill': context.kill() elif option == 'clean': context.clean()
class ClusterContext(BaseObject): def __init__(self, configFile, platformType): baseDir = dirname(os.path.realpath(__file__)) if not configFile.startswith('/'): configFile = baseDir + "/" + configFile configData = open(configFile, 'r').read() config = json.loads(configData, object_hook=lambda d: Namespace(**d)) setattr(config.properties, 'base_dir', baseDir) self.config = config PropertyUtil.traverseAndResolvePropertyExpression( config, config.properties) self.images = {} for name in config.images.__dict__.keys(): imgObj = getattr(config.images, name) image = {"name": name, "repository": imgObj.repository} self.images[name] = image self.serverSets = {} for name in config.servers.__dict__.keys(): serverConfig = getattr(config.servers, name) self.serverSets[name] = ServerSet(name, serverConfig) self.services = {} for name in config.services.__dict__.keys(): serviceConfig = getattr(config.services, name) self.services[name] = Service.createService(name, serviceConfig) self.platform = DockerPlatform() def getImages(self): return self.images def getServerSet(self, name): if self.serverSets[name] is not None: return self.serverSets[name] return None def getServerSets(self): return self.serverSets def getServices(self): return self.serverSets def getZookeeperConnect(self): serverSet = self.getServerSet('zookeeper') if serverSet is None: return "localhost:2181" zkConnect = "" first = True for (serverName, server) in serverSet.getServers().iteritems(): if not first: zkConnect += "," zkConnect += server.hostname + ':2181' first = False return zkConnect def getZookeeperClusterConfig(self): serverSet = self.getServerSet('zookeeper') if serverSet is None: return "" config = "\n\n#Zookeeper cluster configuration\n" for (serverName, server) in serverSet.getServers().iteritems(): config += "server." + str( server.serverId) + "=" + server.hostname + ":2888:3888\n" return config def getHadoopMasterConfig(self): serverSet = self.getServerSet('hadoop_master') if serverSet is None: return "" config = "" for (serverName, server) in serverSet.getServers().iteritems(): config += server.hostname + "\n" return config def getHadoopSlaveConfig(self): serverSet = self.getServerSet('hadoop_worker') if serverSet is None: return "" config = "" for (serverName, server) in serverSet.getServers().iteritems(): config += server.hostname + "\n" return config def status(self): self.title("List the servers and the service status") self.serverResolveIp() account = self.config.properties.account_dev headers = ["Host", "IP", "Service", "Process", "PID"] cells = [] for serverSet in sorted(self.serverSets.itervalues(), key=operator.attrgetter("priority")): servers = serverSet.getServers() for (severName, server) in servers.iteritems(): cells.append([server.hostname, server.ip, "", "", ""]) for serviceName in server.services.__dict__.keys(): service = self.services[serviceName] cells.append(["", "", serviceName, "", ""]) processes = service.processes for (processName, process) in processes.iteritems(): pid = server.getProcessPid(account, self.platform, serviceName, processName, process) if pid is not None: cells.append(["", "", "", processName, pid]) print tabulate(cells, headers) def imageBuild(self): return self.platform.imageBuild(self) def imageClean(self): return self.platform.imageClean(self) def serverResolveIp(self): for (serverSetName, serverSet) in self.serverSets.iteritems(): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): if server.ip is None: self.platform.serverResolveIp(server) def server(self, option): self.title("Server " + option) #sortedServerSet = sorted(self.serverSets.items(), key=lambda x : MyDict[x]) #for (serverSetName, serverSet) in self.serverSets.iteritems(): for serverSet in sorted(self.serverSets.itervalues(), key=operator.attrgetter("priority")): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): if option == 'start': self.platform.serverStart(server) if option == 'stop': self.platform.serverStop(server) if option == 'kill': self.platform.serverKill(server) if option == 'destroy': self.platform.serverDestroy(server) if option == 'start': hostIpMap = self.generateHostsMap() for (serverSetName, serverSet) in self.serverSets.iteritems(): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): cmd = "echo '%s' >> /etc/hosts" % (hostIpMap) self.platform.sshExec("root", server, cmd) def serverDestroyAll(self): self.title("Destroy All The Server In The Cluster ") self.platform.serverDestroyAll() def generateHostsMap(self): BEGIN = "##CLUSTER START##" END = "##CLUSTER END##" hostIpMap = BEGIN + '\n' self.serverResolveIp() for (name, serverSet) in self.serverSets.iteritems(): servers = serverSet.getServers() for (hostname, server) in servers.iteritems(): hostIpMap += server.ip + ' ' + hostname + '\n' hostIpMap += END return hostIpMap def updateHostsFile(self): self.info("Generate /etc/hosts content") hostString = self.generateHostsMap() hostFile = open('/etc/hosts', 'r') hostFileContent = hostFile.read() hostFile.close() regex = re.compile(r"##CLUSTER START##.*##CLUSTER END##", re.DOTALL) hostFileContent = regex.sub('', hostFileContent) hostFileContent += '\n' + hostString print hostFileContent def service(self, option): self.title("Service " + option) account = self.config.properties.account_dev for serverSet in sorted(self.serverSets.itervalues(), key=operator.attrgetter("priority")): servers = serverSet.getServers() for (serverName, server) in servers.iteritems(): for serviceName in server.services.__dict__.keys(): service = self.services[serviceName] context = ServiceContext(account, self, self.platform, server, service) if option == 'install': context.install() elif option == 'configure': context.configure() elif option == 'start': context.start() elif option == 'stop': context.stop() elif option == 'kill': context.kill() elif option == 'clean': context.clean()