def createScreenPath(self): """ set the screen path, change the screenrc files """ # get the current screenrc info screenrc = Command.executeSsh("tail -5 /etc/screenrc", self.client) # check if screen log setted. path = Command.checkPath(self.screenlogpath) if screenrc.find("logfile") == -1: Command.executeSsh("echo \"logfile " + path + "screen_%t.log\" >> /etc/screenrc", self.client) #create folder. Command.createFolder(path,self.client)
def execute(self, State): #init the variables. self.init(State) if self.logflag == True: self.logger.log(self.info) pass if self.OS == "Linux": # 1.connect to Linux machine self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED # 2.check Iserver path buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/" if not Command.checkPathExist(buildpath, self.client): self.info = "directory "+buildpath+" does not exist." self.stat = self.ABORT return self.ABORT # 3.stop Iserver if it is running cmd = buildpath+"BIN/Linux/bin/mstrctl -s IntelligenceServer gs" result = Command.executeSsh(cmd, self.client) serverStatus = re.search(r">\w+<\/state>", result).group(0) if serverStatus[1:-8] == "running": cmd = buildpath + "BIN/Linux/bin/mstrctl -s IntelligenceServer stop" Command.executeSsh(cmd, self.client) time.sleep(120) # 4.delete Iserver build cmd = buildpath+"BIN/Linux/bin/mstrctl -s IntelligenceServer gs" result = Command.executeSsh(cmd, self.client) serverStatus = re.search(r">\w+<\/state>", result).group(0) if (serverStatus[1:-8] == "stopped" or serverStatus[1:-8] == "terminated"): cmd = "rm -rf "+Command.checkPath(self.buildInstallPath)+self.buildno result = Command.executeSsh(cmd, self.client) if result != "": self.info = "Error in delete directory "+Command.checkPath(self.buildInstallPath)+self.buildno self.stat = self.FAILED else: self.info = "Successfully delete directory "+Command.checkPath(self.buildInstallPath)+self.buildno self.stat = self.PASS else: self.info = "Cannot delete build when IServer is not stopped." self.stat = self.FAILED else: # if os is windows. c = rpyc.classic.connect(State.get("server"))
def execute(self,State): """ install linux build """ self.init(State) if self.logflag == True: self.logger.log(self.info) # 1.connect to Linux machine self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED #2. get official build path copyToPath = Command.checkPath(self.buildLocation)+self.buildNo+"/"+self.buildVersion+"/BIN/" print "version:"+self.buildVersion print "copyto path:" +copyToPath targetPath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/" #3. check if copytopath and installpath exist if Command.checkPathExist(copyToPath,self.client) == False: self.stat = self.FAILED self.info = "can not open build folder, the test build may not be ready" if self.logflag == True: self.logger.log(self.info) return self.FAILED if Command.checkPathExist(self.buildInstallPath,self.client) == False: self.stat = self.FAILED self.info = "can not open build folder, please provide the right install path" if self.logflag == True: self.logger.log(self.info) return self.FAILED #4. Check if there are enough disk self = Command.getUnixOS(self) buildsize = Command.getFolderSize(self.client,copyToPath+self.OS); diskspace = Command.getDiskSpace(self.client,self.buildInstallPath) if int(diskspace) < int(buildsize): self.stat = self.FAILED self.info = "there is not enough disk space to install build, please clean the disk space" if self.logflag == True: self.logger.log(self.info) return self.FAILED #2. install build. cmd = "perl "+copyToPath+"copyto.pl "+targetPath result = Command.executeSsh(cmd, self.client) if result.lower().find("command not found") != -1: self.stat = self.FAILED self.info = "target build not found, please check if the build ready is ready" if self.logflag == True: self.logger.log(self.info) return self.FAILED else: self.stat = self.PASS self.info ="build install success: "+targetPath if self.logflag == True: self.logger.log(self.info) return self.PASS
def execute(self, State): #init the variables. self.init(State) if self.logflag == True: self.logger.log(self.info) pass # 1.connect to Linux machine self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED # 2.clear system caches command = "echo 3 > /proc/sys/vm/drop_caches" result = Command.executeSsh(command, self.client) if result != "": self.stat = self.FAILED self.info ="clear system cache failed." +result+command if self.logflag == True: self.logger.log(self.info) Command.restoreFiles(self,smpath) return self.FAILED else: self.stat = self.PASS self.info = "clear system cache successfully." if self.logflag == True: self.logger.log(task.info) return self.PASS
def checkServerStatus(self): buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/BIN/"+self.OS+"/bin/" cmd = buildpath + "mstrctl -s IntelligenceServer gs" result = Command.executeSsh(cmd, self.client) serverStatus = re.search(r">\w+<\/state>", result) strStatus = serverStatus.group(0) return strStatus[1:-8]
def getProjectIDList(self, sourcetype): proj_ID = [] if sourcetype == "Cache": if not Command.checkPathExist(self.sourcepath+"Caches/", self.client): return proj_ID res = Command.executeSsh("ls "+self.sourcepath+"Caches/", self.client) elif sourcetype == "Cube": if not Command.checkPathExist(self.sourcepath+"Cube/", self.client): return proj_ID res = Command.executeSsh("ls "+self.sourcepath+"Cube/", self.client) elif sourcetype == "Inbox": if not Command.checkPathExist(self.sourcepath+"Inbox/", self.client): return proj_ID res = Command.executeSsh("ls "+self.sourcepath+"Inbox/", self.client) else: return proj_ID proj_ID = res.split(" ") return proj_ID
def stopServer(self): """ stop iserver """ buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/BIN/"+self.OS+"/bin/" cmd = buildpath + "mstrctl -s IntelligenceServer stop" result = Command.executeSsh(cmd, self.client) #operation not permitted. need check user privilege if "Error" in result: self.info = "Error in stopServer. StopServerTask failed." self.stat = self.FAILED pass
def execute(self, State): ''' this method write smart heap variables to mstrexec-iserver. ''' #init the variables. self.init(State) if self.logflag == True: self.logger.log(self.info) pass # 1.connect to Linux machine self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED #2. backup the files smpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/mstrexec-iserver" self = Command.backupFiles(self,smpath) if self.stat == self.FAILED: return self.FAILED # 3.read smart heap variables from config.ini configlocation = "Config\config.ini" cf = ConfigParser.ConfigParser() cf.read(configlocation) parameters=cf.items("SmartHeap") smstr = "" for item in parameters: smstr = smstr+"export "+item[0].upper()+"="+item[1]+"\\n" # 4.write smart heap variables to mstrexec-iserver smpos = "mstr_check_max_semaphore" command = "sed -i '/"+smpos+"/a "+smstr+"' "+smpath result = Command.executeSsh(command, self.client) if result != "": self.stat = self.FAILED self.info ="modify mstrexec-iserver failed." +result+command if self.logflag == True: self.logger.log(self.info) Command.restoreFiles(self,smpath) return self.FAILED if self.stat == self.FAILED: Command.restoreFiles(self,smpath) return self.FAILED else: self.stat = self.PASS self.info = "tune smart heap successfully." if self.logflag == True: self.logger.log(task.info) return self.PASS
def execute(self,State): """ download log file. """ self.init(State) if self.logflag == True: self.logger.log(self.info) self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED #get process id self = Command.getUnixOS(self) binpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/" getStatusCommnad = binpath+"mstrctl -s IntelligenceServer gs" statusInfo = Command.executeSsh(getStatusCommnad,self.client) xmlInfo = xml.dom.minidom.parseString(statusInfo) root=xmlInfo.documentElement process_id = "" node= root.getElementsByTagName('process_id')[0] for node in node.childNodes: process_id = node.data try: t=paramiko.Transport((self.server,22)) t.connect(username=self.serverLogin,password=self.serverPwd) sftp=paramiko.SFTPClient.from_transport(t) DSSErrorsFilesPath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/DSSErrors.log" DSSPerformanceFilesPath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/DSSPerformanceMonitor"+process_id+".csv" os.makedirs(self.localLogPath+self.buildNo+"/"+self.server) sftp.get(DSSErrorsFilesPath,self.localLogPath+self.buildNo+"/"+self.server+"/"+"DSSError.log") sftp.get(DSSPerformanceFilesPath,self.localLogPath+self.buildNo+"/"+self.server+"/"+"DSSPerformanceMonitor"+process_id+".csv") t.close() except Exception as e: self.stat = self.FAILED self.info ="log download failed"+str(e)+DSSErrorsFilesPath+DSSPerformanceFilesPath if self.logflag == True: self.logger.log(self.info) return self.FAILED self.stat=self.PASS self.info="download log file task has completed!" if self.logflag == True: self.logger.log(self.info) return self.PASS
def checkCoreDiskSpace(self): """ check if there are enough diskspace of the build path, if core_pattern are default. """ coredumppath = Command.checkPath(self.coredumppath) #get coredump disk space diskspace = Command.getDiskSpace(self.client,coredumppath) #get system RSS mem = Command.executeSsh("free -m | awk \'NR==2{print $2}\'", self.client) if int(mem) > int(diskspace): self.stat = self.ABORT self.info = "not enough disk space for coredump" if self.logflag == True: self.logger.log(self.info) return self.ABORT else: return self.RUNNING
def createScreen(self): """ create screen """ output = Command.executeSsh("screen -ls", self.client) if output.find(self.server) == -1: # create screen with name=server # record screen log to [default path set in /etc/screenrc]/screen_server-date.log, use date to separate the log # -L: use the log # -t: set the screen title # -d:detach the screen after create the screen # -m:force to create screen # -S:create screen with name path = Command.checkPath(self.screenlogpath) #[TODO][@Yanhong]need add file existence check here, to avoid error output of rm command in case no target files to remove. Command.executeSsh(" \\rm " + path + "screen_*.log", self.client) Command.executeSsh("screen -L -t " + self.server + " -dmS " + self.server, self.client) else: path = Command.checkPath(self.screenlogpath) #[TODO][@Yanhong]need add file existence check here, to avoid error output of rm command in case no target files to remove. Command.executeSsh(" \\rm " + path + "screen_*.log", self.client) print "screen exist"
def execute(self,State): """ monitor Linux server """ self.init(State) if self.logflag == True: self.logger.log(self.info) # 1.connect to Linux machine self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED # record time starttime = time.time() duration = 0 self = Command.getUnixOS(self) binpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/" getStatusCommnad = binpath+"mstrctl -s IntelligenceServer gs" # 2.monitor #check state :./mstrctl -s IntelligenceServer gs while duration < int(self.testTimeLimit)*60: #1. check if server crash #get server status statusInfo = Command.executeSsh(getStatusCommnad,self.client) xmlInfo = xml.dom.minidom.parseString(statusInfo) root=xmlInfo.documentElement process_id = "" node= root.getElementsByTagName('process_id')[0] for node in node.childNodes: process_id = node.data state = "" node= root.getElementsByTagName('state')[0] for node in node.childNodes: state = node.data if state.lower().find("stopped") != -1 : self.stat = self.FAILED self.info ="iserver is stopped, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("starting")!= -1: self.info ="iserver is starting" elif state.lower().find("running")!= -1: self.info ="iserver is running fine." elif state.lower().find("stopping")!= -1: self.stat = self.FAILED self.info ="iserver is stopping, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("terminated")!= -1: self.stat = self.FAILED self.info ="iserver is terminated, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("pausing")!= -1: self.stat = self.FAILED self.info ="iserver is pausing, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("paused")!= -1: self.stat = self.FAILED self.info ="iserver is paused, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED elif state.lower().find("resuming")!= -1: self.info ="iserver is resuming, please check if server crashed" elif state.lower().find("hanging")!= -1: self.stat = self.FAILED self.info ="iserver is hanging, please check if server crashed" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED #2. check if server hit assertion screeninfo = self.getScreenInfo() if screeninfo.lower().find("assertion") != -1: self.stat = self.FAILED self.info = "iserver hit assertion" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+"hit assertion:"+screeninfo, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.FAILED # print duration # print self.info # print state time.sleep(int(self.testCheckTime)*60) currenttime = time.time() duration = int(currenttime-starttime) self.stat=self.PASS self.info="monitor Linux iserver task has completed!" if self.logflag == True: self.logger.log(self.info) if self.emailflag == True: es = EmailSender(self.emailSender, self.server+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None) es.send() return self.PASS
def getScreenInfo(self): """ get screenlog info """ screenlog = Command.executeSsh("tail -100 " + Command.checkPath(self.screenlogpath) + "screen_" + self.server+".log", self.client) return screenlog
def execute(self, State): ''' this method copies all caches/cube/inbox from backup location. ''' #init the variables. self.init(State) if self.logflag == True: self.logger.log(self.info) pass if self.OS == "Linux": # 1.connect to Linux machine self = Command.getClient(self) if self.stat == self.FAILED: return self.FAILED # 2.check cache source path existing if not Command.checkPathExist(self.sourcepath, self.client): self.info = "source directory "+self.sourcepath+" does not exist. Please check inputs in config.ini." self.stat = self.ABORT return self.ABORT else: res = Command.executeSsh("ls "+self.sourcepath, self.client) if res == "": self.info = "no caches/cube/inbox directory in "+self.sourcepath+". Please check if it's mounted." self.stat = self.ABORT return self.ABORT # 3.copy cache/cube/inbox accordingly buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/BIN/"+self.OS +"/" localcachepath = buildpath+"Caches/"+self.serverdef+"/" localcubepath = buildpath+"Cube/"+self.serverdef+"/" localinboxpath =buildpath+"Inbox/"+self.serverdef+"/" projIDlist = self.getProjectIDList("Cache") if not projIDlist: print "No caches to copy." else: if Command.checkPathExist(localcachepath, self.client): for id in projIDlist: cachesourcepath = self.sourcepath+"Caches/"+id+"/*" localcacheprojectpath = localcachepath+"Server"+self.server+"_P"+id+"/" if not Command.checkPathExist(localcacheprojectpath, self.client): Command.executeSsh("mkdir "+localcacheprojectpath, self.client) result = Command.executeSsh("cp -fr "+cachesourcepath+" "+localcacheprojectpath, self.client) if result != "": print "something wrong when copy cache of project "+id else: print "local cache path does not exist. Please check if IServer is installed correctly." projIDlist = self.getProjectIDList("Cube") if not projIDlist: print "No cube to copy." else: if Command.checkPathExist(localcubepath, self.client): for id in projIDlist: cubesourcepath = self.sourcepath+"Cube/"+id+"/*" localcubeprojectpath = localcubepath+"Server_"+self.server+"_P"+id+"/" if not Command.checkPathExist(localcubeprojectpath, self.client): Command.executeSsh("mkdir "+localcubeprojectpath, self.client) result = Command.executeSsh("cp -fr "+cubesourcepath+" "+localcubeprojectpath, self.client) if result != "": print "something wrong when copy cube of project "+id else: print "local cube path does not exist. Please check if IServer is installed correctly." projIDlist = self.getProjectIDList("Inbox") if not projIDlist: print "No inbox to copy." else: if Command.checkPathExist(localinboxpath, self.client): for id in projIDlist: inboxsourcepath = self.sourcepath+"Inbox/"+id+"/*" #localinboxprojectpath = localinboxpath+"Server_"+self.server+"_P"+id+"/" localinboxprojectpath = localinboxpath if not Command.checkPathExist(localinboxprojectpath, self.client): Command.executeSsh("mkdir "+localinboxprojectpath, self.client) result = Command.executeSsh("cp -fr "+inboxsourcepath+" "+localinboxprojectpath, self.client) if result != "": print "something wrong when copy inbox of project "+id else: print "local inbox path does not exist. Please check if IServer is installed correctly." self.info = "copy cache task is completed." self.stat = self.PASS else: # if os is windows. c = rpyc.classic.connect(State.get("server")) rsub = c.modules.subprocess # copy caches on windows iserver