def createScreenPath(self):
"""
set the screen path, change the screenrc files
"""
# get the current screenrc info
screenrc = Command.executeSsh("tail -5 /etc/screenrc", self.client)
# check if screen log setted.
path = Command.checkPath(self.screenlogpath)
if screenrc.find("logfile") == -1:
Command.executeSsh("echo \"logfile " + path + "screen_%t.log\" >> /etc/screenrc", self.client)
#create folder.
Command.createFolder(path,self.client)
def execute(self, State):
#init the variables.
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
pass
if self.OS == "Linux":
# 1.connect to Linux machine
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
# 2.check Iserver path
buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/"
if not Command.checkPathExist(buildpath, self.client):
self.info = "directory "+buildpath+" does not exist."
self.stat = self.ABORT
return self.ABORT
# 3.stop Iserver if it is running
cmd = buildpath+"BIN/Linux/bin/mstrctl -s IntelligenceServer gs"
result = Command.executeSsh(cmd, self.client)
serverStatus = re.search(r">\w+<\/state>", result).group(0)
if serverStatus[1:-8] == "running":
cmd = buildpath + "BIN/Linux/bin/mstrctl -s IntelligenceServer stop"
Command.executeSsh(cmd, self.client)
time.sleep(120)
# 4.delete Iserver build
cmd = buildpath+"BIN/Linux/bin/mstrctl -s IntelligenceServer gs"
result = Command.executeSsh(cmd, self.client)
serverStatus = re.search(r">\w+<\/state>", result).group(0)
if (serverStatus[1:-8] == "stopped" or serverStatus[1:-8] == "terminated"):
cmd = "rm -rf "+Command.checkPath(self.buildInstallPath)+self.buildno
result = Command.executeSsh(cmd, self.client)
if result != "":
self.info = "Error in delete directory "+Command.checkPath(self.buildInstallPath)+self.buildno
self.stat = self.FAILED
else:
self.info = "Successfully delete directory "+Command.checkPath(self.buildInstallPath)+self.buildno
self.stat = self.PASS
else:
self.info = "Cannot delete build when IServer is not stopped."
self.stat = self.FAILED
else:
# if os is windows.
c = rpyc.classic.connect(State.get("server"))
def execute(self,State):
"""
install linux build
"""
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
# 1.connect to Linux machine
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
#2. get official build path
copyToPath = Command.checkPath(self.buildLocation)+self.buildNo+"/"+self.buildVersion+"/BIN/"
print "version:"+self.buildVersion
print "copyto path:" +copyToPath
targetPath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/"
#3. check if copytopath and installpath exist
if Command.checkPathExist(copyToPath,self.client) == False:
self.stat = self.FAILED
self.info = "can not open build folder, the test build may not be ready"
if self.logflag == True:
self.logger.log(self.info)
return self.FAILED
if Command.checkPathExist(self.buildInstallPath,self.client) == False:
self.stat = self.FAILED
self.info = "can not open build folder, please provide the right install path"
if self.logflag == True:
self.logger.log(self.info)
return self.FAILED
#4. Check if there are enough disk
self = Command.getUnixOS(self)
buildsize = Command.getFolderSize(self.client,copyToPath+self.OS);
diskspace = Command.getDiskSpace(self.client,self.buildInstallPath)
if int(diskspace) < int(buildsize):
self.stat = self.FAILED
self.info = "there is not enough disk space to install build, please clean the disk space"
if self.logflag == True:
self.logger.log(self.info)
return self.FAILED
#2. install build.
cmd = "perl "+copyToPath+"copyto.pl "+targetPath
result = Command.executeSsh(cmd, self.client)
if result.lower().find("command not found") != -1:
self.stat = self.FAILED
self.info = "target build not found, please check if the build ready is ready"
if self.logflag == True:
self.logger.log(self.info)
return self.FAILED
else:
self.stat = self.PASS
self.info ="build install success: "+targetPath
if self.logflag == True:
self.logger.log(self.info)
return self.PASS
def execute(self, State):
#init the variables.
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
pass
# 1.connect to Linux machine
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
# 2.clear system caches
command = "echo 3 > /proc/sys/vm/drop_caches"
result = Command.executeSsh(command, self.client)
if result != "":
self.stat = self.FAILED
self.info ="clear system cache failed." +result+command
if self.logflag == True:
self.logger.log(self.info)
Command.restoreFiles(self,smpath)
return self.FAILED
else:
self.stat = self.PASS
self.info = "clear system cache successfully."
if self.logflag == True:
self.logger.log(task.info)
return self.PASS
def checkServerStatus(self):
buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/BIN/"+self.OS+"/bin/"
cmd = buildpath + "mstrctl -s IntelligenceServer gs"
result = Command.executeSsh(cmd, self.client)
serverStatus = re.search(r">\w+<\/state>", result)
strStatus = serverStatus.group(0)
return strStatus[1:-8]
def getProjectIDList(self, sourcetype):
proj_ID = []
if sourcetype == "Cache":
if not Command.checkPathExist(self.sourcepath+"Caches/", self.client):
return proj_ID
res = Command.executeSsh("ls "+self.sourcepath+"Caches/", self.client)
elif sourcetype == "Cube":
if not Command.checkPathExist(self.sourcepath+"Cube/", self.client):
return proj_ID
res = Command.executeSsh("ls "+self.sourcepath+"Cube/", self.client)
elif sourcetype == "Inbox":
if not Command.checkPathExist(self.sourcepath+"Inbox/", self.client):
return proj_ID
res = Command.executeSsh("ls "+self.sourcepath+"Inbox/", self.client)
else:
return proj_ID
proj_ID = res.split(" ")
return proj_ID
def stopServer(self):
"""
stop iserver
"""
buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/BIN/"+self.OS+"/bin/"
cmd = buildpath + "mstrctl -s IntelligenceServer stop"
result = Command.executeSsh(cmd, self.client)
#operation not permitted. need check user privilege
if "Error" in result:
self.info = "Error in stopServer. StopServerTask failed."
self.stat = self.FAILED
pass
def execute(self, State):
'''
this method write smart heap variables to mstrexec-iserver.
'''
#init the variables.
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
pass
# 1.connect to Linux machine
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
#2. backup the files
smpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/mstrexec-iserver"
self = Command.backupFiles(self,smpath)
if self.stat == self.FAILED:
return self.FAILED
# 3.read smart heap variables from config.ini
configlocation = "Config\config.ini"
cf = ConfigParser.ConfigParser()
cf.read(configlocation)
parameters=cf.items("SmartHeap")
smstr = ""
for item in parameters:
smstr = smstr+"export "+item[0].upper()+"="+item[1]+"\\n"
# 4.write smart heap variables to mstrexec-iserver
smpos = "mstr_check_max_semaphore"
command = "sed -i '/"+smpos+"/a "+smstr+"' "+smpath
result = Command.executeSsh(command, self.client)
if result != "":
self.stat = self.FAILED
self.info ="modify mstrexec-iserver failed." +result+command
if self.logflag == True:
self.logger.log(self.info)
Command.restoreFiles(self,smpath)
return self.FAILED
if self.stat == self.FAILED:
Command.restoreFiles(self,smpath)
return self.FAILED
else:
self.stat = self.PASS
self.info = "tune smart heap successfully."
if self.logflag == True:
self.logger.log(task.info)
return self.PASS
def execute(self,State):
"""
download log file.
"""
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
#get process id
self = Command.getUnixOS(self)
binpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/"
getStatusCommnad = binpath+"mstrctl -s IntelligenceServer gs"
statusInfo = Command.executeSsh(getStatusCommnad,self.client)
xmlInfo = xml.dom.minidom.parseString(statusInfo)
root=xmlInfo.documentElement
process_id = ""
node= root.getElementsByTagName('process_id')[0]
for node in node.childNodes:
process_id = node.data
try:
t=paramiko.Transport((self.server,22))
t.connect(username=self.serverLogin,password=self.serverPwd)
sftp=paramiko.SFTPClient.from_transport(t)
DSSErrorsFilesPath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/DSSErrors.log"
DSSPerformanceFilesPath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/DSSPerformanceMonitor"+process_id+".csv"
os.makedirs(self.localLogPath+self.buildNo+"/"+self.server)
sftp.get(DSSErrorsFilesPath,self.localLogPath+self.buildNo+"/"+self.server+"/"+"DSSError.log")
sftp.get(DSSPerformanceFilesPath,self.localLogPath+self.buildNo+"/"+self.server+"/"+"DSSPerformanceMonitor"+process_id+".csv")
t.close()
except Exception as e:
self.stat = self.FAILED
self.info ="log download failed"+str(e)+DSSErrorsFilesPath+DSSPerformanceFilesPath
if self.logflag == True:
self.logger.log(self.info)
return self.FAILED
self.stat=self.PASS
self.info="download log file task has completed!"
if self.logflag == True:
self.logger.log(self.info)
return self.PASS
def checkCoreDiskSpace(self):
"""
check if there are enough diskspace of the build path, if core_pattern are default.
"""
coredumppath = Command.checkPath(self.coredumppath)
#get coredump disk space
diskspace = Command.getDiskSpace(self.client,coredumppath)
#get system RSS
mem = Command.executeSsh("free -m | awk \'NR==2{print $2}\'", self.client)
if int(mem) > int(diskspace):
self.stat = self.ABORT
self.info = "not enough disk space for coredump"
if self.logflag == True:
self.logger.log(self.info)
return self.ABORT
else:
return self.RUNNING
def createScreen(self):
"""
create screen
"""
output = Command.executeSsh("screen -ls", self.client)
if output.find(self.server) == -1:
# create screen with name=server
# record screen log to [default path set in /etc/screenrc]/screen_server-date.log, use date to separate the log
# -L: use the log
# -t: set the screen title
# -d:detach the screen after create the screen
# -m:force to create screen
# -S:create screen with name
path = Command.checkPath(self.screenlogpath)
#[TODO][@Yanhong]need add file existence check here, to avoid error output of rm command in case no target files to remove.
Command.executeSsh(" \\rm " + path + "screen_*.log", self.client)
Command.executeSsh("screen -L -t " + self.server + " -dmS " + self.server, self.client)
else:
path = Command.checkPath(self.screenlogpath)
#[TODO][@Yanhong]need add file existence check here, to avoid error output of rm command in case no target files to remove.
Command.executeSsh(" \\rm " + path + "screen_*.log", self.client)
print "screen exist"
def execute(self,State):
"""
monitor Linux server
"""
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
# 1.connect to Linux machine
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
# record time
starttime = time.time()
duration = 0
self = Command.getUnixOS(self)
binpath = Command.checkPath(self.buildInstallPath)+self.buildNo+"/"+self.buildVersion+"/BIN/"+self.OS+"/bin/"
getStatusCommnad = binpath+"mstrctl -s IntelligenceServer gs"
# 2.monitor
#check state :./mstrctl -s IntelligenceServer gs
while duration < int(self.testTimeLimit)*60:
#1. check if server crash
#get server status
statusInfo = Command.executeSsh(getStatusCommnad,self.client)
xmlInfo = xml.dom.minidom.parseString(statusInfo)
root=xmlInfo.documentElement
process_id = ""
node= root.getElementsByTagName('process_id')[0]
for node in node.childNodes:
process_id = node.data
state = ""
node= root.getElementsByTagName('state')[0]
for node in node.childNodes:
state = node.data
if state.lower().find("stopped") != -1 :
self.stat = self.FAILED
self.info ="iserver is stopped, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("starting")!= -1:
self.info ="iserver is starting"
elif state.lower().find("running")!= -1:
self.info ="iserver is running fine."
elif state.lower().find("stopping")!= -1:
self.stat = self.FAILED
self.info ="iserver is stopping, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("terminated")!= -1:
self.stat = self.FAILED
self.info ="iserver is terminated, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("pausing")!= -1:
self.stat = self.FAILED
self.info ="iserver is pausing, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("paused")!= -1:
self.stat = self.FAILED
self.info ="iserver is paused, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
elif state.lower().find("resuming")!= -1:
self.info ="iserver is resuming, please check if server crashed"
elif state.lower().find("hanging")!= -1:
self.stat = self.FAILED
self.info ="iserver is hanging, please check if server crashed"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+": "+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
#2. check if server hit assertion
screeninfo = self.getScreenInfo()
if screeninfo.lower().find("assertion") != -1:
self.stat = self.FAILED
self.info = "iserver hit assertion"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+"hit assertion:"+screeninfo, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.FAILED
# print duration
# print self.info
# print state
time.sleep(int(self.testCheckTime)*60)
currenttime = time.time()
duration = int(currenttime-starttime)
self.stat=self.PASS
self.info="monitor Linux iserver task has completed!"
if self.logflag == True:
self.logger.log(self.info)
if self.emailflag == True:
es = EmailSender(self.emailSender, self.server+self.info, self.server+": "+self.info,self.emailToList, "","", attachment=None)
es.send()
return self.PASS
def getScreenInfo(self):
"""
get screenlog info
"""
screenlog = Command.executeSsh("tail -100 " + Command.checkPath(self.screenlogpath) + "screen_" + self.server+".log", self.client)
return screenlog
def execute(self, State):
'''
this method copies all caches/cube/inbox from backup location.
'''
#init the variables.
self.init(State)
if self.logflag == True:
self.logger.log(self.info)
pass
if self.OS == "Linux":
# 1.connect to Linux machine
self = Command.getClient(self)
if self.stat == self.FAILED:
return self.FAILED
# 2.check cache source path existing
if not Command.checkPathExist(self.sourcepath, self.client):
self.info = "source directory "+self.sourcepath+" does not exist. Please check inputs in config.ini."
self.stat = self.ABORT
return self.ABORT
else:
res = Command.executeSsh("ls "+self.sourcepath, self.client)
if res == "":
self.info = "no caches/cube/inbox directory in "+self.sourcepath+". Please check if it's mounted."
self.stat = self.ABORT
return self.ABORT
# 3.copy cache/cube/inbox accordingly
buildpath = Command.checkPath(self.buildInstallPath)+self.buildno+"/"+self.buildversion+"/BIN/"+self.OS +"/"
localcachepath = buildpath+"Caches/"+self.serverdef+"/"
localcubepath = buildpath+"Cube/"+self.serverdef+"/"
localinboxpath =buildpath+"Inbox/"+self.serverdef+"/"
projIDlist = self.getProjectIDList("Cache")
if not projIDlist:
print "No caches to copy."
else:
if Command.checkPathExist(localcachepath, self.client):
for id in projIDlist:
cachesourcepath = self.sourcepath+"Caches/"+id+"/*"
localcacheprojectpath = localcachepath+"Server"+self.server+"_P"+id+"/"
if not Command.checkPathExist(localcacheprojectpath, self.client):
Command.executeSsh("mkdir "+localcacheprojectpath, self.client)
result = Command.executeSsh("cp -fr "+cachesourcepath+" "+localcacheprojectpath, self.client)
if result != "":
print "something wrong when copy cache of project "+id
else:
print "local cache path does not exist. Please check if IServer is installed correctly."
projIDlist = self.getProjectIDList("Cube")
if not projIDlist:
print "No cube to copy."
else:
if Command.checkPathExist(localcubepath, self.client):
for id in projIDlist:
cubesourcepath = self.sourcepath+"Cube/"+id+"/*"
localcubeprojectpath = localcubepath+"Server_"+self.server+"_P"+id+"/"
if not Command.checkPathExist(localcubeprojectpath, self.client):
Command.executeSsh("mkdir "+localcubeprojectpath, self.client)
result = Command.executeSsh("cp -fr "+cubesourcepath+" "+localcubeprojectpath, self.client)
if result != "":
print "something wrong when copy cube of project "+id
else:
print "local cube path does not exist. Please check if IServer is installed correctly."
projIDlist = self.getProjectIDList("Inbox")
if not projIDlist:
print "No inbox to copy."
else:
if Command.checkPathExist(localinboxpath, self.client):
for id in projIDlist:
inboxsourcepath = self.sourcepath+"Inbox/"+id+"/*"
#localinboxprojectpath = localinboxpath+"Server_"+self.server+"_P"+id+"/"
localinboxprojectpath = localinboxpath
if not Command.checkPathExist(localinboxprojectpath, self.client):
Command.executeSsh("mkdir "+localinboxprojectpath, self.client)
result = Command.executeSsh("cp -fr "+inboxsourcepath+" "+localinboxprojectpath, self.client)
if result != "":
print "something wrong when copy inbox of project "+id
else:
print "local inbox path does not exist. Please check if IServer is installed correctly."
self.info = "copy cache task is completed."
self.stat = self.PASS
else:
# if os is windows.
c = rpyc.classic.connect(State.get("server"))
rsub = c.modules.subprocess
# copy caches on windows iserver
