def run(self, arglist=[]): if self.updateMachineWithAutoFlaggerTag: params = xenrt.APIFactory().get_machine(self.machineName)['params'] if "AUTOFLAGGERTAG" in params and params["AUTOFLAGGERTAG"] == self.updateMachineWithAutoFlaggerTag: comment("Machine %s has already been checked. Exiting." % self.machineName) return for flag,flagData in self.flags.iteritems(): if "seqFile" in flagData: seqFile=flagData["seqFile"] elif "productType" in flagData or "productVersion" in flagData or "version" in flagData: seqFile = self.createTempSeq(**flagData) else: warning("Unimplemented") log("Using Temp Seq File : %s" % seqFile) passed = False try: self.doSequence(seqFile) passed = True except Exception, e: warning(str(e)) if passed == flagData["isSetIfPass"] and not self.isPropAlreadySet(flag): comment("Adding flag '%s' to machine '%s'" % (flag, self.machineName)) xenrt.APIFactory().update_machine(self.machineName, addflags=[flag]) elif passed != flagData["isSetIfPass"] and self.isPropAlreadySet(flag): comment("Removing flag '%s' from machine '%s'" % (flag, self.machineName)) xenrt.APIFactory().update_machine(self.machineName, delflags=[flag]) else: comment("Machine '%s' %s flag '%s'" % (self.machineName,"is already having required" if self.isPropAlreadySet(flag) else "neither need nor has", flag))
def __writeOnDom0(self): """Run writing concurrently.""" # prepare scripts if self.sequential: cmd = "dd if=/dev/zero of=/dev/\\${DEVICE} bs=1M count=%d" % \ (self.vdisize / xenrt.MEGA) chkcmd = "ps -efl | grep '[d]d if=/dev/zero'" else: cmd = "%s/remote/patterns.py /dev/\\${DEVICE} %d write 3" % \ (xenrt.TEC().lookup("REMOTE_SCRIPTDIR"), self.vdisize) chkcmd = "ps -efl | grep '[p]atterns.py'" cmd = "while [ -f /tmp/hold_running ] ; do sleep 0.1 ; done ; %s" % cmd self.host.execdom0("echo \"%s\" > /tmp/cmd.sh" % cmd) self.host.execdom0("chmod u+x /tmp/cmd.sh") # Run all commands with bash scripts first. self.host.execdom0("touch /tmp/hold_running") for vdi in self.vdis: self.host.execdom0("(/opt/xensource/debug/with-vdi %s /tmp/cmd.sh) >/dev/null 2>&1 </dev/null &" % vdi) # Triggering all the process at once. self.host.execdom0("rm -f /tmp/hold_running") waiting = 120 # mins. while waiting > 0: xenrt.sleep(300, log=False) waiting -= 5 try: pslist = self.host.execdom0(chkcmd).strip().splitlines() except: pslist = [] if len(pslist) == 0: log("Cannot find running process.") break log("Found %d processes running." % len(pslist)) log("Checking processes.") ptotal = [] pkilled = [] pfailed = [] try: pslist = self.host.execdom0(chkcmd).strip().splitlines() except: palist = [] for line in pslist: pid = line.split()[3] ptotal.append(pid) try: self.host.dom0("kill -9 %s" % pid) pkilled.append(pid) except: pfailed.append(pid) if ptotal: warning("Found %d processes are still running." % len(ptotal)) log("Killed: %s" % pkilled) log("Failed to killed: %s" % pfailed)
def lacpCleanUp(hostname): # get the list of clean-up actions from clean-up flags file # and perform the actions. cleanUpFlagsFile = xenrt.TEC().lookup( "CLEANUP_FLAGS_PATH") + '/' + hostname + "/LACP" if not os.path.exists(cleanUpFlagsFile): raise xenrt.XRTError("Expected file not found: '%s'" % cleanUpFlagsFile) flagFileHandle = open(cleanUpFlagsFile, 'r') content = flagFileHandle.read().strip() flagFileHandle.close() lines = content.split('\n') switches = {} for line in lines: if len(line) == 0: continue fields = line.split() if len(fields) < 3: warning("Ignoring malformatted line '%s' in clean-up file" % line) continue if len(fields) > 3: warning( "Malformatted line '%s' in clean-up file, will attempt to process anyway" % line) switchName, field, value = fields if not switches.has_key(switchName): switches[switchName] = [] switches[switchName].append([field, value]) for switchName in switches.keys(): sw = createSwitch(switchName, hostname) for field, value in switches[switchName]: if field == 'PORT': sw.unsetLagOnPort(port=value) elif field == 'LAG': sw.portChannelInterfaceClean(lag=value) else: raise xenrt.XRTError("Unsupported field '%s' in file '%s'" % (field, cleanUpFlagsFile)) sw.disconnect() # check if the cleanUpFlags file is empty now flagFileHandle = open(cleanUpFlagsFile, 'r') content = flagFileHandle.read().strip() flagFileHandle.close() if len(content) > 0: raise xenrt.XRTError( "Clean-up of LACP seems to have failed, following content still exist in the cleanup-flags file: %s", content) os.remove(cleanUpFlagsFile)
def lacpCleanUp(hostname): # get the list of clean-up actions from clean-up flags file # and perform the actions. cleanUpFlagsFile = xenrt.TEC().lookup("CLEANUP_FLAGS_PATH") + '/' + hostname + "/LACP" if not os.path.exists(cleanUpFlagsFile): raise xenrt.XRTError("Expected file not found: '%s'" % cleanUpFlagsFile) flagFileHandle = open(cleanUpFlagsFile, 'r') content = flagFileHandle.read().strip() flagFileHandle.close() lines = content.split('\n') switches = {} for line in lines: if len(line) == 0 : continue fields = line.split() if len(fields) < 3: warning("Ignoring malformatted line '%s' in clean-up file" % line) continue if len(fields) > 3: warning("Malformatted line '%s' in clean-up file, will attempt to process anyway" % line) switchName, field, value = fields if not switches.has_key(switchName): switches[switchName] = [] switches[switchName].append([field, value]) for switchName in switches.keys(): sw = createSwitch(switchName, hostname) for field, value in switches[switchName]: if field == 'PORT': sw.unsetLagOnPort(port=value) elif field == 'LAG': sw.portChannelInterfaceClean(lag=value) else: raise xenrt.XRTError("Unsupported field '%s' in file '%s'" % (field, cleanUpFlagsFile) ) sw.disconnect() # check if the cleanUpFlags file is empty now flagFileHandle = open(cleanUpFlagsFile, 'r') content = flagFileHandle.read().strip() flagFileHandle.close() if len(content) > 0: raise xenrt.XRTError( "Clean-up of LACP seems to have failed, following content still exist in the cleanup-flags file: %s", content ) os.remove(cleanUpFlagsFile)
def run(self, arglist=[]): sruuid = self.host.lookupDefaultSR() log("SR param: %s" % self.host.genParamGet("sr", sruuid, "sm-config")) exceptions = [] for i in xrange(self.iteration): log("Iteration %d..." % (i + 1)) prev = self.getPhysicalUtil() log("Initial physical utilisation: %d" % prev) # If prepare fails, iteration won't run properly. # Allow raising exception and halt running. self.prepareIteration() try: self.runWritingTest() # Check VDIs' size are increased as expected. final = self.getPhysicalUtil() log("Final physical utilisation: %d" % final) expected = prev + (self.vdicount * self.vdisize) log("Expected Utilisation: %d" % expected) if final < expected: raise xenrt.XRTFailure("After writing, SR utilisation has not increased as expected." \ "Expected: >= %d Found: %d" % (expected, final)) self.verifyWrittenData() except Exception as e: warning("Iteration %d failed due to exception: %s" % ((i + 1), str(e))) exceptions.append(str(e)) finally: # If finalizing fails, coming iteration won't run properly anyway. # Allow raising exception and halt running. self.finalizeIteration() if exceptions: log("Found following errors:") for exception in exceptions: log(exception) raise xenrt.XRTFailure( "Failed to run %d (out of %d) iteration(s)." % (len(exceptions), self.iteration))
def _checkContent(self, content): # Check that we have three fields in every line # This check was implemented, as occasionally we see malformatted lines # It can be removed once we get to root cause of this behaviour candidateContent = content content = content.rstrip() if len(content) == 0: return lines = content.rstrip().split("\n") for l in lines: fields = l.split() if len(fields) != 3: warning('Problem in switch.py library - attempt to write a malformatted line') current = self._readAll() log('current data:\n%s' % current) log('candidate data:\n"%s"' % candidateContent) raise xenrt.XRTError('Attempted to write a malformatted line "%s" ' 'in LACP clean-up flags file.' % l)
def run(self, arglist=[]): sruuid = self.host.lookupDefaultSR() log("SR param: %s" % self.host.genParamGet("sr", sruuid, "sm-config")) exceptions = [] for i in xrange(self.iteration): log("Iteration %d..." % (i + 1)) prev = self.getPhysicalUtil() log("Initial physical utilisation: %d" % prev) # If prepare fails, iteration won't run properly. # Allow raising exception and halt running. self.prepareIteration() try: self.runWritingTest() # Check VDIs' size are increased as expected. final = self.getPhysicalUtil() log("Final physical utilisation: %d" % final) expected = prev + (self.vdicount * self.vdisize) log("Expected Utilisation: %d" % expected) if final < expected: raise xenrt.XRTFailure("After writing, SR utilisation has not increased as expected." \ "Expected: >= %d Found: %d" % (expected, final)) self.verifyWrittenData() except Exception as e: warning("Iteration %d failed due to exception: %s" % ((i + 1), str(e))) exceptions.append(str(e)) finally: # If finalizing fails, coming iteration won't run properly anyway. # Allow raising exception and halt running. self.finalizeIteration() if exceptions: log("Found following errors:") for exception in exceptions: log(exception) raise xenrt.XRTFailure("Failed to run %d (out of %d) iteration(s)." % (len(exceptions), self.iteration))
def _checkContent(self, content): # Check that we have three fields in every line # This check was implemented, as occasionally we see malformatted lines # It can be removed once we get to root cause of this behaviour candidateContent = content content = content.rstrip() if len(content) == 0: return lines = content.rstrip().split("\n") for l in lines: fields = l.split() if len(fields) != 3: warning( 'Problem in switch.py library - attempt to write a malformatted line' ) current = self._readAll() log('current data:\n%s' % current) log('candidate data:\n"%s"' % candidateContent) raise xenrt.XRTError( 'Attempted to write a malformatted line "%s" ' 'in LACP clean-up flags file.' % l)
def run(self, arglist=[]): if self.updateMachineWithAutoFlaggerTag: params = xenrt.APIFactory().get_machine(self.machineName)["params"] if "AUTOFLAGGERTAG" in params and params["AUTOFLAGGERTAG"] == self.updateMachineWithAutoFlaggerTag: comment("Machine %s has already been checked. Exiting." % self.machineName) return for flag, flagData in self.flags.iteritems(): if "seqFile" in flagData: seqFile = flagData["seqFile"] elif "productType" in flagData or "productVersion" in flagData or "version" in flagData: seqFile = self.createTempSeq(**flagData) else: warning("Unimplemented") log("Using Temp Seq File : %s" % seqFile) passed = False try: self.doSequence(seqFile) passed = True except Exception, e: warning(str(e)) if passed == flagData["isSetIfPass"] and not self.isPropAlreadySet(flag): comment("Adding flag '%s' to machine '%s'" % (flag, self.machineName)) xenrt.APIFactory().update_machine(self.machineName, addflags=[flag]) elif passed != flagData["isSetIfPass"] and self.isPropAlreadySet(flag): comment("Removing flag '%s' from machine '%s'" % (flag, self.machineName)) xenrt.APIFactory().update_machine(self.machineName, delflags=[flag]) else: comment( "Machine '%s' %s flag '%s'" % ( self.machineName, "is already having required" if self.isPropAlreadySet(flag) else "neither need nor has", flag, ) )
def __writeOnDom0(self): """Run writing concurrently.""" # prepare scripts if self.sequential: cmd = "dd if=/dev/zero of=/dev/\\${DEVICE} bs=1M count=%d" % \ (self.vdisize / xenrt.MEGA) chkcmd = "ps -efl | grep '[d]d if=/dev/zero'" else: cmd = "%s/remote/patterns.py /dev/\\${DEVICE} %d write 3" % \ (xenrt.TEC().lookup("REMOTE_SCRIPTDIR"), self.vdisize) chkcmd = "ps -efl | grep '[p]atterns.py'" cmd = "while [ -f /tmp/hold_running ] ; do sleep 0.1 ; done ; %s" % cmd self.host.execdom0("echo \"%s\" > /tmp/cmd.sh" % cmd) self.host.execdom0("chmod u+x /tmp/cmd.sh") # Run all commands with bash scripts first. self.host.execdom0("touch /tmp/hold_running") for vdi in self.vdis: self.host.execdom0( "(/opt/xensource/debug/with-vdi %s /tmp/cmd.sh) >/dev/null 2>&1 </dev/null &" % vdi) # Triggering all the process at once. self.host.execdom0("rm -f /tmp/hold_running") waiting = 120 # mins. while waiting > 0: xenrt.sleep(300, log=False) waiting -= 5 try: pslist = self.host.execdom0(chkcmd).strip().splitlines() except: pslist = [] if len(pslist) == 0: log("Cannot find running process.") break log("Found %d processes running." % len(pslist)) log("Checking processes.") ptotal = [] pkilled = [] pfailed = [] try: pslist = self.host.execdom0(chkcmd).strip().splitlines() except: palist = [] for line in pslist: pid = line.split()[3] ptotal.append(pid) try: self.host.dom0("kill -9 %s" % pid) pkilled.append(pid) except: pfailed.append(pid) if ptotal: warning("Found %d processes are still running." % len(ptotal)) log("Killed: %s" % pkilled) log("Failed to killed: %s" % pfailed)
except Exception, e: errors.append(clone.getName() + ":" + str(e)) if len(errors) > 1: xenrt.TEC().logverbose("One or many guests failed to start with error messages %s" % errors) raise xenrt.XRTFailure("One or many guests failed to start.") # Now collect iolatency metrics for every cloned guests parallely. results = xenrt.pfarm([xenrt.PTask(self.collectMetrics, clone) for clone in self.clones], exception=False) log("Threads returned: %s" % results) exceptions = 0 for result in results: if result: exceptions += 1 warning("Found exception: %s" % result) if exceptions: raise xenrt.XRTFailure("Failed to run %d / %d io latency tests." % (exceptions, len(results))) def postRun(self, arglist=None): # Do not remove VMs after the test or otherwise we will not be able to collect the logs in it if False: #TODO: perhaps, in the future, add option to remove the vms # Removing all cloned VMs after the test run. errors = [] for clone in self.clones: xenrt.TEC().progress("Uninstalling VM %s" % clone.getName()) try: clone.uninstall() except Exception, e: errors.append(clone.getName() + ":" + str(e))