def run(self, arglist=[]):
if self.updateMachineWithAutoFlaggerTag:
params = xenrt.APIFactory().get_machine(self.machineName)['params']
if "AUTOFLAGGERTAG" in params and params["AUTOFLAGGERTAG"] == self.updateMachineWithAutoFlaggerTag:
comment("Machine %s has already been checked. Exiting." % self.machineName)
return
for flag,flagData in self.flags.iteritems():
if "seqFile" in flagData:
seqFile=flagData["seqFile"]
elif "productType" in flagData or "productVersion" in flagData or "version" in flagData:
seqFile = self.createTempSeq(**flagData)
else:
warning("Unimplemented")
log("Using Temp Seq File : %s" % seqFile)
passed = False
try:
self.doSequence(seqFile)
passed = True
except Exception, e:
warning(str(e))
if passed == flagData["isSetIfPass"] and not self.isPropAlreadySet(flag):
comment("Adding flag '%s' to machine '%s'" % (flag, self.machineName))
xenrt.APIFactory().update_machine(self.machineName, addflags=[flag])
elif passed != flagData["isSetIfPass"] and self.isPropAlreadySet(flag):
comment("Removing flag '%s' from machine '%s'" % (flag, self.machineName))
xenrt.APIFactory().update_machine(self.machineName, delflags=[flag])
else:
comment("Machine '%s' %s flag '%s'" % (self.machineName,"is already having required" if self.isPropAlreadySet(flag) else "neither need nor has", flag))
def __writeOnDom0(self):
"""Run writing concurrently."""
# prepare scripts
if self.sequential:
cmd = "dd if=/dev/zero of=/dev/\\${DEVICE} bs=1M count=%d" % \
(self.vdisize / xenrt.MEGA)
chkcmd = "ps -efl | grep '[d]d if=/dev/zero'"
else:
cmd = "%s/remote/patterns.py /dev/\\${DEVICE} %d write 3" % \
(xenrt.TEC().lookup("REMOTE_SCRIPTDIR"), self.vdisize)
chkcmd = "ps -efl | grep '[p]atterns.py'"
cmd = "while [ -f /tmp/hold_running ] ; do sleep 0.1 ; done ; %s" % cmd
self.host.execdom0("echo \"%s\" > /tmp/cmd.sh" % cmd)
self.host.execdom0("chmod u+x /tmp/cmd.sh")
# Run all commands with bash scripts first.
self.host.execdom0("touch /tmp/hold_running")
for vdi in self.vdis:
self.host.execdom0("(/opt/xensource/debug/with-vdi %s /tmp/cmd.sh) >/dev/null 2>&1 </dev/null &" % vdi)
# Triggering all the process at once.
self.host.execdom0("rm -f /tmp/hold_running")
waiting = 120 # mins.
while waiting > 0:
xenrt.sleep(300, log=False)
waiting -= 5
try:
pslist = self.host.execdom0(chkcmd).strip().splitlines()
except:
pslist = []
if len(pslist) == 0:
log("Cannot find running process.")
break
log("Found %d processes running." % len(pslist))
log("Checking processes.")
ptotal = []
pkilled = []
pfailed = []
try:
pslist = self.host.execdom0(chkcmd).strip().splitlines()
except:
palist = []
for line in pslist:
pid = line.split()[3]
ptotal.append(pid)
try:
self.host.dom0("kill -9 %s" % pid)
pkilled.append(pid)
except:
pfailed.append(pid)
if ptotal:
warning("Found %d processes are still running." % len(ptotal))
log("Killed: %s" % pkilled)
log("Failed to killed: %s" % pfailed)
def lacpCleanUp(hostname):
# get the list of clean-up actions from clean-up flags file
# and perform the actions.
cleanUpFlagsFile = xenrt.TEC().lookup(
"CLEANUP_FLAGS_PATH") + '/' + hostname + "/LACP"
if not os.path.exists(cleanUpFlagsFile):
raise xenrt.XRTError("Expected file not found: '%s'" %
cleanUpFlagsFile)
flagFileHandle = open(cleanUpFlagsFile, 'r')
content = flagFileHandle.read().strip()
flagFileHandle.close()
lines = content.split('\n')
switches = {}
for line in lines:
if len(line) == 0:
continue
fields = line.split()
if len(fields) < 3:
warning("Ignoring malformatted line '%s' in clean-up file" % line)
continue
if len(fields) > 3:
warning(
"Malformatted line '%s' in clean-up file, will attempt to process anyway"
% line)
switchName, field, value = fields
if not switches.has_key(switchName):
switches[switchName] = []
switches[switchName].append([field, value])
for switchName in switches.keys():
sw = createSwitch(switchName, hostname)
for field, value in switches[switchName]:
if field == 'PORT':
sw.unsetLagOnPort(port=value)
elif field == 'LAG':
sw.portChannelInterfaceClean(lag=value)
else:
raise xenrt.XRTError("Unsupported field '%s' in file '%s'" %
(field, cleanUpFlagsFile))
sw.disconnect()
# check if the cleanUpFlags file is empty now
flagFileHandle = open(cleanUpFlagsFile, 'r')
content = flagFileHandle.read().strip()
flagFileHandle.close()
if len(content) > 0:
raise xenrt.XRTError(
"Clean-up of LACP seems to have failed, following content still exist in the cleanup-flags file: %s",
content)
os.remove(cleanUpFlagsFile)
def lacpCleanUp(hostname):
# get the list of clean-up actions from clean-up flags file
# and perform the actions.
cleanUpFlagsFile = xenrt.TEC().lookup("CLEANUP_FLAGS_PATH") + '/' + hostname + "/LACP"
if not os.path.exists(cleanUpFlagsFile):
raise xenrt.XRTError("Expected file not found: '%s'" % cleanUpFlagsFile)
flagFileHandle = open(cleanUpFlagsFile, 'r')
content = flagFileHandle.read().strip()
flagFileHandle.close()
lines = content.split('\n')
switches = {}
for line in lines:
if len(line) == 0 :
continue
fields = line.split()
if len(fields) < 3:
warning("Ignoring malformatted line '%s' in clean-up file" % line)
continue
if len(fields) > 3:
warning("Malformatted line '%s' in clean-up file, will attempt to process anyway"
% line)
switchName, field, value = fields
if not switches.has_key(switchName):
switches[switchName] = []
switches[switchName].append([field, value])
for switchName in switches.keys():
sw = createSwitch(switchName, hostname)
for field, value in switches[switchName]:
if field == 'PORT':
sw.unsetLagOnPort(port=value)
elif field == 'LAG':
sw.portChannelInterfaceClean(lag=value)
else:
raise xenrt.XRTError("Unsupported field '%s' in file '%s'" % (field, cleanUpFlagsFile) )
sw.disconnect()
# check if the cleanUpFlags file is empty now
flagFileHandle = open(cleanUpFlagsFile, 'r')
content = flagFileHandle.read().strip()
flagFileHandle.close()
if len(content) > 0:
raise xenrt.XRTError(
"Clean-up of LACP seems to have failed, following content still exist in the cleanup-flags file: %s",
content )
os.remove(cleanUpFlagsFile)
def run(self, arglist=[]):
sruuid = self.host.lookupDefaultSR()
log("SR param: %s" % self.host.genParamGet("sr", sruuid, "sm-config"))
exceptions = []
for i in xrange(self.iteration):
log("Iteration %d..." % (i + 1))
prev = self.getPhysicalUtil()
log("Initial physical utilisation: %d" % prev)
# If prepare fails, iteration won't run properly.
# Allow raising exception and halt running.
self.prepareIteration()
try:
self.runWritingTest()
# Check VDIs' size are increased as expected.
final = self.getPhysicalUtil()
log("Final physical utilisation: %d" % final)
expected = prev + (self.vdicount * self.vdisize)
log("Expected Utilisation: %d" % expected)
if final < expected:
raise xenrt.XRTFailure("After writing, SR utilisation has not increased as expected." \
"Expected: >= %d Found: %d" % (expected, final))
self.verifyWrittenData()
except Exception as e:
warning("Iteration %d failed due to exception: %s" %
((i + 1), str(e)))
exceptions.append(str(e))
finally:
# If finalizing fails, coming iteration won't run properly anyway.
# Allow raising exception and halt running.
self.finalizeIteration()
if exceptions:
log("Found following errors:")
for exception in exceptions:
log(exception)
raise xenrt.XRTFailure(
"Failed to run %d (out of %d) iteration(s)." %
(len(exceptions), self.iteration))
def _checkContent(self, content):
# Check that we have three fields in every line
# This check was implemented, as occasionally we see malformatted lines
# It can be removed once we get to root cause of this behaviour
candidateContent = content
content = content.rstrip()
if len(content) == 0:
return
lines = content.rstrip().split("\n")
for l in lines:
fields = l.split()
if len(fields) != 3:
warning('Problem in switch.py library - attempt to write a malformatted line')
current = self._readAll()
log('current data:\n%s' % current)
log('candidate data:\n"%s"' % candidateContent)
raise xenrt.XRTError('Attempted to write a malformatted line "%s" '
'in LACP clean-up flags file.' % l)
def run(self, arglist=[]):
sruuid = self.host.lookupDefaultSR()
log("SR param: %s" % self.host.genParamGet("sr", sruuid, "sm-config"))
exceptions = []
for i in xrange(self.iteration):
log("Iteration %d..." % (i + 1))
prev = self.getPhysicalUtil()
log("Initial physical utilisation: %d" % prev)
# If prepare fails, iteration won't run properly.
# Allow raising exception and halt running.
self.prepareIteration()
try:
self.runWritingTest()
# Check VDIs' size are increased as expected.
final = self.getPhysicalUtil()
log("Final physical utilisation: %d" % final)
expected = prev + (self.vdicount * self.vdisize)
log("Expected Utilisation: %d" % expected)
if final < expected:
raise xenrt.XRTFailure("After writing, SR utilisation has not increased as expected." \
"Expected: >= %d Found: %d" % (expected, final))
self.verifyWrittenData()
except Exception as e:
warning("Iteration %d failed due to exception: %s" % ((i + 1), str(e)))
exceptions.append(str(e))
finally:
# If finalizing fails, coming iteration won't run properly anyway.
# Allow raising exception and halt running.
self.finalizeIteration()
if exceptions:
log("Found following errors:")
for exception in exceptions:
log(exception)
raise xenrt.XRTFailure("Failed to run %d (out of %d) iteration(s)." % (len(exceptions), self.iteration))
def _checkContent(self, content):
# Check that we have three fields in every line
# This check was implemented, as occasionally we see malformatted lines
# It can be removed once we get to root cause of this behaviour
candidateContent = content
content = content.rstrip()
if len(content) == 0:
return
lines = content.rstrip().split("\n")
for l in lines:
fields = l.split()
if len(fields) != 3:
warning(
'Problem in switch.py library - attempt to write a malformatted line'
)
current = self._readAll()
log('current data:\n%s' % current)
log('candidate data:\n"%s"' % candidateContent)
raise xenrt.XRTError(
'Attempted to write a malformatted line "%s" '
'in LACP clean-up flags file.' % l)
def run(self, arglist=[]):
if self.updateMachineWithAutoFlaggerTag:
params = xenrt.APIFactory().get_machine(self.machineName)["params"]
if "AUTOFLAGGERTAG" in params and params["AUTOFLAGGERTAG"] == self.updateMachineWithAutoFlaggerTag:
comment("Machine %s has already been checked. Exiting." % self.machineName)
return
for flag, flagData in self.flags.iteritems():
if "seqFile" in flagData:
seqFile = flagData["seqFile"]
elif "productType" in flagData or "productVersion" in flagData or "version" in flagData:
seqFile = self.createTempSeq(**flagData)
else:
warning("Unimplemented")
log("Using Temp Seq File : %s" % seqFile)
passed = False
try:
self.doSequence(seqFile)
passed = True
except Exception, e:
warning(str(e))
if passed == flagData["isSetIfPass"] and not self.isPropAlreadySet(flag):
comment("Adding flag '%s' to machine '%s'" % (flag, self.machineName))
xenrt.APIFactory().update_machine(self.machineName, addflags=[flag])
elif passed != flagData["isSetIfPass"] and self.isPropAlreadySet(flag):
comment("Removing flag '%s' from machine '%s'" % (flag, self.machineName))
xenrt.APIFactory().update_machine(self.machineName, delflags=[flag])
else:
comment(
"Machine '%s' %s flag '%s'"
% (
self.machineName,
"is already having required" if self.isPropAlreadySet(flag) else "neither need nor has",
flag,
)
)
def __writeOnDom0(self):
"""Run writing concurrently."""
# prepare scripts
if self.sequential:
cmd = "dd if=/dev/zero of=/dev/\\${DEVICE} bs=1M count=%d" % \
(self.vdisize / xenrt.MEGA)
chkcmd = "ps -efl | grep '[d]d if=/dev/zero'"
else:
cmd = "%s/remote/patterns.py /dev/\\${DEVICE} %d write 3" % \
(xenrt.TEC().lookup("REMOTE_SCRIPTDIR"), self.vdisize)
chkcmd = "ps -efl | grep '[p]atterns.py'"
cmd = "while [ -f /tmp/hold_running ] ; do sleep 0.1 ; done ; %s" % cmd
self.host.execdom0("echo \"%s\" > /tmp/cmd.sh" % cmd)
self.host.execdom0("chmod u+x /tmp/cmd.sh")
# Run all commands with bash scripts first.
self.host.execdom0("touch /tmp/hold_running")
for vdi in self.vdis:
self.host.execdom0(
"(/opt/xensource/debug/with-vdi %s /tmp/cmd.sh) >/dev/null 2>&1 </dev/null &"
% vdi)
# Triggering all the process at once.
self.host.execdom0("rm -f /tmp/hold_running")
waiting = 120 # mins.
while waiting > 0:
xenrt.sleep(300, log=False)
waiting -= 5
try:
pslist = self.host.execdom0(chkcmd).strip().splitlines()
except:
pslist = []
if len(pslist) == 0:
log("Cannot find running process.")
break
log("Found %d processes running." % len(pslist))
log("Checking processes.")
ptotal = []
pkilled = []
pfailed = []
try:
pslist = self.host.execdom0(chkcmd).strip().splitlines()
except:
palist = []
for line in pslist:
pid = line.split()[3]
ptotal.append(pid)
try:
self.host.dom0("kill -9 %s" % pid)
pkilled.append(pid)
except:
pfailed.append(pid)
if ptotal:
warning("Found %d processes are still running." % len(ptotal))
log("Killed: %s" % pkilled)
log("Failed to killed: %s" % pfailed)
except Exception, e:
errors.append(clone.getName() + ":" + str(e))
if len(errors) > 1:
xenrt.TEC().logverbose("One or many guests failed to start with error messages %s" % errors)
raise xenrt.XRTFailure("One or many guests failed to start.")
# Now collect iolatency metrics for every cloned guests parallely.
results = xenrt.pfarm([xenrt.PTask(self.collectMetrics, clone) for clone in self.clones], exception=False)
log("Threads returned: %s" % results)
exceptions = 0
for result in results:
if result:
exceptions += 1
warning("Found exception: %s" % result)
if exceptions:
raise xenrt.XRTFailure("Failed to run %d / %d io latency tests." % (exceptions, len(results)))
def postRun(self, arglist=None):
# Do not remove VMs after the test or otherwise we will not be able to collect the logs in it
if False: #TODO: perhaps, in the future, add option to remove the vms
# Removing all cloned VMs after the test run.
errors = []
for clone in self.clones:
xenrt.TEC().progress("Uninstalling VM %s" % clone.getName())
try:
clone.uninstall()
except Exception, e:
errors.append(clone.getName() + ":" + str(e))
