def _spawnChild(self, sAction):
"""
Spawns the child process, returning success indicator + child object.
"""
# Argument list.
asArgs = self._assembleArguments(sAction)
if asArgs is None:
self._log('Malformed command line: "%s"' % (self._sScriptCmdLine,));
return (False, None);
# Spawn child.
try:
oChild = subprocess.Popen(asArgs,
shell = False,
bufsize = -1,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
cwd = self._oTestBoxScript.getPathSpill(),
universal_newlines = True,
close_fds = (False if utils.getHostOs() == 'win' else True),
preexec_fn = (None if utils.getHostOs() in ['win', 'os2']
else os.setsid)); # pylint: disable=E1101
except Exception, oXcpt:
self._log('Error creating child process %s: %s' % (asArgs, oXcpt));
return (False, None);
def testRunUnitTestsSet(self, sTestCasePattern, sTestCaseSubDir):
"""
Run subset of the unit tests set.
"""
# Open /dev/null for use as stdin further down.
try:
oDevNull = open(os.path.devnull, 'w+');
except:
oDevNull = None;
# Determin the host OS specific exclusion lists.
dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(utils.getHostOs(), []);
dTestCasesBuggyForHostOs.update(self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []));
#
# Process the file list and run everything looking like a testcase.
#
for sFilename in sorted(os.listdir(os.path.join(self.sUnitTestsPathBase, sTestCaseSubDir))):
# Separate base and suffix and morph the base into something we
# can use for reporting and array lookups.
sName, sSuffix = os.path.splitext(sFilename);
if sTestCaseSubDir != '.':
sName = sTestCaseSubDir + '/' + sName;
# Basic exclusion.
if not re.match(sTestCasePattern, sFilename) \
or sSuffix in self.kasSuffixBlackList:
reporter.log('"%s" is not a test case.' % (sFilename,))
continue
# Check if the testcase is black listed or buggy before executing it.
if self._isExcluded(sName, self.kdTestCasesBlackList):
# (No testStart/Done or accounting here!)
reporter.log('%s: SKIPPED (blacklisted)' % (sName,));
elif self._isExcluded(sName, self.kdTestCasesBuggy):
reporter.testStart(sName);
reporter.log('%s: Skipping, buggy in general.' % (sName,));
reporter.testDone(fSkipped = True);
self.cSkipped += 1;
elif self._isExcluded(sName, dTestCasesBuggyForHostOs):
reporter.testStart(sName);
reporter.log('%s: Skipping, buggy on %s.' % (sName, utils.getHostOs(),));
reporter.testDone(fSkipped = True);
self.cSkipped += 1;
else:
sFullPath = os.path.normpath(os.path.join(self.sUnitTestsPathBase, os.path.join(sTestCaseSubDir, sFilename)));
reporter.testStart(sName);
try:
fSkipped = self._executeTestCase(sName, sFullPath, sTestCaseSubDir, oDevNull);
except:
reporter.errorXcpt('!*!');
self.cFailed += 1;
fSkipped = False;
reporter.testDone(fSkipped);
def postRequestRaw(self, sAction, dParams):
"""
Posts a request to the test manager and gets the response. The dParams
argument is a dictionary of unencoded key-value pairs (will be
modified).
Raises exception on failure.
"""
dHeader = \
{
'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
'User-Agent': 'TestBoxScript/%s.0 (%s, %s)' % (__version__, utils.getHostOs(), utils.getHostArch()),
'Accept': 'text/plain,application/x-www-form-urlencoded',
'Accept-Encoding': 'identity',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
};
sServerPath = '/%s/testboxdisp.py' % (self._oParsedUrl.path.strip('/'),); # pylint: disable=E1101
dParams[constants.tbreq.ALL_PARAM_ACTION] = sAction;
sBody = urllib.urlencode(dParams);
##testboxcommons.log2('sServerPath=%s' % (sServerPath,));
try:
self._oConn.request('POST', sServerPath, sBody, dHeader);
oResponse = self._oConn.getresponse();
oResponse2 = TestBoxResponse(oResponse);
except:
testboxcommons.log2Xcpt();
raise
return oResponse2;
def _getVBoxInstallPath(self, fFailIfNotFound):
""" Returns the default VBox installation path. """
sHost = utils.getHostOs()
if sHost == 'win':
sProgFiles = os.environ.get('ProgramFiles', 'C:\\Program Files')
asLocs = [
os.path.join(sProgFiles, 'Oracle', 'VirtualBox'),
os.path.join(sProgFiles, 'OracleVM', 'VirtualBox'),
os.path.join(sProgFiles, 'Sun', 'VirtualBox'),
]
elif sHost in (
'linux',
'solaris',
):
asLocs = [
'/opt/VirtualBox', '/opt/VirtualBox-3.2',
'/opt/VirtualBox-3.1', '/opt/VirtualBox-3.0'
]
elif sHost == 'darwin':
asLocs = ['/Applications/VirtualBox.app/Contents/MacOS']
else:
asLocs = ['/opt/VirtualBox']
if 'VBOX_INSTALL_PATH' in os.environ:
asLocs.insert(0, os.environ.get('VBOX_INSTALL_PATH', None))
for sLoc in asLocs:
if os.path.isdir(sLoc):
return sLoc
if fFailIfNotFound:
reporter.error('Failed to locate VirtualBox installation: %s' %
(asLocs, ))
else:
reporter.log2('Failed to locate VirtualBox installation: %s' %
(asLocs, ))
return None
def actionCleanupAfter(self):
"""
Forward this to the sub testdriver, then uninstall VBox.
"""
fRc = True
if 'execute' not in self.asActions and 'all' not in self.asActions:
fRc = self._executeSubDriver([
'cleanup-after',
], fMaySkip=False)
if not self._killAllVBoxProcesses():
fRc = False
if not self._uninstallVBox(
self._persistentVarExists(self.ksVar_Skipped)):
fRc = False
if utils.getHostOs() == 'darwin':
self._darwinUnmountDmg(fIgnoreError=True)
# paranoia
if not TestDriverBase.actionCleanupAfter(self):
fRc = False
return fRc
def postRequestRaw(self, sAction, dParams):
"""
Posts a request to the test manager and gets the response. The dParams
argument is a dictionary of unencoded key-value pairs (will be
modified).
Raises exception on failure.
"""
dHeader = \
{
'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
'User-Agent': 'TestBoxScript/%s.0 (%s, %s)' % (__version__, utils.getHostOs(), utils.getHostArch()),
'Accept': 'text/plain,application/x-www-form-urlencoded',
'Accept-Encoding': 'identity',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
}
sServerPath = '/%s/testboxdisp.py' % (
self._oParsedUrl.path.strip('/'), )
# pylint: disable=E1101
dParams[constants.tbreq.ALL_PARAM_ACTION] = sAction
sBody = urllib.urlencode(dParams)
##testboxcommons.log2('sServerPath=%s' % (sServerPath,));
try:
self._oConn.request('POST', sServerPath, sBody, dHeader)
oResponse = self._oConn.getresponse()
oResponse2 = TestBoxResponse(oResponse)
except:
testboxcommons.log2Xcpt()
raise
return oResponse2
def doReboot(self):
"""
Worker common to _cmdReboot and _doUpgrade that performs a system reboot.
"""
# !! Not more exceptions beyond this point !!
testboxcommons.log('Rebooting');
# Stop anything that might be executing at this point.
oCurTask = self._getCurTask();
if oCurTask is not None:
oCurTask.terminate();
oCurTask.wait(self.kcSecStopBeforeRebootTimeout);
# Invoke shutdown command line utility.
sOs = utils.getHostOs();
asCmd2 = None;
if sOs == 'win':
asCmd = ['shutdown', '/r', '/t', '0'];
elif sOs == 'os2':
asCmd = ['setboot', '/B'];
elif sOs in ('solaris',):
asCmd = ['/usr/sbin/reboot', '-p'];
asCmd2 = ['/usr/sbin/reboot']; # Hack! S10 doesn't have -p, but don't know how to reliably detect S10.
else:
asCmd = ['/sbin/shutdown', '-r', 'now'];
try:
utils.sudoProcessOutputChecked(asCmd);
except Exception, oXcpt:
if asCmd2 is not None:
try:
utils.sudoProcessOutputChecked(asCmd2);
except Exception, oXcpt:
testboxcommons.log('Error executing reboot command "%s" as well as "%s": %s' % (asCmd, asCmd2, oXcpt));
return False;
def _getVBoxInstallPath(self, fFailIfNotFound):
""" Returns the default VBox installation path. """
sHost = utils.getHostOs();
if sHost == 'win':
sProgFiles = os.environ.get('ProgramFiles', 'C:\\Program Files');
asLocs = [
os.path.join(sProgFiles, 'Oracle', 'VirtualBox'),
os.path.join(sProgFiles, 'OracleVM', 'VirtualBox'),
os.path.join(sProgFiles, 'Sun', 'VirtualBox'),
];
elif sHost == 'linux' or sHost == 'solaris':
asLocs = [ '/opt/VirtualBox', '/opt/VirtualBox-3.2', '/opt/VirtualBox-3.1', '/opt/VirtualBox-3.0'];
elif sHost == 'darwin':
asLocs = [ '/Applications/VirtualBox.app/Contents/MacOS' ];
else:
asLocs = [ '/opt/VirtualBox' ];
if 'VBOX_INSTALL_PATH' in os.environ:
asLocs.insert(0, os.environ.get('VBOX_INSTALL_PATH', None));
for sLoc in asLocs:
if os.path.isdir(sLoc):
return sLoc;
if fFailIfNotFound:
reporter.error('Failed to locate VirtualBox installation: %s' % (asLocs,));
else:
reporter.log2('Failed to locate VirtualBox installation: %s' % (asLocs,));
return None;
def __init__(self):
ReporterBase.__init__(self)
self.sTestManagerUrl = os.environ.get('TESTBOX_MANAGER_URL')
self.sTestBoxUuid = os.environ.get('TESTBOX_UUID')
self.idTestBox = int(os.environ.get('TESTBOX_ID'))
self.idTestSet = int(os.environ.get('TESTBOX_TEST_SET_ID'))
self._asXml = []
self._secTsXmlFlush = utils.timestampSecond()
self._secTsXmlLast = self._secTsXmlFlush
self._fXmlFlushing = False
self.oOutput = sys.stdout
# Hack for __del__ output.
self.fFlushEachLine = True
self.fDebugXml = 'TESTDRIVER_REPORTER_DEBUG_XML' in os.environ
# Prepare the TM connecting.
import urlparse
import httplib
import urllib
from common import constants
self._fnUrlEncode = urllib.urlencode
self._fnUrlParseQs = urlparse.parse_qs
self._oParsedTmUrl = urlparse.urlparse(self.sTestManagerUrl)
if sys.version_info[0] >= 3 \
or (sys.version_info[0] == 2 and sys.version_info[1] >= 6):
if self._oParsedTmUrl.scheme == 'https': # pylint: disable=E1101
self._fnTmConnect = lambda: httplib.HTTPSConnection(
self._oParsedTmUrl.hostname,
timeout=self.kcSecTestManagerRequestTimeout)
else:
self._fnTmConnect = lambda: httplib.HTTPConnection(
self._oParsedTmUrl.hostname,
timeout=self.kcSecTestManagerRequestTimeout)
else:
if self._oParsedTmUrl.scheme == 'https': # pylint: disable=E1101
self._fnTmConnect = lambda: httplib.HTTPSConnection(
self._oParsedTmUrl.hostname)
else:
self._fnTmConnect = lambda: httplib.HTTPConnection(
self._oParsedTmUrl.hostname)
self._dHttpHeader = \
{
'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
'User-Agent': 'TestDriverReporter/%s.0 (%s, %s)' % (__version__, utils.getHostOs(), utils.getHostArch(),),
'Accept': 'text/plain,application/x-www-form-urlencoded',
'Accept-Encoding': 'identity',
'Cache-Control': 'max-age=0',
#'Connection': 'keep-alive',
}
dParams = {
constants.tbreq.ALL_PARAM_TESTBOX_UUID: self.sTestBoxUuid,
constants.tbreq.ALL_PARAM_TESTBOX_ID: self.idTestBox,
constants.tbreq.RESULT_PARAM_TEST_SET_ID: self.idTestSet,
}
self._sTmServerPath = '/%s/testboxdisp.py?%s' \
% ( self._oParsedTmUrl.path.strip('/'), # pylint: disable=E1101
urllib.urlencode(dParams), )
def _createVmDoIt(self, oTestDrv, eNic0AttachType, sDvdImage):
#
# Use HostOnly networking for ubuntu and debian VMs to prevent them from
# downloading updates and doing database updates during installation.
# We want predicable results.
#
if eNic0AttachType is None:
if self.isLinux() \
and ( 'ubuntu' in self.sKind.lower()
or 'debian' in self.sKind.lower()):
eNic0AttachType = vboxcon.NetworkAttachmentType_HostOnly;
# Also use it for windows xp to prevent it from ever going online.
if self.sKind in ('WindowsXP','WindowsXP_64',):
eNic0AttachType = vboxcon.NetworkAttachmentType_HostOnly;
#
# Use host-only networks instead of host-only adapters for trunk builds on Mac OS.
#
if eNic0AttachType == vboxcon.NetworkAttachmentType_HostOnly \
and utils.getHostOs() == 'darwin' \
and oTestDrv.fpApiVer >= 7.0:
eNic0AttachType = vboxcon.NetworkAttachmentType_HostOnlyNetwork;
return vboxtestvms.BaseTestVm._createVmDoIt(self, oTestDrv, eNic0AttachType, sDvdImage);
def _installVBox(self):
"""
Download / copy the build files into the scratch area and install them.
"""
reporter.testStart('Installing VirtualBox');
reporter.log('CWD=%s' % (os.getcwd(),)); # curious
#
# Download the build files.
#
for i in range(len(self._asBuildUrls)):
if webutils.downloadFile(self._asBuildUrls[i], self._asBuildFiles[i],
self.sBuildPath, reporter.log, reporter.log) is not True:
reporter.testDone(fSkipped = True);
return None; # Failed to get binaries, probably deleted. Skip the test run.
#
# Unpack anything we know what is and append it to the build files
# list. This allows us to use VBoxAll*.tar.gz files.
#
for sFile in list(self._asBuildFiles):
if self._maybeUnpackArchive(sFile, fNonFatal = True) is not True:
reporter.testDone(fSkipped = True);
return None; # Failed to unpack. Probably local error, like busy
# DLLs on windows, no reason for failing the build.
#
# Go to system specific installation code.
#
sHost = utils.getHostOs()
if sHost == 'darwin': fRc = self._installVBoxOnDarwin();
elif sHost == 'linux': fRc = self._installVBoxOnLinux();
elif sHost == 'solaris': fRc = self._installVBoxOnSolaris();
elif sHost == 'win': fRc = self._installVBoxOnWindows();
else:
reporter.error('Unsupported host "%s".' % (sHost,));
if fRc is False:
reporter.testFailure('Installation error.');
elif fRc is not True:
reporter.log('Seems installation was skipped. Old version lurking behind? Not the fault of this build/test run!');
#
# Install the extension pack.
#
if fRc is True and self._fAutoInstallPuelExtPack:
fRc = self._installExtPack();
if fRc is False:
reporter.testFailure('Extension pack installation error.');
# Some debugging...
try:
cMbFreeSpace = utils.getDiskUsage(self.sScratchPath);
reporter.log('Disk usage after VBox install: %d MB available at %s' % (cMbFreeSpace, self.sScratchPath,));
except:
reporter.logXcpt('Unable to get disk free space. Ignored. Continuing.');
reporter.testDone(fRc is None);
return fRc;
def _installVBox(self):
"""
Download / copy the build files into the scratch area and install them.
"""
reporter.testStart('Installing VirtualBox');
reporter.log('CWD=%s' % (os.getcwd(),)); # curious
#
# Download the build files.
#
for i in range(len(self._asBuildUrls)):
if webutils.downloadFile(self._asBuildUrls[i], self._asBuildFiles[i],
self.sBuildPath, reporter.log, reporter.log) is not True:
reporter.testDone(fSkipped = True);
return None; # Failed to get binaries, probably deleted. Skip the test run.
#
# Unpack anything we know what is and append it to the build files
# list. This allows us to use VBoxAll*.tar.gz files.
#
for sFile in list(self._asBuildFiles):
if self._maybeUnpackArchive(sFile, fNonFatal = True) is not True:
reporter.testDone(fSkipped = True);
return None; # Failed to unpack. Probably local error, like busy
# DLLs on windows, no reason for failing the build.
#
# Go to system specific installation code.
#
sHost = utils.getHostOs()
if sHost == 'darwin': fRc = self._installVBoxOnDarwin();
elif sHost == 'linux': fRc = self._installVBoxOnLinux();
elif sHost == 'solaris': fRc = self._installVBoxOnSolaris();
elif sHost == 'win': fRc = self._installVBoxOnWindows();
else:
reporter.error('Unsupported host "%s".' % (sHost,));
if fRc is False:
reporter.testFailure('Installation error.');
elif fRc is not True:
reporter.log('Seems installation was skipped. Old version lurking behind? Not the fault of this build/test run!');
#
# Install the extension pack.
#
if fRc is True and self._fAutoInstallPuelExtPack:
fRc = self._installExtPack();
if fRc is False:
reporter.testFailure('Extension pack installation error.');
# Some debugging...
try:
cMbFreeSpace = utils.getDiskUsage(self.sScratchPath);
reporter.log('Disk usage after VBox install: %d MB available at %s' % (cMbFreeSpace, self.sScratchPath,));
except:
reporter.logXcpt('Unable to get disk free space. Ignored. Continuing.');
reporter.testDone(fRc is None);
return fRc;
def _reinitScratch(self, fnLog, fUseTheForce):
"""
Wipes the scratch directories and re-initializes them.
No exceptions raise, returns success indicator instead.
"""
if fUseTheForce is None:
fUseTheForce = self._fFirstSignOn;
class ErrorCallback(object): # pylint: disable=R0903
"""
Callbacks + state for the cleanup.
"""
def __init__(self):
self.fRc = True;
def onErrorCallback(self, sFnName, sPath, aXcptInfo):
""" Logs error during shutil.rmtree operation. """
fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
self.fRc = False;
oRc = ErrorCallback();
#
# Cleanup.
#
for sName in os.listdir(self._oOptions.sScratchRoot):
sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
try:
if os.path.isdir(sFullName):
shutil.rmtree(sFullName, False, oRc.onErrorCallback);
else:
os.remove(sFullName);
if os.path.exists(sFullName):
raise Exception('Still exists after deletion, weird.');
except Exception, oXcpt:
if fUseTheForce is True \
and utils.getHostOs() not in ['win', 'os2'] \
and len(sFullName) >= 8 \
and sFullName[0] == '/' \
and sFullName[1] != '/' \
and sFullName.find('/../') < 0:
fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
try:
if os.path.isdir(sFullName):
iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
else:
iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
if iRc != 0:
raise Exception('exit code %s' % iRc);
if os.path.exists(sFullName):
raise Exception('Still exists after forced deletion, weird^2.');
except:
fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
oRc.fRc = False;
else:
fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
oRc.fRc = False;
def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None):
"""
Wipes the scratch directories and re-initializes them.
No exceptions raise, returns success indicator instead.
"""
if fUseTheForce is None:
fUseTheForce = self._fFirstSignOn;
class ErrorCallback(object): # pylint: disable=R0903
"""
Callbacks + state for the cleanup.
"""
def __init__(self):
self.fRc = True;
def onErrorCallback(self, sFnName, sPath, aXcptInfo):
""" Logs error during shutil.rmtree operation. """
fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
self.fRc = False;
oRc = ErrorCallback();
#
# Cleanup.
#
for sName in os.listdir(self._oOptions.sScratchRoot):
sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
try:
if os.path.isdir(sFullName):
shutil.rmtree(sFullName, False, oRc.onErrorCallback);
else:
os.remove(sFullName);
if os.path.exists(sFullName):
raise Exception('Still exists after deletion, weird.');
except Exception, oXcpt:
if fUseTheForce is True \
and utils.getHostOs() not in ['win', 'os2'] \
and len(sFullName) >= 8 \
and sFullName[0] == '/' \
and sFullName[1] != '/' \
and sFullName.find('/../') < 0:
fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
try:
if os.path.isdir(sFullName):
iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
else:
iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
if iRc != 0:
raise Exception('exit code %s' % iRc);
if os.path.exists(sFullName):
raise Exception('Still exists after forced deletion, weird^2.');
except:
fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
oRc.fRc = False;
else:
fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
oRc.fRc = False;
def _getHelperOutput(self, sCmd):
"""
Invokes TestBoxHelper to obtain information hard to access from python.
"""
if self._sTestBoxHelper is None:
if not utils.isRunningFromCheckout():
# See VBoxTestBoxScript.zip for layout.
self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
'TestBoxHelper');
else: # Only for in-tree testing, so don't bother be too accurate right now.
sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
utils.getHostOsDotArch(), sType, 'testboxscript', \
utils.getHostOs(), utils.getHostArch(), \
'TestBoxHelper');
if utils.getHostOs() in ['win', 'os2']:
self._sTestBoxHelper += '.exe';
return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
def _getHelperOutput(self, sCmd):
"""
Invokes TestBoxHelper to obtain information hard to access from python.
"""
if self._sTestBoxHelper is None:
if not utils.isRunningFromCheckout():
# See VBoxTestBoxScript.zip for layout.
self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
'TestBoxHelper');
else: # Only for in-tree testing, so don't bother be too accurate right now.
sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
utils.getHostOsDotArch(), sType, 'testboxscript', \
utils.getHostOs(), utils.getHostArch(), \
'TestBoxHelper');
if utils.getHostOs() in ['win', 'os2']:
self._sTestBoxHelper += '.exe';
return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
def __init__(self):
ReporterBase.__init__(self);
self.sTestManagerUrl = os.environ.get('TESTBOX_MANAGER_URL');
self.sTestBoxUuid = os.environ.get('TESTBOX_UUID');
self.idTestBox = int(os.environ.get('TESTBOX_ID'));
self.idTestSet = int(os.environ.get('TESTBOX_TEST_SET_ID'));
self._asXml = [];
self._secTsXmlFlush = utils.timestampSecond();
self._secTsXmlLast = self._secTsXmlFlush;
self._fXmlFlushing = False;
self.oOutput = sys.stdout; # Hack for __del__ output.
self.fFlushEachLine = True;
self.fDebugXml = 'TESTDRIVER_REPORTER_DEBUG_XML' in os.environ;
# Prepare the TM connecting.
import urlparse;
import httplib;
import urllib;
from common import constants;
self._fnUrlEncode = urllib.urlencode;
self._fnUrlParseQs = urlparse.parse_qs;
self._oParsedTmUrl = urlparse.urlparse(self.sTestManagerUrl);
if sys.version_info[0] >= 3 \
or (sys.version_info[0] == 2 and sys.version_info[1] >= 6):
if self._oParsedTmUrl.scheme == 'https': # pylint: disable=E1101
self._fnTmConnect = lambda: httplib.HTTPSConnection(self._oParsedTmUrl.hostname,
timeout = self.kcSecTestManagerRequestTimeout);
else:
self._fnTmConnect = lambda: httplib.HTTPConnection( self._oParsedTmUrl.hostname,
timeout = self.kcSecTestManagerRequestTimeout);
else:
if self._oParsedTmUrl.scheme == 'https': # pylint: disable=E1101
self._fnTmConnect = lambda: httplib.HTTPSConnection(self._oParsedTmUrl.hostname);
else:
self._fnTmConnect = lambda: httplib.HTTPConnection( self._oParsedTmUrl.hostname);
self._dHttpHeader = \
{
'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
'User-Agent': 'TestDriverReporter/%s.0 (%s, %s)' % (__version__, utils.getHostOs(), utils.getHostArch(),),
'Accept': 'text/plain,application/x-www-form-urlencoded',
'Accept-Encoding': 'identity',
'Cache-Control': 'max-age=0',
#'Connection': 'keep-alive',
};
dParams = {
constants.tbreq.ALL_PARAM_TESTBOX_UUID: self.sTestBoxUuid,
constants.tbreq.ALL_PARAM_TESTBOX_ID: self.idTestBox,
constants.tbreq.RESULT_PARAM_TEST_SET_ID: self.idTestSet,
};
self._sTmServerPath = '/%s/testboxdisp.py?%s' \
% ( self._oParsedTmUrl.path.strip('/'), # pylint: disable=E1101
urllib.urlencode(dParams), );
def _spawnChild(self, sAction):
"""
Spawns the child process, returning success indicator + child object.
"""
# Argument list.
asArgs = self._assembleArguments(sAction)
if asArgs is None:
self._log('Malformed command line: "%s"' %
(self._sScriptCmdLine, ))
return (False, None)
# Spawn child.
try:
oChild = utils.processPopenSafe(
asArgs,
shell=False,
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=self._oTestBoxScript.getPathSpill(),
universal_newlines=True,
close_fds=(False if utils.getHostOs() == 'win' else True),
preexec_fn=(None if utils.getHostOs() in ['win', 'os2'] else
os.setsid))
# pylint: disable=E1101
except Exception as oXcpt:
self._log('Error creating child process %s: %s' % (asArgs, oXcpt))
return (False, None)
oChild.sTestBoxScriptAction = sAction
# Start output thread, extending the child object to keep track of it.
oChild.oOutputThread = threading.Thread(target=self._outputThreadProc,
args=(oChild, oChild.stdout,
sAction))
oChild.oOutputThread.daemon = True
oChild.oOutputThread.fPleaseQuit = False
# Our extension.
oChild.oOutputThread.start()
return (True, oChild)
def _hardenedMkDir(self, sPath):
"""
Creates the directory specified sPath (including parents).
"""
reporter.log('_hardenedMkDir: %s' % (sPath,));
if utils.getHostOs() in [ 'win', 'os2' ]:
os.makedirs(sPath, 0755);
else:
fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
if fRc is not True:
raise Exception('Failed to create dir "%s".' % (sPath,));
return True;
def _hardenedMkDir(self, sPath):
"""
Creates the directory specified sPath (including parents).
"""
reporter.log('_hardenedMkDir: %s' % (sPath,));
if utils.getHostOs() in [ 'win', 'os2' ]:
os.makedirs(sPath, 0755);
else:
fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
if fRc is not True:
raise Exception('Failed to create dir "%s".' % (sPath,));
return True;
def _hardenedMkDir(self, sPath):
"""
Creates the directory specified sPath (including parents).
"""
reporter.log("_hardenedMkDir: %s" % (sPath,))
if utils.getHostOs() in ["win", "os2"]:
os.makedirs(sPath, 0755)
else:
fRc = self._sudoExecuteSync(["/bin/mkdir", "-p", "-m", "0755", sPath])
if fRc is not True:
raise Exception('Failed to create dir "%s".' % (sPath,))
return True
def test1(self):
"""
Executes test #1.
"""
fRc = True;
oDiskCfg = self.kdStorageCfgs.get(socket.gethostname().lower());
# Test the host first if requested
if oDiskCfg is not None:
lstBinaryPaths = ['/bin', '/sbin', '/usr/bin', '/usr/sbin', \
'/opt/csw/bin', '/usr/ccs/bin', '/usr/sfw/bin'];
oExecutor = remoteexecutor.RemoteExecutor(None, lstBinaryPaths, self.sScratchPath);
self.oStorCfg = storagecfg.StorageCfg(oExecutor, utils.getHostOs(), oDiskCfg);
if self.fTestHost:
reporter.testStart('Host');
if self.fUseScratch:
sMountPoint = self.sScratchPath;
else:
sMountPoint = self.prepareStorage(self.oStorCfg);
if sMountPoint is not None:
self.testBenchmarks(utils.getHostOs(), sMountPoint, oExecutor);
self.cleanupStorage(self.oStorCfg);
else:
reporter.testFailure('Failed to prepare host storage');
fRc = False;
reporter.testDone();
else:
# Loop thru the test VMs.
for sVM in self.asTestVMs:
# run test on the VM.
if not self.testBenchmarkOneVM(sVM):
fRc = False;
else:
fRc = True;
else:
fRc = False;
return fRc;
def _hardenedDeleteFile(self, sPath):
"""
Deletes a file.
"""
reporter.log('_hardenedDeleteFile: %s' % (sPath,));
if os.path.exists(sPath):
if utils.getHostOs() in [ 'win', 'os2' ]:
os.remove(sPath);
else:
fRc = self._sudoExecuteSync(['/bin/rm', sPath]);
if fRc is not True:
raise Exception('Failed to remove "%s".' % (sPath,));
return True;
def _hardenedRemoveDir(self, sPath):
"""
Removes a directory.
"""
reporter.log('_hardenedRemoveDir: %s' % (sPath,));
if os.path.exists(sPath):
if utils.getHostOs() in [ 'win', 'os2' ]:
os.rmdir(sPath);
else:
fRc = self._sudoExecuteSync(['/bin/rmdir', sPath]);
if fRc is not True:
raise Exception('Failed to remove "%s".' % (sPath,));
return True;
def _hardenedRemoveDir(self, sPath):
"""
Removes a directory.
"""
reporter.log("_hardenedRemoveDir: %s" % (sPath,))
if os.path.exists(sPath):
if utils.getHostOs() in ["win", "os2"]:
os.rmdir(sPath)
else:
fRc = self._sudoExecuteSync(["/bin/rmdir", sPath])
if fRc is not True:
raise Exception('Failed to remove "%s".' % (sPath,))
return True
def _hardenedRemoveDir(self, sPath):
"""
Removes a directory.
"""
reporter.log('_hardenedRemoveDir: %s' % (sPath, ))
if os.path.exists(sPath):
if utils.getHostOs() in ['win', 'os2']:
os.rmdir(sPath)
else:
fRc = self._sudoExecuteSync(['/bin/rmdir', sPath])
if fRc is not True:
raise Exception('Failed to remove "%s".' % (sPath, ))
return True
def _hardenedDeleteFile(self, sPath):
"""
Deletes a file.
"""
reporter.log('_hardenedDeleteFile: %s' % (sPath, ))
if os.path.exists(sPath):
if utils.getHostOs() in ['win', 'os2']:
os.remove(sPath)
else:
fRc = self._sudoExecuteSync(['/bin/rm', sPath])
if fRc is not True:
raise Exception('Failed to remove "%s".' % (sPath, ))
return True
def _hardenedDeleteFile(self, sPath):
"""
Deletes a file.
"""
reporter.log("_hardenedDeleteFile: %s" % (sPath,))
if os.path.exists(sPath):
if utils.getHostOs() in ["win", "os2"]:
os.remove(sPath)
else:
fRc = self._sudoExecuteSync(["/bin/rm", sPath])
if fRc is not True:
raise Exception('Failed to remove "%s".' % (sPath,))
return True
def _spawnChild(self, sAction):
"""
Spawns the child process, returning success indicator + child object.
"""
# Argument list.
asArgs = self._assembleArguments(sAction)
if asArgs is None:
self._log('Malformed command line: "%s"' % (self._sScriptCmdLine,));
return (False, None);
# Spawn child.
try:
oChild = utils.processPopenSafe(asArgs,
shell = False,
bufsize = -1,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
cwd = self._oTestBoxScript.getPathSpill(),
universal_newlines = True,
close_fds = (False if utils.getHostOs() == 'win' else True),
preexec_fn = (None if utils.getHostOs() in ['win', 'os2']
else os.setsid)); # pylint: disable=E1101
except Exception as oXcpt:
self._log('Error creating child process %s: %s' % (asArgs, oXcpt));
return (False, None);
oChild.sTestBoxScriptAction = sAction;
# Start output thread, extending the child object to keep track of it.
oChild.oOutputThread = threading.Thread(target=self._outputThreadProc, args=(oChild, oChild.stdout, sAction))
oChild.oOutputThread.daemon = True;
oChild.oOutputThread.fPleaseQuit = False; # Our extension.
oChild.oOutputThread.start();
return (True, oChild);
def doReboot(self):
"""
Worker common to _cmdReboot and _doUpgrade that performs a system reboot.
"""
# !! Not more exceptions beyond this point !!
testboxcommons.log('Rebooting')
# Stop anything that might be executing at this point.
oCurTask = self._getCurTask()
if oCurTask is not None:
oCurTask.terminate()
oCurTask.wait(self.kcSecStopBeforeRebootTimeout)
# Invoke shutdown command line utility.
sOs = utils.getHostOs()
asCmd2 = None
if sOs == 'win':
asCmd = [
'shutdown', '/r', '/t', '0', '/c',
'"ValidationKit triggered reboot"', '/d', '4:1'
]
elif sOs == 'os2':
asCmd = ['setboot', '/B']
elif sOs in ('solaris', ):
asCmd = ['/usr/sbin/reboot', '-p']
asCmd2 = ['/usr/sbin/reboot']
# Hack! S10 doesn't have -p, but don't know how to reliably detect S10.
else:
asCmd = ['/sbin/shutdown', '-r', 'now']
try:
utils.sudoProcessOutputChecked(asCmd)
except Exception as oXcpt:
if asCmd2 is not None:
try:
utils.sudoProcessOutputChecked(asCmd2)
except Exception as oXcpt:
testboxcommons.log(
'Error executing reboot command "%s" as well as "%s": %s'
% (asCmd, asCmd2, oXcpt))
return False
testboxcommons.log('Error executing reboot command "%s": %s' %
(asCmd, oXcpt))
return False
# Quit the script.
while True:
sys.exit(32)
return True
def _hardenedCopyFile(self, sSrc, sDst, iMode):
"""
Copies a file.
"""
reporter.log('_hardenedCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
if utils.getHostOs() in [ 'win', 'os2' ]:
utils.copyFileSimple(sSrc, sDst);
os.chmod(sDst, iMode);
else:
fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
if fRc is not True:
raise Exception('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
if fRc is not True:
raise Exception('Failed to chmod "%s".' % (sDst,));
return True;
def _hardenedCopyFile(self, sSrc, sDst, iMode):
"""
Copies a file.
"""
reporter.log("_hardenedCopyFile: %s -> %s (mode: %o)" % (sSrc, sDst, iMode))
if utils.getHostOs() in ["win", "os2"]:
shutil.copyfile(sSrc, sDst)
os.chmod(sDst, iMode)
else:
fRc = self._sudoExecuteSync(["/bin/cp", sSrc, sDst])
if fRc is not True:
raise Exception('Failed to copy "%s" to "%s".' % (sSrc, sDst))
fRc = self._sudoExecuteSync(["/bin/chmod", "%o" % (iMode,), sDst])
if fRc is not True:
raise Exception('Failed to chmod "%s".' % (sDst,))
return True
def _hardenedCopyFile(self, sSrc, sDst, iMode):
"""
Copies a file.
"""
reporter.log('_hardenedCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
if utils.getHostOs() in [ 'win', 'os2' ]:
shutil.copyfile(sSrc, sDst);
os.chmod(sDst, iMode);
else:
fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
if fRc is not True:
raise Exception('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
if fRc is not True:
raise Exception('Failed to chmod "%s".' % (sDst,));
return True;
def _threadProc(self):
"""
Perform the actual clean up on script startup.
"""
#
# First make sure we won't repeat this exercise should it turn out to
# trigger another reboot/panic/whatever.
#
sScriptCmdLine = os.path.join(self._oTestBoxScript.getPathState(),
'script-cmdline.txt')
try:
os.remove(sScriptCmdLine)
oFile = open(sScriptCmdLine, 'wb')
oFile.close()
except Exception as oXcpt:
self._log('Error truncating "%s": %s' % (sScriptCmdLine, oXcpt))
#
# Report the incident.
#
self._log('Seems we rebooted!')
self._log('script-cmdline="%s"' % (self._sScriptCmdLine))
self._log('result-id=%d' % (self._idResult))
self._log('testbox-id=%d' % (self.idTestBox))
self._log('testbox-name=%s' % (self.sTestBoxName))
self._logFlush()
# System specific info.
sOs = utils.getHostOs()
if sOs == 'darwin':
self._log('NVRAM Panic Info:\n%s\n' %
(self.darwinGetPanicInfo(), ))
self._logFlush()
## @todo Add some special command for reporting this situation so we get something
# useful in the event log.
#
# Do the cleaning up.
#
self._cleanupAfter()
self._reportDone(constants.result.REBOOTED)
return False
def _threadProc(self):
"""
Perform the actual clean up on script startup.
"""
#
# First make sure we won't repeat this exercise should it turn out to
# trigger another reboot/panic/whatever.
#
sScriptCmdLine = os.path.join(self._oTestBoxScript.getPathState(), 'script-cmdline.txt');
try:
os.remove(sScriptCmdLine);
oFile = open(sScriptCmdLine, 'wb');
oFile.close();
except Exception as oXcpt:
self._log('Error truncating "%s": %s' % (sScriptCmdLine, oXcpt));
#
# Report the incident.
#
self._log('Seems we rebooted!');
self._log('script-cmdline="%s"' % (self._sScriptCmdLine));
self._log('result-id=%d' % (self._idResult));
self._log('testbox-id=%d' % (self.idTestBox));
self._log('testbox-name=%s' % (self.sTestBoxName));
self._logFlush();
# System specific info.
sOs = utils.getHostOs();
if sOs == 'darwin':
self._log('NVRAM Panic Info:\n%s\n' % (self.darwinGetPanicInfo(),));
self._logFlush();
## @todo Add some special command for reporting this situation so we get something
# useful in the event log.
#
# Do the cleaning up.
#
self._cleanupAfter();
self._reportDone(constants.result.REBOOTED);
return False;
def __init__(self):
"""
Reinitialize child class instance.
"""
vbox.TestDriver.__init__(self)
self.oTestVmSet = None
self.sVBoxInstallRoot = None
self.cSkipped = 0
self.cPassed = 0
self.cFailed = 0
self.sUnitTestsPathBase = None
self.sExeSuff = ".exe" if utils.getHostOs() in ["win", "dos", "os2"] else ""
self.aiVBoxVer = (4, 3, 0, 0)
# For testing testcase logic.
self.fDryRun = False
def __init__(self):
"""
Reinitialize child class instance.
"""
vbox.TestDriver.__init__(self)
self.oTestVmSet = None;
self.sVBoxInstallRoot = None
self.cSkipped = 0
self.cPassed = 0
self.cFailed = 0
self.sUnitTestsPathBase = None
self.sExeSuff = '.exe' if utils.getHostOs() in [ 'win', 'dos', 'os2' ] else '';
self.aiVBoxVer = (4, 3, 0, 0);
# For testing testcase logic.
self.fDryRun = False;
def __init__(self):
"""
Reinitialize child class instance.
"""
vbox.TestDriver.__init__(self)
self.oTestVmSet = None;
self.sVBoxInstallRoot = None
self.cSkipped = 0
self.cPassed = 0
self.cFailed = 0
self.sUnitTestsPathBase = None
self.sExeSuff = '.exe' if utils.getHostOs() in [ 'win', 'dos', 'os2' ] else '';
self.aiVBoxVer = (4, 3, 0, 0);
# For testing testcase logic.
self.fDryRun = False;
def actionCleanupAfter(self):
"""
Forward this to the sub testdriver, then uninstall VBox.
"""
fRc = True;
if 'execute' not in self.asActions and 'all' not in self.asActions:
fRc = self._executeSubDriver([ 'cleanup-after', ]);
if not self._killAllVBoxProcesses():
fRc = False;
if not self._uninstallVBox(self._persistentVarExists(self.ksVar_Skipped)):
fRc = False;
if utils.getHostOs() == 'darwin':
self._darwinUnmountDmg(fIgnoreError = True); # paranoia
if not TestDriverBase.actionCleanupAfter(self):
fRc = False;
return fRc;
def executeHstLoop(self, sWhat, asArgs, asEnv=None, fAsAdmin=False):
"""
Inner loop which handles the execution of a host binary.
"""
fRc = False
asEnvTmp = os.environ.copy()
if asEnv:
for sEnv in asEnv:
sKey, sValue = sEnv.split('=')
reporter.log2('Setting env var \"%s\" -> \"%s\"' %
(sKey, sValue))
os.environ[sKey] = sValue
# Also apply it to the current environment.
asEnvTmp[sKey] = sValue
if fAsAdmin \
and utils.getHostOs() != 'win':
oProcess = utils.sudoProcessPopen(asArgs,
env=asEnvTmp,
stdout=sys.stdout,
stderr=sys.stdout,
close_fds=False)
else:
oProcess = utils.processPopenSafe(asArgs,
env=asEnvTmp,
stdout=sys.stdout,
stderr=sys.stdout)
if oProcess:
self.pidFileAdd(oProcess.pid, sWhat, fSudo=fAsAdmin)
iRc = oProcess.wait()
self.pidFileRemove(oProcess.pid)
if iRc == 0:
reporter.log('*** %s: exit code %d' % (sWhat, iRc))
fRc = True
else:
reporter.log('!*! %s: exit code %d' % (sWhat, iRc))
return fRc
def _uninstallVBox(self, fIgnoreError=False):
"""
Uninstall VirtualBox.
"""
reporter.testStart('Uninstalling VirtualBox')
sHost = utils.getHostOs()
if sHost == 'darwin': fRc = self._uninstallVBoxOnDarwin()
elif sHost == 'linux': fRc = self._uninstallVBoxOnLinux()
elif sHost == 'solaris': fRc = self._uninstallVBoxOnSolaris(True)
elif sHost == 'win': fRc = self._uninstallVBoxOnWindows(True)
else:
reporter.error('Unsupported host "%s".' % (sHost, ))
if fRc is False and not fIgnoreError:
reporter.testFailure('Uninstallation failed.')
fRc2 = self._uninstallAllExtPacks()
if not fRc2 and fRc:
fRc = fRc2
reporter.testDone(fSkipped=(fRc is None))
return fRc
def _uninstallVBox(self, fIgnoreError = False):
"""
Uninstall VirtualBox.
"""
reporter.testStart('Uninstalling VirtualBox');
sHost = utils.getHostOs()
if sHost == 'darwin': fRc = self._uninstallVBoxOnDarwin();
elif sHost == 'linux': fRc = self._uninstallVBoxOnLinux();
elif sHost == 'solaris': fRc = self._uninstallVBoxOnSolaris();
elif sHost == 'win': fRc = self._uninstallVBoxOnWindows();
else:
reporter.error('Unsupported host "%s".' % (sHost,));
if fRc is False and not fIgnoreError:
reporter.testFailure('Uninstallation failed.');
fRc2 = self._uninstallAllExtPacks();
if not fRc2 and fRc:
fRc = fRc2;
reporter.testDone(fSkipped = (fRc is None));
return fRc;
def __init__(
self,
oTestVm,
sBasePath,
sSubDir, # pylint: disable=too-many-arguments
oRngFileSizes=xrange(0, 16384),
oRngManyFiles=xrange(128, 512),
oRngTreeFiles=xrange(128, 384),
oRngTreeDepth=xrange(92, 256),
oRngTreeDirs=xrange(2, 16),
cchMaxPath=230,
cchMaxName=230,
asCompatibleWith=None,
uSeed=None):
asCompOses = [
oTestVm.getGuestOs(),
]
sHostOs = utils.getHostOs()
if sHostOs not in asCompOses:
asCompOses.append(sHostOs)
testfileset.TestFileSet.__init__(self,
fDosStyle=oTestVm.isWindows()
or oTestVm.isOS2(),
asCompatibleWith=asCompOses,
sBasePath=sBasePath,
sSubDir=sSubDir,
oRngFileSizes=oRngFileSizes,
oRngManyFiles=oRngManyFiles,
oRngTreeFiles=oRngTreeFiles,
oRngTreeDepth=oRngTreeDepth,
oRngTreeDirs=oRngTreeDirs,
cchMaxPath=cchMaxPath,
cchMaxName=cchMaxName,
uSeed=uSeed)
self.oTestVm = oTestVm
def main():
"""
Main function a la C/C++. Returns exit code.
"""
#
# Parse arguments.
#
if utils.getHostOs() in ('win', 'os2'):
sDefTestRsrc = 'T:';
sDefBuilds = 'U:';
elif utils.getHostOs() == 'darwin':
sDefTestRsrc = '/Volumes/testrsrc';
sDefBuilds = '/Volumes/builds';
else:
sDefTestRsrc = '/mnt/testrsrc';
sDefBuilds = '/mnt/builds';
class MyOptionParser(OptionParser):
""" We need to override the exit code on --help, error and so on. """
def __init__(self, *args, **kwargs):
OptionParser.__init__(self, *args, **kwargs);
def exit(self, status = 0, msg = None):
OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
parser = MyOptionParser(version=__version__[11:-1].strip());
for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
sLower = sMixed.lower();
sPrefix = 's' + sMixed;
parser.add_option('--' + sLower + '-path',
dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
help='Where ' + sDesc + ' can be found');
parser.add_option('--' + sLower + '-server-type',
dest=sPrefix + 'ServerType', metavar='<nfs|cifs>', default=None,
help='The type of server, cifs or nfs. If empty (default), we won\'t try mount anything.');
parser.add_option('--' + sLower + '-server-name',
dest=sPrefix + 'ServerName', metavar='<server>', default='solserv.de.oracle.com',
help='The name of the server with the builds.');
parser.add_option('--' + sLower + '-server-share',
dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
help='The name of the builds share.');
parser.add_option('--' + sLower + '-server-user',
dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
help='The user name to use when accessing the ' + sDesc + ' share.');
parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
help='The password to use when accessing the ' + sDesc + ' share.');
parser.add_option("--test-manager", metavar="<url>",
dest="sTestManagerUrl",
help="Test Manager URL",
default="http://tindertux.de.oracle.com/testmanager")
parser.add_option("--scratch-root", metavar="<abs-path>",
dest="sScratchRoot",
help="Path to the scratch directory",
default=None)
parser.add_option("--system-uuid", metavar="<uuid>",
dest="sSystemUuid",
help="The system UUID of the testbox, used for uniquely identifiying the machine",
default=None)
parser.add_option("--hwvirt",
dest="fHasHwVirt", action="store_true", default=None,
help="Hardware virtualization available in the CPU");
parser.add_option("--no-hwvirt",
dest="fHasHwVirt", action="store_false", default=None,
help="Hardware virtualization not available in the CPU");
parser.add_option("--nested-paging",
dest="fHasNestedPaging", action="store_true", default=None,
help="Nested paging is available");
parser.add_option("--no-nested-paging",
dest="fHasNestedPaging", action="store_false", default=None,
help="Nested paging is not available");
parser.add_option("--64-bit-guest",
dest="fCan64BitGuest", action="store_true", default=None,
help="Host can execute 64-bit guests");
parser.add_option("--no-64-bit-guest",
dest="fCan64BitGuest", action="store_false", default=None,
help="Host cannot execute 64-bit guests");
parser.add_option("--io-mmu",
dest="fHasIoMmu", action="store_true", default=None,
help="I/O MMU available");
parser.add_option("--no-io-mmu",
dest="fHasIoMmu", action="store_false", default=None,
help="No I/O MMU available");
parser.add_option("--raw-mode",
dest="fWithRawMode", action="store_true", default=None,
help="Use raw-mode on this host.");
parser.add_option("--no-raw-mode",
dest="fWithRawMode", action="store_false", default=None,
help="Disables raw-mode tests on this host.");
parser.add_option("--pidfile",
dest="sPidFile", default=None,
help="For the parent script, ignored.");
parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
dest = "asEnvVars", default = [],
help = "Sets an environment variable. Can be repeated.");
(oOptions, args) = parser.parse_args()
# Check command line
if args != []:
parser.print_help();
return TBS_EXITCODE_SYNTAX;
if oOptions.sSystemUuid is not None:
uuid.UUID(oOptions.sSystemUuid);
if not oOptions.sTestManagerUrl.startswith('http://') \
and not oOptions.sTestManagerUrl.startswith('https://'):
print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
return TBS_EXITCODE_SYNTAX;
for sPrefix in ['sBuilds', 'sTestRsrc']:
sType = getattr(oOptions, sPrefix + 'ServerType');
if sType is None or len(sType.strip()) == 0:
setattr(oOptions, sPrefix + 'ServerType', None);
elif sType not in ['cifs', 'nfs']:
print('Syntax error: Invalid server type "%s"' % (sType,));
return TBS_EXITCODE_SYNTAX;
#
# Instantiate the testbox script and start dispatching work.
#
try:
oTestBoxScript = TestBoxScript(oOptions);
except TestBoxScriptException, oXcpt:
print('Error: %s' % (oXcpt,));
return TBS_EXITCODE_SYNTAX;
def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sWhat):
"""
Mounts the specified share if needed.
Raises exception on failure.
"""
# Only mount if the type is specified.
if sType is None:
return True;
# Test if already mounted.
sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '.txt');
if os.path.isfile(sTestFile):
return True;
#
# Platform specific mount code.
#
sHostOs = utils.getHostOs()
if sHostOs in ('darwin', 'freebsd'):
utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
if sType == 'cifs':
# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
'-f', '0555', '-d', '0555',
'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
sMountPoint]);
else:
raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
elif sHostOs == 'linux':
utils.sudoProcessCall(['/bin/umount', sMountPoint]);
utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
if sType == 'cifs':
utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
'-o',
'user='******',password='******',sec=ntlmv2'
+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
'//%s/%s' % (sServer, sShare),
sMountPoint]);
elif sType == 'nfs':
utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
'-o', 'soft,ro',
'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
sMountPoint]);
else:
raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
elif sHostOs == 'solaris':
utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
if sType == 'cifs':
## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
oPasswdFile = tempfile.TemporaryFile();
oPasswdFile.write(sPassword + '\n');
oPasswdFile.flush();
utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
'-o',
'user='******',uid=' + str(os.getuid()) # pylint: disable=E1101
+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
+ ',fileperms=0555,dirperms=0555,noxattr,ro',
'//%s/%s' % (sServer, sShare),
sMountPoint],
stdin = oPasswdFile);
oPasswdFile.close();
elif sType == 'nfs':
utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
'-o', 'noxattr,ro',
'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
sMountPoint]);
else:
raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
elif sHostOs == 'win':
if sType != 'cifs':
raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
utils.processCall(['net', 'use', sMountPoint, '/d']);
utils.processOutputChecked(['net', 'use', sMountPoint,
'\\\\' + sServer + '\\' + sShare,
sPassword,
'/USER:'******'Unsupported host %s' % (sHostOs,));
#
# Re-test.
#
if not os.path.isfile(sTestFile):
raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
% (sWhat, sServer, sShare, sMountPoint, sTestFile));
return True;
def test1(self):
"""
Executes test #1.
"""
fRc = True
oDiskCfg = self.kdStorageCfgs.get(socket.gethostname().lower())
# Test the host first if requested
if oDiskCfg is not None or self.fUseScratch:
lstBinaryPaths = ['/bin', '/sbin', '/usr/bin', '/usr/sbin', \
'/opt/csw/bin', '/usr/ccs/bin', '/usr/sfw/bin']
oExecutor = remoteexecutor.RemoteExecutor(None, lstBinaryPaths,
self.sScratchPath)
if not self.fUseScratch:
self.oStorCfg = storagecfg.StorageCfg(oExecutor,
utils.getHostOs(),
oDiskCfg)
# Try to cleanup any leftovers from a previous run first.
fRc = self.oStorCfg.cleanupLeftovers()
if not fRc:
reporter.error(
'Failed to cleanup any leftovers from a previous run')
if self.fTestHost:
reporter.testStart('Host')
if self.fUseScratch:
sMountPoint = self.sScratchPath
else:
sMountPoint = self.prepareStorage(self.oStorCfg)
if sMountPoint is not None:
for sIoTest in self.asTests:
reporter.testStart(sIoTest)
for sTestSet in self.asTestSets:
reporter.testStart(sTestSet)
dTestSet = self.kdTestSets.get(sTestSet)
self.testBenchmark(utils.getHostOs(), sIoTest,
sMountPoint, oExecutor,
dTestSet)
reporter.testDone()
reporter.testDone()
self.cleanupStorage(self.oStorCfg)
else:
reporter.testFailure('Failed to prepare host storage')
fRc = False
reporter.testDone()
else:
# Create the storage space first if it is not done before every test.
sMountPoint = None
if self.fUseScratch:
sMountPoint = self.sScratchPath
elif not self.fRecreateStorCfg:
reporter.testStart('Create host storage')
sMountPoint = self.prepareStorage(self.oStorCfg)
if sMountPoint is None:
reporter.testFailure('Failed to prepare host storage')
fRc = False
self.oStorCfg.mkDirOnVolume(sMountPoint, 'test', 0777)
sMountPoint = sMountPoint + '/test'
reporter.testDone()
if fRc:
# Run the storage tests.
if not self.testStorage(sMountPoint):
fRc = False
if not self.fRecreateStorCfg:
self.cleanupStorage(self.oStorCfg)
else:
fRc = False
return fRc
def testIt(self, oTestVm, oSession, oTxsSession):
"""
Executes the test.
Returns fRc, oTxsSession. The latter may have changed.
"""
reporter.log("Active tests: %s" % (self.asTests, ))
#
# Skip the test if before 6.0
#
if self.oTstDrv.fpApiVer < 6.0:
reporter.log('Requires 6.0 or later (for now)')
return (None, oTxsSession)
# Guess a free mount point inside the guest.
if oTestVm.isWindows() or oTestVm.isOS2():
self.sGuestSlash = '\\'
else:
self.sGuestSlash = '/'
#
# Create the host directory to share. Empty except for a 'candle.dir' subdir
# that we use to check that it mounted correctly.
#
sShareName1 = 'shfl1'
sShareHostPath1 = os.path.join(self.oTstDrv.sScratchPath, sShareName1)
reporter.log2('Creating shared host folder "%s"...' %
(sShareHostPath1, ))
if os.path.exists(sShareHostPath1):
try:
shutil.rmtree(sShareHostPath1)
except:
return (reporter.errorXcpt('shutil.rmtree(%s)' %
(sShareHostPath1, )), oTxsSession)
try:
os.mkdir(sShareHostPath1)
except:
return (reporter.errorXcpt('os.mkdir(%s)' % (sShareHostPath1, )),
oTxsSession)
try:
os.mkdir(os.path.join(sShareHostPath1, 'candle.dir'))
except:
return (reporter.errorXcpt('os.mkdir(%s)' % (sShareHostPath1, )),
oTxsSession)
# Guess a free mount point inside the guest.
if oTestVm.isWindows() or oTestVm.isOS2():
sMountPoint1 = 'V:'
else:
sMountPoint1 = '/mnt/' + sShareName1
fRc = self.mountShare(oSession, oTxsSession, sShareName1,
sShareHostPath1, sMountPoint1)
if fRc is not True:
return (False, oTxsSession)
# skip the remainder if we cannot auto mount the folder.
#
# Run FsPerf inside the guest.
#
fSkip = 'fsperf' not in self.asTests
if fSkip is False:
cMbFree = utils.getDiskUsage(sShareHostPath1)
if cMbFree >= 16:
reporter.log2('Free space: %u MBs' % (cMbFree, ))
else:
reporter.log('Skipping FsPerf because only %u MB free on %s' %
(
cMbFree,
sShareHostPath1,
))
fSkip = True
if fSkip is False:
# Common arguments:
asArgs = [
'FsPerf', '-d',
sMountPoint1 + self.sGuestSlash + 'fstestdir-1', '-s8'
]
# Skip part of mmap on older windows systems without CcCoherencyFlushAndPurgeCache (>= w7).
reporter.log2('oTestVm.sGuestOsType=%s' % (oTestVm.sGuestOsType, ))
if oTestVm.getNonCanonicalGuestOsType() \
in [ 'WindowsNT3x', 'WindowsNT4', 'Windows2000', 'WindowsXP', 'WindowsXP_64', 'Windows2003',
'Windows2003_64', 'WindowsVista', 'WindowsVista_64', 'Windows2008', 'Windows2008_64']:
asArgs.append('--no-mmap-coherency')
# Configure I/O block sizes according to guest memory size:
cbMbRam = 128
try:
cbMbRam = oSession.o.machine.memorySize
except:
reporter.errorXcpt()
reporter.log2('cbMbRam=%s' % (cbMbRam, ))
asArgs.append('--set-block-size=1')
asArgs.append('--add-block-size=512')
asArgs.append('--add-block-size=4096')
asArgs.append('--add-block-size=16384')
asArgs.append('--add-block-size=65536')
asArgs.append('--add-block-size=1048576')
# 1 MiB
if cbMbRam >= 512:
asArgs.append('--add-block-size=33554432')
# 32 MiB
if cbMbRam >= 768:
asArgs.append('--add-block-size=134217728')
# 128 MiB
# Putting lots (10000) of files in a single directory causes issues on OS X
# (HFS+ presumably, though could be slow disks) and some linuxes (slow disks,
# maybe ext2/3?). So, generally reduce the file count to 4096 everywhere
# since we're not here to test the host file systems, and 3072 on macs.
if utils.getHostOs() in [
'darwin',
]:
asArgs.append('--many-files=3072')
elif utils.getHostOs() in [
'linux',
]:
asArgs.append('--many-files=4096')
# Add the extra arguments from the command line and kick it off:
asArgs.extend(self.asExtraArgs)
# Run FsPerf:
reporter.log2('Starting guest FsPerf (%s)...' % (asArgs, ))
sFsPerfPath = self._locateGstFsPerf(oTxsSession)
## @todo For some odd reason the combined GA/VaKit .ISO (by IPRT/fs/isomakercmd)
# sometimes (?) contains FsPerf as non-executable (-r--r--r-- 1 root root) on Linux.
#
# So work around this for now by copying the desired FsPerf binary to the temp directory,
# make it executable and execute it from there.
fISOMakerCmdIsBuggy = oTestVm.isLinux()
if fISOMakerCmdIsBuggy:
sFsPerfPathTemp = oTestVm.pathJoin(
self.oTstDrv.getGuestTempDir(oTestVm), 'FsPerf${EXESUFF}')
if oTestVm.isWindows() \
or oTestVm.isOS2():
sCopy = self.oTstDrv.getGuestSystemShell()
sCopyArgs = (sCopy, "/C", "copy", "/Y", sFsPerfPath,
sFsPerfPathTemp)
else:
sCopy = oTestVm.pathJoin(
self.oTstDrv.getGuestSystemDir(oTestVm), 'cp')
sCopyArgs = (sCopy, "-a", "-v", sFsPerfPath,
sFsPerfPathTemp)
fRc = self.oTstDrv.txsRunTest(oTxsSession,
'Copying FsPerf',
60 * 1000,
sCopy,
sCopyArgs,
fCheckSessionStatus=True)
fRc = fRc and oTxsSession.syncChMod(sFsPerfPathTemp, 0o755)
if fRc:
sFsPerfPath = sFsPerfPathTemp
fRc = self.oTstDrv.txsRunTest(oTxsSession,
'Running FsPerf',
90 * 60 * 1000,
sFsPerfPath,
asArgs,
fCheckSessionStatus=True)
reporter.log2('FsPerf -> %s' % (fRc, ))
if fRc:
# Do a bit of diagnosis to find out why this failed.
if not oTestVm.isWindows() \
and not oTestVm.isOS2():
sCmdLs = oTestVm.pathJoin(
self.oTstDrv.getGuestSystemDir(oTestVm), 'ls')
oTxsSession.syncExec(sCmdLs, (sCmdLs, "-al", sFsPerfPath),
fIgnoreErrors=True)
oTxsSession.syncExec(sCmdLs, (sCmdLs, "-al", "-R", "/opt"),
fIgnoreErrors=True)
oTxsSession.syncExec(sCmdLs,
(sCmdLs, "-al", "-R", "/media/cdrom"),
fIgnoreErrors=True)
sTestDir = os.path.join(sShareHostPath1, 'fstestdir-1')
if os.path.exists(sTestDir):
fRc = reporter.errorXcpt('test directory lingers: %s' %
(sTestDir, ))
try:
shutil.rmtree(sTestDir)
except:
fRc = reporter.errorXcpt('shutil.rmtree(%s)' %
(sTestDir, ))
else:
reporter.testStart('FsPerf')
reporter.testDone(fSkip or fRc is None)
#
# Check if auto-unmounting works.
#
if fRc is True:
fRc = self.unmountShare(oSession, oTxsSession, sShareName1,
sMountPoint1)
## @todo Add tests for multiple automount shares, random unmounting, reboot test.
return (fRc, oTxsSession)
def __init__(self, oOptions):
"""
Initialize internals
"""
self._oOptions = oOptions;
self._sTestBoxHelper = None;
# Signed-on state
self._cSignOnAttempts = 0;
self._fSignedOn = False;
self._fNeedReSignOn = False;
self._fFirstSignOn = True;
self._idTestBox = None;
self._sTestBoxName = '';
self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
# Command processor.
self._oCommand = TestBoxCommand(self);
#
# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
# many many GBs for some test scenarios and /tmp can be backed by swap
# or be a fast+small disk of some kind, while /var/tmp is normally
# larger, if slower. /var/tmp is generally not cleaned up on reboot,
# /tmp often is, this would break host panic / triple-fault detection.
#
if self._oOptions.sScratchRoot is None:
if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
# We need *lots* of space, so avoid /tmp as it may be a memory
# file system backed by the swap file, or worse.
self._oOptions.sScratchRoot = tempfile.gettempdir();
else:
self._oOptions.sScratchRoot = '/var/tmp';
sSubDir = 'testbox';
try:
sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
except:
pass;
self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
if not os.path.isdir(sDir):
os.makedirs(sDir, 0700);
# We count consecutive reinitScratch failures and will reboot the
# testbox after a while in the hope that it will correct the issue.
self._cReinitScratchErrors = 0;
#
# Mount builds and test resources if requested.
#
self.mountShares();
#
# Sign-on parameters: Packed into list of records of format:
# { <Parameter ID>: { <Current value>, <Check function> } }
#
self._ddSignOnParams = \
{
constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
}
for sItem in self._ddSignOnParams:
if self._ddSignOnParams[sItem][self.FN] is not None:
self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
for sItem in self._ddSignOnParams:
testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
#
# The System UUID is the primary identification of the machine, so
# refuse to cooperate if it's NULL.
#
self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
if self._sTestBoxUuid == self.ksNullUuid:
raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
#
# Export environment variables, clearing any we don't know yet.
#
for sEnvVar in self._oOptions.asEnvVars:
iEqual = sEnvVar.find('=');
if iEqual == -1: # No '=', remove it.
if sEnvVar in os.environ:
del os.environ[sEnvVar];
elif iEqual > 0: # Set it.
os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
else: # Starts with '=', bad user.
raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
os.environ['TESTBOX_REPORTER'] = 'remote';
os.environ['TESTBOX_NAME'] = '';
os.environ['TESTBOX_ID'] = '';
os.environ['TESTBOX_TEST_SET_ID'] = '';
os.environ['TESTBOX_TIMEOUT'] = '0';
os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
if utils.getHostOs() == 'win':
os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
def main():
"""
Main function a la C/C++. Returns exit code.
"""
#
# Parse arguments.
#
sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
if utils.getHostOs() in ('win', 'os2'):
sDefTestRsrc = 'T:';
sDefBuilds = 'U:';
elif utils.getHostOs() == 'darwin':
sDefTestRsrc = '/Volumes/testrsrc';
sDefBuilds = '/Volumes/builds';
else:
sDefTestRsrc = '/mnt/testrsrc';
sDefBuilds = '/mnt/builds';
class MyOptionParser(OptionParser):
""" We need to override the exit code on --help, error and so on. """
def __init__(self, *args, **kwargs):
OptionParser.__init__(self, *args, **kwargs);
def exit(self, status = 0, msg = None):
OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
parser = MyOptionParser(version=__version__[11:-1].strip());
for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
sLower = sMixed.lower();
sPrefix = 's' + sMixed;
parser.add_option('--' + sLower + '-path',
dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
help='Where ' + sDesc + ' can be found');
parser.add_option('--' + sLower + '-server-type',
dest=sPrefix + 'ServerType', metavar='<nfs|cifs>', default=sDefShareType,
help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
parser.add_option('--' + sLower + '-server-name',
dest=sPrefix + 'ServerName', metavar='<server>',
default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
help='The name of the server with the builds.');
parser.add_option('--' + sLower + '-server-share',
dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
help='The name of the builds share.');
parser.add_option('--' + sLower + '-server-user',
dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
help='The user name to use when accessing the ' + sDesc + ' share.');
parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
help='The password to use when accessing the ' + sDesc + ' share.');
parser.add_option("--test-manager", metavar="<url>",
dest="sTestManagerUrl",
help="Test Manager URL",
default="http://tindertux.de.oracle.com/testmanager")
parser.add_option("--scratch-root", metavar="<abs-path>",
dest="sScratchRoot",
help="Path to the scratch directory",
default=None)
parser.add_option("--system-uuid", metavar="<uuid>",
dest="sSystemUuid",
help="The system UUID of the testbox, used for uniquely identifiying the machine",
default=None)
parser.add_option("--hwvirt",
dest="fHasHwVirt", action="store_true", default=None,
help="Hardware virtualization available in the CPU");
parser.add_option("--no-hwvirt",
dest="fHasHwVirt", action="store_false", default=None,
help="Hardware virtualization not available in the CPU");
parser.add_option("--nested-paging",
dest="fHasNestedPaging", action="store_true", default=None,
help="Nested paging is available");
parser.add_option("--no-nested-paging",
dest="fHasNestedPaging", action="store_false", default=None,
help="Nested paging is not available");
parser.add_option("--64-bit-guest",
dest="fCan64BitGuest", action="store_true", default=None,
help="Host can execute 64-bit guests");
parser.add_option("--no-64-bit-guest",
dest="fCan64BitGuest", action="store_false", default=None,
help="Host cannot execute 64-bit guests");
parser.add_option("--io-mmu",
dest="fHasIoMmu", action="store_true", default=None,
help="I/O MMU available");
parser.add_option("--no-io-mmu",
dest="fHasIoMmu", action="store_false", default=None,
help="No I/O MMU available");
parser.add_option("--raw-mode",
dest="fWithRawMode", action="store_true", default=None,
help="Use raw-mode on this host.");
parser.add_option("--no-raw-mode",
dest="fWithRawMode", action="store_false", default=None,
help="Disables raw-mode tests on this host.");
parser.add_option("--pidfile",
dest="sPidFile", default=None,
help="For the parent script, ignored.");
parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
dest = "asEnvVars", default = [],
help = "Sets an environment variable. Can be repeated.");
(oOptions, args) = parser.parse_args()
# Check command line
if args != []:
parser.print_help();
return TBS_EXITCODE_SYNTAX;
if oOptions.sSystemUuid is not None:
uuid.UUID(oOptions.sSystemUuid);
if not oOptions.sTestManagerUrl.startswith('http://') \
and not oOptions.sTestManagerUrl.startswith('https://'):
print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
return TBS_EXITCODE_SYNTAX;
for sPrefix in ['sBuilds', 'sTestRsrc']:
sType = getattr(oOptions, sPrefix + 'ServerType');
if sType is None or not sType.strip():
setattr(oOptions, sPrefix + 'ServerType', None);
elif sType not in ['cifs', 'nfs']:
print('Syntax error: Invalid server type "%s"' % (sType,));
return TBS_EXITCODE_SYNTAX;
#
# Instantiate the testbox script and start dispatching work.
#
try:
oTestBoxScript = TestBoxScript(oOptions);
except TestBoxScriptException, oXcpt:
print('Error: %s' % (oXcpt,));
return TBS_EXITCODE_SYNTAX;
def _executeTestCase(self, sName, sFullPath, sTestCaseSubDir, oDevNull): # pylint: disable=R0914
"""
Executes a test case.
"""
fSkipped = False
#
# If hardening is enabled, some test cases and their dependencies
# needs to be copied to and execute from the sVBoxInstallRoot
# directory in order to work. They also have to be executed as
# root, i.e. via sudo.
#
fHardened = False
asFilesToRemove = []
# Stuff to clean up.
asDirsToRemove = []
# Ditto.
if sName in self.kasHardened \
and self.sUnitTestsPathBase != self.sVBoxInstallRoot:
sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir)
if not os.path.exists(sDstDir):
self._hardenedMkDir(sDstDir)
asDirsToRemove.append(sDstDir)
sDst = os.path.join(sDstDir, os.path.basename(sFullPath))
self._hardenedCopyFile(sFullPath, sDst, 0o755)
asFilesToRemove.append(sDst)
# Copy any associated .dll/.so/.dylib.
for sSuff in ['.dll', '.so', '.dylib']:
sSrc = os.path.splitext(sFullPath)[0] + sSuff
if os.path.exists(sSrc):
sDst = os.path.join(sDstDir, os.path.basename(sSrc))
self._hardenedCopyFile(sSrc, sDst, 0o644)
asFilesToRemove.append(sDst)
# Copy any associated .r0, .rc and .gc modules.
offDriver = sFullPath.rfind('Driver')
if offDriver > 0:
for sSuff in ['.r0', 'RC.rc', 'RC.gc']:
sSrc = sFullPath[:offDriver] + sSuff
if os.path.exists(sSrc):
sDst = os.path.join(sDstDir, os.path.basename(sSrc))
self._hardenedCopyFile(sSrc, sDst, 0o644)
asFilesToRemove.append(sDst)
sFullPath = os.path.join(sDstDir, os.path.basename(sFullPath))
fHardened = True
#
# Set up arguments and environment.
#
asArgs = [
sFullPath,
]
if sName in self.kdArguments:
asArgs.extend(self.kdArguments[sName])
os.environ['IPRT_TEST_OMIT_TOP_TEST'] = '1'
os.environ['IPRT_TEST_FILE'] = sXmlFile = os.path.join(
self.sScratchPath, 'result.xml')
if os.path.exists(sXmlFile):
try:
os.unlink(sXmlFile)
except:
self._hardenedDeleteFile(sXmlFile)
#
# Execute the test case.
#
# Windows is confusing output. Trying a few things to get rid of this.
# First, flush both stderr and stdout before running the child. Second,
# assign the child stderr to stdout. If this doesn't help, we'll have
# to capture the child output.
#
reporter.log('*** Executing %s%s...' %
(asArgs, ' [hardened]' if fHardened else ''))
try:
sys.stdout.flush()
except:
pass
try:
sys.stderr.flush()
except:
pass
if not self.fDryRun:
try:
if fHardened:
oChild = utils.sudoProcessPopen(asArgs,
stdin=oDevNull,
stdout=sys.stdout,
stderr=sys.stdout)
else:
oChild = utils.processPopenSafe(asArgs,
stdin=oDevNull,
stdout=sys.stdout,
stderr=sys.stdout)
except:
if sName in [
'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
]:
reporter.logXcpt()
fSkipped = True
else:
reporter.errorXcpt()
iRc = 1023
oChild = None
if oChild is not None:
self.pidFileAdd(oChild.pid, sName, fSudo=fHardened)
iRc = oChild.wait()
self.pidFileRemove(oChild.pid)
else:
iRc = 0
#
# Clean up
#
for sPath in asFilesToRemove:
self._hardenedDeleteFile(sPath)
for sPath in asDirsToRemove:
self._hardenedRemoveDir(sPath)
#
# Report.
#
if os.path.exists(sXmlFile):
reporter.addSubXmlFile(sXmlFile)
if fHardened:
self._hardenedDeleteFile(sXmlFile)
else:
os.unlink(sXmlFile)
if iRc == 0:
reporter.log('*** %s: exit code %d' % (sFullPath, iRc))
self.cPassed += 1
elif iRc == 4: # RTEXITCODE_SKIPPED
reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' %
(sFullPath, iRc))
fSkipped = True
self.cSkipped += 1
elif fSkipped:
reporter.log('*** %s: exit code %d (Skipped)' % (sFullPath, iRc))
self.cSkipped += 1
else:
sName = self.kdExitCodeNames.get(iRc, '')
if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
sName = self.kdExitCodeNamesWin[iRc]
if sName != '':
sName = ' (%s)' % (sName)
if iRc != 1:
reporter.testFailure('Exit status: %d%s' % (iRc, sName))
reporter.log('!*! %s: exit code %d%s' %
(sFullPath, iRc, sName))
else:
reporter.error('!*! %s: exit code %d%s' %
(sFullPath, iRc, sName))
self.cFailed += 1
return fSkipped
def _getHostSystemUuid(self):
"""
Get the system UUID string from the System, return null-uuid if
unable to get retrieve it.
"""
if self._oOptions.sSystemUuid is not None:
return self._oOptions.sSystemUuid;
sUuid = self.ksNullUuid;
#
# Try get at the firmware UUID.
#
if utils.getHostOs() == 'linux':
# NOTE: This requires to have kernel option enabled:
# Firmware Drivers -> Export DMI identification via sysfs to userspace
if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
try:
sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
sUuid = str(uuid.UUID(sVar.strip()));
except:
pass;
## @todo consider dmidecoder? What about EFI systems?
elif utils.getHostOs() == 'win':
# Windows: WMI
try:
import win32com.client; # pylint: disable=F0401
oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
if oItem.UUID != None:
sUuid = str(uuid.UUID(oItem.UUID));
except:
pass;
elif utils.getHostOs() == 'darwin':
try:
sVar = utils.processOutputChecked(['/bin/sh', '-c',
'/usr/sbin/ioreg -k IOPlatformUUID' \
+ '| /usr/bin/grep IOPlatformUUID' \
+ '| /usr/bin/head -1']);
sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
sUuid = str(uuid.UUID(sVar));
except:
pass;
elif utils.getHostOs() == 'solaris':
# Solaris: The smbios util.
try:
sVar = utils.processOutputChecked(['/bin/sh', '-c',
'/usr/sbin/smbios ' \
+ '| /usr/xpg4/bin/sed -ne \'s/^.*UUID: *//p\'' \
+ '| /usr/bin/head -1']);
sUuid = str(uuid.UUID(sVar.strip()));
except:
pass;
if self._isUuidGood(sUuid):
return sUuid;
#
# Try add the MAC address.
# uuid.getnode may provide it, or it may return a random number...
#
lMacAddr = uuid.getnode();
sNode = '%012x' % (lMacAddr,)
if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
return sUuid[:-12] + sNode;
return sUuid;
def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sWhat):
"""
Mounts the specified share if needed.
Raises exception on failure.
"""
# Only mount if the type is specified.
if sType is None:
return True;
# Test if already mounted.
sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '-new.txt');
if os.path.isfile(sTestFile):
return True;
#
# Platform specific mount code.
#
sHostOs = utils.getHostOs()
if sHostOs in ('darwin', 'freebsd'):
utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
if sType == 'cifs':
# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
'-f', '0555', '-d', '0555',
'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
sMountPoint]);
else:
raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
elif sHostOs == 'linux':
utils.sudoProcessCall(['/bin/umount', sMountPoint]);
utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
if sType == 'cifs':
utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
'-o',
'user='******',password='******',sec=ntlmv2'
+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
'//%s/%s' % (sServer, sShare),
sMountPoint]);
elif sType == 'nfs':
utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
'-o', 'soft,ro',
'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
sMountPoint]);
else:
raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
elif sHostOs == 'solaris':
utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
if sType == 'cifs':
## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
oPasswdFile = tempfile.TemporaryFile();
oPasswdFile.write(sPassword + '\n');
oPasswdFile.flush();
utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
'-o',
'user='******',uid=' + str(os.getuid()) # pylint: disable=E1101
+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
+ ',fileperms=0555,dirperms=0555,noxattr,ro',
'//%s/%s' % (sServer, sShare),
sMountPoint],
stdin = oPasswdFile);
oPasswdFile.close();
elif sType == 'nfs':
utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
'-o', 'noxattr,ro',
'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
sMountPoint]);
else:
raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
elif sHostOs == 'win':
if sType != 'cifs':
raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
utils.processCall(['net', 'use', sMountPoint, '/d']);
utils.processOutputChecked(['net', 'use', sMountPoint,
'\\\\' + sServer + '\\' + sShare,
sPassword,
'/USER:'******'Unsupported host %s' % (sHostOs,));
#
# Re-test.
#
if not os.path.isfile(sTestFile):
raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
% (sWhat, sServer, sShare, sMountPoint, sTestFile));
return True;
def _monitorChild(self, cSecTimeout, fTryKillCommand = True, oChild = None):
"""
Monitors the child process. If the child executes longer that
cSecTimeout allows, we'll terminate it.
Returns Success indicator and constants.result value.
"""
if oChild is None:
oChild = self._oChild;
iProcGroup = oChild.pid;
if utils.getHostOs() in ['win', 'os2'] or iProcGroup <= 0:
iProcGroup = -2;
#
# Do timeout processing and check the health of the child.
#
sResult = constants.result.PASSED;
seStarted = utils.timestampSecond();
while True:
# Check status.
iRc = oChild.poll();
if iRc is not None:
self._log('Child doing "%s" completed with exit code %d' % (oChild.sTestBoxScriptAction, iRc));
oChild.oOutputThread.join(self.kcSecFinalOutputTimeout);
if oChild is self._oChild:
self._oChild = None;
if iRc == constants.rtexitcode.SKIPPED:
return (True, constants.result.SKIPPED);
if iRc != constants.rtexitcode.SUCCESS:
return (False, constants.result.FAILED);
return (True, constants.result.PASSED);
# Check for abort first, since that has less of a stigma.
if self._shouldTerminate() is True:
sResult = constants.result.ABORTED;
break;
# Check timeout.
cSecElapsed = utils.timestampSecond() - seStarted;
if cSecElapsed > cSecTimeout:
self._log('Timeout: %u secs (limit %u secs)' % (cSecElapsed, cSecTimeout));
sResult = constants.result.TIMED_OUT;
break;
# Wait.
cSecLeft = cSecTimeout - cSecElapsed;
oChild.oOutputThread.join(15 if cSecLeft > 15 else (cSecLeft + 1));
#
# If the child is still alive, try use the abort command to stop it
# very gently. This let's the testdriver clean up daemon processes
# and such that our code below won't catch.
#
if fTryKillCommand and oChild.poll() is None:
self._log('Attempting to abort child...');
(fRc2, oAbortChild) = self._spawnChild('abort');
if oAbortChild is not None and fRc2 is True:
self._monitorChild(self.kcSecAbortTimeout, False, oAbortChild);
oAbortChild = None;
#
# If the child is still alive, try the polite way.
#
if oChild.poll() is None:
self._log('Attempting to terminate child doing "%s"...' % (oChild.sTestBoxScriptAction,));
if iProcGroup > 0:
try:
os.killpg(iProcGroup, signal.SIGTERM); # pylint: disable=E1101
except Exception, oXcpt:
self._log('killpg() failed: %s' % (oXcpt,));
try:
self._oChild.terminate();
oChild.oOutputThread.join(self.kcSecTerminateOutputTimeout);
except Exception, oXcpt:
self._log('terminate() failed: %s' % (oXcpt,));
oFile.close();
except Exception, oXcpt:
self._log('Error truncating "%s": %s' % (sScriptCmdLine, oXcpt));
#
# Report the incident.
#
self._log('Seems we rebooted!');
self._log('script-cmdline="%s"' % (self._sScriptCmdLine));
self._log('result-id=%d' % (self._idResult));
self._log('testbox-id=%d' % (self.idTestBox));
self._log('testbox-name=%s' % (self.sTestBoxName));
self._logFlush();
# System specific info.
sOs = utils.getHostOs();
if sOs == 'darwin':
self._log('NVRAM Panic Info:\n%s\n' % (self.darwinGetPanicInfo(),));
self._logFlush();
## @todo Add some special command for reporting this situation so we get something
# useful in the event log.
#
# Do the cleaning up.
#
self._cleanupAfter();
self._reportDone(constants.result.REBOOTED);
return False;
def _executeTestCase(self, sName, sFullPath, sTestCaseSubDir, oDevNull): # pylint: disable=R0914
"""
Executes a test case.
"""
fSkipped = False;
#
# If hardening is enabled, some test cases and their dependencies
# needs to be copied to and execute from the sVBoxInstallRoot
# directory in order to work. They also have to be executed as
# root, i.e. via sudo.
#
fHardened = False;
asFilesToRemove = []; # Stuff to clean up.
asDirsToRemove = []; # Ditto.
if sName in self.kasHardened \
and self.sUnitTestsPathBase != self.sVBoxInstallRoot:
sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir);
if not os.path.exists(sDstDir):
self._hardenedMkDir(sDstDir);
asDirsToRemove.append(sDstDir);
sDst = os.path.join(sDstDir, os.path.basename(sFullPath));
self._hardenedCopyFile(sFullPath, sDst, 0755);
asFilesToRemove.append(sDst);
# Copy any associated .dll/.so/.dylib.
for sSuff in [ '.dll', '.so', '.dylib' ]:
sSrc = os.path.splitext(sFullPath)[0] + sSuff;
if os.path.exists(sSrc):
sDst = os.path.join(sDstDir, os.path.basename(sSrc));
self._hardenedCopyFile(sSrc, sDst, 0644);
asFilesToRemove.append(sDst);
# Copy any associated .r0, .rc and .gc modules.
offDriver = sFullPath.rfind('Driver')
if offDriver > 0:
for sSuff in [ '.r0', 'RC.rc', 'RC.gc' ]:
sSrc = sFullPath[:offDriver] + sSuff;
if os.path.exists(sSrc):
sDst = os.path.join(sDstDir, os.path.basename(sSrc));
self._hardenedCopyFile(sSrc, sDst, 0644);
asFilesToRemove.append(sDst);
sFullPath = os.path.join(sDstDir, os.path.basename(sFullPath));
fHardened = True;
#
# Set up arguments and environment.
#
asArgs = [sFullPath,]
if sName in self.kdArguments:
asArgs.extend(self.kdArguments[sName]);
os.environ['IPRT_TEST_OMIT_TOP_TEST'] = '1';
os.environ['IPRT_TEST_FILE'] = sXmlFile = os.path.join(self.sScratchPath, 'result.xml');
if os.path.exists(sXmlFile):
try: os.unlink(sXmlFile);
except: self._hardenedDeleteFile(sXmlFile);
#
# Execute the test case.
#
# Windows is confusing output. Trying a few things to get rid of this.
# First, flush both stderr and stdout before running the child. Second,
# assign the child stderr to stdout. If this doesn't help, we'll have
# to capture the child output.
#
reporter.log('*** Executing %s%s...' % (asArgs, ' [hardened]' if fHardened else ''));
try: sys.stdout.flush();
except: pass;
try: sys.stderr.flush();
except: pass;
if not self.fDryRun:
try:
if fHardened:
oChild = utils.sudoProcessPopen(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
else:
oChild = subprocess.Popen( asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
except:
if sName in [ 'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
]:
reporter.logXcpt();
fSkipped = True;
else:
reporter.errorXcpt();
iRc = 1023;
oChild = None;
if oChild is not None:
self.pidFileAdd(oChild.pid, fSudo = fHardened);
iRc = oChild.wait();
self.pidFileRemove(oChild.pid);
else:
iRc = 0;
#
# Clean up
#
for sPath in asFilesToRemove:
self._hardenedDeleteFile(sPath);
for sPath in asDirsToRemove:
self._hardenedRemoveDir(sPath);
#
# Report.
#
if os.path.exists(sXmlFile):
reporter.addSubXmlFile(sXmlFile);
if fHardened:
self._hardenedDeleteFile(sXmlFile);
else:
os.unlink(sXmlFile);
if iRc == 0:
reporter.log('*** %s: exit code %d' % (sFullPath, iRc));
self.cPassed += 1
elif iRc == 4: # RTEXITCODE_SKIPPED
reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' % (sFullPath, iRc));
fSkipped = True;
self.cSkipped += 1;
elif fSkipped:
reporter.log('*** %s: exit code %d (Skipped)' % (sFullPath, iRc));
self.cSkipped += 1;
else:
sName = self.kdExitCodeNames.get(iRc, '');
if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
sName = self.kdExitCodeNamesWin[iRc];
if sName != '':
sName = ' (%s)' % (sName);
if iRc != 1:
reporter.testFailure('Exit status: %d%s' % (iRc, sName));
reporter.log( '!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
else:
reporter.error('!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
self.cFailed += 1
return fSkipped;
def testRunUnitTestsSet(self, sTestCasePattern, sTestCaseSubDir):
"""
Run subset of the unit tests set.
"""
# Open /dev/null for use as stdin further down.
try:
oDevNull = open(os.path.devnull, 'w+')
except:
oDevNull = None
# Determin the host OS specific exclusion lists.
dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(
utils.getHostOs(), [])
dTestCasesBuggyForHostOs.update(
self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []))
#
# Process the file list and run everything looking like a testcase.
#
for sFilename in sorted(
os.listdir(
os.path.join(self.sUnitTestsPathBase, sTestCaseSubDir))):
# Separate base and suffix and morph the base into something we
# can use for reporting and array lookups.
sName, sSuffix = os.path.splitext(sFilename)
if sTestCaseSubDir != '.':
sName = sTestCaseSubDir + '/' + sName
# Basic exclusion.
if not re.match(sTestCasePattern, sFilename) \
or sSuffix in self.kasSuffixBlackList:
reporter.log('"%s" is not a test case.' % (sFilename, ))
continue
# Check if the testcase is black listed or buggy before executing it.
if self._isExcluded(sName, self.kdTestCasesBlackList):
# (No testStart/Done or accounting here!)
reporter.log('%s: SKIPPED (blacklisted)' % (sName, ))
elif self._isExcluded(sName, self.kdTestCasesBuggy):
reporter.testStart(sName)
reporter.log('%s: Skipping, buggy in general.' % (sName, ))
reporter.testDone(fSkipped=True)
self.cSkipped += 1
elif self._isExcluded(sName, dTestCasesBuggyForHostOs):
reporter.testStart(sName)
reporter.log('%s: Skipping, buggy on %s.' % (
sName,
utils.getHostOs(),
))
reporter.testDone(fSkipped=True)
self.cSkipped += 1
else:
sFullPath = os.path.normpath(
os.path.join(self.sUnitTestsPathBase,
os.path.join(sTestCaseSubDir, sFilename)))
reporter.testStart(sName)
try:
fSkipped = self._executeTestCase(sName, sFullPath,
sTestCaseSubDir, oDevNull)
except:
reporter.errorXcpt('!*!')
self.cFailed += 1
fSkipped = False
reporter.testDone(fSkipped)
def _getHostSystemUuid(self):
"""
Get the system UUID string from the System, return null-uuid if
unable to get retrieve it.
"""
if self._oOptions.sSystemUuid is not None:
return self._oOptions.sSystemUuid;
sUuid = self.ksNullUuid;
#
# Try get at the firmware UUID.
#
if utils.getHostOs() == 'linux':
# NOTE: This requires to have kernel option enabled:
# Firmware Drivers -> Export DMI identification via sysfs to userspace
if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
try:
sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
sUuid = str(uuid.UUID(sVar.strip()));
except:
pass;
## @todo consider dmidecoder? What about EFI systems?
elif utils.getHostOs() == 'win':
# Windows: WMI
try:
import win32com.client; # pylint: disable=F0401
oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
if oItem.UUID != None:
sUuid = str(uuid.UUID(oItem.UUID));
except:
pass;
elif utils.getHostOs() == 'darwin':
try:
sVar = utils.processOutputChecked(['/bin/sh', '-c',
'/usr/sbin/ioreg -k IOPlatformUUID' \
+ '| /usr/bin/grep IOPlatformUUID' \
+ '| /usr/bin/head -1']);
sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
sUuid = str(uuid.UUID(sVar));
except:
pass;
elif utils.getHostOs() == 'solaris':
# Solaris: The smbios util.
try:
sVar = utils.processOutputChecked(['/bin/sh', '-c',
'/usr/sbin/smbios ' \
+ '| /usr/xpg4/bin/sed -ne \'s/^.*UUID: *//p\'' \
+ '| /usr/bin/head -1']);
sUuid = str(uuid.UUID(sVar.strip()));
except:
pass;
if self._isUuidGood(sUuid):
return sUuid;
#
# Try add the MAC address.
# uuid.getnode may provide it, or it may return a random number...
#
lMacAddr = uuid.getnode();
sNode = '%012x' % (lMacAddr,)
if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
return sUuid[:-12] + sNode;
return sUuid;
def _detectPaths(self):
"""
Internal worker for actionVerify and actionExecute that detects paths.
This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
"""
#
# We need a VBox install (/ build) to test.
#
if False:
if not self.importVBoxApi():
reporter.error('Unabled to import the VBox Python API.')
return False
else:
self._detectBuild();
if self.oBuild is None:
reporter.error('Unabled to detect the VBox build.');
return False;
#
# Where are the files installed?
# Solaris requires special handling because of it's multi arch subdirs.
#
self.sVBoxInstallRoot = self.oBuild.sInstallPath
if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
sArchDir = utils.getHostArch();
if sArchDir == 'x86': sArchDir = 'i386';
self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot, sArchDir);
# Add the installation root to the PATH on windows so we can get DLLs from it.
if utils.getHostOs() == 'win':
sPathName = 'PATH';
if not sPathName in os.environ:
sPathName = 'Path';
sPath = os.environ.get(sPathName, '.');
if len(sPath) > 0 and sPath[-1] != ';':
sPath += ';';
os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';';
#
# The unittests are generally not installed, so look for them.
#
sBinOrDist = 'dist' if utils.getHostOs() in [ 'darwin', ] else 'bin';
asCandidates = [
self.oBuild.sInstallPath,
os.path.join(self.sScratchPath, sBinOrDist + '.' + utils.getHostArch()),
os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
os.path.join(self.sScratchPath, sBinOrDist),
];
if utils.getHostOs() == 'darwin':
for i in range(1, len(asCandidates)):
asCandidates[i] = os.path.join(asCandidates[i], 'VirtualBox.app', 'Contents', 'MacOS');
for sCandidat in asCandidates:
if os.path.exists(os.path.join(sCandidat, 'testcase', 'tstVMStructSize' + self.sExeSuff)):
self.sUnitTestsPathBase = sCandidat;
return True;
reporter.error('Unable to find unit test dir. Candidates: %s' % (asCandidates,))
return False;
def __init__(self, oOptions):
"""
Initialize internals
"""
self._oOptions = oOptions;
self._sTestBoxHelper = None;
# Signed-on state
self._cSignOnAttempts = 0;
self._fSignedOn = False;
self._fNeedReSignOn = False;
self._fFirstSignOn = True;
self._idTestBox = None;
self._sTestBoxName = '';
self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
# Command processor.
self._oCommand = TestBoxCommand(self);
#
# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
# many many GBs for some test scenarios and /tmp can be backed by swap
# or be a fast+small disk of some kind, while /var/tmp is normally
# larger, if slower. /var/tmp is generally not cleaned up on reboot,
# /tmp often is, this would break host panic / triple-fault detection.
#
if self._oOptions.sScratchRoot is None:
if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
# We need *lots* of space, so avoid /tmp as it may be a memory
# file system backed by the swap file, or worse.
self._oOptions.sScratchRoot = tempfile.gettempdir();
else:
self._oOptions.sScratchRoot = '/var/tmp';
sSubDir = 'testbox';
try:
sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
except:
pass;
self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
if not os.path.isdir(sDir):
os.makedirs(sDir, 0700);
# We count consecutive reinitScratch failures and will reboot the
# testbox after a while in the hope that it will correct the issue.
self._cReinitScratchErrors = 0;
#
# Mount builds and test resources if requested.
#
self.mountShares();
#
# Sign-on parameters: Packed into list of records of format:
# { <Parameter ID>: { <Current value>, <Check function> } }
#
self._ddSignOnParams = \
{
constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
}
for sItem in self._ddSignOnParams:
if self._ddSignOnParams[sItem][self.FN] is not None:
self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
testboxcommons.log('Starting Test Box script (%s)' % __version__)
testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
for sItem in self._ddSignOnParams:
testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
#
# The System UUID is the primary identification of the machine, so
# refuse to cooperate if it's NULL.
#
self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
if self._sTestBoxUuid == self.ksNullUuid:
raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
#
# Export environment variables, clearing any we don't know yet.
#
for sEnvVar in self._oOptions.asEnvVars:
iEqual = sEnvVar.find('=');
if iEqual == -1: # No '=', remove it.
if sEnvVar in os.environ:
del os.environ[sEnvVar];
elif iEqual > 0: # Set it.
os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
else: # Starts with '=', bad user.
raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
os.environ['TESTBOX_WITH_RAW_MODE'] = self._withRawModeSupport();
os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
os.environ['TESTBOX_REPORTER'] = 'remote';
os.environ['TESTBOX_NAME'] = '';
os.environ['TESTBOX_ID'] = '';
os.environ['TESTBOX_TEST_SET_ID'] = '';
os.environ['TESTBOX_TIMEOUT'] = '0';
os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
if utils.getHostOs() is 'win':
os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
def _detectPaths(self):
"""
Internal worker for actionVerify and actionExecute that detects paths.
This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
"""
#
# We need a VBox install (/ build) to test.
#
if False is True:
if not self.importVBoxApi():
reporter.error('Unabled to import the VBox Python API.')
return False
else:
self._detectBuild()
if self.oBuild is None:
reporter.error('Unabled to detect the VBox build.')
return False
#
# Where are the files installed?
# Solaris requires special handling because of it's multi arch subdirs.
#
self.sVBoxInstallRoot = self.oBuild.sInstallPath
if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
sArchDir = utils.getHostArch()
if sArchDir == 'x86': sArchDir = 'i386'
self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot,
sArchDir)
# Add the installation root to the PATH on windows so we can get DLLs from it.
if utils.getHostOs() == 'win':
sPathName = 'PATH'
if not sPathName in os.environ:
sPathName = 'Path'
sPath = os.environ.get(sPathName, '.')
if sPath and sPath[-1] != ';':
sPath += ';'
os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';'
#
# The unittests are generally not installed, so look for them.
#
sBinOrDist = 'dist' if utils.getHostOs() in [
'darwin',
] else 'bin'
asCandidates = [
self.oBuild.sInstallPath,
os.path.join(self.sScratchPath, utils.getHostOsDotArch(),
self.oBuild.sType, sBinOrDist),
os.path.join(self.sScratchPath, utils.getHostOsDotArch(),
'release', sBinOrDist),
os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'debug',
sBinOrDist),
os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'strict',
sBinOrDist),
os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'dbgopt',
sBinOrDist),
os.path.join(self.sScratchPath, utils.getHostOsDotArch(),
'profile', sBinOrDist),
os.path.join(self.sScratchPath,
sBinOrDist + '.' + utils.getHostArch()),
os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
os.path.join(self.sScratchPath, sBinOrDist),
]
if utils.getHostOs() == 'darwin':
for i in range(1, len(asCandidates)):
asCandidates[i] = os.path.join(asCandidates[i],
'VirtualBox.app', 'Contents',
'MacOS')
for sCandidat in asCandidates:
if os.path.exists(
os.path.join(sCandidat, 'testcase',
'tstVMStructSize' + self.sExeSuff)):
self.sUnitTestsPathBase = sCandidat
return True
reporter.error('Unable to find unit test dir. Candidates: %s' %
(asCandidates, ))
return False
