def main(self):
""" Main method. """
oDb = TMDatabaseConnection();
# Get a list of orphans.
oLogic = TestSetLogic(oDb);
aoOrphans = oLogic.fetchOrphaned();
if len(aoOrphans) > 0:
# Complete them.
if self.oConfig.fJustDoIt:
print 'Completing %u test sets as abandond:' % (len(aoOrphans),);
for oTestSet in aoOrphans:
print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
% (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone);
oLogic.completeAsAbandond(oTestSet.idTestSet);
print 'Committing...';
oDb.commit();
else:
for oTestSet in aoOrphans:
print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
% (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone);
print 'Not completing any testsets without seeing the --just-do-it option.'
else:
print 'No orphaned test sets.\n'
return 0;
def main(self):
""" Main method. """
oDb = TMDatabaseConnection()
# Get a list of orphans.
oLogic = TestSetLogic(oDb)
aoOrphans = oLogic.fetchOrphaned()
if len(aoOrphans) > 0:
# Complete them.
if self.oConfig.fJustDoIt:
print 'Completing %u test sets as abandoned:' % (
len(aoOrphans), )
for oTestSet in aoOrphans:
print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
% (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)
oLogic.completeAsAbandoned(oTestSet.idTestSet)
print 'Committing...'
oDb.commit()
else:
for oTestSet in aoOrphans:
print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
% (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)
print 'Not completing any testsets without seeing the --just-do-it option.'
else:
print 'No orphaned test sets.\n'
return 0
def _actionExecCompleted(self):
"""
Implement EXEC completion.
Because the action is request by the worker thread of the testbox
script we cannot pass pending commands back to it like originally
planned. So, we just complete the test set and update the status.
"""
#
# Parameter validation.
#
sStatus = self._getStringParam(
constants.tbreq.EXEC_COMPLETED_PARAM_RESULT,
TestBoxController.kasValidResults)
self._checkForUnknownParameters()
(oDb, oStatusData, _) = self._connectToDbAndValidateTb([
TestBoxStatusData.ksTestBoxState_Testing,
TestBoxStatusData.ksTestBoxState_GangTesting
])
if oStatusData is None:
return False
#
# Complete the status.
#
oDb.rollback()
oDb.begin()
oTestSetLogic = TestSetLogic(oDb)
idTestSetGangLeader = oTestSetLogic.complete(
oStatusData.idTestSet,
self.kadTbResultToStatus[sStatus],
fCommit=False)
oStatusLogic = TestBoxStatusLogic(oDb)
if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Testing:
assert idTestSetGangLeader is None
GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(
self._idTestBox)
oStatusLogic.updateState(self._idTestBox,
TestBoxStatusData.ksTestBoxState_Idle,
fCommit=False)
else:
assert idTestSetGangLeader is not None
oStatusLogic.updateState(
self._idTestBox,
TestBoxStatusData.ksTestBoxState_GangCleanup,
oStatusData.idTestSet,
fCommit=False)
if oStatusLogic.isWholeGangDoneTesting(idTestSetGangLeader):
GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(
self._idTestBox)
oStatusLogic.updateState(self._idTestBox,
TestBoxStatusData.ksTestBoxState_Idle,
fCommit=False)
oDb.commit()
return self._resultResponse(constants.tbresp.STATUS_ACK)
def __init__(self, oOptions):
"""
Parse command line
"""
self.fVerbose = oOptions.fVerbose
self.sSrcDir = config.g_ksFileAreaRootDir
self.sDstDir = config.g_ksZipFileAreaRootDir
#self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None))
self.fDryRun = oOptions.fDryRun
def _cleanupOldTest(self, oDb, oStatusData):
"""
Cleans up any old test set that may be left behind and changes the
state to 'idle'. See scenario #9:
file://../../docs/AutomaticTestingRevamp.html#cleaning-up-abandoned-testcase
Note. oStatusData.enmState is set to idle, but tsUpdated is not changed.
"""
# Cleanup any abandoned test.
if oStatusData.idTestSet is not None:
SystemLogLogic(oDb).addEntry(
SystemLogData.ksEvent_TestSetAbandoned,
"idTestSet=%u idTestBox=%u enmState=%s %s" %
(oStatusData.idTestSet, oStatusData.idTestBox,
oStatusData.enmState, self._sAction),
fCommit=False)
TestSetLogic(oDb).completeAsAbandoned(oStatusData.idTestSet,
fCommit=False)
GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(
self._idTestBox, fCommit=False)
# Change to idle status
if oStatusData.enmState != TestBoxStatusData.ksTestBoxState_Idle:
TestBoxStatusLogic(oDb).updateState(
self._idTestBox,
TestBoxStatusData.ksTestBoxState_Idle,
fCommit=False)
oStatusData.tsUpdated = oDb.getCurrentTimestamp()
oStatusData.enmState = TestBoxStatusData.ksTestBoxState_Idle
# Commit.
oDb.commit()
return True
def __init__(self, oOptions):
"""
Parse command line
"""
self.fVerbose = oOptions.fVerbose;
self.sSrcDir = config.g_ksFileAreaRootDir;
self.sDstDir = config.g_ksZipFileAreaRootDir;
#self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));
self.fDryRun = oOptions.fDryRun;
def _actionExecCompleted(self):
"""
Implement EXEC completion.
Because the action is request by the worker thread of the testbox
script we cannot pass pending commands back to it like originally
planned. So, we just complete the test set and update the status.
"""
#
# Parameter validation.
#
sStatus = self._getStringParam(constants.tbreq.EXEC_COMPLETED_PARAM_RESULT, TestBoxController.kasValidResults);
self._checkForUnknownParameters();
(oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing,
TestBoxStatusData.ksTestBoxState_GangTesting]);
if oStatusData is None:
return False;
#
# Complete the status.
#
oDb.rollback();
oDb.begin();
oTestSetLogic = TestSetLogic(oDb);
idTestSetGangLeader = oTestSetLogic.complete(oStatusData.idTestSet, self.kadTbResultToStatus[sStatus], fCommit = False);
oStatusLogic = TestBoxStatusLogic(oDb);
if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Testing:
assert idTestSetGangLeader is None;
GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox);
oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False);
else:
assert idTestSetGangLeader is not None;
oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_GangCleanup, oStatusData.idTestSet,
fCommit = False);
if oStatusLogic.isWholeGangDoneTesting(idTestSetGangLeader):
GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox);
oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False);
oDb.commit();
return self._resultResponse(constants.tbresp.STATUS_ACK);
def _doGangGatheringTimedOut(self, oDb, oStatusData):
"""
_doRequestCommand worker for handling a box in gang-gathering-timed-out state.
This will do clean-ups similar to _cleanupOldTest and update the state likewise.
"""
oDb.begin();
TestSetLogic(oDb).completeAsGangGatheringTimeout(oStatusData.idTestSet, fCommit = False);
GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox, fCommit = False);
TestBoxStatusLogic(oDb).updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False);
oStatusData.tsUpdated = oDb.getCurrentTimestamp();
oStatusData.enmState = TestBoxStatusData.ksTestBoxState_Idle;
oDb.commit();
return None;
def _actionUpload(self):
""" Implement uploading of files. """
#
# Parameter validation.
#
sName = self._getStringParam(constants.tbreq.UPLOAD_PARAM_NAME)
sMime = self._getStringParam(constants.tbreq.UPLOAD_PARAM_MIME)
sKind = self._getStringParam(constants.tbreq.UPLOAD_PARAM_KIND)
sDesc = self._getStringParam(constants.tbreq.UPLOAD_PARAM_DESC)
self._checkForUnknownParameters()
(oDb, oStatusData, _) = self._connectToDbAndValidateTb([
TestBoxStatusData.ksTestBoxState_Testing,
TestBoxStatusData.ksTestBoxState_GangTesting
])
if oStatusData is None:
return False
if len(sName) > 128 or len(sName) < 3:
raise TestBoxControllerException('Invalid file name "%s"' %
(sName, ))
if re.match(r'^[a-zA-Z0-9_\-(){}#@+,.=]*$', sName) is None:
raise TestBoxControllerException('Invalid file name "%s"' %
(sName, ))
if sMime not in [
'text/plain', #'text/html', 'text/xml',
'application/octet-stream',
'image/png', #'image/gif', 'image/jpeg',
#'video/webm', 'video/mpeg', 'video/mpeg4-generic',
]:
raise TestBoxControllerException('Invalid MIME type "%s"' %
(sMime, ))
if sKind not in TestResultFileData.kasKinds:
raise TestBoxControllerException('Invalid kind "%s"' % (sKind, ))
if len(sDesc) > 256:
raise TestBoxControllerException('Invalid description "%s"' %
(sDesc, ))
if not set(sDesc).issubset(set(string.printable)):
raise TestBoxControllerException('Invalid description "%s"' %
(sDesc, ))
if ('application/octet-stream', {}) != self._oSrvGlue.getContentType():
raise TestBoxControllerException(
'Unexpected content type: %s; %s' %
self._oSrvGlue.getContentType())
cbFile = self._oSrvGlue.getContentLength()
if cbFile <= 0:
raise TestBoxControllerException(
'File "%s" is empty or negative in size (%s)' %
(sName, cbFile))
if (cbFile + 1048575) / 1048576 > config.g_kcMbMaxUploadSingle:
raise TestBoxControllerException(
'File "%s" is too big %u bytes (max %u MiB)' % (
sName,
cbFile,
config.g_kcMbMaxUploadSingle,
))
#
# Write the text to the log file.
#
oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet)
oDstFile = TestSetLogic(oDb).createFile(oTestSet,
sName=sName,
sMime=sMime,
sKind=sKind,
sDesc=sDesc,
cbFile=cbFile,
fCommit=True)
offFile = 0
oSrcFile = self._oSrvGlue.getBodyIoStream()
while offFile < cbFile:
cbToRead = cbFile - offFile
if cbToRead > 256 * 1024:
cbToRead = 256 * 1024
offFile += cbToRead
abBuf = oSrcFile.read(cbToRead)
oDstFile.write(abBuf)
# pylint: disable=E1103
del abBuf
oDstFile.close()
# pylint: disable=E1103
# Done.
return self._resultResponse(constants.tbresp.STATUS_ACK)
def _doRequestCommand(self, fIdle):
"""
Common code for handling command request.
"""
(oDb, oStatusData, oTestBoxData) = self._connectToDbAndValidateTb()
if oDb is None:
return False
#
# Status clean up.
#
# Only when BUSY will the TestBox Script request and execute commands
# concurrently. So, it must be idle when sending REQUEST_COMMAND_IDLE.
#
if fIdle:
if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGathering:
self._doGangGathering(oDb, oStatusData)
elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut:
self._doGangGatheringTimedOut(oDb, oStatusData)
elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangTesting:
dResponse = SchedulerBase.composeExecResponse(
oDb, oTestBoxData.idTestBox, self._oSrvGlue.getBaseUrl())
if dResponse is not None:
return dResponse
elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangCleanup:
self._doGangCleanup(oDb, oStatusData)
elif oStatusData.enmState != TestBoxStatusData.ksTestBoxState_Idle: # (includes ksTestBoxState_GangGatheringTimedOut)
self._cleanupOldTest(oDb, oStatusData)
#
# Check for pending command.
#
if oTestBoxData.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
asValidCmds = TestBoxController.kasIdleCmds if fIdle else TestBoxController.kasBusyCmds
if oTestBoxData.enmPendingCmd in asValidCmds:
dResponse = {
constants.tbresp.ALL_PARAM_RESULT:
TestBoxController.kdCmdToTbRespCmd[
oTestBoxData.enmPendingCmd]
}
if oTestBoxData.enmPendingCmd in [
TestBoxData.ksTestBoxCmd_Upgrade,
TestBoxData.ksTestBoxCmd_UpgradeAndReboot
]:
dResponse[constants.tbresp.
UPGRADE_PARAM_URL] = self._oSrvGlue.getBaseUrl(
) + TestBoxController.ksUpgradeZip
return self._writeResponse(dResponse)
if oTestBoxData.enmPendingCmd == TestBoxData.ksTestBoxCmd_Abort and fIdle:
TestBoxLogic(oDb).setCommand(
self._idTestBox,
sOldCommand=oTestBoxData.enmPendingCmd,
sNewCommand=TestBoxData.ksTestBoxCmd_None,
fCommit=True)
#
# If doing gang stuff, return 'CMD_WAIT'.
#
## @todo r=bird: Why is GangTesting included here? Figure out when testing gang testing.
if oStatusData.enmState in [
TestBoxStatusData.ksTestBoxState_GangGathering,
TestBoxStatusData.ksTestBoxState_GangTesting,
TestBoxStatusData.ksTestBoxState_GangCleanup
]:
return self._resultResponse(constants.tbresp.CMD_WAIT)
#
# If idling and enabled try schedule a new task.
#
if fIdle \
and oTestBoxData.fEnabled \
and not TestSetLogic(oDb).isTestBoxExecutingToRapidly(oTestBoxData.idTestBox) \
and oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Idle: # (paranoia)
dResponse = SchedulerBase.scheduleNewTask(
oDb, oTestBoxData, oStatusData.iWorkItem,
self._oSrvGlue.getBaseUrl())
if dResponse is not None:
return self._writeResponse(dResponse)
#
# Touch the status row every couple of mins so we can tell that the box is alive.
#
oStatusLogic = TestBoxStatusLogic(oDb)
if oStatusData.enmState != TestBoxStatusData.ksTestBoxState_GangGathering \
and oStatusLogic.timeSinceLastChangeInSecs(oStatusData) >= TestBoxStatusLogic.kcSecIdleTouchStatus:
oStatusLogic.touchStatus(oTestBoxData.idTestBox, fCommit=True)
return self._idleResponse()
class FileArchiverBatchJob(object): # pylint: disable=too-few-public-methods
"""
Log+files comp
"""
def __init__(self, oOptions):
"""
Parse command line
"""
self.fVerbose = oOptions.fVerbose
self.sSrcDir = config.g_ksFileAreaRootDir
self.sDstDir = config.g_ksZipFileAreaRootDir
#self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None))
self.fDryRun = oOptions.fDryRun
def dprint(self, sText):
""" Verbose output. """
if self.fVerbose:
print(sText)
return True
def warning(self, sText):
"""Prints a warning."""
print(sText)
return True
def _processTestSet(self, idTestSet, asFiles, sCurDir):
"""
Worker for processDir.
Same return codes as processDir.
"""
sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet, ))
if sBaseFilename[0:2] == ('.' + os.path.sep):
sBaseFilename = sBaseFilename[2:]
sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-')
#
# Skip the file if the test set is still running.
# But delete them if the testset is not found.
#
oTestSet = self.oTestSetLogic.tryFetch(idTestSet)
if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename:
self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \
% (idTestSet, sBaseFilename, oTestSet.sBaseFilename,))
oTestSet = None
if oTestSet is not None:
if oTestSet.enmStatus == TestSetData.ksTestStatus_Running:
self.dprint('Skipping test set #%d, still running' %
(idTestSet, ))
return True
#
# If we have a zip file already, don't try recreate it as we might
# have had trouble removing the source files.
#
sDstDirPath = os.path.join(self.sDstDir, sCurDir)
sZipFileNm = os.path.join(sDstDirPath,
'TestSet-%d.zip' % (idTestSet, ))
if not os.path.exists(sZipFileNm):
#
# Create zip file with all testset files as members.
#
self.dprint('TestSet %d: Creating %s...' % (
idTestSet,
sZipFileNm,
))
if not self.fDryRun:
if not os.path.exists(sDstDirPath):
os.makedirs(sDstDirPath, 0o755)
utils.noxcptDeleteFile(sZipFileNm + '.tmp')
oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp',
'w',
zipfile.ZIP_DEFLATED,
allowZip64=True)
for sFile in asFiles:
sSuff = os.path.splitext(sFile)[1]
if sSuff in [
'.png', '.webm', '.gz', '.bz2', '.zip', '.mov',
'.avi', '.mpg', '.gif', '.jpg'
]:
## @todo Consider storing these files outside the zip if they are a little largish.
self.dprint('TestSet %d: Storing %s...' %
(idTestSet, sFile))
oZipFile.write(sSrcFileBase + sFile, sFile,
zipfile.ZIP_STORED)
else:
self.dprint('TestSet %d: Deflating %s...' %
(idTestSet, sFile))
oZipFile.write(sSrcFileBase + sFile, sFile,
zipfile.ZIP_DEFLATED)
oZipFile.close()
#
# .zip.tmp -> .zip.
#
utils.noxcptDeleteFile(sZipFileNm)
os.rename(sZipFileNm + '.tmp', sZipFileNm)
#else: Dry run.
else:
self.dprint('TestSet %d: zip file exists already (%s)' % (
idTestSet,
sZipFileNm,
))
#
# Delete the files.
#
fRc = True
if self.fVerbose:
self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles))
if not self.fDryRun:
for sFile in asFiles:
if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False:
self.warning('TestSet %d: Failed to delete "%s" (%s)' % (
idTestSet,
sFile,
sSrcFileBase + sFile,
))
fRc = False
return fRc
def processDir(self, sCurDir):
"""
Process the given directory (relative to sSrcDir and sDstDir).
Returns success indicator.
"""
if self.fVerbose:
self.dprint('processDir: %s' % (sCurDir, ))
#
# Sift thought the directory content, collecting subdirectories and
# sort relevant files by test set.
# Generally there will either be subdirs or there will be files.
#
asSubDirs = []
dTestSets = {}
sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir))
for sFile in os.listdir(sCurPath):
if os.path.isdir(os.path.join(sCurPath, sFile)):
if sFile not in ['.', '..']:
asSubDirs.append(sFile)
elif sFile.startswith('TestSet-'):
# Parse the file name. ASSUMES 'TestSet-%d-filename' format.
iSlash1 = sFile.find('-')
iSlash2 = sFile.find('-', iSlash1 + 1)
if iSlash2 <= iSlash1:
self.warning('Bad filename (1): "%s"' % (sFile, ))
continue
try:
idTestSet = int(sFile[(iSlash1 + 1):iSlash2])
except:
self.warning('Bad filename (2): "%s"' % (sFile, ))
if self.fVerbose:
self.dprint('\n'.join(utils.getXcptInfo(4)))
continue
if idTestSet <= 0:
self.warning('Bad filename (3): "%s"' % (sFile, ))
continue
if iSlash2 + 2 >= len(sFile):
self.warning('Bad filename (4): "%s"' % (sFile, ))
continue
sName = sFile[(iSlash2 + 1):]
# Add it.
if idTestSet not in dTestSets:
dTestSets[idTestSet] = []
asTestSet = dTestSets[idTestSet]
asTestSet.append(sName)
#
# Test sets.
#
fRc = True
for idTestSet in dTestSets:
try:
if self._processTestSet(idTestSet, dTestSets[idTestSet],
sCurDir) is not True:
fRc = False
except:
self.warning('TestSet %d: Exception in _processTestSet:\n%s' %
(
idTestSet,
'\n'.join(utils.getXcptInfo()),
))
fRc = False
#
# Sub dirs.
#
for sSubDir in asSubDirs:
if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:
fRc = False
#
# Try Remove the directory iff it's not '.' and it's been unmodified
# for the last 24h (race protection).
#
if sCurDir != '.':
try:
fpModTime = float(os.path.getmtime(sCurPath))
if fpModTime + (24 * 3600) <= time.time():
if utils.noxcptRmDir(sCurPath) is True:
self.dprint('Removed "%s".' % (sCurPath, ))
except:
pass
return fRc
@staticmethod
def main():
""" C-style main(). """
#
# Parse options.
#
oParser = OptionParser()
oParser.add_option('-v',
'--verbose',
dest='fVerbose',
action='store_true',
default=False,
help='Verbose output.')
oParser.add_option('-q',
'--quiet',
dest='fVerbose',
action='store_false',
default=False,
help='Quiet operation.')
oParser.add_option('-d',
'--dry-run',
dest='fDryRun',
action='store_true',
default=False,
help='Dry run, do not make any changes.')
(oOptions, asArgs) = oParser.parse_args()
if asArgs != []:
oParser.print_help()
return 1
#
# Do the work.
#
oBatchJob = FileArchiverBatchJob(oOptions)
fRc = oBatchJob.processDir('.')
return 0 if fRc is True else 1
def _actionUpload(self):
""" Implement uploading of files. """
#
# Parameter validation.
#
sName = self._getStringParam(constants.tbreq.UPLOAD_PARAM_NAME);
sMime = self._getStringParam(constants.tbreq.UPLOAD_PARAM_MIME);
sKind = self._getStringParam(constants.tbreq.UPLOAD_PARAM_KIND);
sDesc = self._getStringParam(constants.tbreq.UPLOAD_PARAM_DESC);
self._checkForUnknownParameters();
(oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing,
TestBoxStatusData.ksTestBoxState_GangTesting]);
if oStatusData is None:
return False;
if len(sName) > 128 or len(sName) < 3:
raise TestBoxControllerException('Invalid file name "%s"' % (sName,));
if re.match(r'^[a-zA-Z0-9_\-(){}#@+,.=]*$', sName) is None:
raise TestBoxControllerException('Invalid file name "%s"' % (sName,));
if sMime not in [ 'text/plain', #'text/html', 'text/xml',
'application/octet-stream',
'image/png', #'image/gif', 'image/jpeg',
#'video/webm', 'video/mpeg', 'video/mpeg4-generic',
]:
raise TestBoxControllerException('Invalid MIME type "%s"' % (sMime,));
if sKind not in [ 'log/release/vm',
'log/debug/vm',
'log/release/svc',
'log/debug/svc',
'log/release/client',
'log/debug/client',
'log/installer',
'log/uninstaller',
'crash/report/vm',
'crash/dump/vm',
'crash/report/svc',
'crash/dump/svc',
'crash/report/client',
'crash/dump/client',
'misc/other',
'screenshot/failure',
'screenshot/success',
#'screencapture/failure',
]:
raise TestBoxControllerException('Invalid kind "%s"' % (sKind,));
if len(sDesc) > 256:
raise TestBoxControllerException('Invalid description "%s"' % (sDesc,));
if not set(sDesc).issubset(set(string.printable)):
raise TestBoxControllerException('Invalid description "%s"' % (sDesc,));
if ('application/octet-stream', {}) != self._oSrvGlue.getContentType():
raise TestBoxControllerException('Unexpected content type: %s; %s' % self._oSrvGlue.getContentType());
cbFile = self._oSrvGlue.getContentLength();
if cbFile <= 0:
raise TestBoxControllerException('File "%s" is empty or negative in size (%s)' % (sName, cbFile));
if (cbFile + 1048575) / 1048576 > config.g_kcMbMaxUploadSingle:
raise TestBoxControllerException('File "%s" is too big %u bytes (max %u MiB)'
% (sName, cbFile, config.g_kcMbMaxUploadSingle,));
#
# Write the text to the log file.
#
oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet);
oDstFile = TestSetLogic(oDb).createFile(oTestSet, sName = sName, sMime = sMime, sKind = sKind, sDesc = sDesc,
cbFile = cbFile, fCommit = True);
offFile = 0;
oSrcFile = self._oSrvGlue.getBodyIoStream();
while offFile < cbFile:
cbToRead = cbFile - offFile;
if cbToRead > 256*1024:
cbToRead = 256*1024;
offFile += cbToRead;
abBuf = oSrcFile.read(cbToRead);
oDstFile.write(abBuf); # pylint: disable=E1103
del abBuf;
oDstFile.close(); # pylint: disable=E1103
# Done.
return self._resultResponse(constants.tbresp.STATUS_ACK);
class FileArchiverBatchJob(object): # pylint: disable=R0903
"""
Log+files comp
"""
def __init__(self, oOptions):
"""
Parse command line
"""
self.fVerbose = oOptions.fVerbose;
self.sSrcDir = config.g_ksFileAreaRootDir;
self.sDstDir = config.g_ksZipFileAreaRootDir;
#self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));
self.fDryRun = oOptions.fDryRun;
def dprint(self, sText):
""" Verbose output. """
if self.fVerbose:
print(sText);
return True;
def warning(self, sText):
"""Prints a warning."""
print(sText);
return True;
def _processTestSet(self, idTestSet, asFiles, sCurDir):
"""
Worker for processDir.
Same return codes as processDir.
"""
sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,));
if sBaseFilename[0:2] == ('.' + os.path.sep):
sBaseFilename = sBaseFilename[2:];
sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-');
#
# Skip the file if the test set is still running.
# But delete them if the testset is not found.
#
oTestSet = self.oTestSetLogic.tryFetch(idTestSet);
if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename:
self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \
% (idTestSet, sBaseFilename, oTestSet.sBaseFilename,));
oTestSet = None;
if oTestSet is not None:
if oTestSet.enmStatus == TestSetData.ksTestStatus_Running:
self.dprint('Skipping test set #%d, still running' % (idTestSet,));
return True;
#
# If we have a zip file already, don't try recreate it as we might
# have had trouble removing the source files.
#
sDstDirPath = os.path.join(self.sDstDir, sCurDir);
sZipFileNm = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,));
if not os.path.exists(sZipFileNm):
#
# Create zip file with all testset files as members.
#
self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,));
if not self.fDryRun:
if not os.path.exists(sDstDirPath):
os.makedirs(sDstDirPath, 0o755);
utils.noxcptDeleteFile(sZipFileNm + '.tmp');
oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True);
for sFile in asFiles:
sSuff = os.path.splitext(sFile)[1];
if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]:
## @todo Consider storing these files outside the zip if they are a little largish.
self.dprint('TestSet %d: Storing %s...' % (idTestSet, sFile));
oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED);
else:
self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile));
oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED);
oZipFile.close();
#
# .zip.tmp -> .zip.
#
utils.noxcptDeleteFile(sZipFileNm);
os.rename(sZipFileNm + '.tmp', sZipFileNm);
#else: Dry run.
else:
self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,));
#
# Delete the files.
#
fRc = True;
if self.fVerbose:
self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles));
if not self.fDryRun:
for sFile in asFiles:
if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False:
self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,));
fRc = False;
return fRc;
def processDir(self, sCurDir):
"""
Process the given directory (relative to sSrcDir and sDstDir).
Returns success indicator.
"""
if self.fVerbose:
self.dprint('processDir: %s' % (sCurDir,));
#
# Sift thought the directory content, collecting subdirectories and
# sort relevant files by test set.
# Generally there will either be subdirs or there will be files.
#
asSubDirs = [];
dTestSets = {};
sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir));
for sFile in os.listdir(sCurPath):
if os.path.isdir(os.path.join(sCurPath, sFile)):
if sFile not in [ '.', '..' ]:
asSubDirs.append(sFile);
elif sFile.startswith('TestSet-'):
# Parse the file name. ASSUMES 'TestSet-%d-filename' format.
iSlash1 = sFile.find('-');
iSlash2 = sFile.find('-', iSlash1 + 1);
if iSlash2 <= iSlash1:
self.warning('Bad filename (1): "%s"' % (sFile,));
continue;
try: idTestSet = int(sFile[(iSlash1 + 1):iSlash2]);
except:
self.warning('Bad filename (2): "%s"' % (sFile,));
if self.fVerbose:
self.dprint('\n'.join(utils.getXcptInfo(4)));
continue;
if idTestSet <= 0:
self.warning('Bad filename (3): "%s"' % (sFile,));
continue;
if iSlash2 + 2 >= len(sFile):
self.warning('Bad filename (4): "%s"' % (sFile,));
continue;
sName = sFile[(iSlash2 + 1):];
# Add it.
if idTestSet not in dTestSets:
dTestSets[idTestSet] = [];
asTestSet = dTestSets[idTestSet];
asTestSet.append(sName);
#
# Test sets.
#
fRc = True;
for idTestSet in dTestSets:
try:
if self._processTestSet(idTestSet, dTestSets[idTestSet], sCurDir) is not True:
fRc = False;
except:
self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),));
fRc = False;
#
# Sub dirs.
#
for sSubDir in asSubDirs:
if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:
fRc = False;
#
# Try Remove the directory iff it's not '.' and it's been unmodified
# for the last 24h (race protection).
#
if sCurDir != '.':
try:
fpModTime = float(os.path.getmtime(sCurPath));
if fpModTime + (24*3600) <= time.time():
if utils.noxcptRmDir(sCurPath) is True:
self.dprint('Removed "%s".' % (sCurPath,));
except:
pass;
return fRc;
@staticmethod
def main():
""" C-style main(). """
#
# Parse options.
#
oParser = OptionParser();
oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False,
help = 'Verbose output.');
oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False,
help = 'Quiet operation.');
oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False,
help = 'Dry run, do not make any changes.');
(oOptions, asArgs) = oParser.parse_args()
if asArgs != []:
oParser.print_help();
return 1;
#
# Do the work.
#
oBatchJob = FileArchiverBatchJob(oOptions);
fRc = oBatchJob.processDir('.');
return 0 if fRc is True else 1;