Example #1
0
    def main(self):
        """ Main method. """
        oDb = TMDatabaseConnection();

        # Get a list of orphans.
        oLogic = TestSetLogic(oDb);
        aoOrphans = oLogic.fetchOrphaned();
        if len(aoOrphans) > 0:
            # Complete them.
            if self.oConfig.fJustDoIt:
                print 'Completing %u test sets as abandond:' % (len(aoOrphans),);
                for oTestSet in aoOrphans:
                    print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
                        % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone);
                    oLogic.completeAsAbandond(oTestSet.idTestSet);
                print 'Committing...';
                oDb.commit();
            else:
                for oTestSet in aoOrphans:
                    print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
                        % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone);
                print 'Not completing any testsets without seeing the --just-do-it option.'
        else:
            print 'No orphaned test sets.\n'
        return 0;
Example #2
0
    def main(self):
        """ Main method. """
        oDb = TMDatabaseConnection()

        # Get a list of orphans.
        oLogic = TestSetLogic(oDb)
        aoOrphans = oLogic.fetchOrphaned()
        if len(aoOrphans) > 0:
            # Complete them.
            if self.oConfig.fJustDoIt:
                print 'Completing %u test sets as abandoned:' % (
                    len(aoOrphans), )
                for oTestSet in aoOrphans:
                    print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
                        % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)
                    oLogic.completeAsAbandoned(oTestSet.idTestSet)
                print 'Committing...'
                oDb.commit()
            else:
                for oTestSet in aoOrphans:
                    print '#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' \
                        % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)
                print 'Not completing any testsets without seeing the --just-do-it option.'
        else:
            print 'No orphaned test sets.\n'
        return 0
    def _actionExecCompleted(self):
        """
        Implement EXEC completion.

        Because the action is request by the worker thread of the testbox
        script we cannot pass pending commands back to it like originally
        planned.  So, we just complete the test set and update the status.
        """
        #
        # Parameter validation.
        #
        sStatus = self._getStringParam(
            constants.tbreq.EXEC_COMPLETED_PARAM_RESULT,
            TestBoxController.kasValidResults)
        self._checkForUnknownParameters()

        (oDb, oStatusData, _) = self._connectToDbAndValidateTb([
            TestBoxStatusData.ksTestBoxState_Testing,
            TestBoxStatusData.ksTestBoxState_GangTesting
        ])
        if oStatusData is None:
            return False

        #
        # Complete the status.
        #
        oDb.rollback()
        oDb.begin()
        oTestSetLogic = TestSetLogic(oDb)
        idTestSetGangLeader = oTestSetLogic.complete(
            oStatusData.idTestSet,
            self.kadTbResultToStatus[sStatus],
            fCommit=False)

        oStatusLogic = TestBoxStatusLogic(oDb)
        if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Testing:
            assert idTestSetGangLeader is None
            GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(
                self._idTestBox)
            oStatusLogic.updateState(self._idTestBox,
                                     TestBoxStatusData.ksTestBoxState_Idle,
                                     fCommit=False)
        else:
            assert idTestSetGangLeader is not None
            oStatusLogic.updateState(
                self._idTestBox,
                TestBoxStatusData.ksTestBoxState_GangCleanup,
                oStatusData.idTestSet,
                fCommit=False)
            if oStatusLogic.isWholeGangDoneTesting(idTestSetGangLeader):
                GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(
                    self._idTestBox)
                oStatusLogic.updateState(self._idTestBox,
                                         TestBoxStatusData.ksTestBoxState_Idle,
                                         fCommit=False)

        oDb.commit()
        return self._resultResponse(constants.tbresp.STATUS_ACK)
Example #4
0
 def __init__(self, oOptions):
     """
     Parse command line
     """
     self.fVerbose = oOptions.fVerbose
     self.sSrcDir = config.g_ksFileAreaRootDir
     self.sDstDir = config.g_ksZipFileAreaRootDir
     #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
     self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None))
     self.fDryRun = oOptions.fDryRun
    def _cleanupOldTest(self, oDb, oStatusData):
        """
        Cleans up any old test set that may be left behind and changes the
        state to 'idle'.  See scenario #9:
        file://../../docs/AutomaticTestingRevamp.html#cleaning-up-abandoned-testcase

        Note. oStatusData.enmState is set to idle, but tsUpdated is not changed.
        """

        # Cleanup any abandoned test.
        if oStatusData.idTestSet is not None:
            SystemLogLogic(oDb).addEntry(
                SystemLogData.ksEvent_TestSetAbandoned,
                "idTestSet=%u idTestBox=%u enmState=%s %s" %
                (oStatusData.idTestSet, oStatusData.idTestBox,
                 oStatusData.enmState, self._sAction),
                fCommit=False)
            TestSetLogic(oDb).completeAsAbandoned(oStatusData.idTestSet,
                                                  fCommit=False)
            GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(
                self._idTestBox, fCommit=False)

        # Change to idle status
        if oStatusData.enmState != TestBoxStatusData.ksTestBoxState_Idle:
            TestBoxStatusLogic(oDb).updateState(
                self._idTestBox,
                TestBoxStatusData.ksTestBoxState_Idle,
                fCommit=False)
            oStatusData.tsUpdated = oDb.getCurrentTimestamp()
            oStatusData.enmState = TestBoxStatusData.ksTestBoxState_Idle

        # Commit.
        oDb.commit()

        return True
 def __init__(self, oOptions):
     """
     Parse command line
     """
     self.fVerbose      = oOptions.fVerbose;
     self.sSrcDir       = config.g_ksFileAreaRootDir;
     self.sDstDir       = config.g_ksZipFileAreaRootDir;
     #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
     self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));
     self.fDryRun       = oOptions.fDryRun;
    def _actionExecCompleted(self):
        """
        Implement EXEC completion.

        Because the action is request by the worker thread of the testbox
        script we cannot pass pending commands back to it like originally
        planned.  So, we just complete the test set and update the status.
        """
        #
        # Parameter validation.
        #
        sStatus = self._getStringParam(constants.tbreq.EXEC_COMPLETED_PARAM_RESULT, TestBoxController.kasValidResults);
        self._checkForUnknownParameters();

        (oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing,
                                                                TestBoxStatusData.ksTestBoxState_GangTesting]);
        if oStatusData is None:
            return False;

        #
        # Complete the status.
        #
        oDb.rollback();
        oDb.begin();
        oTestSetLogic = TestSetLogic(oDb);
        idTestSetGangLeader = oTestSetLogic.complete(oStatusData.idTestSet, self.kadTbResultToStatus[sStatus], fCommit = False);

        oStatusLogic = TestBoxStatusLogic(oDb);
        if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Testing:
            assert idTestSetGangLeader is None;
            GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox);
            oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False);
        else:
            assert idTestSetGangLeader is not None;
            oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_GangCleanup, oStatusData.idTestSet,
                                     fCommit = False);
            if oStatusLogic.isWholeGangDoneTesting(idTestSetGangLeader):
                GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox);
                oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False);

        oDb.commit();
        return self._resultResponse(constants.tbresp.STATUS_ACK);
Example #8
0
    def _doGangGatheringTimedOut(self, oDb, oStatusData):
        """
        _doRequestCommand worker for handling a box in gang-gathering-timed-out state.
        This will do clean-ups similar to _cleanupOldTest and update the state likewise.
        """
        oDb.begin();

        TestSetLogic(oDb).completeAsGangGatheringTimeout(oStatusData.idTestSet, fCommit = False);
        GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox, fCommit = False);
        TestBoxStatusLogic(oDb).updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False);

        oStatusData.tsUpdated = oDb.getCurrentTimestamp();
        oStatusData.enmState  = TestBoxStatusData.ksTestBoxState_Idle;

        oDb.commit();
        return None;
    def _actionUpload(self):
        """ Implement uploading of files. """
        #
        # Parameter validation.
        #
        sName = self._getStringParam(constants.tbreq.UPLOAD_PARAM_NAME)
        sMime = self._getStringParam(constants.tbreq.UPLOAD_PARAM_MIME)
        sKind = self._getStringParam(constants.tbreq.UPLOAD_PARAM_KIND)
        sDesc = self._getStringParam(constants.tbreq.UPLOAD_PARAM_DESC)
        self._checkForUnknownParameters()

        (oDb, oStatusData, _) = self._connectToDbAndValidateTb([
            TestBoxStatusData.ksTestBoxState_Testing,
            TestBoxStatusData.ksTestBoxState_GangTesting
        ])
        if oStatusData is None:
            return False

        if len(sName) > 128 or len(sName) < 3:
            raise TestBoxControllerException('Invalid file name "%s"' %
                                             (sName, ))
        if re.match(r'^[a-zA-Z0-9_\-(){}#@+,.=]*$', sName) is None:
            raise TestBoxControllerException('Invalid file name "%s"' %
                                             (sName, ))

        if sMime not in [
                'text/plain',  #'text/html', 'text/xml',
                'application/octet-stream',
                'image/png',  #'image/gif', 'image/jpeg',
                #'video/webm', 'video/mpeg', 'video/mpeg4-generic',
        ]:
            raise TestBoxControllerException('Invalid MIME type "%s"' %
                                             (sMime, ))

        if sKind not in TestResultFileData.kasKinds:
            raise TestBoxControllerException('Invalid kind "%s"' % (sKind, ))

        if len(sDesc) > 256:
            raise TestBoxControllerException('Invalid description "%s"' %
                                             (sDesc, ))
        if not set(sDesc).issubset(set(string.printable)):
            raise TestBoxControllerException('Invalid description "%s"' %
                                             (sDesc, ))

        if ('application/octet-stream', {}) != self._oSrvGlue.getContentType():
            raise TestBoxControllerException(
                'Unexpected content type: %s; %s' %
                self._oSrvGlue.getContentType())

        cbFile = self._oSrvGlue.getContentLength()
        if cbFile <= 0:
            raise TestBoxControllerException(
                'File "%s" is empty or negative in size (%s)' %
                (sName, cbFile))
        if (cbFile + 1048575) / 1048576 > config.g_kcMbMaxUploadSingle:
            raise TestBoxControllerException(
                'File "%s" is too big %u bytes (max %u MiB)' % (
                    sName,
                    cbFile,
                    config.g_kcMbMaxUploadSingle,
                ))

        #
        # Write the text to the log file.
        #
        oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet)
        oDstFile = TestSetLogic(oDb).createFile(oTestSet,
                                                sName=sName,
                                                sMime=sMime,
                                                sKind=sKind,
                                                sDesc=sDesc,
                                                cbFile=cbFile,
                                                fCommit=True)

        offFile = 0
        oSrcFile = self._oSrvGlue.getBodyIoStream()
        while offFile < cbFile:
            cbToRead = cbFile - offFile
            if cbToRead > 256 * 1024:
                cbToRead = 256 * 1024
            offFile += cbToRead

            abBuf = oSrcFile.read(cbToRead)
            oDstFile.write(abBuf)
            # pylint: disable=E1103
            del abBuf

        oDstFile.close()
        # pylint: disable=E1103

        # Done.
        return self._resultResponse(constants.tbresp.STATUS_ACK)
    def _doRequestCommand(self, fIdle):
        """
        Common code for handling command request.
        """

        (oDb, oStatusData, oTestBoxData) = self._connectToDbAndValidateTb()
        if oDb is None:
            return False

        #
        # Status clean up.
        #
        # Only when BUSY will the TestBox Script request and execute commands
        # concurrently.  So, it must be idle when sending REQUEST_COMMAND_IDLE.
        #
        if fIdle:
            if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGathering:
                self._doGangGathering(oDb, oStatusData)
            elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut:
                self._doGangGatheringTimedOut(oDb, oStatusData)
            elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangTesting:
                dResponse = SchedulerBase.composeExecResponse(
                    oDb, oTestBoxData.idTestBox, self._oSrvGlue.getBaseUrl())
                if dResponse is not None:
                    return dResponse
            elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangCleanup:
                self._doGangCleanup(oDb, oStatusData)
            elif oStatusData.enmState != TestBoxStatusData.ksTestBoxState_Idle:  # (includes ksTestBoxState_GangGatheringTimedOut)
                self._cleanupOldTest(oDb, oStatusData)

        #
        # Check for pending command.
        #
        if oTestBoxData.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
            asValidCmds = TestBoxController.kasIdleCmds if fIdle else TestBoxController.kasBusyCmds
            if oTestBoxData.enmPendingCmd in asValidCmds:
                dResponse = {
                    constants.tbresp.ALL_PARAM_RESULT:
                    TestBoxController.kdCmdToTbRespCmd[
                        oTestBoxData.enmPendingCmd]
                }
                if oTestBoxData.enmPendingCmd in [
                        TestBoxData.ksTestBoxCmd_Upgrade,
                        TestBoxData.ksTestBoxCmd_UpgradeAndReboot
                ]:
                    dResponse[constants.tbresp.
                              UPGRADE_PARAM_URL] = self._oSrvGlue.getBaseUrl(
                              ) + TestBoxController.ksUpgradeZip
                return self._writeResponse(dResponse)

            if oTestBoxData.enmPendingCmd == TestBoxData.ksTestBoxCmd_Abort and fIdle:
                TestBoxLogic(oDb).setCommand(
                    self._idTestBox,
                    sOldCommand=oTestBoxData.enmPendingCmd,
                    sNewCommand=TestBoxData.ksTestBoxCmd_None,
                    fCommit=True)

        #
        # If doing gang stuff, return 'CMD_WAIT'.
        #
        ## @todo r=bird: Why is GangTesting included here? Figure out when testing gang testing.
        if oStatusData.enmState in [
                TestBoxStatusData.ksTestBoxState_GangGathering,
                TestBoxStatusData.ksTestBoxState_GangTesting,
                TestBoxStatusData.ksTestBoxState_GangCleanup
        ]:
            return self._resultResponse(constants.tbresp.CMD_WAIT)

        #
        # If idling and enabled try schedule a new task.
        #
        if    fIdle \
          and oTestBoxData.fEnabled \
          and not TestSetLogic(oDb).isTestBoxExecutingToRapidly(oTestBoxData.idTestBox) \
          and oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Idle: # (paranoia)
            dResponse = SchedulerBase.scheduleNewTask(
                oDb, oTestBoxData, oStatusData.iWorkItem,
                self._oSrvGlue.getBaseUrl())
            if dResponse is not None:
                return self._writeResponse(dResponse)

        #
        # Touch the status row every couple of mins so we can tell that the box is alive.
        #
        oStatusLogic = TestBoxStatusLogic(oDb)
        if    oStatusData.enmState != TestBoxStatusData.ksTestBoxState_GangGathering \
          and oStatusLogic.timeSinceLastChangeInSecs(oStatusData) >= TestBoxStatusLogic.kcSecIdleTouchStatus:
            oStatusLogic.touchStatus(oTestBoxData.idTestBox, fCommit=True)

        return self._idleResponse()
Example #11
0
class FileArchiverBatchJob(object):  # pylint: disable=too-few-public-methods
    """
    Log+files comp
    """
    def __init__(self, oOptions):
        """
        Parse command line
        """
        self.fVerbose = oOptions.fVerbose
        self.sSrcDir = config.g_ksFileAreaRootDir
        self.sDstDir = config.g_ksZipFileAreaRootDir
        #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
        self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None))
        self.fDryRun = oOptions.fDryRun

    def dprint(self, sText):
        """ Verbose output. """
        if self.fVerbose:
            print(sText)
        return True

    def warning(self, sText):
        """Prints a warning."""
        print(sText)
        return True

    def _processTestSet(self, idTestSet, asFiles, sCurDir):
        """
        Worker for processDir.
        Same return codes as processDir.
        """

        sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet, ))
        if sBaseFilename[0:2] == ('.' + os.path.sep):
            sBaseFilename = sBaseFilename[2:]
        sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-')

        #
        # Skip the file if the test set is still running.
        # But delete them if the testset is not found.
        #
        oTestSet = self.oTestSetLogic.tryFetch(idTestSet)
        if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename:
            self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \
                         % (idTestSet, sBaseFilename, oTestSet.sBaseFilename,))
            oTestSet = None

        if oTestSet is not None:
            if oTestSet.enmStatus == TestSetData.ksTestStatus_Running:
                self.dprint('Skipping test set #%d, still running' %
                            (idTestSet, ))
                return True

            #
            # If we have a zip file already, don't try recreate it as we might
            # have had trouble removing the source files.
            #
            sDstDirPath = os.path.join(self.sDstDir, sCurDir)
            sZipFileNm = os.path.join(sDstDirPath,
                                      'TestSet-%d.zip' % (idTestSet, ))
            if not os.path.exists(sZipFileNm):
                #
                # Create zip file with all testset files as members.
                #
                self.dprint('TestSet %d: Creating %s...' % (
                    idTestSet,
                    sZipFileNm,
                ))
                if not self.fDryRun:

                    if not os.path.exists(sDstDirPath):
                        os.makedirs(sDstDirPath, 0o755)

                    utils.noxcptDeleteFile(sZipFileNm + '.tmp')
                    oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp',
                                               'w',
                                               zipfile.ZIP_DEFLATED,
                                               allowZip64=True)

                    for sFile in asFiles:
                        sSuff = os.path.splitext(sFile)[1]
                        if sSuff in [
                                '.png', '.webm', '.gz', '.bz2', '.zip', '.mov',
                                '.avi', '.mpg', '.gif', '.jpg'
                        ]:
                            ## @todo Consider storing these files outside the zip if they are a little largish.
                            self.dprint('TestSet %d: Storing   %s...' %
                                        (idTestSet, sFile))
                            oZipFile.write(sSrcFileBase + sFile, sFile,
                                           zipfile.ZIP_STORED)
                        else:
                            self.dprint('TestSet %d: Deflating %s...' %
                                        (idTestSet, sFile))
                            oZipFile.write(sSrcFileBase + sFile, sFile,
                                           zipfile.ZIP_DEFLATED)

                    oZipFile.close()

                    #
                    # .zip.tmp -> .zip.
                    #
                    utils.noxcptDeleteFile(sZipFileNm)
                    os.rename(sZipFileNm + '.tmp', sZipFileNm)

                #else: Dry run.
            else:
                self.dprint('TestSet %d: zip file exists already (%s)' % (
                    idTestSet,
                    sZipFileNm,
                ))

        #
        # Delete the files.
        #
        fRc = True
        if self.fVerbose:
            self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles))
        if not self.fDryRun:
            for sFile in asFiles:
                if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False:
                    self.warning('TestSet %d: Failed to delete "%s" (%s)' % (
                        idTestSet,
                        sFile,
                        sSrcFileBase + sFile,
                    ))
                    fRc = False

        return fRc

    def processDir(self, sCurDir):
        """
        Process the given directory (relative to sSrcDir and sDstDir).
        Returns success indicator.
        """
        if self.fVerbose:
            self.dprint('processDir: %s' % (sCurDir, ))

        #
        # Sift thought the directory content, collecting subdirectories and
        # sort relevant files by test set.
        # Generally there will either be subdirs or there will be files.
        #
        asSubDirs = []
        dTestSets = {}
        sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir))
        for sFile in os.listdir(sCurPath):
            if os.path.isdir(os.path.join(sCurPath, sFile)):
                if sFile not in ['.', '..']:
                    asSubDirs.append(sFile)
            elif sFile.startswith('TestSet-'):
                # Parse the file name. ASSUMES 'TestSet-%d-filename' format.
                iSlash1 = sFile.find('-')
                iSlash2 = sFile.find('-', iSlash1 + 1)
                if iSlash2 <= iSlash1:
                    self.warning('Bad filename (1): "%s"' % (sFile, ))
                    continue

                try:
                    idTestSet = int(sFile[(iSlash1 + 1):iSlash2])
                except:
                    self.warning('Bad filename (2): "%s"' % (sFile, ))
                    if self.fVerbose:
                        self.dprint('\n'.join(utils.getXcptInfo(4)))
                    continue

                if idTestSet <= 0:
                    self.warning('Bad filename (3): "%s"' % (sFile, ))
                    continue

                if iSlash2 + 2 >= len(sFile):
                    self.warning('Bad filename (4): "%s"' % (sFile, ))
                    continue
                sName = sFile[(iSlash2 + 1):]

                # Add it.
                if idTestSet not in dTestSets:
                    dTestSets[idTestSet] = []
                asTestSet = dTestSets[idTestSet]
                asTestSet.append(sName)

        #
        # Test sets.
        #
        fRc = True
        for idTestSet in dTestSets:
            try:
                if self._processTestSet(idTestSet, dTestSets[idTestSet],
                                        sCurDir) is not True:
                    fRc = False
            except:
                self.warning('TestSet %d: Exception in _processTestSet:\n%s' %
                             (
                                 idTestSet,
                                 '\n'.join(utils.getXcptInfo()),
                             ))
                fRc = False

        #
        # Sub dirs.
        #
        for sSubDir in asSubDirs:
            if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:
                fRc = False

        #
        # Try Remove the directory iff it's not '.' and it's been unmodified
        # for the last 24h (race protection).
        #
        if sCurDir != '.':
            try:
                fpModTime = float(os.path.getmtime(sCurPath))
                if fpModTime + (24 * 3600) <= time.time():
                    if utils.noxcptRmDir(sCurPath) is True:
                        self.dprint('Removed "%s".' % (sCurPath, ))
            except:
                pass

        return fRc

    @staticmethod
    def main():
        """ C-style main(). """
        #
        # Parse options.
        #
        oParser = OptionParser()
        oParser.add_option('-v',
                           '--verbose',
                           dest='fVerbose',
                           action='store_true',
                           default=False,
                           help='Verbose output.')
        oParser.add_option('-q',
                           '--quiet',
                           dest='fVerbose',
                           action='store_false',
                           default=False,
                           help='Quiet operation.')
        oParser.add_option('-d',
                           '--dry-run',
                           dest='fDryRun',
                           action='store_true',
                           default=False,
                           help='Dry run, do not make any changes.')
        (oOptions, asArgs) = oParser.parse_args()
        if asArgs != []:
            oParser.print_help()
            return 1

        #
        # Do the work.
        #
        oBatchJob = FileArchiverBatchJob(oOptions)
        fRc = oBatchJob.processDir('.')
        return 0 if fRc is True else 1
    def _actionUpload(self):
        """ Implement uploading of files. """
        #
        # Parameter validation.
        #
        sName = self._getStringParam(constants.tbreq.UPLOAD_PARAM_NAME);
        sMime = self._getStringParam(constants.tbreq.UPLOAD_PARAM_MIME);
        sKind = self._getStringParam(constants.tbreq.UPLOAD_PARAM_KIND);
        sDesc = self._getStringParam(constants.tbreq.UPLOAD_PARAM_DESC);
        self._checkForUnknownParameters();

        (oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing,
                                                                TestBoxStatusData.ksTestBoxState_GangTesting]);
        if oStatusData is None:
            return False;

        if len(sName) > 128 or len(sName) < 3:
            raise TestBoxControllerException('Invalid file name "%s"' % (sName,));
        if re.match(r'^[a-zA-Z0-9_\-(){}#@+,.=]*$', sName) is None:
            raise TestBoxControllerException('Invalid file name "%s"' % (sName,));

        if sMime not in [ 'text/plain', #'text/html', 'text/xml',
                          'application/octet-stream',
                          'image/png', #'image/gif', 'image/jpeg',
                          #'video/webm', 'video/mpeg', 'video/mpeg4-generic',
                          ]:
            raise TestBoxControllerException('Invalid MIME type "%s"' % (sMime,));

        if sKind not in [ 'log/release/vm',
                          'log/debug/vm',
                          'log/release/svc',
                          'log/debug/svc',
                          'log/release/client',
                          'log/debug/client',
                          'log/installer',
                          'log/uninstaller',
                          'crash/report/vm',
                          'crash/dump/vm',
                          'crash/report/svc',
                          'crash/dump/svc',
                          'crash/report/client',
                          'crash/dump/client',
                          'misc/other',
                          'screenshot/failure',
                          'screenshot/success',
                          #'screencapture/failure',
                          ]:
            raise TestBoxControllerException('Invalid kind "%s"' % (sKind,));

        if len(sDesc) > 256:
            raise TestBoxControllerException('Invalid description "%s"' % (sDesc,));
        if not set(sDesc).issubset(set(string.printable)):
            raise TestBoxControllerException('Invalid description "%s"' % (sDesc,));

        if ('application/octet-stream', {}) != self._oSrvGlue.getContentType():
            raise TestBoxControllerException('Unexpected content type: %s; %s' % self._oSrvGlue.getContentType());

        cbFile = self._oSrvGlue.getContentLength();
        if cbFile <= 0:
            raise TestBoxControllerException('File "%s" is empty or negative in size (%s)' % (sName, cbFile));
        if (cbFile + 1048575) / 1048576 > config.g_kcMbMaxUploadSingle:
            raise TestBoxControllerException('File "%s" is too big %u bytes (max %u MiB)'
                                             % (sName, cbFile, config.g_kcMbMaxUploadSingle,));

        #
        # Write the text to the log file.
        #
        oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet);
        oDstFile = TestSetLogic(oDb).createFile(oTestSet, sName = sName, sMime = sMime, sKind = sKind, sDesc = sDesc,
                                                cbFile = cbFile, fCommit = True);

        offFile  = 0;
        oSrcFile = self._oSrvGlue.getBodyIoStream();
        while offFile < cbFile:
            cbToRead = cbFile - offFile;
            if cbToRead > 256*1024:
                cbToRead = 256*1024;
            offFile += cbToRead;

            abBuf = oSrcFile.read(cbToRead);
            oDstFile.write(abBuf); # pylint: disable=E1103
            del abBuf;

        oDstFile.close(); # pylint: disable=E1103

        # Done.
        return self._resultResponse(constants.tbresp.STATUS_ACK);
class FileArchiverBatchJob(object): # pylint: disable=R0903
    """
    Log+files comp
    """

    def __init__(self, oOptions):
        """
        Parse command line
        """
        self.fVerbose      = oOptions.fVerbose;
        self.sSrcDir       = config.g_ksFileAreaRootDir;
        self.sDstDir       = config.g_ksZipFileAreaRootDir;
        #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
        self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));
        self.fDryRun       = oOptions.fDryRun;

    def dprint(self, sText):
        """ Verbose output. """
        if self.fVerbose:
            print(sText);
        return True;

    def warning(self, sText):
        """Prints a warning."""
        print(sText);
        return True;

    def _processTestSet(self, idTestSet, asFiles, sCurDir):
        """
        Worker for processDir.
        Same return codes as processDir.
        """

        sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,));
        if sBaseFilename[0:2] == ('.' + os.path.sep):
            sBaseFilename = sBaseFilename[2:];
        sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-');

        #
        # Skip the file if the test set is still running.
        # But delete them if the testset is not found.
        #
        oTestSet = self.oTestSetLogic.tryFetch(idTestSet);
        if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename:
            self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \
                         % (idTestSet, sBaseFilename, oTestSet.sBaseFilename,));
            oTestSet = None;

        if oTestSet is not None:
            if oTestSet.enmStatus == TestSetData.ksTestStatus_Running:
                self.dprint('Skipping test set #%d, still running' % (idTestSet,));
                return True;

            #
            # If we have a zip file already, don't try recreate it as we might
            # have had trouble removing the source files.
            #
            sDstDirPath = os.path.join(self.sDstDir, sCurDir);
            sZipFileNm  = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,));
            if not os.path.exists(sZipFileNm):
                #
                # Create zip file with all testset files as members.
                #
                self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,));
                if not self.fDryRun:

                    if not os.path.exists(sDstDirPath):
                        os.makedirs(sDstDirPath, 0o755);

                    utils.noxcptDeleteFile(sZipFileNm + '.tmp');
                    oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True);

                    for sFile in asFiles:
                        sSuff = os.path.splitext(sFile)[1];
                        if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]:
                            ## @todo Consider storing these files outside the zip if they are a little largish.
                            self.dprint('TestSet %d: Storing   %s...' % (idTestSet, sFile));
                            oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED);
                        else:
                            self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile));
                            oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED);

                    oZipFile.close();

                    #
                    # .zip.tmp -> .zip.
                    #
                    utils.noxcptDeleteFile(sZipFileNm);
                    os.rename(sZipFileNm + '.tmp', sZipFileNm);

                #else: Dry run.
            else:
                self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,));

        #
        # Delete the files.
        #
        fRc = True;
        if self.fVerbose:
            self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles));
        if not self.fDryRun:
            for sFile in asFiles:
                if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False:
                    self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,));
                    fRc = False;

        return fRc;


    def processDir(self, sCurDir):
        """
        Process the given directory (relative to sSrcDir and sDstDir).
        Returns success indicator.
        """
        if self.fVerbose:
            self.dprint('processDir: %s' % (sCurDir,));

        #
        # Sift thought the directory content, collecting subdirectories and
        # sort relevant files by test set.
        # Generally there will either be subdirs or there will be files.
        #
        asSubDirs = [];
        dTestSets = {};
        sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir));
        for sFile in os.listdir(sCurPath):
            if os.path.isdir(os.path.join(sCurPath, sFile)):
                if sFile not in [ '.', '..' ]:
                    asSubDirs.append(sFile);
            elif sFile.startswith('TestSet-'):
                # Parse the file name. ASSUMES 'TestSet-%d-filename' format.
                iSlash1 = sFile.find('-');
                iSlash2 = sFile.find('-', iSlash1 + 1);
                if iSlash2 <= iSlash1:
                    self.warning('Bad filename (1): "%s"' % (sFile,));
                    continue;

                try:    idTestSet = int(sFile[(iSlash1 + 1):iSlash2]);
                except:
                    self.warning('Bad filename (2): "%s"' % (sFile,));
                    if self.fVerbose:
                        self.dprint('\n'.join(utils.getXcptInfo(4)));
                    continue;

                if idTestSet <= 0:
                    self.warning('Bad filename (3): "%s"' % (sFile,));
                    continue;

                if iSlash2 + 2 >= len(sFile):
                    self.warning('Bad filename (4): "%s"' % (sFile,));
                    continue;
                sName = sFile[(iSlash2 + 1):];

                # Add it.
                if idTestSet not in dTestSets:
                    dTestSets[idTestSet] = [];
                asTestSet = dTestSets[idTestSet];
                asTestSet.append(sName);

        #
        # Test sets.
        #
        fRc = True;
        for idTestSet in dTestSets:
            try:
                if self._processTestSet(idTestSet, dTestSets[idTestSet], sCurDir) is not True:
                    fRc = False;
            except:
                self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),));
                fRc = False;

        #
        # Sub dirs.
        #
        for sSubDir in asSubDirs:
            if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:
                fRc = False;

        #
        # Try Remove the directory iff it's not '.' and it's been unmodified
        # for the last 24h (race protection).
        #
        if sCurDir != '.':
            try:
                fpModTime = float(os.path.getmtime(sCurPath));
                if fpModTime + (24*3600) <= time.time():
                    if utils.noxcptRmDir(sCurPath) is True:
                        self.dprint('Removed "%s".' % (sCurPath,));
            except:
                pass;

        return fRc;

    @staticmethod
    def main():
        """ C-style main(). """
        #
        # Parse options.
        #
        oParser = OptionParser();
        oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true',  default = False,
                           help = 'Verbose output.');
        oParser.add_option('-q', '--quiet',   dest = 'fVerbose', action = 'store_false', default = False,
                           help = 'Quiet operation.');
        oParser.add_option('-d', '--dry-run', dest = 'fDryRun',  action = 'store_true',  default = False,
                           help = 'Dry run, do not make any changes.');
        (oOptions, asArgs) = oParser.parse_args()
        if asArgs != []:
            oParser.print_help();
            return 1;

        #
        # Do the work.
        #
        oBatchJob = FileArchiverBatchJob(oOptions);
        fRc = oBatchJob.processDir('.');
        return 0 if fRc is True else 1;