示例#1
0
def main():
    dropBoxRuns = calculateOldDropBoxRuns()

    with open('/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfo.json', 'rb') as f:
        runInfo = json.load(f)

    # Ask the frontend to clean up the files and database
    (username, account, password) = netrc.netrc().authenticators('newOffDb')
    frontendHttp = http.HTTP()
    frontendHttp.setBaseUrl(doUpload.frontendUrlTemplate % doUpload.frontendHost)

    logging.info('Signing in the frontend...')
    frontendHttp.query('signIn', {
        'username': username,
        'password': password,
    })

    logging.info('Asking the frontend to clean up files and database...')
    frontendHttp.query('cleanUp')

    logging.info('Signing out the frontend...')
    frontendHttp.query('signOut')

    logging.info('Removing files in the backend...')
    execute('rm -rf ../NewOfflineDropBoxBaseDir/TestDropBox/*/*')

    conf = config.replay()

    logging.info('Cleaning up backend database...')
    execute('cmscond_schema_manager -c %s -P %s --dropAll' % (conf.destinationDB, conf.authpath))

    logging.info('Setting up backend database...')
    execute('cmscond_export_database -s sqlite_file:%s -d %s -P %s' % (replayMasterDB, conf.destinationDB, conf.authpath), 'Y\n')

    dropBoxBE = Dropbox.Dropbox( conf )

    # Replay all the runs
    _fwLoad = conditionDatabase.condDB.FWIncantation()

    i = 0
    for runTimestamp in sorted(dropBoxRuns):
        i += 1
        (hltRun, fcsRun) = runInfo[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]]
        logging.info('[%s/%s] %s: Replaying run with hltRun %s and fcsRun %s...', i, len(dropBoxRuns), runTimestamp, hltRun, fcsRun)

        j = 0
        for fileName in dropBoxRuns[runTimestamp]:
            j += 1
            logging.info('  [%s/%s] %s: Converting...', j, len(dropBoxRuns[runTimestamp]), fileName)

            tarFile = tarfile.open(os.path.join(dropBoxReplayFilesFolder, fileName))

            names = tarFile.getnames()
            if len(names) != 2:
                raise Exception('%s: Invalid number of files in tar file.', fileName)

            baseFileName = names[0].rsplit('.', 1)[0]
            dbFileName = '%s.db' % baseFileName
            txtFileName = '%s.txt' % baseFileName
            if set([dbFileName, txtFileName]) != set(names):
                raise Exception('%s: Invalid file names in tar file.', fileName)

            with open('/tmp/replayRequest.txt', 'wb') as f:
                f.write(metadata.port(tarFile.extractfile(txtFileName).read(), fileName))

            with open('/tmp/replayRequest.db', 'wb') as f:
                f.write(tarFile.extractfile(dbFileName).read())

            tarFile.close()

            logging.info('  [%s/%s] %s: Uploading...', j, len(dropBoxRuns[runTimestamp]), fileName)

            try:
                doUpload.upload('/tmp/replayRequest', 'private')
            except doUpload.UploadError as e:
                # If it is a error from the server (i.e. UploadError),
                # we can continue with the next files.
                # If it is another kind, we do not catch it since in that case
                # it is a real problem with the upload.py script.
                logging.info('  [%s/%s] %s: Upload error: %s', j, len(dropBoxRuns[runTimestamp]), fileName, str(e))

        dropBoxBE.reprocess(runTimestamp, hltRun, fcsRun)

        if runTimestamp in truncates:
            for runNumber in truncates[runTimestamp]:
                for tag in truncates[runTimestamp][runNumber]:
                    logging.info('[%s/%s] %s: Truncating up to %s tag %s...', i, len(dropBoxRuns), runTimestamp, runNumber, tag)

                    while True:
                        # FIXME: Why can't we instantiate the RDBMS once?
                        db = conditionDatabase.condDB.RDBMS(conf.authpath).getReadOnlyDB(conf.destinationDB)
                        iov = conditionDatabase.IOVChecker(db)
                        iov.load(tag)

                        lastSince = iov.lastSince()
                        if iov.timetype() == 'lumiid':
                            lastSince >>= 32

                        db.closeSession()

                        logging.info('[%s/%s] %s: lastSince now is %s...', i, len(dropBoxRuns), runTimestamp, lastSince)

                        if lastSince < runNumber:
                            break

                        execute('cmscond_truncate_iov -c %s -P %s -t %s' % (conf.destinationDB, conf.authpath, tag))
示例#2
0
def main():

    # The skipped tags are the ones from prep at the moment
    prepTags = []

    with open('replayTags.json', 'rb') as f:
        replayTags = json.load(f)

    for destinationDatabase in replayTags:
        if destinationDatabase.startswith('oracle://cms_orcoff_prep/'):
            for t in replayTags[destinationDatabase]:
                prepTags.append(str(t))
                print "WARNING: Tag %s has been targeted to PREP DB in the replay files." % (
                    t)

    conf = config.replay()
    _fwLoad = conditionDatabase.condDB.FWIncantation()
    rdbms = conditionDatabase.condDB.RDBMS(conf.authpath)
    # reference
    print 'opening reference DB...'
    referenceDB = rdbms.getReadOnlyDB(referenceDBConnStr)
    referenceDB.startReadOnlyTransaction()
    refTagList = referenceDB.allTags().strip().split()
    referenceDB.commitTransaction()
    print 'Found %d tags in reference.' % (len(refTagList))
    #replay
    print 'opening replay DB...'
    replayDB = rdbms.getReadOnlyDB(replayDBConnStr)
    replayDB.startReadOnlyTransaction()
    replayTagList = replayDB.allTags().strip().split()
    #replayTagList = [x for x in replayTagList if x not in skippedTags]
    replayDB.commitTransaction()
    print 'Found %d tags in replay.' % (len(replayTagList))

    if not len(refTagList) == len(replayTagList):
        print 'ERROR: Tag size content is different'
    missingTags = []
    for tag in refTagList:
        if not tag in replayTagList:
            print "ERROR: Reference Tag %s has not been found in the replay database." % (
                tag)
            missingTags.append(tag)
    for tag in replayTagList:
        if not tag in refTagList:
            if not tag in prepTags:
                print "ERROR: Replay tag %s has not been found in the reference database." % (
                    tag)

    refIov = conditionDatabase.IOVChecker(referenceDB)
    replayIov = conditionDatabase.IOVChecker(replayDB)
    numberOfErrors = 0
    numberOfComparison = 0
    for tag in refTagList:
        if tag in missingTags:
            continue
        numberOfComparison += 1
        refIov.load(tag)
        refSize = refIov.size()
        replayIov.load(tag)
        replaySize = replayIov.size()
        print "**** Checking Tag %s" % (tag)
        print "     Size in ref DB: %d; in replay: %d" % (refSize, replaySize)
        refIovs = refIov.getAllSinceValues()
        replayIovs = replayIov.getAllSinceValues()
        if not refIovs == replayIovs:
            timeType = refIov.timetype()
            error = True
            print '     ERROR: IOV content is different'
            print "     List of different elements:"
            numberOfErrors += 1
            printElem = False
            kLowMask = 0XFFFFFFFF
            for i in range(max(refSize, replaySize)):
                ref = ''
                rep = ''
                if i < refSize:
                    refElem = refIovs[i]
                    if str(timeType) == 'lumiid':
                        lumiBlock = refElem & kLowMask
                        refElem = refElem >> 32
                        ref = "%s/%s" % (refElem, lumiBlock)
                    else:
                        ref = str(refElem)
                if i < replaySize:
                    repElem = replayIovs[i]
                    if str(timeType) == 'lumiid':
                        lumiBlock = repElem & kLowMask
                        repElem = repElem >> 32
                        rep = "%s/%s" % (repElem, lumiBlock)
                    else:
                        rep = str(repElem)
                if not ref == rep:
                    printElem = True
                if printElem:
                    print '        Elem[%d] ref=%s     replay=%s' % (i, ref,
                                                                     rep)
        else:
            print '     Comparison OK.'
    print "\n"
    print "Tags compared:%d. Found with difference:%s" % (numberOfComparison,
                                                          numberOfErrors)
    referenceDB.closeSession()
    replayDB.closeSession()
    return 0
示例#3
0
def main():

    # The skipped tags are the ones from prep at the moment
    prepTags = []

    with open('replayTags.json', 'rb') as f:
        replayTags = json.load(f)

    for destinationDatabase in replayTags:
        if destinationDatabase.startswith('oracle://cms_orcoff_prep/'):
            for t in replayTags[destinationDatabase]:
                prepTags.append(str(t))
                print "WARNING: Tag %s has been targeted to PREP DB in the replay files." %(t)

    conf = config.replay()
    _fwLoad = conditionDatabase.condDB.FWIncantation()
    rdbms = conditionDatabase.condDB.RDBMS( conf.authpath )
    # reference
    print 'opening reference DB...'
    referenceDB = rdbms.getReadOnlyDB( referenceDBConnStr )
    referenceDB.startReadOnlyTransaction()
    refTagList = referenceDB.allTags().strip().split()
    referenceDB.commitTransaction()
    print 'Found %d tags in reference.' %(len(refTagList)) 
    #replay
    print 'opening replay DB...'
    replayDB = rdbms.getReadOnlyDB( replayDBConnStr )
    replayDB.startReadOnlyTransaction()
    replayTagList = replayDB.allTags().strip().split()
    #replayTagList = [x for x in replayTagList if x not in skippedTags]
    replayDB.commitTransaction()
    print 'Found %d tags in replay.' %(len(replayTagList)) 

    if not len(refTagList)==len(replayTagList):
        print 'ERROR: Tag size content is different'
    missingTags = []
    for tag in refTagList:
        if not tag in replayTagList:
            print "ERROR: Reference Tag %s has not been found in the replay database." %(tag)
            missingTags.append( tag )
    for tag in replayTagList:
        if not tag in refTagList:
            if not tag in prepTags:
                print "ERROR: Replay tag %s has not been found in the reference database." %(tag)

    refIov = conditionDatabase.IOVChecker( referenceDB )
    replayIov = conditionDatabase.IOVChecker( replayDB  )
    numberOfErrors = 0
    numberOfComparison = 0
    for tag in refTagList:
        if tag in missingTags:
            continue
        numberOfComparison += 1
        refIov.load( tag )
        refSize = refIov.size()
        replayIov.load( tag )
        replaySize = replayIov.size()
        print "**** Checking Tag %s" %(tag)
        print "     Size in ref DB: %d; in replay: %d" %(refSize,replaySize)
        refIovs = refIov.getAllSinceValues()
        replayIovs = replayIov.getAllSinceValues()
        if not refIovs == replayIovs:
            timeType = refIov.timetype()
            error = True
            print '     ERROR: IOV content is different'
            print "     List of different elements:"
            numberOfErrors += 1
            printElem = False
            kLowMask = 0XFFFFFFFF
            for i in range(max(refSize,replaySize)):
                ref = ''
                rep = ''
                if i<refSize:
                    refElem = refIovs[i]
                    if str(timeType) == 'lumiid':
                        lumiBlock = refElem & kLowMask
                        refElem = refElem >> 32
                        ref = "%s/%s" %(refElem,lumiBlock)
                    else:
                        ref = str(refElem)
                if i<replaySize:
                    repElem = replayIovs[i]
                    if str(timeType) == 'lumiid':
                        lumiBlock = repElem & kLowMask
                        repElem = repElem >> 32
                        rep = "%s/%s" %(repElem,lumiBlock)
                    else:
                        rep = str(repElem)
                if not ref == rep:
                    printElem = True
                if printElem:
                    print '        Elem[%d] ref=%s     replay=%s' %(i,ref,rep)
        else:
            print '     Comparison OK.'
    print "\n" 
    print "Tags compared:%d. Found with difference:%s" %(numberOfComparison,numberOfErrors)
    referenceDB.closeSession()
    replayDB.closeSession()
    return 0