def main():
dropBoxRuns = calculateOldDropBoxRuns()
with open('/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfo.json', 'rb') as f:
runInfo = json.load(f)
# Ask the frontend to clean up the files and database
(username, account, password) = netrc.netrc().authenticators('newOffDb')
frontendHttp = http.HTTP()
frontendHttp.setBaseUrl(doUpload.frontendUrlTemplate % doUpload.frontendHost)
logging.info('Signing in the frontend...')
frontendHttp.query('signIn', {
'username': username,
'password': password,
})
logging.info('Asking the frontend to clean up files and database...')
frontendHttp.query('cleanUp')
logging.info('Signing out the frontend...')
frontendHttp.query('signOut')
logging.info('Removing files in the backend...')
execute('rm -rf ../NewOfflineDropBoxBaseDir/TestDropBox/*/*')
conf = config.replay()
logging.info('Cleaning up backend database...')
execute('cmscond_schema_manager -c %s -P %s --dropAll' % (conf.destinationDB, conf.authpath))
logging.info('Setting up backend database...')
execute('cmscond_export_database -s sqlite_file:%s -d %s -P %s' % (replayMasterDB, conf.destinationDB, conf.authpath), 'Y\n')
dropBoxBE = Dropbox.Dropbox( conf )
# Replay all the runs
_fwLoad = conditionDatabase.condDB.FWIncantation()
i = 0
for runTimestamp in sorted(dropBoxRuns):
i += 1
(hltRun, fcsRun) = runInfo[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]]
logging.info('[%s/%s] %s: Replaying run with hltRun %s and fcsRun %s...', i, len(dropBoxRuns), runTimestamp, hltRun, fcsRun)
j = 0
for fileName in dropBoxRuns[runTimestamp]:
j += 1
logging.info(' [%s/%s] %s: Converting...', j, len(dropBoxRuns[runTimestamp]), fileName)
tarFile = tarfile.open(os.path.join(dropBoxReplayFilesFolder, fileName))
names = tarFile.getnames()
if len(names) != 2:
raise Exception('%s: Invalid number of files in tar file.', fileName)
baseFileName = names[0].rsplit('.', 1)[0]
dbFileName = '%s.db' % baseFileName
txtFileName = '%s.txt' % baseFileName
if set([dbFileName, txtFileName]) != set(names):
raise Exception('%s: Invalid file names in tar file.', fileName)
with open('/tmp/replayRequest.txt', 'wb') as f:
f.write(metadata.port(tarFile.extractfile(txtFileName).read(), fileName))
with open('/tmp/replayRequest.db', 'wb') as f:
f.write(tarFile.extractfile(dbFileName).read())
tarFile.close()
logging.info(' [%s/%s] %s: Uploading...', j, len(dropBoxRuns[runTimestamp]), fileName)
try:
doUpload.upload('/tmp/replayRequest', 'private')
except doUpload.UploadError as e:
# If it is a error from the server (i.e. UploadError),
# we can continue with the next files.
# If it is another kind, we do not catch it since in that case
# it is a real problem with the upload.py script.
logging.info(' [%s/%s] %s: Upload error: %s', j, len(dropBoxRuns[runTimestamp]), fileName, str(e))
dropBoxBE.reprocess(runTimestamp, hltRun, fcsRun)
if runTimestamp in truncates:
for runNumber in truncates[runTimestamp]:
for tag in truncates[runTimestamp][runNumber]:
logging.info('[%s/%s] %s: Truncating up to %s tag %s...', i, len(dropBoxRuns), runTimestamp, runNumber, tag)
while True:
# FIXME: Why can't we instantiate the RDBMS once?
db = conditionDatabase.condDB.RDBMS(conf.authpath).getReadOnlyDB(conf.destinationDB)
iov = conditionDatabase.IOVChecker(db)
iov.load(tag)
lastSince = iov.lastSince()
if iov.timetype() == 'lumiid':
lastSince >>= 32
db.closeSession()
logging.info('[%s/%s] %s: lastSince now is %s...', i, len(dropBoxRuns), runTimestamp, lastSince)
if lastSince < runNumber:
break
execute('cmscond_truncate_iov -c %s -P %s -t %s' % (conf.destinationDB, conf.authpath, tag))
def main():
# The skipped tags are the ones from prep at the moment
prepTags = []
with open('replayTags.json', 'rb') as f:
replayTags = json.load(f)
for destinationDatabase in replayTags:
if destinationDatabase.startswith('oracle://cms_orcoff_prep/'):
for t in replayTags[destinationDatabase]:
prepTags.append(str(t))
print "WARNING: Tag %s has been targeted to PREP DB in the replay files." % (
t)
conf = config.replay()
_fwLoad = conditionDatabase.condDB.FWIncantation()
rdbms = conditionDatabase.condDB.RDBMS(conf.authpath)
# reference
print 'opening reference DB...'
referenceDB = rdbms.getReadOnlyDB(referenceDBConnStr)
referenceDB.startReadOnlyTransaction()
refTagList = referenceDB.allTags().strip().split()
referenceDB.commitTransaction()
print 'Found %d tags in reference.' % (len(refTagList))
#replay
print 'opening replay DB...'
replayDB = rdbms.getReadOnlyDB(replayDBConnStr)
replayDB.startReadOnlyTransaction()
replayTagList = replayDB.allTags().strip().split()
#replayTagList = [x for x in replayTagList if x not in skippedTags]
replayDB.commitTransaction()
print 'Found %d tags in replay.' % (len(replayTagList))
if not len(refTagList) == len(replayTagList):
print 'ERROR: Tag size content is different'
missingTags = []
for tag in refTagList:
if not tag in replayTagList:
print "ERROR: Reference Tag %s has not been found in the replay database." % (
tag)
missingTags.append(tag)
for tag in replayTagList:
if not tag in refTagList:
if not tag in prepTags:
print "ERROR: Replay tag %s has not been found in the reference database." % (
tag)
refIov = conditionDatabase.IOVChecker(referenceDB)
replayIov = conditionDatabase.IOVChecker(replayDB)
numberOfErrors = 0
numberOfComparison = 0
for tag in refTagList:
if tag in missingTags:
continue
numberOfComparison += 1
refIov.load(tag)
refSize = refIov.size()
replayIov.load(tag)
replaySize = replayIov.size()
print "**** Checking Tag %s" % (tag)
print " Size in ref DB: %d; in replay: %d" % (refSize, replaySize)
refIovs = refIov.getAllSinceValues()
replayIovs = replayIov.getAllSinceValues()
if not refIovs == replayIovs:
timeType = refIov.timetype()
error = True
print ' ERROR: IOV content is different'
print " List of different elements:"
numberOfErrors += 1
printElem = False
kLowMask = 0XFFFFFFFF
for i in range(max(refSize, replaySize)):
ref = ''
rep = ''
if i < refSize:
refElem = refIovs[i]
if str(timeType) == 'lumiid':
lumiBlock = refElem & kLowMask
refElem = refElem >> 32
ref = "%s/%s" % (refElem, lumiBlock)
else:
ref = str(refElem)
if i < replaySize:
repElem = replayIovs[i]
if str(timeType) == 'lumiid':
lumiBlock = repElem & kLowMask
repElem = repElem >> 32
rep = "%s/%s" % (repElem, lumiBlock)
else:
rep = str(repElem)
if not ref == rep:
printElem = True
if printElem:
print ' Elem[%d] ref=%s replay=%s' % (i, ref,
rep)
else:
print ' Comparison OK.'
print "\n"
print "Tags compared:%d. Found with difference:%s" % (numberOfComparison,
numberOfErrors)
referenceDB.closeSession()
replayDB.closeSession()
return 0
def main():
# The skipped tags are the ones from prep at the moment
prepTags = []
with open('replayTags.json', 'rb') as f:
replayTags = json.load(f)
for destinationDatabase in replayTags:
if destinationDatabase.startswith('oracle://cms_orcoff_prep/'):
for t in replayTags[destinationDatabase]:
prepTags.append(str(t))
print "WARNING: Tag %s has been targeted to PREP DB in the replay files." %(t)
conf = config.replay()
_fwLoad = conditionDatabase.condDB.FWIncantation()
rdbms = conditionDatabase.condDB.RDBMS( conf.authpath )
# reference
print 'opening reference DB...'
referenceDB = rdbms.getReadOnlyDB( referenceDBConnStr )
referenceDB.startReadOnlyTransaction()
refTagList = referenceDB.allTags().strip().split()
referenceDB.commitTransaction()
print 'Found %d tags in reference.' %(len(refTagList))
#replay
print 'opening replay DB...'
replayDB = rdbms.getReadOnlyDB( replayDBConnStr )
replayDB.startReadOnlyTransaction()
replayTagList = replayDB.allTags().strip().split()
#replayTagList = [x for x in replayTagList if x not in skippedTags]
replayDB.commitTransaction()
print 'Found %d tags in replay.' %(len(replayTagList))
if not len(refTagList)==len(replayTagList):
print 'ERROR: Tag size content is different'
missingTags = []
for tag in refTagList:
if not tag in replayTagList:
print "ERROR: Reference Tag %s has not been found in the replay database." %(tag)
missingTags.append( tag )
for tag in replayTagList:
if not tag in refTagList:
if not tag in prepTags:
print "ERROR: Replay tag %s has not been found in the reference database." %(tag)
refIov = conditionDatabase.IOVChecker( referenceDB )
replayIov = conditionDatabase.IOVChecker( replayDB )
numberOfErrors = 0
numberOfComparison = 0
for tag in refTagList:
if tag in missingTags:
continue
numberOfComparison += 1
refIov.load( tag )
refSize = refIov.size()
replayIov.load( tag )
replaySize = replayIov.size()
print "**** Checking Tag %s" %(tag)
print " Size in ref DB: %d; in replay: %d" %(refSize,replaySize)
refIovs = refIov.getAllSinceValues()
replayIovs = replayIov.getAllSinceValues()
if not refIovs == replayIovs:
timeType = refIov.timetype()
error = True
print ' ERROR: IOV content is different'
print " List of different elements:"
numberOfErrors += 1
printElem = False
kLowMask = 0XFFFFFFFF
for i in range(max(refSize,replaySize)):
ref = ''
rep = ''
if i<refSize:
refElem = refIovs[i]
if str(timeType) == 'lumiid':
lumiBlock = refElem & kLowMask
refElem = refElem >> 32
ref = "%s/%s" %(refElem,lumiBlock)
else:
ref = str(refElem)
if i<replaySize:
repElem = replayIovs[i]
if str(timeType) == 'lumiid':
lumiBlock = repElem & kLowMask
repElem = repElem >> 32
rep = "%s/%s" %(repElem,lumiBlock)
else:
rep = str(repElem)
if not ref == rep:
printElem = True
if printElem:
print ' Elem[%d] ref=%s replay=%s' %(i,ref,rep)
else:
print ' Comparison OK.'
print "\n"
print "Tags compared:%d. Found with difference:%s" %(numberOfComparison,numberOfErrors)
referenceDB.closeSession()
replayDB.closeSession()
return 0