def upload(self, folder, loggingPrefix = ''):
tests = [ x.partition( '.txt' )[ 0 ] for x in glob.glob( os.path.join( 'testFiles', folder, '*.txt' ) ) ]
logging.info( '%s Uploading %s files...', loggingPrefix, len( tests ) )
i = 0
for test in tests :
i += 1
logging.info( '%s [%s/%s] %s: Uploading...', loggingPrefix, i, len( tests ), os.path.basename( test ) )
doUpload.upload(test, 'private')
def main():
dropBoxRuns = calculateOldDropBoxRuns()
with open('/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfo.json', 'rb') as f:
runInfo = json.load(f)
# Ask the frontend to clean up the files and database
(username, account, password) = netrc.netrc().authenticators('newOffDb')
frontendHttp = http.HTTP()
frontendHttp.setBaseUrl(doUpload.frontendUrlTemplate % doUpload.frontendHost)
logging.info('Signing in the frontend...')
frontendHttp.query('signIn', {
'username': username,
'password': password,
})
logging.info('Asking the frontend to clean up files and database...')
frontendHttp.query('cleanUp')
logging.info('Signing out the frontend...')
frontendHttp.query('signOut')
logging.info('Removing files in the backend...')
execute('rm -rf ../NewOfflineDropBoxBaseDir/TestDropBox/*/*')
conf = config.replay()
logging.info('Cleaning up backend database...')
execute('cmscond_schema_manager -c %s -P %s --dropAll' % (conf.destinationDB, conf.authpath))
logging.info('Setting up backend database...')
execute('cmscond_export_database -s sqlite_file:%s -d %s -P %s' % (replayMasterDB, conf.destinationDB, conf.authpath), 'Y\n')
dropBoxBE = Dropbox.Dropbox( conf )
# Replay all the runs
_fwLoad = conditionDatabase.condDB.FWIncantation()
i = 0
for runTimestamp in sorted(dropBoxRuns):
i += 1
(hltRun, fcsRun) = runInfo[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]]
logging.info('[%s/%s] %s: Replaying run with hltRun %s and fcsRun %s...', i, len(dropBoxRuns), runTimestamp, hltRun, fcsRun)
j = 0
for fileName in dropBoxRuns[runTimestamp]:
j += 1
logging.info(' [%s/%s] %s: Converting...', j, len(dropBoxRuns[runTimestamp]), fileName)
tarFile = tarfile.open(os.path.join(dropBoxReplayFilesFolder, fileName))
names = tarFile.getnames()
if len(names) != 2:
raise Exception('%s: Invalid number of files in tar file.', fileName)
baseFileName = names[0].rsplit('.', 1)[0]
dbFileName = '%s.db' % baseFileName
txtFileName = '%s.txt' % baseFileName
if set([dbFileName, txtFileName]) != set(names):
raise Exception('%s: Invalid file names in tar file.', fileName)
with open('/tmp/replayRequest.txt', 'wb') as f:
f.write(metadata.port(tarFile.extractfile(txtFileName).read(), fileName))
with open('/tmp/replayRequest.db', 'wb') as f:
f.write(tarFile.extractfile(dbFileName).read())
tarFile.close()
logging.info(' [%s/%s] %s: Uploading...', j, len(dropBoxRuns[runTimestamp]), fileName)
try:
doUpload.upload('/tmp/replayRequest', 'private')
except doUpload.UploadError as e:
# If it is a error from the server (i.e. UploadError),
# we can continue with the next files.
# If it is another kind, we do not catch it since in that case
# it is a real problem with the upload.py script.
logging.info(' [%s/%s] %s: Upload error: %s', j, len(dropBoxRuns[runTimestamp]), fileName, str(e))
dropBoxBE.reprocess(runTimestamp, hltRun, fcsRun)
if runTimestamp in truncates:
for runNumber in truncates[runTimestamp]:
for tag in truncates[runTimestamp][runNumber]:
logging.info('[%s/%s] %s: Truncating up to %s tag %s...', i, len(dropBoxRuns), runTimestamp, runNumber, tag)
while True:
# FIXME: Why can't we instantiate the RDBMS once?
db = conditionDatabase.condDB.RDBMS(conf.authpath).getReadOnlyDB(conf.destinationDB)
iov = conditionDatabase.IOVChecker(db)
iov.load(tag)
lastSince = iov.lastSince()
if iov.timetype() == 'lumiid':
lastSince >>= 32
db.closeSession()
logging.info('[%s/%s] %s: lastSince now is %s...', i, len(dropBoxRuns), runTimestamp, lastSince)
if lastSince < runNumber:
break
execute('cmscond_truncate_iov -c %s -P %s -t %s' % (conf.destinationDB, conf.authpath, tag))
