def main(): replayTags = {} files = replay.getFiles() i = 0 for fileName in files: i += 1 logging.info('[%s/%s] %s: Getting replay tags...', i, len(files), fileName) tarFile = tarfile.open( os.path.join(replay.dropBoxReplayFilesFolder, fileName)) names = tarFile.getnames() if len(names) != 2: raise Exception('%s: Invalid number of files in tar file.', fileName) baseFileName = names[0].rsplit('.', 1)[0] dbFileName = '%s.db' % baseFileName txtFileName = '%s.txt' % baseFileName if set([dbFileName, txtFileName]) != set(names): raise Exception('%s: Invalid file names in tar file.', fileName) newMetadata = json.loads( metadata.port(tarFile.extractfile(txtFileName).read(), fileName)) tarFile.close() for tag in newMetadata['destinationTags']: replayTags.setdefault(newMetadata['destinationDatabase'], set([])).add(tag) for dependentTag in newMetadata['destinationTags'][tag][ 'dependencies']: replayTags[newMetadata['destinationDatabase']].add( dependentTag) # Convert the sets to a sorted list. This allows to dump the JSON and # also makes future changes easier to compare (since the lists are sorted, # when taking a diff we will not see movement of tags) outputDict = {} for destDB in replayTags: outputDict[destDB] = sorted(replayTags[destDB]) logging.info('Writing output file %s...', outputFile) with open(outputFile, 'w') as f: json.dump(outputDict, f, sort_keys=True, indent=4)
def main(): replayTags = {} files = replay.getFiles() i = 0 for fileName in files: i += 1 logging.info('[%s/%s] %s: Getting replay tags...', i, len(files), fileName) tarFile = tarfile.open(os.path.join(replay.dropBoxReplayFilesFolder, fileName)) names = tarFile.getnames() if len(names) != 2: raise Exception('%s: Invalid number of files in tar file.', fileName) baseFileName = names[0].rsplit('.', 1)[0] dbFileName = '%s.db' % baseFileName txtFileName = '%s.txt' % baseFileName if set([dbFileName, txtFileName]) != set(names): raise Exception('%s: Invalid file names in tar file.', fileName) newMetadata = json.loads(metadata.port(tarFile.extractfile(txtFileName).read(), fileName)) tarFile.close() for tag in newMetadata['destinationTags']: replayTags.setdefault(newMetadata['destinationDatabase'], set([])).add(tag) for dependentTag in newMetadata['destinationTags'][tag]['dependencies']: replayTags[newMetadata['destinationDatabase']].add(dependentTag) # Convert the sets to a sorted list. This allows to dump the JSON and # also makes future changes easier to compare (since the lists are sorted, # when taking a diff we will not see movement of tags) outputDict = {} for destDB in replayTags: outputDict[destDB] = sorted(replayTags[destDB]) logging.info('Writing output file %s...', outputFile) with open(outputFile, 'w') as f: json.dump(outputDict, f, sort_keys = True, indent = 4)
def main(): dropBoxRuns = calculateOldDropBoxRuns() with open('/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfo.json', 'rb') as f: runInfo = json.load(f) # Ask the frontend to clean up the files and database (username, account, password) = netrc.netrc().authenticators('newOffDb') frontendHttp = http.HTTP() frontendHttp.setBaseUrl(doUpload.frontendUrlTemplate % doUpload.frontendHost) logging.info('Signing in the frontend...') frontendHttp.query('signIn', { 'username': username, 'password': password, }) logging.info('Asking the frontend to clean up files and database...') frontendHttp.query('cleanUp') logging.info('Signing out the frontend...') frontendHttp.query('signOut') logging.info('Removing files in the backend...') execute('rm -rf ../NewOfflineDropBoxBaseDir/TestDropBox/*/*') conf = config.replay() logging.info('Cleaning up backend database...') execute('cmscond_schema_manager -c %s -P %s --dropAll' % (conf.destinationDB, conf.authpath)) logging.info('Setting up backend database...') execute('cmscond_export_database -s sqlite_file:%s -d %s -P %s' % (replayMasterDB, conf.destinationDB, conf.authpath), 'Y\n') dropBoxBE = Dropbox.Dropbox( conf ) # Replay all the runs _fwLoad = conditionDatabase.condDB.FWIncantation() i = 0 for runTimestamp in sorted(dropBoxRuns): i += 1 (hltRun, fcsRun) = runInfo[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]] logging.info('[%s/%s] %s: Replaying run with hltRun %s and fcsRun %s...', i, len(dropBoxRuns), runTimestamp, hltRun, fcsRun) j = 0 for fileName in dropBoxRuns[runTimestamp]: j += 1 logging.info(' [%s/%s] %s: Converting...', j, len(dropBoxRuns[runTimestamp]), fileName) tarFile = tarfile.open(os.path.join(dropBoxReplayFilesFolder, fileName)) names = tarFile.getnames() if len(names) != 2: raise Exception('%s: Invalid number of files in tar file.', fileName) baseFileName = names[0].rsplit('.', 1)[0] dbFileName = '%s.db' % baseFileName txtFileName = '%s.txt' % baseFileName if set([dbFileName, txtFileName]) != set(names): raise Exception('%s: Invalid file names in tar file.', fileName) with open('/tmp/replayRequest.txt', 'wb') as f: f.write(metadata.port(tarFile.extractfile(txtFileName).read(), fileName)) with open('/tmp/replayRequest.db', 'wb') as f: f.write(tarFile.extractfile(dbFileName).read()) tarFile.close() logging.info(' [%s/%s] %s: Uploading...', j, len(dropBoxRuns[runTimestamp]), fileName) try: doUpload.upload('/tmp/replayRequest', 'private') except doUpload.UploadError as e: # If it is a error from the server (i.e. UploadError), # we can continue with the next files. # If it is another kind, we do not catch it since in that case # it is a real problem with the upload.py script. logging.info(' [%s/%s] %s: Upload error: %s', j, len(dropBoxRuns[runTimestamp]), fileName, str(e)) dropBoxBE.reprocess(runTimestamp, hltRun, fcsRun) if runTimestamp in truncates: for runNumber in truncates[runTimestamp]: for tag in truncates[runTimestamp][runNumber]: logging.info('[%s/%s] %s: Truncating up to %s tag %s...', i, len(dropBoxRuns), runTimestamp, runNumber, tag) while True: # FIXME: Why can't we instantiate the RDBMS once? db = conditionDatabase.condDB.RDBMS(conf.authpath).getReadOnlyDB(conf.destinationDB) iov = conditionDatabase.IOVChecker(db) iov.load(tag) lastSince = iov.lastSince() if iov.timetype() == 'lumiid': lastSince >>= 32 db.closeSession() logging.info('[%s/%s] %s: lastSince now is %s...', i, len(dropBoxRuns), runTimestamp, lastSince) if lastSince < runNumber: break execute('cmscond_truncate_iov -c %s -P %s -t %s' % (conf.destinationDB, conf.authpath, tag))