def getMaskedBlocks(self, task, dbs, datasetPath): """ Get the blocks which pass the lumi mask restrictions. For each block return the list of lumis which were ok (given the lumi mask). The data structure returned is the following: { "block1" : {"file1" : LumiList(), "file5" : LumiList(), ...} "block2" : {"file2" : LumiList(), "file7" : LumiList(), ...} } """ # Get mask and convert to LumiList to make operations easier maskedBlocks = {} lumiMask = task.getLumiMask() taskMask = LumiList(compactList=lumiMask) # Find all the files that have runs and lumis we are interested in, # fill block lfn part of maskedBlocks for run, lumis in lumiMask.items(): files = [] for slumis in Lexicon.slicedIterator(lumis, 50): slicedFiles = dbs.dbs.listFileArray(dataset=datasetPath, run_num=run, lumi_list=slumis, detail=True) files.extend(slicedFiles) for file in files: blockName = file['block_name'] fileName = file['logical_file_name'] if blockName not in maskedBlocks: maskedBlocks[blockName] = {} if fileName not in maskedBlocks[blockName]: maskedBlocks[blockName][fileName] = LumiList() # Fill maskedLumis part of maskedBlocks for block in maskedBlocks: fileLumis = dbs.dbs.listFileLumis(block_name=block, validFileOnly=1) for fileLumi in fileLumis: lfn = fileLumi['logical_file_name'] # For each run : [lumis] mask by needed lumis, append to maskedBlocks if maskedBlocks[block].get(lfn, None) is not None: lumiList = LumiList( runsAndLumis={ fileLumi['run_num']: fileLumi['lumi_section_num'] }) maskedBlocks[block][lfn] += (lumiList & taskMask) return maskedBlocks
def getMaskedBlocks(self, task, dbs, datasetPath): """ Get the blocks which pass the lumi mask restrictions. For each block return the list of lumis which were ok (given the lumi mask). The data structure returned is the following: { "block1" : {"file1" : LumiList(), "file5" : LumiList(), ...} "block2" : {"file2" : LumiList(), "file7" : LumiList(), ...} } """ # Get mask and convert to LumiList to make operations easier maskedBlocks = {} lumiMask = task.getLumiMask() taskMask = LumiList(compactList = lumiMask) # Find all the files that have runs and lumis we are interested in, # fill block lfn part of maskedBlocks for run, lumis in lumiMask.items(): files = [] for slumis in Lexicon.slicedIterator(lumis, 50): slicedFiles = dbs.dbs.listFileArray(dataset=datasetPath, run_num=run, lumi_list=slumis, detail=True) files.extend(slicedFiles) for file in files: blockName = file['block_name'] fileName = file['logical_file_name'] if blockName not in maskedBlocks: maskedBlocks[blockName] = {} if fileName not in maskedBlocks[blockName]: maskedBlocks[blockName][fileName] = LumiList() # Fill maskedLumis part of maskedBlocks for block in maskedBlocks: fileLumis = dbs.dbs.listFileLumis(block_name=block, validFileOnly = 1) for fileLumi in fileLumis: lfn = fileLumi['logical_file_name'] # For each run : [lumis] mask by needed lumis, append to maskedBlocks if maskedBlocks[block].get(lfn, None) is not None: lumiList = LumiList(runsAndLumis = {fileLumi['run_num']: fileLumi['lumi_section_num']}) maskedBlocks[block][lfn] += (lumiList & taskMask) return maskedBlocks
def test(num_lumis): from dbs.apis.dbsClient import DbsApi dbsApi = DbsApi(url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/') #dbsApi = DbsApi(url = 'https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader/') datasetPath = "/SMS-T5qqqqVV_mGluino-1200To1275_mLSP-1to1150_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIIWinter15pLHE-MCRUN2_71_V1-v1/LHE" #datasetPath = "/QDTojWinc_NC_M-1200_TuneZ2star_8TeV-madgraph/Summer12pLHE-DMWM_Validation_DONOTDELETE_Alan_TEST-v1/GEN" run = 1 lumis = range(1, num_lumis+1) files = [] print "Starting queries to listFileArray (in dataset mode) at %s" % datetime.utcnow() for slumis in Lexicon.slicedIterator(lumis, 10): start = datetime.utcnow() print slumis slicedFiles = dbsApi.listFileArray(dataset=datasetPath, run_num=run, lumi_list=slumis, detail=True) files.extend(slicedFiles) end = datetime.utcnow() print " slice completed in %s" % (end - start) # pprint(files) maskedBlocks = {} for lfn in files: blockName = lfn['block_name'] fileName = lfn['logical_file_name'] if blockName not in maskedBlocks: maskedBlocks[blockName] = {} if fileName not in maskedBlocks[blockName]: maskedBlocks[blockName][fileName] = LumiList() # pprint(maskedBlocks) print "\nStarting queries to listFileLumis at %s" % datetime.utcnow() for block in maskedBlocks: print block start = datetime.utcnow() fileLumis = dbsApi.listFileLumis(block_name=block, validFileOnly = 1) end = datetime.utcnow() print " query completed in %s" % (end - start)