def getMaskedBlocks(self, task, dbs, datasetPath):
""" Get the blocks which pass the lumi mask restrictions. For each block return the list of lumis
which were ok (given the lumi mask). The data structure returned is the following:
{
"block1" : {"file1" : LumiList(), "file5" : LumiList(), ...}
"block2" : {"file2" : LumiList(), "file7" : LumiList(), ...}
}
"""
# Get mask and convert to LumiList to make operations easier
maskedBlocks = {}
lumiMask = task.getLumiMask()
taskMask = LumiList(compactList=lumiMask)
# Find all the files that have runs and lumis we are interested in,
# fill block lfn part of maskedBlocks
for run, lumis in lumiMask.items():
files = []
for slumis in Lexicon.slicedIterator(lumis, 50):
slicedFiles = dbs.dbs.listFileArray(dataset=datasetPath,
run_num=run,
lumi_list=slumis,
detail=True)
files.extend(slicedFiles)
for file in files:
blockName = file['block_name']
fileName = file['logical_file_name']
if blockName not in maskedBlocks:
maskedBlocks[blockName] = {}
if fileName not in maskedBlocks[blockName]:
maskedBlocks[blockName][fileName] = LumiList()
# Fill maskedLumis part of maskedBlocks
for block in maskedBlocks:
fileLumis = dbs.dbs.listFileLumis(block_name=block,
validFileOnly=1)
for fileLumi in fileLumis:
lfn = fileLumi['logical_file_name']
# For each run : [lumis] mask by needed lumis, append to maskedBlocks
if maskedBlocks[block].get(lfn, None) is not None:
lumiList = LumiList(
runsAndLumis={
fileLumi['run_num']: fileLumi['lumi_section_num']
})
maskedBlocks[block][lfn] += (lumiList & taskMask)
return maskedBlocks
def getMaskedBlocks(self, task, dbs, datasetPath):
""" Get the blocks which pass the lumi mask restrictions. For each block return the list of lumis
which were ok (given the lumi mask). The data structure returned is the following:
{
"block1" : {"file1" : LumiList(), "file5" : LumiList(), ...}
"block2" : {"file2" : LumiList(), "file7" : LumiList(), ...}
}
"""
# Get mask and convert to LumiList to make operations easier
maskedBlocks = {}
lumiMask = task.getLumiMask()
taskMask = LumiList(compactList = lumiMask)
# Find all the files that have runs and lumis we are interested in,
# fill block lfn part of maskedBlocks
for run, lumis in lumiMask.items():
files = []
for slumis in Lexicon.slicedIterator(lumis, 50):
slicedFiles = dbs.dbs.listFileArray(dataset=datasetPath, run_num=run,
lumi_list=slumis, detail=True)
files.extend(slicedFiles)
for file in files:
blockName = file['block_name']
fileName = file['logical_file_name']
if blockName not in maskedBlocks:
maskedBlocks[blockName] = {}
if fileName not in maskedBlocks[blockName]:
maskedBlocks[blockName][fileName] = LumiList()
# Fill maskedLumis part of maskedBlocks
for block in maskedBlocks:
fileLumis = dbs.dbs.listFileLumis(block_name=block, validFileOnly = 1)
for fileLumi in fileLumis:
lfn = fileLumi['logical_file_name']
# For each run : [lumis] mask by needed lumis, append to maskedBlocks
if maskedBlocks[block].get(lfn, None) is not None:
lumiList = LumiList(runsAndLumis = {fileLumi['run_num']: fileLumi['lumi_section_num']})
maskedBlocks[block][lfn] += (lumiList & taskMask)
return maskedBlocks
def test(num_lumis):
from dbs.apis.dbsClient import DbsApi
dbsApi = DbsApi(url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/')
#dbsApi = DbsApi(url = 'https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader/')
datasetPath = "/SMS-T5qqqqVV_mGluino-1200To1275_mLSP-1to1150_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIIWinter15pLHE-MCRUN2_71_V1-v1/LHE"
#datasetPath = "/QDTojWinc_NC_M-1200_TuneZ2star_8TeV-madgraph/Summer12pLHE-DMWM_Validation_DONOTDELETE_Alan_TEST-v1/GEN"
run = 1
lumis = range(1, num_lumis+1)
files = []
print "Starting queries to listFileArray (in dataset mode) at %s" % datetime.utcnow()
for slumis in Lexicon.slicedIterator(lumis, 10):
start = datetime.utcnow()
print slumis
slicedFiles = dbsApi.listFileArray(dataset=datasetPath, run_num=run, lumi_list=slumis, detail=True)
files.extend(slicedFiles)
end = datetime.utcnow()
print " slice completed in %s" % (end - start)
# pprint(files)
maskedBlocks = {}
for lfn in files:
blockName = lfn['block_name']
fileName = lfn['logical_file_name']
if blockName not in maskedBlocks:
maskedBlocks[blockName] = {}
if fileName not in maskedBlocks[blockName]:
maskedBlocks[blockName][fileName] = LumiList()
# pprint(maskedBlocks)
print "\nStarting queries to listFileLumis at %s" % datetime.utcnow()
for block in maskedBlocks:
print block
start = datetime.utcnow()
fileLumis = dbsApi.listFileLumis(block_name=block, validFileOnly = 1)
end = datetime.utcnow()
print " query completed in %s" % (end - start)
