def shortenJson(jsonFile,minRun=0,maxRun=-1,output=None,debug=False):
from copy import deepcopy
runList = jsonFile
if isinstance(runList,LumiList):
runList = deepcopy(jsonFile)
else:
runList = LumiList (filename = jsonFile) # Read in first JSON file
allRuns = runList.getRuns()
runsToRemove=[]
for run in allRuns:
if int(run) < minRun:
runsToRemove.append (run)
if maxRun > 0 and int(run) > maxRun:
runsToRemove.append (run)
if debug:
print " runsToRemove ",runsToRemove
runList.removeRuns (runsToRemove)
if output:
runList.writeJSON (output)
else:
return runList
def __createSnippet(self,
jsonPath=None,
begin=None,
end=None,
firstRun=None,
lastRun=None,
repMap=None,
crab=False,
parent=False):
if firstRun:
firstRun = int(firstRun)
if lastRun:
lastRun = int(lastRun)
if (begin and firstRun) or (end and lastRun):
msg = (
"The Usage of " +
"'begin' & 'firstRun' " * int(bool(begin and firstRun)) +
"and " * int(bool(
(begin and firstRun) and (end and lastRun))) +
"'end' & 'lastRun' " * int(bool(end and lastRun)) +
"is ambigous.")
raise AllInOneError(msg)
if begin or end:
(firstRun, lastRun) = self.convertTimeToRun(begin=begin,
end=end,
firstRun=firstRun,
lastRun=lastRun)
if (firstRun and lastRun) and (firstRun > lastRun):
msg = ("The lower time/runrange limit ('begin'/'firstRun') "
"chosen is greater than the upper time/runrange limit "
"('end'/'lastRun').")
raise AllInOneError(msg)
if self.predefined() and (jsonPath or begin or end or firstRun
or lastRun):
msg = (
"The parameters 'JSON', 'begin', 'end', 'firstRun', and 'lastRun'"
"only work for official datasets, not predefined _cff.py files"
)
raise AllInOneError(msg)
goodLumiSecStr = ""
lumiStr = ""
lumiSecExtend = ""
if firstRun or lastRun or jsonPath:
goodLumiSecStr = ("lumiSecs = cms.untracked."
"VLuminosityBlockRange()\n")
lumiStr = " lumisToProcess = lumiSecs,\n"
if not jsonPath:
selectedRunList = self.__getRunList()
if firstRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") >= firstRun ]
if lastRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") <= lastRun ]
lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
+ str( self.__findInJson(run, "run_number") ) + ":max" \
for run in selectedRunList ]
splitLumiList = list(self.__chunks(lumiList, 255))
else:
theLumiList = None
try:
theLumiList = LumiList(filename=jsonPath)
except ValueError:
pass
if theLumiList is not None:
allRuns = theLumiList.getRuns()
runsToRemove = []
for run in allRuns:
if firstRun and int(run) < firstRun:
runsToRemove.append(run)
if lastRun and int(run) > lastRun:
runsToRemove.append(run)
theLumiList.removeRuns(runsToRemove)
splitLumiList = list(
self.__chunks(theLumiList.getCMSSWString().split(','),
255))
else:
with open(jsonPath) as f:
jsoncontents = f.read()
if "process.source.lumisToProcess" in jsoncontents:
msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet. Trying to use it" % jsonPath
if firstRun or lastRun:
msg += (
"\n (after applying firstRun and/or lastRun)"
)
msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
msg += "\nCheck your config file to make sure that it worked properly."
print msg
runlist = self.__getRunList()
if firstRun or lastRun:
self.__firstusedrun = -1
self.__lastusedrun = -1
jsoncontents = re.sub(
"\d+:(\d+|max)-\d+:(\d+|max)",
self.getForceRunRangeFunction(
firstRun, lastRun), jsoncontents)
self.__firstusedrun = max(
self.__firstusedrun,
int(
self.__findInJson(
runlist[0], "run_number")))
self.__lastusedrun = min(
self.__lastusedrun,
int(
self.__findInJson(
runlist[-1], "run_number")))
else:
self.__firstusedrun = int(
self.__findInJson(runlist[0],
"run_number"))
self.__lastusedrun = int(
self.__findInJson(runlist[-1],
"run_number"))
lumiSecExtend = jsoncontents
splitLumiList = [[""]]
if splitLumiList and splitLumiList[0]:
if splitLumiList[0][0]:
lumiSecStr = [ "',\n'".join( lumis ) \
for lumis in splitLumiList ]
lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
for lumis in lumiSecStr ]
lumiSecExtend = "\n".join(lumiSecStr)
runlist = self.__getRunList()
self.__firstusedrun = max(
int(splitLumiList[0][0].split(":")[0]),
int(self.__findInJson(runlist[0], "run_number")))
self.__lastusedrun = min(
int(splitLumiList[-1][-1].split(":")[0]),
int(self.__findInJson(runlist[-1], "run_number")))
else:
msg = "You are trying to run a validation without any runs! Check that:"
if firstRun or lastRun:
msg += "\n - firstRun and lastRun are correct for this dataset, and there are runs in between containing data"
if jsonPath:
msg += "\n - your JSON file is correct for this dataset, and the runs contain data"
if (firstRun or lastRun) and jsonPath:
msg += "\n - firstRun and lastRun are consistent with your JSON file"
if begin:
msg = msg.replace("firstRun", "begin")
if end:
msg = msg.replace("lastRun", "end")
raise AllInOneError(msg)
else:
runlist = self.__getRunList()
self.__firstusedrun = int(
self.__findInJson(self.__getRunList()[0], "run_number"))
self.__lastusedrun = int(
self.__findInJson(self.__getRunList()[-1], "run_number"))
if crab:
files = ""
else:
splitFileList = list(self.__chunks(self.fileList(), 255))
fileStr = ["',\n'".join(files) for files in splitFileList]
fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \
for files in fileStr ]
files = "\n".join(fileStr)
if parent:
splitParentFileList = list(
self.__chunks(self.fileList(parent=True), 255))
parentFileStr = [
"',\n'".join(parentFiles)
for parentFiles in splitParentFileList
]
parentFileStr = [ "secFiles.extend( [\n'" + parentFiles + "'\n] )" \
for parentFiles in parentFileStr ]
parentFiles = "\n".join(parentFileStr)
files += "\n\n" + parentFiles
theMap = repMap
theMap["files"] = files
theMap["json"] = jsonPath
theMap["lumiStr"] = lumiStr
theMap["goodLumiSecStr"] = goodLumiSecStr % (theMap)
theMap["lumiSecExtend"] = lumiSecExtend
if crab:
dataset_snippet = self.__dummy_source_template % (theMap)
else:
dataset_snippet = self.__source_template % (theMap)
return dataset_snippet
parser.add_option ('--output', dest='output', type='string',
help='Save output to file OUTPUT')
# required parameters
(options, args) = parser.parse_args()
if len (args) != 1:
raise RuntimeError, "Must provide exactly one input file"
if options.min and options.max and options.min > options.max:
raise RuntimeError, "Minimum value (%d) is greater than maximum value (%d)" % (options.min, options.max)
commaRE = re.compile (r',')
runsToRemove = []
for chunk in options.runs:
runs = commaRE.split (chunk)
runsToRemove.extend (runs)
alphaList = LumiList (filename = args[0]) # Read in first JSON file
allRuns = alphaList.getRuns()
for run in allRuns:
if options.min and int(run) < options.min:
runsToRemove.append (run)
if options.max and int(run) > options.max:
runsToRemove.append (run)
alphaList.removeRuns (runsToRemove)
if options.output:
alphaList.writeJSON (options.output)
else:
print alphaList
def __createSnippet(self,
jsonPath=None,
begin=None,
end=None,
firstRun=None,
lastRun=None,
repMap=None,
crab=False):
if firstRun:
firstRun = int(firstRun)
if lastRun:
lastRun = int(lastRun)
if (begin and firstRun) or (end and lastRun):
msg = (
"The Usage of " +
"'begin' & 'firstRun' " * int(bool(begin and firstRun)) +
"and " * int(bool(
(begin and firstRun) and (end and lastRun))) +
"'end' & 'lastRun' " * int(bool(end and lastRun)) +
"is ambigous.")
raise AllInOneError(msg)
if begin or end:
(firstRun, lastRun) = self.convertTimeToRun(begin=begin,
end=end,
firstRun=firstRun,
lastRun=lastRun)
if (firstRun and lastRun) and (firstRun > lastRun):
msg = ("The lower time/runrange limit ('begin'/'firstRun') "
"chosen is greater than the upper time/runrange limit "
"('end'/'lastRun').")
raise AllInOneError(msg)
goodLumiSecStr = ""
lumiStr = ""
lumiSecExtend = ""
if firstRun or lastRun:
goodLumiSecStr = ("lumiSecs = cms.untracked."
"VLuminosityBlockRange()\n")
lumiStr = " lumisToProcess = lumiSecs,\n"
if not jsonPath:
selectedRunList = self.__getRunList()
if firstRun:
selectedRunList = [ run for run in selectedRunList \
if run["run_number"] >= firstRun ]
if lastRun:
selectedRunList = [ run for run in selectedRunList \
if run["run_number"] <= lastRun ]
lumiList = [ str( run["run_number"] ) + ":1-" \
+ str( run["run_number"] ) + ":max" \
for run in selectedRunList ]
splitLumiList = list(self.__chunks(lumiList, 255))
else:
theLumiList = LumiList(filename=jsonPath)
allRuns = theLumiList.getRuns()
runsToRemove = []
for run in allRuns:
if firstRun and int(run) < firstRun:
runsToRemove.append(run)
if lastRun and int(run) > lastRun:
runsToRemove.append(run)
theLumiList.removeRuns(runsToRemove)
splitLumiList = list(
self.__chunks(theLumiList.getCMSSWString().split(','),
255))
if not len(splitLumiList[0][0]) == 0:
lumiSecStr = [ "',\n'".join( lumis ) \
for lumis in splitLumiList ]
lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
for lumis in lumiSecStr ]
lumiSecExtend = "\n".join(lumiSecStr)
elif jsonPath:
goodLumiSecStr = ("goodLumiSecs = LumiList.LumiList(filename"
"= '%(json)s').getCMSSWString().split(',')\n"
"lumiSecs = cms.untracked"
".VLuminosityBlockRange()\n")
lumiStr = " lumisToProcess = lumiSecs,\n"
lumiSecExtend = "lumiSecs.extend(goodLumiSecs)\n"
if crab:
files = ""
else:
splitFileList = list(self.__chunks(self.fileList(), 255))
fileStr = ["',\n'".join(files) for files in splitFileList]
fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \
for files in fileStr ]
files = "\n".join(fileStr)
theMap = repMap
theMap["files"] = files
theMap["json"] = jsonPath
theMap["lumiStr"] = lumiStr
theMap["goodLumiSecStr"] = goodLumiSecStr % (theMap)
theMap["lumiSecExtend"] = lumiSecExtend
if crab:
dataset_snippet = self.__dummy_source_template % (theMap)
else:
dataset_snippet = self.__source_template % (theMap)
return dataset_snippet
def __lumiSelectionSnippet(self,
jsonPath=None,
firstRun=None,
lastRun=None):
lumiSecExtend = ""
if firstRun or lastRun or jsonPath:
if not jsonPath:
selectedRunList = self.__getRunList()
if firstRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") >= firstRun ]
if lastRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") <= lastRun ]
lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
+ str( self.__findInJson(run, "run_number") ) + ":max" \
for run in selectedRunList ]
splitLumiList = list(self.__chunks(lumiList, 255))
else:
theLumiList = None
try:
theLumiList = LumiList(filename=jsonPath)
except ValueError:
pass
if theLumiList is not None:
allRuns = theLumiList.getRuns()
runsToRemove = []
for run in allRuns:
if firstRun and int(run) < firstRun:
runsToRemove.append(run)
if lastRun and int(run) > lastRun:
runsToRemove.append(run)
theLumiList.removeRuns(runsToRemove)
splitLumiList = list(
self.__chunks(theLumiList.getCMSSWString().split(','),
255))
if not (splitLumiList and splitLumiList[0]
and splitLumiList[0][0]):
splitLumiList = None
else:
with open(jsonPath) as f:
jsoncontents = f.read()
if "process.source.lumisToProcess" in jsoncontents:
msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet. Trying to use it" % jsonPath
if firstRun or lastRun:
msg += (
"\n (after applying firstRun and/or lastRun)"
)
msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
msg += "\nCheck your config file to make sure that it worked properly."
print msg
runlist = self.__getRunList()
if firstRun or lastRun:
self.__firstusedrun = -1
self.__lastusedrun = -1
jsoncontents = re.sub(
r"\d+:(\d+|max)(-\d+:(\d+|max))?",
self.getForceRunRangeFunction(
firstRun, lastRun), jsoncontents)
jsoncontents = (jsoncontents.replace(
"'',\n", "").replace("''\n", "").replace(
'"",\n', '').replace('""\n', ''))
self.__firstusedrun = max(
self.__firstusedrun,
int(
self.__findInJson(
runlist[0], "run_number")))
self.__lastusedrun = min(
self.__lastusedrun,
int(
self.__findInJson(
runlist[-1], "run_number")))
if self.__lastusedrun < self.__firstusedrun:
jsoncontents = None
else:
self.__firstusedrun = int(
self.__findInJson(runlist[0],
"run_number"))
self.__lastusedrun = int(
self.__findInJson(runlist[-1],
"run_number"))
lumiSecExtend = jsoncontents
splitLumiList = None
else:
raise AllInOneError(
"%s is not a valid json file!" % jsonPath)
if splitLumiList and splitLumiList[0] and splitLumiList[0][0]:
lumiSecStr = [ "',\n'".join( lumis ) \
for lumis in splitLumiList ]
lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
for lumis in lumiSecStr ]
lumiSecExtend = "\n".join(lumiSecStr)
runlist = self.__getRunList()
self.__firstusedrun = max(
int(splitLumiList[0][0].split(":")[0]),
int(self.__findInJson(runlist[0], "run_number")))
self.__lastusedrun = min(
int(splitLumiList[-1][-1].split(":")[0]),
int(self.__findInJson(runlist[-1], "run_number")))
elif lumiSecExtend:
pass
else:
msg = "You are trying to run a validation without any runs! Check that:"
if firstRun or lastRun:
msg += "\n - firstRun/begin and lastRun/end are correct for this dataset, and there are runs in between containing data"
if jsonPath:
msg += "\n - your JSON file is correct for this dataset, and the runs contain data"
if (firstRun or lastRun) and jsonPath:
msg += "\n - firstRun/begin and lastRun/end are consistent with your JSON file"
raise AllInOneError(msg)
else:
runlist = self.__getRunList()
self.__firstusedrun = int(
self.__findInJson(self.__getRunList()[0], "run_number"))
self.__lastusedrun = int(
self.__findInJson(self.__getRunList()[-1], "run_number"))
return lumiSecExtend
# required parameters
(options, args) = parser.parse_args()
if len(args) != 1:
raise RuntimeError("Must provide exactly one input file")
if options.min and options.max and options.min > options.max:
raise RuntimeError(
"Minimum value (%d) is greater than maximum value (%d)" %
(options.min, options.max))
commaRE = re.compile(r',')
runsToRemove = []
for chunk in options.runs:
runs = commaRE.split(chunk)
runsToRemove.extend(runs)
alphaList = LumiList(filename=args[0]) # Read in first JSON file
allRuns = alphaList.getRuns()
for run in allRuns:
if options.min and int(run) < options.min:
runsToRemove.append(run)
if options.max and int(run) > options.max:
runsToRemove.append(run)
alphaList.removeRuns(runsToRemove)
if options.output:
alphaList.writeJSON(options.output)
else:
print(alphaList)
def __lumiSelectionSnippet( self, jsonPath = None, firstRun = None, lastRun = None ):
lumiSecExtend = ""
if firstRun or lastRun or jsonPath:
if not jsonPath:
selectedRunList = self.__getRunList()
if firstRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") >= firstRun ]
if lastRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") <= lastRun ]
lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
+ str( self.__findInJson(run, "run_number") ) + ":max" \
for run in selectedRunList ]
splitLumiList = list( self.__chunks( lumiList, 255 ) )
else:
theLumiList = None
try:
theLumiList = LumiList ( filename = jsonPath )
except ValueError:
pass
if theLumiList is not None:
allRuns = theLumiList.getRuns()
runsToRemove = []
for run in allRuns:
if firstRun and int( run ) < firstRun:
runsToRemove.append( run )
if lastRun and int( run ) > lastRun:
runsToRemove.append( run )
theLumiList.removeRuns( runsToRemove )
splitLumiList = list( self.__chunks(
theLumiList.getCMSSWString().split(','), 255 ) )
if not (splitLumiList and splitLumiList[0] and splitLumiList[0][0]):
splitLumiList = None
else:
with open(jsonPath) as f:
jsoncontents = f.read()
if "process.source.lumisToProcess" in jsoncontents:
msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet. Trying to use it" % jsonPath
if firstRun or lastRun:
msg += ("\n (after applying firstRun and/or lastRun)")
msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
msg += "\nCheck your config file to make sure that it worked properly."
print msg
runlist = self.__getRunList()
if firstRun or lastRun:
self.__firstusedrun = -1
self.__lastusedrun = -1
jsoncontents = re.sub(r"\d+:(\d+|max)(-\d+:(\d+|max))?", self.getForceRunRangeFunction(firstRun, lastRun), jsoncontents)
jsoncontents = (jsoncontents.replace("'',\n","").replace("''\n","")
.replace('"",\n','').replace('""\n',''))
self.__firstusedrun = max(self.__firstusedrun, int(self.__findInJson(runlist[0],"run_number")))
self.__lastusedrun = min(self.__lastusedrun, int(self.__findInJson(runlist[-1],"run_number")))
if self.__lastusedrun < self.__firstusedrun:
jsoncontents = None
else:
self.__firstusedrun = int(self.__findInJson(runlist[0],"run_number"))
self.__lastusedrun = int(self.__findInJson(runlist[-1],"run_number"))
lumiSecExtend = jsoncontents
splitLumiList = None
else:
raise AllInOneError("%s is not a valid json file!" % jsonPath)
if splitLumiList and splitLumiList[0] and splitLumiList[0][0]:
lumiSecStr = [ "',\n'".join( lumis ) \
for lumis in splitLumiList ]
lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
for lumis in lumiSecStr ]
lumiSecExtend = "\n".join( lumiSecStr )
runlist = self.__getRunList()
self.__firstusedrun = max(int(splitLumiList[0][0].split(":")[0]), int(self.__findInJson(runlist[0],"run_number")))
self.__lastusedrun = min(int(splitLumiList[-1][-1].split(":")[0]), int(self.__findInJson(runlist[-1],"run_number")))
elif lumiSecExtend:
pass
else:
msg = "You are trying to run a validation without any runs! Check that:"
if firstRun or lastRun:
msg += "\n - firstRun/begin and lastRun/end are correct for this dataset, and there are runs in between containing data"
if jsonPath:
msg += "\n - your JSON file is correct for this dataset, and the runs contain data"
if (firstRun or lastRun) and jsonPath:
msg += "\n - firstRun/begin and lastRun/end are consistent with your JSON file"
raise AllInOneError(msg)
else:
if self.__inputMagneticField is not None:
pass #never need self.__firstusedrun or self.__lastusedrun
else:
runlist = self.__getRunList()
self.__firstusedrun = int(self.__findInJson(self.__getRunList()[0],"run_number"))
self.__lastusedrun = int(self.__findInJson(self.__getRunList()[-1],"run_number"))
return lumiSecExtend
def __createSnippet( self, jsonPath = None, begin = None, end = None,
firstRun = None, lastRun = None, repMap = None,
crab = False ):
if firstRun:
firstRun = int( firstRun )
if lastRun:
lastRun = int( lastRun )
if ( begin and firstRun ) or ( end and lastRun ):
msg = ( "The Usage of "
+ "'begin' & 'firstRun' " * int( bool( begin and
firstRun ) )
+ "and " * int( bool( ( begin and firstRun ) and
( end and lastRun ) ) )
+ "'end' & 'lastRun' " * int( bool( end and lastRun ) )
+ "is ambigous." )
raise AllInOneError( msg )
if begin or end:
( firstRun, lastRun ) = self.convertTimeToRun(
begin = begin, end = end, firstRun = firstRun,
lastRun = lastRun )
if ( firstRun and lastRun ) and ( firstRun > lastRun ):
msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
"chosen is greater than the upper time/runrange limit "
"('end'/'lastRun').")
raise AllInOneError( msg )
goodLumiSecStr = ""
lumiStr = ""
lumiSecExtend = ""
if firstRun or lastRun:
goodLumiSecStr = ( "lumiSecs = cms.untracked."
"VLuminosityBlockRange()\n" )
lumiStr = " lumisToProcess = lumiSecs,\n"
if not jsonPath:
selectedRunList = self.__getRunList()
if firstRun:
selectedRunList = [ run for run in selectedRunList \
if run["run_number"] >= firstRun ]
if lastRun:
selectedRunList = [ run for run in selectedRunList \
if run["run_number"] <= lastRun ]
lumiList = [ str( run["run_number"] ) + ":1-" \
+ str( run["run_number"] ) + ":max" \
for run in selectedRunList ]
splitLumiList = list( self.__chunks( lumiList, 255 ) )
else:
theLumiList = LumiList ( filename = jsonPath )
allRuns = theLumiList.getRuns()
runsToRemove = []
for run in allRuns:
if firstRun and int( run ) < firstRun:
runsToRemove.append( run )
if lastRun and int( run ) > lastRun:
runsToRemove.append( run )
theLumiList.removeRuns( runsToRemove )
splitLumiList = list( self.__chunks(
theLumiList.getCMSSWString().split(','), 255 ) )
if not len(splitLumiList[0][0]) == 0:
lumiSecStr = [ "',\n'".join( lumis ) \
for lumis in splitLumiList ]
lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
for lumis in lumiSecStr ]
lumiSecExtend = "\n".join( lumiSecStr )
elif jsonPath:
goodLumiSecStr = ( "goodLumiSecs = LumiList.LumiList(filename"
"= '%(json)s').getCMSSWString().split(',')\n"
"lumiSecs = cms.untracked"
".VLuminosityBlockRange()\n"
)
lumiStr = " lumisToProcess = lumiSecs,\n"
lumiSecExtend = "lumiSecs.extend(goodLumiSecs)\n"
if crab:
files = ""
else:
splitFileList = list( self.__chunks( self.fileList(), 255 ) )
fileStr = [ "',\n'".join( files ) for files in splitFileList ]
fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \
for files in fileStr ]
files = "\n".join( fileStr )
theMap = repMap
theMap["files"] = files
theMap["json"] = jsonPath
theMap["lumiStr"] = lumiStr
theMap["goodLumiSecStr"] = goodLumiSecStr%( theMap )
theMap["lumiSecExtend"] = lumiSecExtend
if crab:
dataset_snippet = self.__dummy_source_template%( theMap )
else:
dataset_snippet = self.__source_template%( theMap )
return dataset_snippet
# lumis_2=l['number']
#print run, run_lumis
#lumis.append((run,run_lumis))
lumis[run]=run_lumis
# print lumis
ll = LumiList(compactList=lumis)
#print "ll", ll
runrange = sorted(int(x) for x in ll.getCompactList().keys())
dcs_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions16/13TeV/DCSOnly/json_DCSONLY.txt') # JMTBAD import from goodlumis
#print "dcs_ll", dcs_ll
dcs_runrange = sorted(int(x) for x in dcs_ll.getCompactList().keys())
dcs_ll.removeRuns(xrange(dcs_runrange[0], runrange[0]))
dcs_ll.removeRuns(xrange(runrange[-1]+1, dcs_runrange[-1]))
ok = LumiList(compactList={
})#from lumiSummary.json
print 'run range for', latest_dataset, ':', runrange[0], runrange[-1]
print 'these lumis are in the DCS-only JSON but not (yet) in', latest_dataset
print str(dcs_ll - ll - ok)
#elif cmd == 'oklist': not true
# addOk_ll = LumiList('crab/crab_datamc_EGammaRun2015A-Prompt_246958_247068_20150622005921/results/lumiSummary.json')
# print addOk_ll
elif cmd == 'drawall':
# considered in the certification for the latest prompt JSON. So,
# don't drop them from the DCS-only list when combining later.
holes = []
# Order of priorities: rereco, then prompt reco, and then DCS.
runs_to_remove_from_prompt = range(first_run, last_rereco_run+1)
runs_to_remove_from_dcsonly = range(first_run, last_prompt_run+1)
for hole in holes:
print 'goodlumis warning: re-adding "hole" run %i from DCS-only list' % hole
runs_to_remove_from_dcsonly.remove(hole)
#DCSOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions15/13TeV/DCSOnly/json_DCSONLY.txt')
#DCSOnlyForNewRuns_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions15/13TeV/DCSOnly/json_DCSONLY.txt')
DCSOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions15/13TeV/DCSOnly/json_DCSONLY.txt')
DCSOnlyForNewRuns_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions15/13TeV/DCSOnly/json_DCSONLY.txt')
DCSOnlyForNewRuns_ll.removeRuns(runs_to_remove_from_dcsonly)
# Remove runs outside the range [first_run, last_run] since DCS-only
# list includes HI runs, etc. ###### Use later
#for ll in (DCSOnly_ll, DCSOnlyForNewRuns_ll):
# ll.removeRuns(xrange(1, first_run))
# ll.removeRuns(xrange(last_run+1, 300000)) # dummy number
Prompt_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions18/13TeV/PromptReco/Cert_%i-%i_13TeV_PromptReco_Collisions18_JSON%s.txt' % (first_run, last_prompt_run, prompt_version))
PromptMuonsOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions18/13TeV/PromptReco/Cert_%i-%i_13TeV_PromptReco_Collisions18_JSON_MuonPhys%s.txt' % (first_run, last_prompt_run, prompt_version))
#PromptMuonsOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions15/13TeV/Cert_%i-%i_13TeV_PromptReco_Collisions15_50ns_JSON_MuonPhys%s_v2.txt' % (first_run, last_prompt_run, prompt_version))
#Prompt_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/Prompt/Cert_%i-%i_8TeV_PromptReco_Collisions12_JSON%s.txt' % (first_run, last_prompt_run, prompt_version))
#PromptMuonsOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/Prompt/Cert_%i-%i_8TeV_PromptReco_Collisions12_JSON_MuonPhys%s.txt' % (first_run, last_prompt_run, prompt_version))
def __createSnippet( self, jsonPath = None, begin = None, end = None,
firstRun = None, lastRun = None, repMap = None,
crab = False, parent = False ):
if firstRun:
firstRun = int( firstRun )
if lastRun:
lastRun = int( lastRun )
if ( begin and firstRun ) or ( end and lastRun ):
msg = ( "The Usage of "
+ "'begin' & 'firstRun' " * int( bool( begin and
firstRun ) )
+ "and " * int( bool( ( begin and firstRun ) and
( end and lastRun ) ) )
+ "'end' & 'lastRun' " * int( bool( end and lastRun ) )
+ "is ambigous." )
raise AllInOneError( msg )
if begin or end:
( firstRun, lastRun ) = self.convertTimeToRun(
begin = begin, end = end, firstRun = firstRun,
lastRun = lastRun )
if ( firstRun and lastRun ) and ( firstRun > lastRun ):
msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
"chosen is greater than the upper time/runrange limit "
"('end'/'lastRun').")
raise AllInOneError( msg )
if self.predefined() and (jsonPath or begin or end or firstRun or lastRun):
msg = ( "The parameters 'JSON', 'begin', 'end', 'firstRun', and 'lastRun'"
"only work for official datasets, not predefined _cff.py files" )
raise AllInOneError( msg )
goodLumiSecStr = ""
lumiStr = ""
lumiSecExtend = ""
if firstRun or lastRun or jsonPath:
goodLumiSecStr = ( "lumiSecs = cms.untracked."
"VLuminosityBlockRange()\n" )
lumiStr = " lumisToProcess = lumiSecs,\n"
if not jsonPath:
selectedRunList = self.__getRunList()
if firstRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") >= firstRun ]
if lastRun:
selectedRunList = [ run for run in selectedRunList \
if self.__findInJson(run, "run_number") <= lastRun ]
lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
+ str( self.__findInJson(run, "run_number") ) + ":max" \
for run in selectedRunList ]
splitLumiList = list( self.__chunks( lumiList, 255 ) )
else:
theLumiList = None
try:
theLumiList = LumiList ( filename = jsonPath )
except ValueError:
pass
if theLumiList is not None:
allRuns = theLumiList.getRuns()
runsToRemove = []
for run in allRuns:
if firstRun and int( run ) < firstRun:
runsToRemove.append( run )
if lastRun and int( run ) > lastRun:
runsToRemove.append( run )
theLumiList.removeRuns( runsToRemove )
splitLumiList = list( self.__chunks(
theLumiList.getCMSSWString().split(','), 255 ) )
else:
with open(jsonPath) as f:
jsoncontents = f.read()
if "process.source.lumisToProcess" in jsoncontents:
msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet. Trying to use it" % jsonPath
if firstRun or lastRun:
msg += ("\n (after applying firstRun and/or lastRun)")
msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
msg += "\nCheck your config file to make sure that it worked properly."
print msg
runlist = self.__getRunList()
if firstRun or lastRun:
self.__firstusedrun = -1
self.__lastusedrun = -1
jsoncontents = re.sub("\d+:(\d+|max)-\d+:(\d+|max)", self.getForceRunRangeFunction(firstRun, lastRun), jsoncontents)
self.__firstusedrun = max(self.__firstusedrun, int(self.__findInJson(runlist[0],"run_number")))
self.__lastusedrun = min(self.__lastusedrun, int(self.__findInJson(runlist[-1],"run_number")))
else:
self.__firstusedrun = int(self.__findInJson(runlist[0],"run_number"))
self.__lastusedrun = int(self.__findInJson(runlist[-1],"run_number"))
lumiSecExtend = jsoncontents
splitLumiList = [[""]]
if not len(splitLumiList[0][0]) == 0:
lumiSecStr = [ "',\n'".join( lumis ) \
for lumis in splitLumiList ]
lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
for lumis in lumiSecStr ]
lumiSecExtend = "\n".join( lumiSecStr )
runlist = self.__getRunList()
self.__firstusedrun = max(int(splitLumiList[0][0].split(":")[0]), int(self.__findInJson(runlist[0],"run_number")))
self.__lastusedrun = min(int(splitLumiList[-1][-1].split(":")[0]), int(self.__findInJson(runlist[-1],"run_number")))
else:
runlist = self.__getRunList()
self.__firstusedrun = int(self.__findInJson(self.__getRunList()[0],"run_number"))
self.__lastusedrun = int(self.__findInJson(self.__getRunList()[-1],"run_number"))
if crab:
files = ""
else:
splitFileList = list( self.__chunks( self.fileList(), 255 ) )
fileStr = [ "',\n'".join( files ) for files in splitFileList ]
fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \
for files in fileStr ]
files = "\n".join( fileStr )
if parent:
splitParentFileList = list( self.__chunks( self.fileList(parent = True), 255 ) )
parentFileStr = [ "',\n'".join( parentFiles ) for parentFiles in splitParentFileList ]
parentFileStr = [ "secFiles.extend( [\n'" + parentFiles + "'\n] )" \
for parentFiles in parentFileStr ]
parentFiles = "\n".join( parentFileStr )
files += "\n\n" + parentFiles
theMap = repMap
theMap["files"] = files
theMap["json"] = jsonPath
theMap["lumiStr"] = lumiStr
theMap["goodLumiSecStr"] = goodLumiSecStr%( theMap )
theMap["lumiSecExtend"] = lumiSecExtend
if crab:
dataset_snippet = self.__dummy_source_template%( theMap )
else:
dataset_snippet = self.__source_template%( theMap )
return dataset_snippet
# These runs are <= last_prompt_run, but they were not actually
# considered in the certification for the latest prompt JSON. So,
# don't drop them from the DCS-only list when combining later.
holes = []
# Order of priorities: rereco, then prompt reco, and then DCS.
runs_to_remove_from_prompt = range(first_run, last_rereco_run+1)
runs_to_remove_from_dcsonly = range(first_run, last_prompt_run+1)
for hole in holes:
print 'goodlumis warning: re-adding "hole" run %i from DCS-only list' % hole
runs_to_remove_from_dcsonly.remove(hole)
DCSOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/DCSOnly/json_DCSONLY.txt')
DCSOnlyForNewRuns_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/DCSOnly/json_DCSONLY.txt')
DCSOnlyForNewRuns_ll.removeRuns(runs_to_remove_from_dcsonly)
# Remove runs outside the range [first_run, last_run] since DCS-only
# list includes HI runs, etc.
for ll in (DCSOnly_ll, DCSOnlyForNewRuns_ll):
ll.removeRuns(xrange(1, first_run))
ll.removeRuns(xrange(last_run+1, 300000)) # dummy number
# July 13th reprocessing of 2012A and 2012B
Jul13_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/Reprocessing/Cert_190456-196531_8TeV_13Jul2012ReReco_Collisions12_JSON_v2.txt')
Jul13MuonsOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/Reprocessing/Cert_190456-196531_8TeV_13Jul2012ReReco_Collisions12_JSON_MuonPhys_v4.txt')
# August 6th reprocessing of 5 runs of 2012A
Aug06_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/Reprocessing/Cert_190782-190949_8TeV_06Aug2012ReReco_Collisions12_JSON.txt')
Aug06MuonsOnly_ll = LumiList('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/Reprocessing/Cert_190782-190949_8TeV_06Aug2012ReReco_Collisions12_JSON_MuonPhys.txt')
