def checkMessageLoggerSvc(cfgInstance):
"""
_checkMessageLogger_
Check that the message logger is active and provides the default
job report and event logger settings for production
"""
svcs = cfgInstance.services
# //
# // Check Job report service will write the expected
#// FrameworkJobReport.xml file
if not svcs.has_key('MessageLogger'):
cfgInstance.add_(CfgModules.Service("MessageLogger"))
messageLogger = cfgInstance.services['MessageLogger']
if "fwkJobReports" not in messageLogger.parameterNames_():
messageLogger.fwkJobReports = CfgTypes.untracked(CfgTypes.vstring())
if "FrameworkJobReport" not in messageLogger.fwkJobReports:
messageLogger.fwkJobReports.append("FrameworkJobReport")
if "FrameworkJobReport" not in messageLogger.parameterNames_():
messageLogger.FrameworkJobReport = CfgTypes.untracked(CfgTypes.PSet())
messageLogger.FrameworkJobReport.default = CfgTypes.untracked(
CfgTypes.PSet(limit=CfgTypes.untracked(CfgTypes.int32(0))))
messageLogger.FrameworkJobReport.FwkJob = CfgTypes.untracked(
CfgTypes.PSet(limit=CfgTypes.untracked(CfgTypes.int32(10000000))))
# //
# // Install the per event output
#//
if "destinations" not in messageLogger.parameterNames_():
messageLogger.destinations = CfgTypes.untracked(CfgTypes.vstring())
if "EventLogger" not in messageLogger.destinations:
messageLogger.destinations.append("EventLogger")
if "EventLogger" not in messageLogger.parameterNames_():
messageLogger.EventLogger = CfgTypes.untracked(CfgTypes.PSet())
messageLogger.EventLogger.default = CfgTypes.untracked(
CfgTypes.PSet(limit=CfgTypes.untracked(CfgTypes.int32(0))))
messageLogger.FrameworkJobReport.FwkReport = CfgTypes.untracked(
CfgTypes.PSet(limit=CfgTypes.untracked(CfgTypes.int32(10000000)),
reportEvery=CfgTypes.untracked(
CfgTypes.int32(10000000))))
return
def mergeProcess(*inputFiles, **options):
"""
_mergeProcess_
Creates and returns a merge process that will merge the provided
filenames
supported options:
- process_name : name of the process, defaults to Merge
- outputmod_label : label of the output module, defaults to Merged
- newDQMIO : specifies if the new DQM format should be used to merge the files
- output_file : sets the output file name
- output_lfn : sets the output LFN
"""
# //
# // process supported options
#//
processName = options.get("process_name", "Merge")
outputModLabel = options.get("outputmod_label", "Merged")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
dropDQM = options.get("drop_dqm", False)
newDQMIO = options.get("newDQMIO", False)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
if newDQMIO:
process.source = Source("DQMRootSource")
process.add_(Service("DQMStore"))
else:
process.source = Source("PoolSource")
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*')
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in inputFiles:
process.source.fileNames.append(str(entry))
# //
# // output module
#//
if newDQMIO:
outMod = OutputModule("DQMRootOutputModule")
else:
outMod = OutputModule("PoolOutputModule")
outMod.fileName = CfgTypes.untracked.string(outputFilename)
if outputLFN != None:
outMod.logicalFileName = CfgTypes.untracked.string(outputLFN)
setattr(process, outputModLabel, outMod)
process.outputPath = EndPath(outMod)
return process
def mergeProcess(*inputFiles, **options):
"""
_mergeProcess_
Creates and returns a merge process that will merge the provided
filenames
supported options:
- process_name : name of the process, defaults to Merge
- outputmod_label : label of the output module, defaults to Merged
- newDQMIO : specifies if the new DQM format should be used to merge the files
- output_file : sets the output file name
- output_lfn : sets the output LFN
"""
# //
# // process supported options
# //
processName = options.get("process_name", "Merge")
outputModLabel = options.get("outputmod_label", "Merged")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
dropDQM = options.get("drop_dqm", False)
newDQMIO = options.get("newDQMIO", False)
# //
# // build process
# //
process = Process(processName)
# //
# // input source
# //
if newDQMIO:
process.source = Source("DQMRootSource")
process.add_(Service("DQMStore"))
else:
process.source = Source("PoolSource")
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring("keep *", "drop *_EDMtoMEConverter_*_*")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in inputFiles:
process.source.fileNames.append(str(entry))
# //
# // output module
# //
if newDQMIO:
outMod = OutputModule("DQMRootOutputModule")
else:
outMod = OutputModule("PoolOutputModule")
outMod.fileName = CfgTypes.untracked.string(outputFilename)
if outputLFN != None:
outMod.logicalFileName = CfgTypes.untracked.string(outputLFN)
setattr(process, outputModLabel, outMod)
process.outputPath = EndPath(outMod)
return process
def setPileupFilesForModule(self, sourceName, targetModule, *fileList):
"""
_setPileupFilesForSource
Insert the files provided into all target modules
for the secsource named sourceName
"""
for mixMod in self.findMixingModulesByType(targetModule):
print "Processing %s: %s " % (targetModule, mixMod)
secSource = getattr(mixMod, sourceName, None)
if secSource == None:
msg = "==============WARNING================\n"
msg += "No Input PoolRASource found for mixing module:\n"
msg += mixMod.dumpConfig()
msg += "\n With secsource named %s\n" % sourceName
msg += " Cannot add Pileup Files...\n"
msg += "======================================\n"
print msg
continue
oldfileList = getattr(secSource, "fileNames", None)
if oldfileList == None:
print "No existing file list in secsource %s" % sourceName
continue
setattr(secSource, 'fileNames',
CfgTypes.untracked(CfgTypes.vstring()))
for fileName in fileList:
secSource.fileNames.append(str(fileName))
print "Adding %s PileupFile: %s " % (sourceName, str(fileName))
return
def insertPileupFiles(self, *fileList):
"""
_insertPileupFiles_
Insert the files provided into all mixing modules
"""
for mixMod in self.mixingModules():
print "Processing MixingModule: %s " % mixMod
secSource = getattr(mixMod, "input", None)
if secSource == None:
secSource = getattr(mixMod, "secsource", None)
if secSource == None:
msg = "==============WARNING================\n"
msg += "No Input PoolRASource found for mixing module:\n"
msg += mixMod.dumpConfig()
msg += "\nCannot add Pileup Files...\n"
msg += "======================================\n"
print msg
continue
oldfileList = getattr(secSource, "fileNames", None)
if oldfileList == None:
print "No existing file list"
continue
setattr(secSource, 'fileNames',
CfgTypes.untracked(CfgTypes.vstring()))
for fileName in fileList:
secSource.fileNames.append(str(fileName))
print "PileupFile: %s " % str(fileName)
return
def setPileupFilesForModule(self, sourceName, targetModule, *fileList):
"""
_setPileupFilesForSource
Insert the files provided into all target modules
for the secsource named sourceName
"""
for mixMod in self.findMixingModulesByType(targetModule):
print "Processing %s: %s " % (targetModule, mixMod)
secSource = getattr(mixMod, sourceName, None)
if secSource == None:
msg = "==============WARNING================\n"
msg += "No Input PoolRASource found for mixing module:\n"
msg += mixMod.dumpConfig()
msg += "\n With secsource named %s\n" % sourceName
msg += " Cannot add Pileup Files...\n"
msg += "======================================\n"
print msg
continue
oldfileList = getattr(secSource, "fileNames", None)
if oldfileList == None:
print "No existing file list in secsource %s" % sourceName
continue
setattr(secSource, 'fileNames', CfgTypes.untracked(
CfgTypes.vstring()))
for fileName in fileList:
secSource.fileNames.append(str(fileName))
print "Adding %s PileupFile: %s " % (sourceName, str(fileName))
return
def insertPileupFiles(self, *fileList):
"""
_insertPileupFiles_
Insert the files provided into all mixing modules
"""
for mixMod in self.mixingModules():
print "Processing MixingModule: %s " % mixMod
secSource = getattr(mixMod, "input", None)
if secSource == None:
secSource = getattr(mixMod, "secsource", None)
if secSource == None:
msg = "==============WARNING================\n"
msg += "No Input PoolRASource found for mixing module:\n"
msg += mixMod.dumpConfig()
msg += "\nCannot add Pileup Files...\n"
msg += "======================================\n"
print msg
continue
oldfileList = getattr(secSource, "fileNames", None)
if oldfileList == None:
print "No existing file list"
continue
setattr(secSource, 'fileNames', CfgTypes.untracked(
CfgTypes.vstring()))
for fileName in fileList:
secSource.fileNames.append(str(fileName))
print "PileupFile: %s " % str(fileName)
return
def setFileNames(self, *fileNames):
"""set fileNames vector"""
if fileNames:
self.data.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in fileNames:
self.data.fileNames.append(entry)
return
def mergeProcess(*inputFiles, **options):
"""
_mergeProcess_
Creates and returns a merge process that will merge the provided
filenames
supported options:
- process_name : name of the procee, defaults to Merge
- output_file : sets the output file name
- output_lfn : sets the output LFN
"""
# //
# // process supported options
#//
processName = options.get("process_name", "Merge")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in inputFiles:
process.source.fileNames.append(str(entry))
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outputLFN))
process.outputPath = EndPath(process.Merged)
return process
def createMergePSet(self):
"""
_createMergePSet_
Merges are a little different since we have to build the entire
process object from scratch.
"""
print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>."
cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0]
cfgFile = str(cfgFile)
self.jobSpecNode.loadConfiguration()
cfgInt = self.jobSpecNode.cfgInterface
from FWCore.ParameterSet.Config import Process, EndPath
from FWCore.ParameterSet.Modules import OutputModule, Source
import FWCore.ParameterSet.Types as CfgTypes
process = Process("Merge")
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in cfgInt.inputFiles:
process.source.fileNames.append(str(entry))
outMod = cfgInt.outputModules['Merged']
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outMod['fileName']))
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outMod['logicalFileName']))
process.Merged.catalog = CfgTypes.untracked(CfgTypes.string(
outMod['catalog']))
process.outputPath = EndPath(process.Merged)
cfgDump = open("CfgFileDump.log", 'w')
cfgDump.write(process.dumpConfig())
cfgDump.close()
handle = open(cfgFile, 'w')
handle.write("import pickle\n")
handle.write("pickledCfg=\"\"\"%s\"\"\"\n" % pickle.dumps(process))
handle.write("process = pickle.loads(pickledCfg)\n")
handle.close()
return
def mergeProcess(*inputFiles, **options):
"""
_mergeProcess_
Creates and returns a merge process that will merge the provided
filenames
supported options:
- process_name : name of the process, defaults to Merge
- outputmod_label : label of the output module, defaults to Merged
- newDQMIO : specifies if the new DQM format should be used to merge the files
- output_file : sets the output file name
- output_lfn : sets the output LFN
- mergeNANO : to merge NanoAOD
- bypassVersionCheck : to bypass version check in case merging happened in lower version of CMSSW (i.e. UL HLT case). This will be TRUE by default.
"""
# //
# // process supported options
#//
processName = options.get("process_name", "Merge")
outputModLabel = options.get("outputmod_label", "Merged")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
dropDQM = options.get("drop_dqm", False)
newDQMIO = options.get("newDQMIO", False)
mergeNANO = options.get("mergeNANO", False)
bypassVersionCheck = options.get("bypassVersionCheck", True)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
if newDQMIO:
process.source = Source("DQMRootSource")
process.add_(Service("DQMStore"))
else:
process.source = Source("PoolSource")
process.source.bypassVersionCheck = CfgTypes.untracked.bool(bypassVersionCheck)
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*')
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in inputFiles:
process.source.fileNames.append(str(entry))
# //
# // output module
#//
if newDQMIO:
outMod = OutputModule("DQMRootOutputModule")
elif mergeNANO:
import Configuration.EventContent.EventContent_cff
outMod = OutputModule("NanoAODOutputModule",Configuration.EventContent.EventContent_cff.NANOAODEventContent.clone())
process.add_(Service("InitRootHandlers", EnableIMT = CfgTypes.untracked.bool(False)))
else:
outMod = OutputModule("PoolOutputModule")
outMod.fileName = CfgTypes.untracked.string(outputFilename)
if outputLFN != None:
outMod.logicalFileName = CfgTypes.untracked.string(outputLFN)
setattr(process, outputModLabel, outMod)
process.outputPath = EndPath(outMod)
return process
def setSecondaryFileNames(self, *fileNames):
"""set secondary file names vector"""
self.data.secondaryFileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in fileNames:
self.data.secondaryFileNames.append(entry)
return
#//
processName = options.get("process_name", "Merge")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
dropDQM = options.get("drop_dqm", False)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
inputFiles = ["redd://import/test/file1.root", "redd://import/test/file2.root"]
for entry in inputFiles:
process.source.fileNames.append(str(entry))
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*')
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
def mergeProcess(*inputFiles, **options):
"""
_mergeProcess_
Creates and returns a merge process that will merge the provided
filenames
supported options:
- process_name : name of the process, defaults to Merge
- outputmod_label : label of the output module, defaults to Merged
- newDQMIO : specifies if the new DQM format should be used to merge the files
- output_file : sets the output file name
- output_lfn : sets the output LFN
- mergeNANO : to merge NanoAOD
- bypassVersionCheck : to bypass version check in case merging happened in lower version of CMSSW (i.e. UL HLT case). This will be FALSE by default.
"""
# //
# // process supported options
#//
processName = options.get("process_name", "Merge")
outputModLabel = options.get("outputmod_label", "Merged")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
dropDQM = options.get("drop_dqm", False)
newDQMIO = options.get("newDQMIO", False)
mergeNANO = options.get("mergeNANO", False)
bypassVersionCheck = options.get("bypassVersionCheck", False)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
if newDQMIO:
process.source = Source("DQMRootSource")
process.add_(Service("DQMStore", forceResetOnBeginLumi = CfgTypes.untracked.bool(True)))
else:
process.source = Source("PoolSource")
if bypassVersionCheck:
process.source.bypassVersionCheck = CfgTypes.untracked.bool(True)
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*')
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in inputFiles:
process.source.fileNames.append(str(entry))
# //
# // output module
#//
if newDQMIO:
outMod = OutputModule("DQMRootOutputModule")
elif mergeNANO:
import Configuration.EventContent.EventContent_cff
outMod = OutputModule("NanoAODOutputModule",Configuration.EventContent.EventContent_cff.NANOAODEventContent.clone())
process.add_(Service("InitRootHandlers", EnableIMT = CfgTypes.untracked.bool(False)))
else:
outMod = OutputModule("PoolOutputModule")
outMod.fileName = CfgTypes.untracked.string(outputFilename)
if outputLFN != None:
outMod.logicalFileName = CfgTypes.untracked.string(outputLFN)
setattr(process, outputModLabel, outMod)
process.outputPath = EndPath(outMod)
return process
def checkMessageLoggerSvc(cfgInstance):
"""
_checkMessageLogger_
Check that the message logger is active and provides the default
job report and event logger settings for production
"""
svcs = cfgInstance.services
# //
# // Check Job report service will write the expected
#// FrameworkJobReport.xml file
if not svcs.has_key('MessageLogger'):
cfgInstance.add_(CfgModules.Service("MessageLogger"))
messageLogger = cfgInstance.services['MessageLogger']
if "fwkJobReports" not in messageLogger.parameterNames_():
messageLogger.fwkJobReports = CfgTypes.untracked(
CfgTypes.vstring()
)
if "FrameworkJobReport" not in messageLogger.fwkJobReports:
messageLogger.fwkJobReports.append("FrameworkJobReport")
if "FrameworkJobReport" not in messageLogger.parameterNames_():
messageLogger.FrameworkJobReport = CfgTypes.untracked(CfgTypes.PSet())
messageLogger.FrameworkJobReport.default = CfgTypes.untracked(
CfgTypes.PSet(
limit = CfgTypes.untracked(CfgTypes.int32(0))
)
)
messageLogger.FrameworkJobReport.FwkJob = CfgTypes.untracked(
CfgTypes.PSet(
limit = CfgTypes.untracked(CfgTypes.int32(10000000))
) )
# //
# // Install the per event output
#//
if "destinations" not in messageLogger.parameterNames_():
messageLogger.destinations = CfgTypes.untracked(
CfgTypes.vstring()
)
if "EventLogger" not in messageLogger.destinations:
messageLogger.destinations.append("EventLogger")
if "EventLogger" not in messageLogger.parameterNames_():
messageLogger.EventLogger = CfgTypes.untracked(CfgTypes.PSet())
messageLogger.EventLogger.default = CfgTypes.untracked(
CfgTypes.PSet(
limit = CfgTypes.untracked(CfgTypes.int32(0))
)
)
messageLogger.FrameworkJobReport.FwkReport = CfgTypes.untracked(
CfgTypes.PSet(
limit = CfgTypes.untracked(CfgTypes.int32(10000000)),
reportEvery = CfgTypes.untracked(CfgTypes.int32(10000000))
) )
return
def createMergePSet(self):
"""
_createMergePSet_
Merges are a little different since we have to build the entire
process object from scratch.
"""
print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>."
cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0]
cfgFile = str(cfgFile)
self.jobSpecNode.loadConfiguration()
cfgInt = self.jobSpecNode.cfgInterface
# taken from cmssw environment
# pylint: disable-msg=F0401
from FWCore.ParameterSet.Config import Process, EndPath
from FWCore.ParameterSet.Modules import OutputModule, Source
import FWCore.ParameterSet.Types as CfgTypes
import FWCore.ParameterSet.Config as cms
# pylint: enable-msg=F0401
processName = "Merge"
process = Process(processName)
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in cfgInt.inputFiles:
process.source.fileNames.append(str(entry))
# For StoreResults, drop DQM products as they are sometimes left over
if cfgInt.configMetadata.get("annotation",None) \
== "AutoGenerated By StoreResults":
inputCommands = getattr(process.source,'inputCommands',
cms.untracked.vstring('keep *'))
inputCommands.append('drop *_EDMtoMEConverter_*_*')
process.source.inputCommands = inputCommands
outMod = cfgInt.outputModules['Merged']
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outMod['fileName']))
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outMod['logicalFileName']))
process.Merged.catalog = CfgTypes.untracked(CfgTypes.string(
outMod['catalog']))
process.outputPath = EndPath(process.Merged)
# Apply site specific customizations
self.localCustomization(process, merge=True)
pycfgDump = open("PyCfgFileDump.log", 'w')
try:
pycfgDump.write(process.dumpPython())
except Exception, ex:
msg = "Error writing python format cfg dump:\n"
msg += "%s\n" % str(ex)
msg += "This needs to be reported to the framework team"
pycfgDump.write(msg)
