def makeProcess(numEvents = 200):
"""
_makeProcess_
Create a new Process instance
"""
proc = Process("HLT")
proc.include("FWCore/MessageLogger/data/MessageLogger.cfi")
configName = "fake-streamer-config"
configVersion = timestamp()
configAnnot = "auto generated fake streamer config"
proc.configurationMetadata = CmsTypes.untracked(CmsTypes.PSet())
proc.configurationMetadata.name = CmsTypes.untracked(CmsTypes.string(
configName))
proc.configurationMetadata.version = CmsTypes.untracked(CmsTypes.string(
configVersion))
proc.configurationMetadata.annotation = CmsTypes.untracked(CmsTypes.string(
configAnnot))
proc.options = CmsTypes.untracked(CmsTypes.PSet())
proc.options.wantSummary = CmsTypes.untracked(CmsTypes.bool(True))
proc.source = Source("EmptySource")
proc.maxEvents = CmsTypes.untracked(CmsTypes.PSet())
proc.maxEvents.input = CmsTypes.untracked(CmsTypes.int32(numEvents))
proc.prod = EDProducer("StreamThingProducer")
proc.prod.array_size = CmsTypes.int32(2500)
proc.prod.instance_count = CmsTypes.int32(150)
proc.prod.apply_bit_mask = CmsTypes.untracked(CmsTypes.bool(True))
proc.prod.bit_mask = CmsTypes.untracked( CmsTypes.uint32( 16777215))
proc.add_(Service("RandomNumberGeneratorService"))
svc = proc.services["RandomNumberGeneratorService"]
svc.moduleSeeds = CmsTypes.PSet()
proc.makeData = Path(proc.prod)
return proc
def mergeProcess(*inputFiles, **options):
"""
_mergeProcess_
Creates and returns a merge process that will merge the provided
filenames
supported options:
- process_name : name of the procee, defaults to Merge
- output_file : sets the output file name
- output_lfn : sets the output LFN
"""
# //
# // process supported options
#//
processName = options.get("process_name", "Merge")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in inputFiles:
process.source.fileNames.append(str(entry))
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outputLFN))
process.outputPath = EndPath(process.Merged)
return process
def setLogicalFileName(self, lfn):
"""
_setLogicalFileName_
"""
self.data.logicalFileName = CfgTypes.untracked(CfgTypes.string(lfn))
return
def setLogicalFileName(self, lfn):
"""
_setLogicalFileName_
"""
self.data.logicalFileName = CfgTypes.untracked(CfgTypes.string(lfn))
return
def insertSeeds(self, *seeds):
"""
_insertSeeds_
Insert the list of seeds into the RandomNumber Service
"""
seedList = list(seeds)
if "RandomNumberGeneratorService" not in self.data.services.keys():
return
if self.hasOldSeeds():
self.insertOldSeeds(*seeds)
return
# //
# // Use seed service utility to generate seeds on the fly
#//
svc = self.data.services["RandomNumberGeneratorService"]
try:
from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
except ImportError:
msg = "Unable to import RandomeNumberServiceHelper"
print msg
raise RuntimeError, msg
randHelper = RandomNumberServiceHelper(svc)
randHelper.populate()
svc.saveFileName = CfgTypes.untracked(
CfgTypes.string("RandomEngineState.log"))
return
def insertSeeds(self, *seeds):
"""
_insertSeeds_
Insert the list of seeds into the RandomNumber Service
"""
seedList = list(seeds)
if "RandomNumberGeneratorService" not in self.data.services.keys():
return
if self.hasOldSeeds():
self.insertOldSeeds(*seeds)
return
# //
# // Use seed service utility to generate seeds on the fly
#//
svc = self.data.services["RandomNumberGeneratorService"]
try:
from IOMC.RandomEngine.RandomServiceHelper import RandomNumberServiceHelper
except ImportError:
msg = "Unable to import RandomeNumberServiceHelper"
print msg
raise RuntimeError, msg
randHelper = RandomNumberServiceHelper(svc)
randHelper.populate()
svc.saveFileName = CfgTypes.untracked(
CfgTypes.string("RandomEngineState.log"))
return
def createMergePSet(self):
"""
_createMergePSet_
Merges are a little different since we have to build the entire
process object from scratch.
"""
print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>."
cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0]
cfgFile = str(cfgFile)
self.jobSpecNode.loadConfiguration()
cfgInt = self.jobSpecNode.cfgInterface
from FWCore.ParameterSet.Config import Process, EndPath
from FWCore.ParameterSet.Modules import OutputModule, Source
import FWCore.ParameterSet.Types as CfgTypes
process = Process("Merge")
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in cfgInt.inputFiles:
process.source.fileNames.append(str(entry))
outMod = cfgInt.outputModules['Merged']
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outMod['fileName']))
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outMod['logicalFileName']))
process.Merged.catalog = CfgTypes.untracked(CfgTypes.string(
outMod['catalog']))
process.outputPath = EndPath(process.Merged)
cfgDump = open("CfgFileDump.log", 'w')
cfgDump.write(process.dumpConfig())
cfgDump.close()
handle = open(cfgFile, 'w')
handle.write("import pickle\n")
handle.write("pickledCfg=\"\"\"%s\"\"\"\n" % pickle.dumps(process))
handle.write("process = pickle.loads(pickledCfg)\n")
handle.close()
return
def setCatalog(self, newCatalog):
"""
_setCatalog_
Set the catalog name
"""
self.data.catalog = CfgTypes.untracked(CfgTypes.string(newCatalog))
return
def setCatalog(self, newCatalog):
"""
_setCatalog_
Set the catalog name
"""
self.data.catalog = CfgTypes.untracked(CfgTypes.string(newCatalog))
return
def addOutputModule(process, fileName):
"""
_addOutputModule_
"""
outMod = OutputModule("EventStreamFileWriter")
outMod.max_event_size = CmsTypes.untracked(CmsTypes.int32(7000000))
outMod.max_queue_depth = CmsTypes.untracked(CmsTypes.int32(5))
outMod.use_compression = CmsTypes.untracked(CmsTypes.bool(True))
outMod.compression_level = CmsTypes.untracked(CmsTypes.int32(1))
streamer = "%s.dat" % fileName
indexFile = "%s.ind" % fileName
outMod.fileName = CmsTypes.untracked(CmsTypes.string(streamer))
outMod.indexFileName = CmsTypes.untracked(CmsTypes.string(indexFile))
process.out = outMod
process.o = EndPath(process.out)
return
def setConditionsTag(self, condTag):
"""
_setConditionsTag-
Set the conditions glbal tag
"""
globalPSet = getattr(self.data, "GlobalTag", None)
if globalPSet == None:
return
globalTag = getattr(globalPSet, "globaltag", None)
if globalTag == None:
globalPSet.globalTag = CfgTypes.string(condTag)
else:
globalPSet.globaltag = condTag
return
def setConditionsTag(self, condTag):
"""
_setConditionsTag-
Set the conditions glbal tag
"""
globalPSet = getattr(
self.data, "GlobalTag", None)
if globalPSet == None:
return
globalTag = getattr(globalPSet, "globaltag", None)
if globalTag == None:
globalPSet.globalTag = CfgTypes.string(condTag)
else:
globalPSet.globaltag = condTag
return
def createPSet(self):
"""
_createPSet_
Create the PSet cfg File
"""
cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0]
cfgFile = str(cfgFile)
self.jobSpecNode.loadConfiguration()
self.jobSpecNode.cfgInterface.rawCfg = self.workflowNode.cfgInterface.rawCfg
# taken from cmssw environment
# pylint: disable-msg=F0401
import FWCore.ParameterSet.Types as CfgTypes
# pylint: enable-msg=F0401
workingDir = os.path.join(os.getcwd(), 'prestage')
if os.path.exists(workingDir + '/prestageTFC.xml'):
rawCfg = pickle.loads(self.jobSpecNode.cfgInterface.rawCfg)
rawCfg.source.overrideCatalog = CfgTypes.untracked(CfgTypes.string('trivialcatalog_file:%s/prestageTFC.xml?protocol=local-stage-in' % workingDir))
self.jobSpecNode.cfgInterface.rawCfg = pickle.dumps(rawCfg)
# Apply site specific customizations
self.localCustomization(self.jobSpecNode.cfgInterface)
for inpLink in self.jobSpecNode._InputLinks:
# //
# // We have in-job input links to be resolved
#//
self.handleInputLink(self.jobSpecNode.cfgInterface, inpLink)
cmsProcess = self.jobSpecNode.cfgInterface.makeConfiguration()
pycfgDump = open("PyCfgFileDump.log", 'w')
try:
pycfgDump.write(cmsProcess.dumpPython())
except Exception, ex:
msg = "Error writing python format cfg dump:\n"
msg += "%s\n" % str(ex)
msg += "This needs to be reported to the framework team"
pycfgDump.write(msg)
def ioCustomization(self, config, custom_config, merge = False):
"""
Apply site specific io customizations
"""
# Don't do anything if no customization or job has no input files
if not custom_config or (merge is False and not config.inputFiles):
return
import re
version = lambda x: tuple(int(x) for x in re.compile('(\d+)').findall(x))
cmssw_version = version(os.environ['CMSSW_VERSION'])
# Only implemented in CMSSW_2_1_8 and above
if cmssw_version < (2, 1, 8):
return
print "Site specific IO parameters will be used:"
# cacheSize is a property of InputSource
cache_size = custom_config.get('cacheSize', None)
if cache_size:
# Merge pset creates process on fly so can't use CMSSWConfig object
if merge:
from ProdCommon.CMSConfigTools.ConfigAPI.InputSource import InputSource
inputSource = InputSource(config.source)
inputSource.setCacheSize(cache_size)
else:
config.sourceParams['cacheSize'] = cache_size
if merge:
from FWCore.ParameterSet.Modules import Service
config.add_(Service('AdaptorConfig'))
for param in custom_config:
print " %s %s" % (param, custom_config[param])
if param == 'cacheSize':
continue
if merge:
import FWCore.ParameterSet.Types as CfgTypes
adaptor = config.services['AdaptorConfig']
setattr(adaptor, param,
CfgTypes.untracked(CfgTypes.string(str(custom_config[param]))))
else:
config.tFileAdaptorConfig[param] = custom_config[param]
return
def addDataset(self, **datasetParams):
"""
_addDataset_
Add Dataset information to this cfg file. Each key, value pair
provided in dataset Params is added as an untracked string to
the datasets PSet in a PSet named entryName
"""
datasetPSet = getattr(self.data, "dataset", None)
if datasetPSet == None:
self.data.dataset = CfgTypes.untracked(CfgTypes.PSet())
datasetPSet = self.data.dataset
for key, value in datasetParams.items():
setattr(datasetPSet, key,
CfgTypes.untracked(CfgTypes.string(str(value))))
return
def addDataset(self, **datasetParams):
"""
_addDataset_
Add Dataset information to this cfg file. Each key, value pair
provided in dataset Params is added as an untracked string to
the datasets PSet in a PSet named entryName
"""
datasetPSet = getattr(self.data, "dataset", None)
if datasetPSet == None:
self.data.dataset = CfgTypes.untracked(CfgTypes.PSet())
datasetPSet = self.data.dataset
for key, value in datasetParams.items():
setattr(
datasetPSet, key,
CfgTypes.untracked(CfgTypes.string(str(value)))
)
return
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource",
fileNames = cms.untracked.vstring()
)
inputFiles = ["lstore://cms-lstore.vampre/test/file2.root", "lstore://cms-lstore.vampire/test/file1.root"]
for entry in inputFiles:
process.source.fileNames.append(str(entry))
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*')
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outputLFN))
process.outputPath = EndPath(process.Merged)
def setOverrideCatalog(self, catalog, protocol):
"""set overrideCatalog to allow local files to be read via LFN"""
uri = "trivialcatalog_file:%s?protocol=%s" % (catalog, protocol)
self.data.overrideCatalog = CfgTypes.untracked(CfgTypes.string(uri))
def setFileMatchMode(self, matchMode):
"""set file match mode for reading files in same job"""
self.data.fileMatchMode = CfgTypes.untracked(
CfgTypes.string(matchMode))
cacheHint = cms.untracked.string("application-only"),
readHint = cms.untracked.string("direct-unbuffered"))
process.RECO = cms.OutputModule("PoolOutputModule")
process.RECO.dataset = cms.untracked(cms.PSet())
process.RECO.dataset.dataTier = cms.untracked(cms.string("RECO"))
process.RECO.fileName = cms.untracked.string("NOTSET")
process.RECO.logicalFileName = cms.untracked.string("NOTSET")
process.RECO.fastCloning = cms.untracked.bool(False)
process.outpath = cms.EndPath(process.RECO)
configName = "prompt-reco-config"
configVersion = "%s-%s-%s" % (self.cmssw["CMSSWVersion"], self.run,
self.primaryDataset)
configAnnot = "auto generated prompt reco config"
process.configurationMetadata = CmsTypes.untracked(CmsTypes.PSet())
process.configurationMetadata.name = CmsTypes.untracked(CmsTypes.string(configName))
process.configurationMetadata.version = CmsTypes.untracked(CmsTypes.string(configVersion))
process.configurationMetadata.annotation = CmsTypes.untracked(CmsTypes.string(configAnnot))
cfgInt = cfgWrapper.loadConfiguration(process)
cfgWrapper.conditionsTag = "NOTSET"
cfgInt.validateForProduction()
setattr(self.workflow.payload, "cfgInterface", cfgWrapper)
loader.unload()
return
def setFileName(self, fname):
"""
_setFileName_
"""
self.data.fileName = CfgTypes.untracked(CfgTypes.string(fname))
def setFileName(self, fname):
"""
_setFileName_
"""
self.data.fileName = CfgTypes.untracked(CfgTypes.string(fname))
def createMergePSet(self):
"""
_createMergePSet_
Merges are a little different since we have to build the entire
process object from scratch.
"""
print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>."
cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0]
cfgFile = str(cfgFile)
self.jobSpecNode.loadConfiguration()
cfgInt = self.jobSpecNode.cfgInterface
# taken from cmssw environment
# pylint: disable-msg=F0401
from FWCore.ParameterSet.Config import Process, EndPath
from FWCore.ParameterSet.Modules import OutputModule, Source
import FWCore.ParameterSet.Types as CfgTypes
import FWCore.ParameterSet.Config as cms
# pylint: enable-msg=F0401
processName = "Merge"
process = Process(processName)
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in cfgInt.inputFiles:
process.source.fileNames.append(str(entry))
# For StoreResults, drop DQM products as they are sometimes left over
if cfgInt.configMetadata.get("annotation",None) \
== "AutoGenerated By StoreResults":
inputCommands = getattr(process.source,'inputCommands',
cms.untracked.vstring('keep *'))
inputCommands.append('drop *_EDMtoMEConverter_*_*')
process.source.inputCommands = inputCommands
outMod = cfgInt.outputModules['Merged']
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outMod['fileName']))
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outMod['logicalFileName']))
process.Merged.catalog = CfgTypes.untracked(CfgTypes.string(
outMod['catalog']))
process.outputPath = EndPath(process.Merged)
# Apply site specific customizations
self.localCustomization(process, merge=True)
pycfgDump = open("PyCfgFileDump.log", 'w')
try:
pycfgDump.write(process.dumpPython())
except Exception, ex:
msg = "Error writing python format cfg dump:\n"
msg += "%s\n" % str(ex)
msg += "This needs to be reported to the framework team"
pycfgDump.write(msg)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource", fileNames=cms.untracked.vstring())
inputFiles = [
"lstore://cms-lstore.vampre/test/file2.root",
"lstore://cms-lstore.vampire/test/file1.root"
]
for entry in inputFiles:
process.source.fileNames.append(str(entry))
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring(
'keep *', 'drop *_EDMtoMEConverter_*_*')
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(
CfgTypes.string(outputLFN))
process.outputPath = EndPath(process.Merged)