def mergeProcess(*inputFiles, **options):
"""
_mergeProcess_
Creates and returns a merge process that will merge the provided
filenames
supported options:
- process_name : name of the procee, defaults to Merge
- output_file : sets the output file name
- output_lfn : sets the output LFN
"""
# //
# // process supported options
#//
processName = options.get("process_name", "Merge")
outputFilename = options.get("output_file", "Merged.root")
outputLFN = options.get("output_lfn", None)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in inputFiles:
process.source.fileNames.append(str(entry))
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outputLFN))
process.outputPath = EndPath(process.Merged)
return process
def createMergePSet(self):
"""
_createMergePSet_
Merges are a little different since we have to build the entire
process object from scratch.
"""
print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>."
cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0]
cfgFile = str(cfgFile)
self.jobSpecNode.loadConfiguration()
cfgInt = self.jobSpecNode.cfgInterface
from FWCore.ParameterSet.Config import Process, EndPath
from FWCore.ParameterSet.Modules import OutputModule, Source
import FWCore.ParameterSet.Types as CfgTypes
process = Process("Merge")
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in cfgInt.inputFiles:
process.source.fileNames.append(str(entry))
outMod = cfgInt.outputModules['Merged']
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outMod['fileName']))
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outMod['logicalFileName']))
process.Merged.catalog = CfgTypes.untracked(CfgTypes.string(
outMod['catalog']))
process.outputPath = EndPath(process.Merged)
cfgDump = open("CfgFileDump.log", 'w')
cfgDump.write(process.dumpConfig())
cfgDump.close()
handle = open(cfgFile, 'w')
handle.write("import pickle\n")
handle.write("pickledCfg=\"\"\"%s\"\"\"\n" % pickle.dumps(process))
handle.write("process = pickle.loads(pickledCfg)\n")
handle.close()
return
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource",
fileNames = cms.untracked.vstring()
)
inputFiles = ["lstore://cms-lstore.vampre/test/file2.root", "lstore://cms-lstore.vampire/test/file1.root"]
for entry in inputFiles:
process.source.fileNames.append(str(entry))
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*')
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outputLFN))
process.outputPath = EndPath(process.Merged)
# //
# // build process
#//
process = Process(processName)
# //
# // input source
#//
process.source = Source("PoolSource", fileNames=cms.untracked.vstring())
inputFiles = [
"lstore://cms-lstore.vampre/test/file2.root",
"lstore://cms-lstore.vampire/test/file1.root"
]
for entry in inputFiles:
process.source.fileNames.append(str(entry))
if dropDQM:
process.source.inputCommands = CfgTypes.untracked.vstring(
'keep *', 'drop *_EDMtoMEConverter_*_*')
# //
# // output module
#//
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(outputFilename))
if outputLFN != None:
process.Merged.logicalFileName = CfgTypes.untracked(
CfgTypes.string(outputLFN))
process.outputPath = EndPath(process.Merged)
def createMergePSet(self):
"""
_createMergePSet_
Merges are a little different since we have to build the entire
process object from scratch.
"""
print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>."
cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0]
cfgFile = str(cfgFile)
self.jobSpecNode.loadConfiguration()
cfgInt = self.jobSpecNode.cfgInterface
# taken from cmssw environment
# pylint: disable-msg=F0401
from FWCore.ParameterSet.Config import Process, EndPath
from FWCore.ParameterSet.Modules import OutputModule, Source
import FWCore.ParameterSet.Types as CfgTypes
import FWCore.ParameterSet.Config as cms
# pylint: enable-msg=F0401
processName = "Merge"
process = Process(processName)
process.source = Source("PoolSource")
process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
for entry in cfgInt.inputFiles:
process.source.fileNames.append(str(entry))
# For StoreResults, drop DQM products as they are sometimes left over
if cfgInt.configMetadata.get("annotation",None) \
== "AutoGenerated By StoreResults":
inputCommands = getattr(process.source,'inputCommands',
cms.untracked.vstring('keep *'))
inputCommands.append('drop *_EDMtoMEConverter_*_*')
process.source.inputCommands = inputCommands
outMod = cfgInt.outputModules['Merged']
process.Merged = OutputModule("PoolOutputModule")
process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(
outMod['fileName']))
process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string(
outMod['logicalFileName']))
process.Merged.catalog = CfgTypes.untracked(CfgTypes.string(
outMod['catalog']))
process.outputPath = EndPath(process.Merged)
# Apply site specific customizations
self.localCustomization(process, merge=True)
pycfgDump = open("PyCfgFileDump.log", 'w')
try:
pycfgDump.write(process.dumpPython())
except Exception, ex:
msg = "Error writing python format cfg dump:\n"
msg += "%s\n" % str(ex)
msg += "This needs to be reported to the framework team"
pycfgDump.write(msg)
