def mergeProcess(*inputFiles, **options): """ _mergeProcess_ Creates and returns a merge process that will merge the provided filenames supported options: - process_name : name of the procee, defaults to Merge - output_file : sets the output file name - output_lfn : sets the output LFN """ # // # // process supported options #// processName = options.get("process_name", "Merge") outputFilename = options.get("output_file", "Merged.root") outputLFN = options.get("output_lfn", None) # // # // build process #// process = Process(processName) # // # // input source #// process.source = Source("PoolSource") process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring()) for entry in inputFiles: process.source.fileNames.append(str(entry)) # // # // output module #// process.Merged = OutputModule("PoolOutputModule") process.Merged.fileName = CfgTypes.untracked(CfgTypes.string( outputFilename)) if outputLFN != None: process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string( outputLFN)) process.outputPath = EndPath(process.Merged) return process
def createMergePSet(self): """ _createMergePSet_ Merges are a little different since we have to build the entire process object from scratch. """ print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>." cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0] cfgFile = str(cfgFile) self.jobSpecNode.loadConfiguration() cfgInt = self.jobSpecNode.cfgInterface from FWCore.ParameterSet.Config import Process, EndPath from FWCore.ParameterSet.Modules import OutputModule, Source import FWCore.ParameterSet.Types as CfgTypes process = Process("Merge") process.source = Source("PoolSource") process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring()) for entry in cfgInt.inputFiles: process.source.fileNames.append(str(entry)) outMod = cfgInt.outputModules['Merged'] process.Merged = OutputModule("PoolOutputModule") process.Merged.fileName = CfgTypes.untracked(CfgTypes.string( outMod['fileName'])) process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string( outMod['logicalFileName'])) process.Merged.catalog = CfgTypes.untracked(CfgTypes.string( outMod['catalog'])) process.outputPath = EndPath(process.Merged) cfgDump = open("CfgFileDump.log", 'w') cfgDump.write(process.dumpConfig()) cfgDump.close() handle = open(cfgFile, 'w') handle.write("import pickle\n") handle.write("pickledCfg=\"\"\"%s\"\"\"\n" % pickle.dumps(process)) handle.write("process = pickle.loads(pickledCfg)\n") handle.close() return
# // build process #// process = Process(processName) # // # // input source #// process.source = Source("PoolSource", fileNames = cms.untracked.vstring() ) inputFiles = ["lstore://cms-lstore.vampre/test/file2.root", "lstore://cms-lstore.vampire/test/file1.root"] for entry in inputFiles: process.source.fileNames.append(str(entry)) if dropDQM: process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*') # // # // output module #// process.Merged = OutputModule("PoolOutputModule") process.Merged.fileName = CfgTypes.untracked(CfgTypes.string( outputFilename)) if outputLFN != None: process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string( outputLFN)) process.outputPath = EndPath(process.Merged)
# // # // build process #// process = Process(processName) # // # // input source #// process.source = Source("PoolSource", fileNames=cms.untracked.vstring()) inputFiles = [ "lstore://cms-lstore.vampre/test/file2.root", "lstore://cms-lstore.vampire/test/file1.root" ] for entry in inputFiles: process.source.fileNames.append(str(entry)) if dropDQM: process.source.inputCommands = CfgTypes.untracked.vstring( 'keep *', 'drop *_EDMtoMEConverter_*_*') # // # // output module #// process.Merged = OutputModule("PoolOutputModule") process.Merged.fileName = CfgTypes.untracked(CfgTypes.string(outputFilename)) if outputLFN != None: process.Merged.logicalFileName = CfgTypes.untracked( CfgTypes.string(outputLFN)) process.outputPath = EndPath(process.Merged)
def createMergePSet(self): """ _createMergePSet_ Merges are a little different since we have to build the entire process object from scratch. """ print "<<<<<<<<<<<<<<<<<<<<Merge>>>>>>>>>>>>>>>>>>>>>." cfgFile = self.config['Configuration'].get("CfgFile", "PSet.py")[0] cfgFile = str(cfgFile) self.jobSpecNode.loadConfiguration() cfgInt = self.jobSpecNode.cfgInterface # taken from cmssw environment # pylint: disable-msg=F0401 from FWCore.ParameterSet.Config import Process, EndPath from FWCore.ParameterSet.Modules import OutputModule, Source import FWCore.ParameterSet.Types as CfgTypes import FWCore.ParameterSet.Config as cms # pylint: enable-msg=F0401 processName = "Merge" process = Process(processName) process.source = Source("PoolSource") process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring()) for entry in cfgInt.inputFiles: process.source.fileNames.append(str(entry)) # For StoreResults, drop DQM products as they are sometimes left over if cfgInt.configMetadata.get("annotation",None) \ == "AutoGenerated By StoreResults": inputCommands = getattr(process.source,'inputCommands', cms.untracked.vstring('keep *')) inputCommands.append('drop *_EDMtoMEConverter_*_*') process.source.inputCommands = inputCommands outMod = cfgInt.outputModules['Merged'] process.Merged = OutputModule("PoolOutputModule") process.Merged.fileName = CfgTypes.untracked(CfgTypes.string( outMod['fileName'])) process.Merged.logicalFileName = CfgTypes.untracked(CfgTypes.string( outMod['logicalFileName'])) process.Merged.catalog = CfgTypes.untracked(CfgTypes.string( outMod['catalog'])) process.outputPath = EndPath(process.Merged) # Apply site specific customizations self.localCustomization(process, merge=True) pycfgDump = open("PyCfgFileDump.log", 'w') try: pycfgDump.write(process.dumpPython()) except Exception, ex: msg = "Error writing python format cfg dump:\n" msg += "%s\n" % str(ex) msg += "This needs to be reported to the framework team" pycfgDump.write(msg)