def testPSetFixup(self): """ _testPSetFixup_ Verify that all necessary parameters are set in the PSet. """ from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset if os.environ.get('CMSSW_VERSION', None): del os.environ['CMSSW_VERSION'] setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = self.testDir setupScript.job = self.createTestJob() setupScript() fixedPSet = self.loadProcessFromPSet() self.assertEqual(len(fixedPSet.source.fileNames.value), 2, "Error: Wrong number of files.") self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2, "Error: Wrong number of secondary files.") self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.maxEvents.input.value, -1, "Error: Wrong maxEvents.")
def testChainedProcesing(self): """ test for chained CMSSW processing - check the overriden TFC, its values and that input files is / are set correctly. """ from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset os.environ['CMSSW_VERSION'] = "CMSSW_7_6_0" setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = self.testDir setupScript.job = self.createTestJob() setupScript.step.setupChainedProcessing("my_first_step", "my_input_module") setupScript() # test if the overriden TFC is right self.assertTrue(hasattr(setupScript.step.data.application, "overrideCatalog"), "Error: overriden TFC was not set") tfc = loadTFC(setupScript.step.data.application.overrideCatalog) inputFile = "../my_first_step/my_input_module.root" self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile) self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile) self.assertEqual(setupScript.process.source.fileNames.value, [inputFile])
def testEventsPerLumi(self): """ _testEventsPerLumi_ Verify that you can put in events per lumi in the process. """ from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.step.setEventsPerLumi(500) setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = self.testDir setupScript.job = self.createTestJob() setupScript() fixedPSet = self.loadProcessFromPSet() self.assertEqual(len(fixedPSet.source.fileNames.value), 2, "Error: Wrong number of files.") self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2, "Error: Wrong number of secondary files.") self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock.value, 500, "Error: Wrong number of events per luminosity block") self.assertEqual(fixedPSet.maxEvents.input.value, -1, "Error: Wrong maxEvents.")
def testPSetFixup(self): """ _testPSetFixup_ Verify that all necessary parameters are set in the PSet. """ from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = self.testDir setupScript.job = self.createTestJob() setupScript() fixedPSetHandle = open(os.path.join(self.testDir, "PSet.pkl")) fixedPSet = pickle.load(fixedPSetHandle) fixedPSetHandle.close() self.assertEqual(len(fixedPSet.source.fileNames.value), 2, "Error: Wrong number of files.") self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2, "Error: Wrong number of secondary files.") self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.maxEvents.input.value, -1, "Error: Wrong maxEvents.")
def testChainedProcesing(self): """ test for chained CMSSW processing - check the overriden TFC, its values and that input files is / are set correctly. """ from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset os.environ['CMSSW_VERSION'] = "CMSSW_7_6_0" setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name="stepSpace") setupScript.stepSpace.location = self.testDir setupScript.job = self.createTestJob() setupScript.step.setupChainedProcessing("my_first_step", "my_input_module") setupScript() # test if the overriden TFC is right self.assertTrue( hasattr(setupScript.step.data.application, "overrideCatalog"), "Error: overriden TFC was not set") tfc = loadTFC(setupScript.step.data.application.overrideCatalog) inputFile = "../my_first_step/my_input_module.root" self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile) self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile) self.assertEqual(setupScript.process.source.fileNames.value, [inputFile])
def testPSetFixup(self): """ _testPSetFixup_ Verify that all necessary parameters are set in the PSet. """ from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset if os.environ.get('CMSSW_VERSION', None): del os.environ['CMSSW_VERSION'] setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name="stepSpace") setupScript.stepSpace.location = self.testDir setupScript.job = self.createTestJob() setupScript() fixedPSet = self.loadProcessFromPSet() self.assertEqual(len(fixedPSet.source.fileNames.value), 2, "Error: Wrong number of files.") self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2, "Error: Wrong number of secondary files.") self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.maxEvents.input.value, -1, "Error: Wrong maxEvents.")
def __init__(self, location, inputFiles, runAndLumis, agentNumber, lfnBase, outputMods, firstEvent=0, lastEvent=-1, firstLumi=None,\ firstRun=None, seeding=None, lheInputFiles=False, oneEventMode=False, eventsPerLumi=None, maxRuntime=None): SetupCMSSWPset.__init__(self, crabPSet=True) self.stepSpace = ConfigSection() self.stepSpace.location = location self.step = StepConfiguration(lfnBase, outputMods) self.step.section_("data") self.step.data._internal_name = "cmsRun" self.step.data.section_("application") self.step.data.application.section_("configuration") self.step.data.application.section_("command") self.step.data.application.section_("multicore") self.step.data.application.command.configuration = "PSet.py" self.step.data.application.command.oneEventMode = oneEventMode in [ "1", "True", True ] self.step.data.application.command.memoryCheck = False self.step.data.application.command.silentMemoryCheck = True # self.step.data.application.configuration.pickledarguments.globalTag/globalTagTransaction if eventsPerLumi: self.step.data.application.configuration.eventsPerLumi = eventsPerLumi if maxRuntime: self.step.data.application.configuration.maxSecondsUntilRampdown = maxRuntime self.step.data.section_("input") self.job = jobDict(lheInputFiles, seeding) self.job["input_files"] = [] for inputF in inputFiles: if isinstance(inputF, basestring): self.job["input_files"].append({"lfn": inputF, "parents": ""}) else: self.job["input_files"].append(inputF) self.job['mask'] = Mask() self.job['mask']["FirstEvent"] = firstEvent self.job['mask']["LastEvent"] = lastEvent self.job['mask']["FirstRun"] = firstRun self.job['mask']["FirstLumi"] = firstLumi self.job['mask']["runAndLumis"] = runAndLumis self.job['agentNumber'] = agentNumber self.job['counter'] = 0
def __init__(self, location, inputFiles, runAndLumis, agentNumber, lfnBase, outputMods, firstEvent=0, lastEvent=-1, firstLumi=None,\ firstRun=None, seeding=None, lheInputFiles=False, oneEventMode=False, eventsPerLumi=None, maxRuntime=None): SetupCMSSWPset.__init__(self, crabPSet=True) self.stepSpace = ConfigSection() self.stepSpace.location = location self.step = StepConfiguration(lfnBase, outputMods) self.step.section_("data") self.step.data._internal_name = "cmsRun" self.step.data.section_("application") self.step.data.application.section_("configuration") self.step.data.application.section_("command") self.step.data.application.section_("multicore") self.step.data.application.command.configuration = "PSet.py" self.step.data.application.command.oneEventMode = oneEventMode in ["1", "True", True] self.step.data.application.command.memoryCheck = False self.step.data.application.command.silentMemoryCheck = True # self.step.data.application.configuration.pickledarguments.globalTag/globalTagTransaction if eventsPerLumi: self.step.data.application.configuration.eventsPerLumi = eventsPerLumi if maxRuntime: self.step.data.application.configuration.maxSecondsUntilRampdown = maxRuntime self.step.data.section_("input") self.job = jobDict(lheInputFiles, seeding) self.job["input_files"] = [] for inputF in inputFiles: if isinstance(inputF, basestring): self.job["input_files"].append({"lfn" : inputF, "parents" : ""}) else: self.job["input_files"].append(inputF) self.job['mask'] = Mask() self.job['mask']["FirstEvent"] = firstEvent self.job['mask']["LastEvent"] = lastEvent self.job['mask']["FirstRun"] = firstRun self.job['mask']["FirstLumi"] = firstLumi self.job['mask']["runAndLumis"] = runAndLumis self.job['agentNumber'] = agentNumber self.job['counter'] = 0
def testEventsPerLumi(self): """ _testEventsPerLumi_ Verify that you can put in events per lumi in the process. """ from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset os.environ['CMSSW_VERSION'] = "CMSSW_7_4_0" setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.step.setEventsPerLumi(500) setupScript.stepSpace = ConfigSection(name="stepSpace") setupScript.stepSpace.location = self.testDir setupScript.job = self.createTestJob() setupScript() fixedPSet = self.loadProcessFromPSet() self.assertEqual(len(fixedPSet.source.fileNames.value), 2, "Error: Wrong number of files.") self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2, "Error: Wrong number of secondary files.") self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two", "Error: Wrong input file.") self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock.value, 500, "Error: Wrong number of events per luminosity block") self.assertEqual(fixedPSet.maxEvents.input.value, -1, "Error: Wrong maxEvents.")
def testPileupSetup(self): """ Test the pileup setting. reference (setupScript.process instance): in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py """ try: from dbs.apis.dbsClient import DbsApi except ImportError as ex: raise unittest.SkipTest # this is modified and shortened version of # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml # since the dataset name in question (below) is only present at # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE siteLocalConfigContent = \ """ <site-local-config> <site name="-SOME-SITE-NAME-"> <event-data> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </event-data> <local-stage-out> <!-- original cmssrm.fnal.gov --> <phedex-node value="T2_CH_CERN"/> <command value="test-copy"/> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </local-stage-out> <calib-data> <frontier-connect> <load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name="stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = { "data": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ], "mc": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ] } dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["phedex-node"] print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher.createPileupConfigFile(setupScript.step) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName)
def testPileupSetup(self): """ Test the pileup setting. reference (setupScript.process instance): in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py """ try: from dbs.apis.dbsClient import DbsApi except ImportError as ex: raise nose.SkipTest # this is modified and shortened version of # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml # since the dataset name in question (below) is only present at # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE siteLocalConfigContent = \ """ <site-local-config> <site name="-SOME-SITE-NAME-"> <event-data> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </event-data> <local-stage-out> <!-- original cmssrm.fnal.gov --> <phedex-node value="T2_CH_CERN"/> <command value="test-copy"/> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </local-stage-out> <calib-data> <frontier-connect> <load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"], "mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]} dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["phedex-node"] print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher._createPileupConfigFile(setupScript.step, fakeSites=['T1_US_FNAL']) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName)
<load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name="stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = { "data": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ], "mc": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ]
<frontier-connect> <load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"], "mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]} dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env.