def testPileupSetup(self): """ Test the pileup setting. reference (setupScript.process instance): in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py """ try: from dbs.apis.dbsClient import DbsApi except ImportError as ex: raise unittest.SkipTest # this is modified and shortened version of # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml # since the dataset name in question (below) is only present at # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE siteLocalConfigContent = \ """ <site-local-config> <site name="-SOME-SITE-NAME-"> <event-data> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </event-data> <local-stage-out> <!-- original cmssrm.fnal.gov --> <phedex-node value="T2_CH_CERN"/> <command value="test-copy"/> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </local-stage-out> <calib-data> <frontier-connect> <load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name="stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = { "data": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ], "mc": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ] } dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["phedex-node"] print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher.createPileupConfigFile(setupScript.step) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName)
siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["se-name"] print "Running on site '%s', local SE name: '%s'" % ( siteConfig.siteName, seLocalName) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher._createPileupConfigFile(setupScript.step) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName) def _mixingModulesInputFilesTest(self, modules, pileupSubDict,
def testPileupSetup(self): """ Test the pileup setting. reference (setupScript.process instance): in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py """ try: from dbs.apis.dbsClient import DbsApi except ImportError as ex: raise nose.SkipTest # this is modified and shortened version of # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml # since the dataset name in question (below) is only present at # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE siteLocalConfigContent = \ """ <site-local-config> <site name="-SOME-SITE-NAME-"> <event-data> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </event-data> <local-stage-out> <!-- original cmssrm.fnal.gov --> <phedex-node value="T2_CH_CERN"/> <command value="test-copy"/> <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/> </local-stage-out> <calib-data> <frontier-connect> <load balance="proxies"/> <proxy url="http://cmsfrontier1.fnal.gov:3128"/> <proxy url="http://cmsfrontier2.fnal.gov:3128"/> </frontier-connect> </calib-data> </site> </site-local-config> """ siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml") f = open(siteLocalConfig, 'w') f.write(siteLocalConfigContent) f.close() from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset setupScript = SetupCMSSWPset() setupScript.step = self.createTestStep() setupScript.stepSpace = ConfigSection(name = "stepSpace") setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1") setupScript.job = self.createTestJob() # define pileup configuration # despite of the implementation considering whichever type of pileup, # only "data" and "mc" types are eventually considered and lead to any # modifications of job input files pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"], "mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]} dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" setupScript.step.setupPileup(pileupConfig, dbsUrl) # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig # to determine StorageElement (SE) name the job is running on # SiteLocalConfig loads the site-local-config.xml file from env. # variable defined location ; if the variable is not defined already, set it # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above # thick with SE name is not effective if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None): os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig # find out local site name from the testing local site config, # will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["phedex-node"] print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName)) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher._createPileupConfigFile(setupScript.step, fakeSites=['T1_US_FNAL']) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName)
# will be needed later siteConfig = loadSiteLocalConfig() seLocalName = siteConfig.localStageOut["se-name"] print "Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName) # before calling the script, SetupCMSSWPset will try to load JSON # pileup configuration file, need to create it in self.testDir fetcher = PileupFetcher() fetcher.setWorkingDirectory(self.testDir) fetcher._createPileupConfigFile(setupScript.step) setupScript() # now test all modifications carried out in SetupCMSSWPset.__call__ # which will also test that CMSSWStepHelper.setupPileup run correctly mixModules, dataMixModules = setupScript._getPileupMixingModules() # load in the pileup configuration in the form of dict which # PileupFetcher previously saved in a JSON file pileupDict = setupScript._getPileupConfigFromJson() # get the sub dict for particular pileup type # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)): # getting KeyError here - above pileupConfig is not correct - need # to have these two types of pile type d = pileupDict[pileupType] self._mixingModulesInputFilesTest(modules, d, seLocalName) def _mixingModulesInputFilesTest(self, modules, pileupSubDict, seLocalName):