def testEventsPerLumi(self):
"""
_testEventsPerLumi_
Verify that you can put in events per lumi in the process.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.step.setEventsPerLumi(500)
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock._value,
500,
"Error: Wrong number of events per luminosity block")
self.assertEqual(fixedPSet.maxEvents.input._value, -1,
"Error: Wrong maxEvents.")
def testChainedProcesing(self):
"""
test for chained CMSSW processing - check the overriden TFC, its values
and that input files is / are set correctly.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript.step.setupChainedProcessing("my_first_step",
"my_input_module")
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
# test if the overriden TFC is right
print("DEBUG override: {0}".format(
setupScript.step.data.application.overrideCatalog))
self.assertTrue(
hasattr(setupScript.step.data.application, "overrideCatalog"),
"Error: overriden TFC was not set")
tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
inputFile = "../my_first_step/my_input_module.root"
self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
def testPSetFixup(self):
"""
_testPSetFixup_
Verify that all necessary parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
if os.environ.get('CMSSW_VERSION', None):
del os.environ['CMSSW_VERSION']
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testPSetFixup(self):
"""
_testPSetFixup_
Verify that all necessary parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
self.assertEqual(fixedPSet.maxEvents.input._value, -1,
"Error: Wrong maxEvents.")
def testChainedProcesing(self):
"""
test for chained CMSSW processing - check the overriden TFC, its values
and that input files is / are set correctly.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
os.environ['CMSSW_VERSION'] = "CMSSW_7_6_0"
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript.step.setupChainedProcessing("my_first_step", "my_input_module")
setupScript()
# test if the overriden TFC is right
self.assertTrue(hasattr(setupScript.step.data.application, "overrideCatalog"),
"Error: overriden TFC was not set")
tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
inputFile = "../my_first_step/my_input_module.root"
self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)
self.assertEqual(setupScript.process.source.fileNames.value, [inputFile])
def testEventsPerLumi(self):
"""
_testEventsPerLumi_
Verify that you can put in events per lumi in the process.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.step.setEventsPerLumi(500)
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0],
"/some/parent/one", "Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1],
"/some/parent/two", "Error: Wrong input file.")
self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock.value,
500,
"Error: Wrong number of events per luminosity block")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testEventsPerLumi(self):
"""
_testEventsPerLumi_
Verify that you can put in events per lumi in the process.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.step.setEventsPerLumi(500)
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock.value,
500, "Error: Wrong number of events per luminosity block")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testPSetFixup(self):
"""
_testPSetFixup_
Verify that all necessary parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSetHandle = open(os.path.join(self.testDir, "PSet.pkl"))
fixedPSet = pickle.load(fixedPSetHandle)
fixedPSetHandle.close()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testPileupSetup(self):
"""
Test the pileup setting.
reference (setupScript.process instance):
in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py
"""
try:
from dbs.apis.dbsClient import DbsApi
except ImportError as ex:
raise unittest.SkipTest
# this is modified and shortened version of
# WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
# since the dataset name in question (below) is only present at
# storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
siteLocalConfigContent = \
"""
<site-local-config>
<site name="-SOME-SITE-NAME-">
<event-data>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</event-data>
<local-stage-out>
<!-- original cmssrm.fnal.gov -->
<phedex-node value="T2_CH_CERN"/>
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir,
"test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {
"data": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"mc": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["phedex-node"]
print("Running on site '%s', local SE name: '%s'" %
(siteConfig.siteName, seLocalName))
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher.createPileupConfigFile(setupScript.step)
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
# load in the pileup configuration in the form of dict which
# PileupFetcher previously saved in a JSON file
pileupDict = setupScript._getPileupConfigFromJson()
# get the sub dict for particular pileup type
# for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
for pileupType, modules in zip(("data", "mc"),
(dataMixModules, mixModules)):
# getting KeyError here - above pileupConfig is not correct - need
# to have these two types of pile type
d = pileupDict[pileupType]
self._mixingModulesInputFilesTest(modules, d, seLocalName)
def testPileupSetup(self):
"""
Test the pileup setting.
reference (setupScript.process instance):
in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py
"""
try:
from dbs.apis.dbsClient import DbsApi
except ImportError as ex:
raise nose.SkipTest
# this is modified and shortened version of
# WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
# since the dataset name in question (below) is only present at
# storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
siteLocalConfigContent = \
"""
<site-local-config>
<site name="-SOME-SITE-NAME-">
<event-data>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</event-data>
<local-stage-out>
<!-- original cmssrm.fnal.gov -->
<phedex-node value="T2_CH_CERN"/>
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
"mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["phedex-node"]
print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName))
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher._createPileupConfigFile(setupScript.step, fakeSites=['T1_US_FNAL'])
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
# load in the pileup configuration in the form of dict which
# PileupFetcher previously saved in a JSON file
pileupDict = setupScript._getPileupConfigFromJson()
# get the sub dict for particular pileup type
# for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)):
# getting KeyError here - above pileupConfig is not correct - need
# to have these two types of pile type
d = pileupDict[pileupType]
self._mixingModulesInputFilesTest(modules, d, seLocalName)
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir,
"test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {
"data": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"mc": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
"mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above