def testEventsPerLumi(self):
"""
_testEventsPerLumi_
Verify that you can put in events per lumi in the process.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.step.setEventsPerLumi(500)
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0],
"/some/parent/one", "Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1],
"/some/parent/two", "Error: Wrong input file.")
self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock.value,
500,
"Error: Wrong number of events per luminosity block")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testCheckLumiInformation(self):
"""
_testCheckLumiInformation_
Test the function that checks if all files
have run lumi information
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
myReport.checkForRunLumiInformation(stepName="cmsRun1")
self.assertNotEqual(myReport.getExitCode(), 70452)
# Remove the lumi information on purpose
myReport2 = Report("cmsRun1")
myReport2.parse(self.xmlPath)
fRefs = myReport2.getAllFileRefsFromStep(step="cmsRun1")
for fRef in fRefs:
fRef.runs = ConfigSection()
myReport2.checkForRunLumiInformation(stepName="cmsRun1")
self.assertFalse(myReport2.stepSuccessful(stepName="cmsRun1"))
self.assertEqual(myReport2.getExitCode(), 70452)
return
def addTier1Skim(config, skimName, dataTier, primaryDataset, cmsswVersion,
processingVersion, configURL, globalTag, twoFileRead = False,
skimNode = None):
"""
_addTier1Skim_
Add the configuration of a skim that is to be run over a particular primary
dataset and data tier at a particular site to the Tier0 configuration. The
skims will be launched as blocks are transfered to the site. The site name
must correspond to the site name in the ProdAgent JobQueue.
"""
datasetConfig = config.Datasets.section_(primaryDataset)
skimConfig = ConfigSection(name = "SomeTier1Skim")
skimConfig.PrimaryDataset = primaryDataset
skimConfig.DataTier = dataTier
skimConfig.SkimName = skimName
skimConfig.CMSSWVersion = cmsswVersion
skimConfig.ConfigURL = configURL
skimConfig.GlobalTag = globalTag
skimConfig.ProcessingVersion = processingVersion
skimConfig.TwoFileRead = twoFileRead
skimConfig.Node = skimNode
datasetConfig.Tier1Skims.append(skimConfig)
return
def testPSetFixup(self):
"""
_testPSetFixup_
Verify that all necessary parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
if os.environ.get('CMSSW_VERSION', None):
del os.environ['CMSSW_VERSION']
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testChainedProcesing(self):
"""
test for chained CMSSW processing - check the overriden TFC, its values
and that input files is / are set correctly.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
os.environ['CMSSW_VERSION'] = "CMSSW_7_6_0"
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript.step.setupChainedProcessing("my_first_step", "my_input_module")
setupScript()
# test if the overriden TFC is right
self.assertTrue(hasattr(setupScript.step.data.application, "overrideCatalog"),
"Error: overriden TFC was not set")
tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
inputFile = "../my_first_step/my_input_module.root"
self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)
self.assertEqual(setupScript.process.source.fileNames.value, [inputFile])
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel=logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = ConfigSection("file")
self.config.outputfile = os.path.join(self.testDir,
"FileSinkTestNew.json")
def testF(self):
"""
Test internal functions pythonise_, listSections_
"""
config = ConfigSection("config")
config.section_("SectionA")
config.section_("SectionB")
config.SectionA.section_("Section1")
config.SectionA.section_("Section2")
config.SectionA.Section1.x = 100
config.SectionA.Section1.y = 100
pythonise = config.pythonise_()
assert "config.section_('SectionA')" in pythonise, "Pythonise failed: Could not find SectionA"
assert "config.SectionA.Section1.x = 100" in pythonise, "Pythonise failed: Could not find x"
pythonise = config.SectionA.pythonise_()
assert "SectionA.section_('Section1')" in pythonise, "Pythonise failed: Could not find Section1"
assert "SectionA.Section1.x = 100" in pythonise, "Pythonise failed: Could not find x"
self.assertEqual(config.listSections_(), ['SectionB', 'SectionA'])
self.assertEqual(config.SectionA.listSections_(),
['Section2', 'Section1'])
def testPSetFixup(self):
"""
_testPSetFixup_
Verify that all necessary parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
self.assertEqual(fixedPSet.maxEvents.input._value, -1,
"Error: Wrong maxEvents.")
def reportWorkflowToDashboard(self, dashboardActivity):
"""
_reportWorkflowToDashboard_
Gathers workflow information from the arguments and reports it to the
dashboard
"""
try:
#Create a fake config
conf = ConfigSection()
conf.section_('DashboardReporter')
conf.DashboardReporter.dashboardHost = self.dashboardHost
conf.DashboardReporter.dashboardPort = self.dashboardPort
#Create the reporter
reporter = DashboardReporter(conf)
#Assemble the info
workflow = {}
workflow['name'] = self.workloadName
workflow['application'] = self.frameworkVersion
workflow['TaskType'] = dashboardActivity
#Let's try to build information about the inputDataset
dataset = 'DoesNotApply'
if hasattr(self, 'inputDataset'):
dataset = self.inputDataset
workflow['datasetFull'] = dataset
workflow['user'] = '******'
#Send the workflow info
reporter.addTask(workflow)
except:
#This is not critical, if it fails just leave it be
logging.error("There was an error with dashboard reporting")
def testEventsPerLumi(self):
"""
_testEventsPerLumi_
Verify that you can put in events per lumi in the process.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.step.setEventsPerLumi(500)
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock._value,
500,
"Error: Wrong number of events per luminosity block")
self.assertEqual(fixedPSet.maxEvents.input._value, -1,
"Error: Wrong maxEvents.")
def testChainedProcesing(self):
"""
test for chained CMSSW processing - check the overriden TFC, its values
and that input files is / are set correctly.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript.step.setupChainedProcessing("my_first_step",
"my_input_module")
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
# test if the overriden TFC is right
print("DEBUG override: {0}".format(
setupScript.step.data.application.overrideCatalog))
self.assertTrue(
hasattr(setupScript.step.data.application, "overrideCatalog"),
"Error: overriden TFC was not set")
tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
inputFile = "../my_first_step/my_input_module.root"
self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
def __init__(self, name=None, files=None):
"""
A job has a jobgroup which gives it its subscription and workflow.
inputFiles is a list containing files associated to a job
last_update is the time the job last changed
"""
dict.__init__(self)
self.baggage = ConfigSection("baggage")
if files == None:
self["input_files"] = []
else:
self["input_files"] = files
self["id"] = None
self["jobgroup"] = None
self["name"] = name
self["state"] = 'new'
self["state_time"] = int(time.time())
self["outcome"] = "failure"
self["retry_count"] = 0
self["location"] = None
self["mask"] = Mask()
self["task"] = None
self["fwjr"] = None
self["fwjr_path"] = None
self["workflow"] = None
self["owner"] = None
return
def __init__(self, lfnBase, outputMods):
Configuration.__init__(self)
for out in outputMods:
setattr(self, out, ConfigSection("output"))
getattr(self, out)._internal_name = "output"
getattr(self, out).lfnBase = lfnBase #'/store/temp/user/mmascher/RelValProdTTbar/mc/v6'
StepConfiguration.outputMods = outputMods
def testForwardSinkBasic(self):
config = ConfigSection("forward")
# address of the Processor, resp. Receiver to forward Alerts to
config.address = self.address1
config.controlAddr = self.controlAddr1
config.label = "ForwardSinkTest"
forwarder = ForwardSink(config)
def setUp(self):
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "alerts-rest_sink"
self.testInit.setupCouch(dbName)
self.config = ConfigSection("rest")
self.config.uri = self.testInit.couchUrl + "/" + dbName
def setUp(self):
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "couch_sink"
self.testInit.setupCouch(dbName)
self.config = ConfigSection("couch")
self.config.url = self.testInit.couchUrl
self.config.database = self.testInit.couchDbName
def __init__(self, reportname=None):
self.data = ConfigSection("FrameworkJobReport")
self.data.steps = []
self.data.workload = "Unknown"
if reportname:
self.addStep(reportname=reportname)
return
def setupPromptSkimConfigObject(self):
"""
_setupPromptSkimConfigObject_
Creates a custom config object for testing
of the skim functionality
"""
self.promptSkim = ConfigSection(name="Tier1Skim")
self.promptSkim.SkimName = "TestSkim1"
self.promptSkim.DataTier = "RECO"
self.promptSkim.TwoFileRead = False
self.promptSkim.ProcessingVersion = "PromptSkim-v1"
self.promptSkim.ConfigURL = "http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi/CMSSW/Configuration/DataOps/python/prescaleskimmer.py?revision=1.1"
def getOutputModulesForStep(self, stepName):
"""
_getOutputModulesForStep_
Retrieve all the output modules for the particular step.
"""
step = self.getStep(stepName)
if hasattr(step.data, "output"):
if hasattr(step.data.output, "modules"):
return step.data.output.modules
return ConfigSection()
def setUp(self):
self.config = ConfigSection("email")
self.config.fromAddr = "*****@*****.**"
self.config.toAddr = ["*****@*****.**", "*****@*****.**"]
self.config.smtpServer = "smtp.gov"
self.config.smtpUser = None
self.config.smtpPass = None
# now we want to mock smtp emailing stuff - via pymox - no actual
# email sending to happen
self.mox = mox.Mox()
self.smtpReal = EmailSinkMod.smtplib
EmailSinkMod.smtplib = self.mox.CreateMock(EmailSinkMod.smtplib)
self.smtp = self.mox.CreateMockAnything()
def testH_ConfigSectionDictionariseInternalChildren(self):
"""
The test checks if any item of the dictionary_whole_tree_()
result is not unexpanded instance of ConfigSection.
"""
config = ConfigSection("config")
config.value1 = "MyValue1"
config.section_("Task1")
config.Task1.value2 = "MyValue2"
config.Task1.section_("subSection")
config.Task1.subSection.value3 = "MyValue3"
d = config.dictionary_whole_tree_()
for values in d.values():
self.assertFalse(isinstance(values, ConfigSection))
self.assertEqual(d["Task1"]["subSection"]["value3"], "MyValue3")
def __init__(self):
self.numCores = 0
self.sections = {}
self.values = {}
self.report = ConfigSection("performance")
#
# populate the aggregator with the list of expected keys
# based on the functions map above
# create a combined performance report with the appropriate sections
for red in AggrFunctions.keys():
self.values[red] = []
sect, param = red.split(".")
if not self.sections.has_key(sect):
self.sections[sect] = []
self.report.section_(sect)
if param not in self.sections[sect]:
self.sections[sect].append(param)
def __init__(self, location, inputFiles, runAndLumis, agentNumber, lfnBase, outputMods, firstEvent=0, lastEvent=-1, firstLumi=None,\
firstRun=None, seeding=None, lheInputFiles=False, oneEventMode=False, eventsPerLumi=None, maxRuntime=None):
SetupCMSSWPset.__init__(self, crabPSet=True)
self.stepSpace = ConfigSection()
self.stepSpace.location = location
self.step = StepConfiguration(lfnBase, outputMods)
self.step.section_("data")
self.step.data._internal_name = "cmsRun"
self.step.data.section_("application")
self.step.data.application.section_("configuration")
self.step.data.application.section_("command")
self.step.data.application.section_("multicore")
self.step.data.application.command.configuration = "PSet.py"
self.step.data.application.command.oneEventMode = oneEventMode in [
"1", "True", True
]
self.step.data.application.command.memoryCheck = False
self.step.data.application.command.silentMemoryCheck = True
# self.step.data.application.configuration.pickledarguments.globalTag/globalTagTransaction
if eventsPerLumi:
self.step.data.application.configuration.eventsPerLumi = eventsPerLumi
if maxRuntime:
self.step.data.application.configuration.maxSecondsUntilRampdown = maxRuntime
self.step.data.section_("input")
self.job = jobDict(lheInputFiles, seeding)
self.job["input_files"] = []
for inputF in inputFiles:
if isinstance(inputF, basestring):
self.job["input_files"].append({"lfn": inputF, "parents": ""})
else:
self.job["input_files"].append(inputF)
self.job['mask'] = Mask()
self.job['mask']["FirstEvent"] = firstEvent
self.job['mask']["LastEvent"] = lastEvent
self.job['mask']["FirstRun"] = firstRun
self.job['mask']["FirstLumi"] = firstLumi
self.job['mask']["runAndLumis"] = runAndLumis
self.job['agentNumber'] = agentNumber
self.job['counter'] = 0
def testBuildPset(self):
"""
_testBuildPset_
Verify that multicore parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWMulticore import SetupCMSSWMulticore
setupScript = SetupCMSSWMulticore()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.files = {'file1': {'events': 1000}}
setupScript.buildPSet()
fixedPSet = self.loadProcessFromPSet()
self.assertTrue(
int(fixedPSet.options.multiProcesses.maxChildProcesses.value) > 0)
self.assertTrue(
int(fixedPSet.options.multiProcesses.maxSequentialEventsPerChild.
value) > 0)
def getBaggage(self):
confSect = ConfigSection()
confSect.seeding = self.seeding
confSect.lheInputFiles = self.lheInputFiles
return confSect
def testPileupSetup(self):
"""
Test the pileup setting.
reference (setupScript.process instance):
in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py
"""
try:
from dbs.apis.dbsClient import DbsApi
except ImportError as ex:
raise unittest.SkipTest
# this is modified and shortened version of
# WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
# since the dataset name in question (below) is only present at
# storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
siteLocalConfigContent = \
"""
<site-local-config>
<site name="-SOME-SITE-NAME-">
<event-data>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</event-data>
<local-stage-out>
<!-- original cmssrm.fnal.gov -->
<phedex-node value="T2_CH_CERN"/>
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir,
"test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {
"data": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"mc": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["phedex-node"]
print("Running on site '%s', local SE name: '%s'" %
(siteConfig.siteName, seLocalName))
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher.createPileupConfigFile(setupScript.step)
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
# load in the pileup configuration in the form of dict which
# PileupFetcher previously saved in a JSON file
pileupDict = setupScript._getPileupConfigFromJson()
# get the sub dict for particular pileup type
# for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
for pileupType, modules in zip(("data", "mc"),
(dataMixModules, mixModules)):
# getting KeyError here - above pileupConfig is not correct - need
# to have these two types of pile type
d = pileupDict[pileupType]
self._mixingModulesInputFilesTest(modules, d, seLocalName)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__ErrorSample__
Example of a report from a job that failed
Created on Fri Jun 8 13:22:11 2012
@author: dballest
"""
from WMCore.Configuration import ConfigSection
from WMCore.FwkJobReport.Report import Report
FrameworkJobReport = ConfigSection("FrameworkJobReport")
FrameworkJobReport.task = '/Run195530-PhotonHad-Run2012B-PromptReco-v1-PhotonHad/DataProcessing'
FrameworkJobReport.workload = 'Unknown'
FrameworkJobReport.section_('cmsRun1')
FrameworkJobReport.cmsRun1.status = 1
FrameworkJobReport.cmsRun1.counter = 1
FrameworkJobReport.cmsRun1.section_('errors')
FrameworkJobReport.cmsRun1.errors.section_('error0')
FrameworkJobReport.cmsRun1.errors.error0.type = 'CMSSWStepFailure'
FrameworkJobReport.cmsRun1.errors.error0.details = ''
FrameworkJobReport.cmsRun1.errors.error0.exitCode = 84
FrameworkJobReport.cmsRun1.errors.section_('error1')
FrameworkJobReport.cmsRun1.errors.error1.type = 'Fatal Exception'
FrameworkJobReport.cmsRun1.errors.error1.details = ''
FrameworkJobReport.cmsRun1.errors.error1.exitCode = '8020'
FrameworkJobReport.cmsRun1.errors.section_('error2')
FrameworkJobReport.cmsRun1.errors.error2.type = 'ErrorLoggingAddition'
def configure(self):
"""Configure server, CherryPy and the rest."""
config = self.config['web_server']
cpconfig["engine.autoreload_on"] = False
cpconfig["engine.timeout_monitor.on"] = config.get(
"timeout_monitor", False)
cpconfig["server.environment"] = config.get("environment",
"production")
cpconfig["server.thread_pool"] = int(config.get("thread_pool", 30))
cpconfig["server.socket_port"] = int(config.get("port", 8080))
self.pid = config.get('pid', '/tmp/das_web.pid')
cpconfig["server.socket_host"] = config.get("host", "0.0.0.0")
cpconfig["server.socket_queue_size"] = \
int(config.get("socket_queue_size", 100))
cpconfig["tools.expires.secs"] = int(config.get("expires", 300))
cpconfig["tools.sessions.timeout"] = int(
config.get("session_timeout", 60))
cpconfig["log.screen"] = bool(config.get("log_screen", True))
cpconfig["log.access_file"] = config.get("access_log_file", None)
cpconfig["log.error_file"] = config.get("error_log_file", None)
cpconfig['request.show_tracebacks'] = False
log.error_log.setLevel(config.get("error_log_level", logging.DEBUG))
log.access_log.setLevel(config.get("access_log_level", logging.DEBUG))
# SecurityModule config
# Registers secmodv2 into cherrypy.tools so it can be used through
# decorators
class SecConfig(object):
pass
security = self.config['security']
secconfig = SecConfig()
try:
secsection = ConfigSection('security')
except NameError:
pass
for key, val in security.items():
setattr(secconfig, key, val)
if security.get('module', '') == '':
cherrypy.tools.secmodv2 = NullAuth(secconfig)
print("### DAS behind NullAuth, should NOT be used in production")
else:
try:
cherrypy.tools.secmodv2 = FrontEndAuth(secconfig)
print("### DAS behind FrontEndAuth")
except ImportError:
cherrypy.tools.secmodv2 = NullAuth(secconfig)
print(
"### DAS behind NullAuth, should NOT be used in production"
)
cherrypy.config.update({
'tools.secmodv2.on':
True,
'tools.secmodv2.role':
security.get('role', ''),
'tools.secmodv2.group':
security.get('group'
''),
'tools.secmodv2.site':
security.get('site', '')
})
cpconfig.update ({\
'tools.expires.on': True,\
'tools.response_headers.on':True,\
'tools.etags.on':True,\
'tools.etags.autotags':True,\
'tools.encode.on': True,\
'tools.proxy.on': True,\
'tools.gzip.on': True,\
})
log("loading config: %s" % cpconfig,\
context='web',\
severity=logging.DEBUG,\
traceback=False)
def testForwardSinkEntireChain(self):
"""
The test chain looks as follows:
worker -> Receiver1(+its Processor configured to do ForwardSink) -> Receiver2 whose
address as the destination the ForwardSink is configured with -> Receiver2 will
do FileSink so that it's possible to verify the chain.
"""
# configuration for the Receiver+Processor+ForwardSink 1 (group)
config1 = Configuration()
config1.component_("AlertProcessor")
config1.AlertProcessor.section_("critical")
config1.AlertProcessor.section_("soft")
config1.AlertProcessor.critical.level = 5
config1.AlertProcessor.soft.level = 0
config1.AlertProcessor.soft.bufferSize = 0
config1.AlertProcessor.critical.section_("sinks")
config1.AlertProcessor.soft.section_("sinks")
config1.AlertProcessor.critical.sinks.section_("forward")
config1.AlertProcessor.soft.sinks.section_("forward")
# address of the Receiver2
config1.AlertProcessor.critical.sinks.forward.address = self.address2
config1.AlertProcessor.critical.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.critical.sinks.forward.label = "ForwardSinkTest"
config1.AlertProcessor.soft.sinks.forward.address = self.address2
config1.AlertProcessor.soft.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.soft.sinks.forward.label = "ForwardSinkTest"
# 1) first item of the chain is source of Alerts: worker()
# 2) second item is Receiver1 + its Processor + its ForwardSink
processor1 = Processor(config1.AlertProcessor)
# ForwardSink will be created automatically by the Processor
receiver1 = Receiver(self.address1, processor1, self.controlAddr1)
receiver1.startReceiver() # non blocking call
# 3) third group is Receiver2 with its Processor and final FileSink
config2 = Configuration()
config2.component_("AlertProcessor")
config2.AlertProcessor.section_("critical")
config2.AlertProcessor.section_("soft")
config2.AlertProcessor.critical.level = 5
config2.AlertProcessor.soft.level = 0
config2.AlertProcessor.soft.bufferSize = 0
config2.AlertProcessor.critical.section_("sinks")
config2.AlertProcessor.soft.section_("sinks")
config2.AlertProcessor.critical.sinks.section_("file")
config2.AlertProcessor.soft.sinks.section_("file")
# configuration of the final sink
config2.AlertProcessor.critical.sinks.file.outputfile = self.outputfileCritical
config2.AlertProcessor.soft.sinks.file.outputfile = self.outputfileSoft
processor2 = Processor(config2.AlertProcessor)
# final FileSink will be automatically created by the Processor
receiver2 = Receiver(self.address2, processor2, self.controlAddr2)
receiver2.startReceiver() # non blocking call
# now send the Alert messages via worker() and eventually shut the receiver1
worker(self.address1, self.controlAddr1, 10)
# wait until receiver1 shuts
while receiver1.isReady():
time.sleep(0.4)
print "%s waiting for Receiver1 to shut ..." % inspect.stack(
)[0][3]
# shut down receiver2 - need to sendShutdown() to it
s = Sender(self.address2, self.controlAddr2, "some_id")
s.sendShutdown()
# wait until receiver2 shuts
while receiver2.isReady():
time.sleep(0.4)
print "%s waiting for Receiver2 to shut ..." % inspect.stack(
)[0][3]
# check the result in the files
# the bufferSize for soft-level Alerts was set to 0 so all
# Alerts should be present also in the soft-level type file
# initial 10 Alerts (Level 0 .. 9) gets distributed though a cascade
# of two Receivers. soft alerts with level 0 .. 4 are considered
# so Receiver1 forwards through its ForwardSink 0 .. 4 Alerts as soft and
# 5 .. 9 level Alerts through 'critical'. order is not guaranteed
# critical Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileCritical
sink = FileSink(fileConfig)
expectedLevels = range(5, 10) # that is 5 .. 9
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
d = dict(very="interesting")
for a in loadAlerts:
self.assertEqual(a["Details"], d)
# soft Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileSoft
sink = FileSink(fileConfig)
expectedLevels = range(0, 5) # that is 0 .. 4
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
for a in loadAlerts:
self.assertEqual(a["Details"], d)
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir,
"test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {
"data": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"mc": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
