def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.failUnless(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.failUnless(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception, ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def testInit(self):
wrongconfig = Configuration()
wrongconfig.section_("BossAir")
try:
mp = MockPlugin(wrongconfig)
except BossAirPluginException:
# The config does not contain MockPlugin section
pass
else:
fail("Expected exception")
wrongconfig.BossAir.section_("MockPlugin")
try:
mp = MockPlugin(wrongconfig)
except BossAirPluginException:
# The config does not contain fakeReport parameter
pass
else:
fail("Expected exception")
wrongconfig.BossAir.MockPlugin.fakeReport = "asdf"
try:
mp = MockPlugin(wrongconfig)
except BossAirPluginException:
# The fakeReport does not exist
pass
else:
fail("Expected exception")
mp = MockPlugin(config)
def __init__(self, db = None, authkey = None, nthreads = 5, port = 8051):
"""
:arg str db: Location of database configuration, "module.object". Defaults
to "SiteDBAuth.dbparam".
:arg str authkey: Location of wmcore security header authentication key.
:arg integer nthreads: Number of server threads to create.
:arg integer port: Server port."""
Configuration.__init__(self)
main = self.section_('main')
srv = main.section_('server')
srv.thread_pool = nthreads
main.application = 'sitedb'
main.port = port
main.index = 'ui'
main.authz_defaults = { 'role': None, 'group': None, 'site': None }
sec = main.section_('tools').section_("cms_auth")
sec.key_file = authkey
app = self.section_('sitedb')
app.admin = '*****@*****.**'
app.description = 'A database of sites known to CMS'
app.title = 'CMS SiteDB'
views = self.section_('views')
ui = views.section_('ui')
ui.object = 'SiteDB.FrontPage.FrontPage'
data = views.section_('data')
data.object = 'SiteDB.Data.Data'
data.db = db or 'SiteDBAuth.dbparam'
def testLumiSubmit(self):
"""
Test submission with the lumiMask parameter"
"""
sections = ["General", "User", "Data", "Site" , "JobType"]
lumiConf = Configuration()
for sec in sections:
lumiConf.section_(sec)
lumiConf.General.serverUrl = "localhost:8518"
lumiConf.JobType.externalPluginFile = os.path.join( os.path.dirname(__file__), "TestPlugin.py")
lumiConf.Site.storageSite = 'T2_XXX'
lumiInput = os.path.join( os.path.dirname(__file__), "../../../data/lumiInput.json")
lumiConf.Data.splitting = 'LumiBased'
lumiConf.Data.lumiMask = 'lumiInput.json'
sub = submit(self.logger, self.maplistopt + ["-c", lumiConf,
"-p", "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mmascher/CN=720897/CN=Marco Mascheroni",
"-s", "127.0.0.1:8518"])
res = sub()
expRes = CommandResult(0, None)
self.assertEquals(res, expRes)
def __init__(self, lfnBase, outputMods):
Configuration.__init__(self)
for out in outputMods:
setattr(self, out, ConfigSection("output"))
getattr(self, out)._internal_name = "output"
getattr(self,out).lfnBase = lfnBase #'/store/temp/user/mmascher/RelValProdTTbar/mc/v6'
StepConfiguration.outputMods = outputMods
def prepareObjects():
#Creting the input parameters
requestname = '130911_093053_mmascher_crab_tmp2'
task = {'tm_taskname':requestname} #just using some locations for this dataset
locations = {'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9':[u'ganymede.hep.kbfi.ee', u'srm.ihepa.ufl.edu', u'storm-se-01.ba.infn.it']}
#and just using some files
datasetfiles = {}
datasetfiles['/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/626F3DC8-1EAE-E111-A64C-0025902CB6B0.root'] = {'NumberOfEvents':1700L, 'BlockName':'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
'Lumis':{1L:[670994L, 670986L, 670958L, 671001L, 670995L, 671018L, 670988L, 671009L, 671038L, 671028L, 670942L, 670960L, 670948L, 670987L,
670991L, 671006L, 671012L, 671014L, 671040L, 670998L, 670983L, 670999L, 671027L, 671052L, 670984L, 671003L, 670996L, 671031L,
671007L, 671011L, 671022L, 670985L, 670962L, 671021L],
2L:[1L, 2L, 3L, 8L, 9L],
4L:[5L, 6L, 7L, 1L, 2L, 100L]},
'Parents':['/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/C880E78D-1EAE-E111-897B-0025902CB6AE.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8C7BB8D5-1EAE-E111-B269-0025901AD638.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8215258E-1EAE-E111-8A4E-0025904B11D4.root'],
'Checksums':{'Checksum':'922282544', 'Adler32':'39f2938b', 'Md5':'NOTSET'}, 'Size':975355735L}
datasetfiles['/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/7A5634E8-03AE-E111-9363-5404A63886EC.root'] = {'NumberOfEvents':1700L, 'BlockName':'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
'Lumis':{1L:[670165L, 670174L, 670199L, 670081L, 670192L, 670170L, 670029L, 670093L, 670173L, 670164L, 670146L, 670079L, 670200L, 670065L,
670062L, 670216L, 670202L, 670150L, 670176L, 670207L, 670087L, 670179L, 670191L, 670185L, 670160L, 670182L, 670177L, 670203L, 670154L,
670187L, 670138L, 670162L, 670066L, 670085L],
2L:[4L, 5L, 20L, 21L, 22L],
3L:[11L, 12L, 13L]},
'Parents':['/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/BAC6F21E-04AE-E111-A591-003048D3C90E.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/781547E6-03AE-E111-B1C0-BCAEC5329717.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/38694250-03AE-E111-B618-003048D37580.root'],
'Checksums':{'Checksum':'2628876232', 'Adler32':'21d69fc7', 'Md5':'NOTSET'}, 'Size':974920274L}
#Instantiate DataDiscovery
config = Configuration()
config.section_("MyProxy")
config.MyProxy.serverhostcert, config.MyProxy.serverhostkey = Requests().getKeyCert()
obj = DataDiscovery(config, None, None)
return obj, task, requestname, datasetfiles, locations
def testAgentConfigurationRetrieving(self):
"""
Test that getting some agent details (config values from config.Agent
section) will be correctly propagated into Alert instances.
Alert instance is obtained via API.getPredefinedAlert factory.
"""
d = dict(Additional = "detail")
# instantiate just plain Alert, no configuration to take
# into account at this point
a = Alert(**d)
self.assertEqual(a["HostName"], None)
self.assertEqual(a["Contact"], None)
self.assertEqual(a["TeamName"], None)
self.assertEqual(a["AgentName"], None)
self.assertEqual(a["Additional"], "detail")
# instantiate via factory which reads configuration instance
config = Configuration()
config.section_("Agent")
config.Agent.hostName = "some1"
config.Agent.contact = "some2"
config.Agent.teamName = "some3"
config.Agent.agentName = "some4"
a = alertAPI.getPredefinedAlert(**d)
self.assertEqual(a["HostName"], "some1")
self.assertEqual(a["Contact"], "some2")
self.assertEqual(a["TeamName"], "some3")
self.assertEqual(a["AgentName"], "some4")
self.assertEqual(a["Additional"], "detail")
def __init__(self, db = None, authkey = None, nthreads = 5, port = 8308):
"""
:arg str db: Location of database configuration, "module.object". Defaults
to "t0auth.dbparam".
:arg str authkey: Location of wmcore security header authentication key.
:arg integer nthreads: Number of server threads to create.
:arg integer port: Server port."""
Configuration.__init__(self)
main = self.section_('main')
srv = main.section_('server')
srv.thread_pool = nthreads
main.application = 't0wmadatasvc'
main.port = port
main.index = 'data'
main.authz_defaults = { 'role': None, 'group': None, 'site': None }
sec = main.section_('tools').section_("cms_auth")
sec.key_file = authkey
app = self.section_('t0wmadatasvc')
app.admin = '*****@*****.**'
app.description = 'Access to the CMS Tier0 database'
app.title = 'CMS T0 WMAgent Data Service'
views = self.section_('views')
data = views.section_('data')
data.object = 'T0WmaDataSvc.Data.Data'
data.db = db or 't0auth.dbparam'
def configure(self, configfile, service, dbinstance):
cfg = loadConfigurationFile(configfile)
wconfig = cfg.section_("Webtools")
app = wconfig.application
appconfig = cfg.section_(app)
dbsconfig = getattr(appconfig.views.active, service)
# Either we change formatter
# OR change the 'Accept' type to application/json (which we don't know how to do at the moment)
dbsconfig.formatter.object="WMCore.WebTools.RESTFormatter"
config = Configuration()
config.component_('SecurityModule')
config.SecurityModule.dangerously_insecure = True
config.component_('DBS')
config.DBS.application = app
config.DBS.model = dbsconfig.model
config.DBS.formatter = dbsconfig.formatter
#Does not support instances
#config.DBS.instances = cfg.dbs.instances
#config.DBS.database = dbsconfig.database
if self.migration_test:
#Use one specific database cms_dbs3_dev_phys02@int2r for migration unittests
from DBSSecrets import dbs3_dp2_i2
config.DBS.section_('database')
config.DBS.database.connectUrl = dbs3_dp2_i2['connectUrl']['writer']
config.DBS.database.dbowner = dbs3_dp2_i2['databaseOwner']
config.DBS.database.engineParameters = { 'pool_size' : 15, 'max_overflow' : 10, 'pool_timeout' : 200 }
version = getattr(dbsconfig.database.instances, dbinstance).version
config.DBS.database.version = version if version else '3.99.98'
config.DBS.section_('security')
config.DBS.security.params = {}
else:
#Use dev/global from dbs configuration for the reader, writer and dao unittests
dbconfig = getattr(dbsconfig.database.instances, dbinstance)
config.DBS.section_('database')
config.DBS.database.connectUrl = dbconfig.connectUrl
config.DBS.database.dbowner = dbconfig.dbowner
config.DBS.database.engineParameters = dbconfig.engineParameters
config.DBS.database.version = dbconfig.version if dbconfig.version else '3.99.98'
#config.DBS.database.instance = dbconfig.instance
try:
secconfig = getattr(dbsconfig.security.instances, dbinstance)
except AttributeError:
pass
else:
config.DBS.section_('security')
config.DBS.security.params = secconfig.params
config.DBS.default_expires = 900
return config
def __init__(self):
BaseConfiguration.__init__(self)
self._internal_sections.append("Agent")
self.Agent = ConfigSection("Agent")
self.Agent.agentName = None
self.Agent.teamName = None
self.Agent.hostName = None
self.Agent.contact = None
def make_crab_config(self):
if self.misc["crab_config"] is not None:
self.do_log("crab config already made, not remaking")
return
config = Configuration()
config.section_('General')
config.General.workArea = self.misc["pfx_crab"] # all crab output goes into crab/
config.General.transferOutputs = True
config.General.transferLogs = True
config.General.requestName = self.sample["crab"]["requestname"]
config.section_('JobType')
config.JobType.inputFiles = params.jecs
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = "%s/%s_cfg.py" % (self.misc["pfx_pset"], self.sample["shortname"])
config.section_('Data')
config.Data.allowNonValidInputDataset = True
config.Data.publication = False
config.Data.inputDataset = self.sample["dataset"]
config.Data.unitsPerJob = 1
config.Data.ignoreLocality = True
config.Data.splitting = 'FileBased'
config.Data.inputDBS = "phys03" if self.sample["dataset"].endswith("/USER") else "global"
config.section_('User')
config.section_('Site')
config.Site.storageSite = 'T2_US_UCSD'
config.Site.whitelist = ['T2_US_*']
self.misc["crab_config"] = config
def _wmAgentConfig(configFile):
config = Configuration()
config.section_("JobStateMachine")
#Waring setting couchDB to None will cause the ERROR:
# but that should be ignored, if you want to test couchDB
# set the real couchDB information here
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = os.getenv("COUCHDB")
config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.section_("Agent")
# User specific parameter
config.Agent.hostName = "cmssrv52.fnal.gov"
# User specific parameter
config.Agent.contact = "*****@*****.**"
# User specific parameter
config.Agent.teamName = "DMWM"
# User specific parameter
config.Agent.agentName = "WMAgentCommissioning"
config.Agent.useMsgService = False
config.Agent.useTrigger = False
# BossAir setup
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
#TaskArchive setup (JobSubmitter needs this)
config.component_("TaskArchiver")
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
saveConfigurationFile(config, configFile)
def _wmAgentConfig(configFile):
config = Configuration()
config.section_("General")
config.General.logdb_name = "unittest_logdb"
config.General.central_logdb_url = "http://localhost/central_logdb"
config.General.ReqMgr2ServiceURL = "http://localhost/reqmgr2"
config.section_("JobStateMachine")
# Waring setting couchDB to None will cause the ERROR:
# but that should be ignored, if you want to test couchDB
# set the real couchDB information here
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = os.getenv("COUCHDB")
config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.section_("Agent")
# User specific parameter
config.Agent.hostName = "cmssrv52.fnal.gov"
# User specific parameter
config.Agent.contact = "*****@*****.**"
# User specific parameter
config.Agent.teamName = "DMWM"
# User specific parameter
config.Agent.agentName = "WMAgentCommissioning"
config.Agent.useMsgService = False
config.Agent.useTrigger = False
# BossAir setup
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'SimpleCondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
saveConfigurationFile(config, configFile)
def testC(self):
"""add components"""
config = Configuration()
config.component_("Component1")
config.component_("Component2")
config.component_("Component3")
comp1 = getattr(config, "Component1", None)
self.failUnless(comp1 != None)
comp2 = getattr(config, "Component2", None)
self.failUnless(comp2 != None)
def testInit(self):
wrongconfig = Configuration()
wrongconfig.section_('BossAir')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
wrongconfig.BossAir.section_('MockPlugin')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The config does not contain fakeReport parameter
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The fakeReport does not exist
wrongconfig.BossAir.MockPlugin.fakeReport = 'asdf'
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
def testG_testStaticReferenceToConfigurationInstance(self):
"""
test Configuration.getInstance() which returns reference
to the Configuration object instance.
"""
config = Configuration()
instance = Configuration.getInstance()
self.assertFalse(hasattr(instance, "testsection"))
config.section_("testsection")
self.assertTrue(hasattr(instance, "testsection"))
config.testsection.var = 10
self.assertEquals(instance.testsection.var, 10)
def _emulatorCofig(phedex, dbs, siteDB, requestMgr, configFile):
config = Configuration()
config.section_("Emulator")
config.Emulator.PhEDEx = phedex
config.Emulator.DBSReader = dbs
config.Emulator.RequestMgr = requestMgr
config.Emulator.SiteDB = siteDB
saveConfigurationFile(config, configFile)
print "create config file:%s, PhEDEx: %s, DBS: %s, RequestManager: %s, SiteDB %s with flag" \
% (configFile, phedex, dbs, siteDB, requestMgr)
def config():
"""
Return a Configuration object containing all the sections that CRAB recognizes.
"""
config = Configuration()
config.section_("General")
config.section_("JobType")
config.section_("Data")
config.section_("Site")
config.section_("User")
config.section_("Debug")
return config
def setUp(self):
"""
Set up for tests.
"""
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.addr = "tcp://127.0.0.1:5557"
self.ctrl = "tcp://127.0.0.1:5559"
self.softOutputFile = "/tmp/ProcessorTestSoftAlerts.json"
self.criticalOutputFile = "/tmp/ProcessorTestCriticalAlerts.json"
self.config = Configuration()
self.config.component_("AlertProcessor")
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 1
self.config.AlertProcessor.soft.bufferSize = 3
self.config.AlertProcessor.critical.section_("sinks")
self.config.AlertProcessor.soft.section_("sinks")
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.Agent.Database',
"WMCore.ResourceControl"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir()
self.config = Configuration()
self.config.section_("Agent")
self.config.Agent.useMsgService = False
self.config.Agent.useTrigger = False
self.config.component_("AlertProcessor")
self.config.AlertProcessor.componentDir = self.testDir
self.config.AlertProcessor.address = "tcp://127.0.0.1:5557"
self.config.AlertProcessor.controlAddr = "tcp://127.0.0.1:5559"
self.config.section_("CoreDatabase")
self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 0
self.config.AlertProcessor.soft.bufferSize = 3
self.config.AlertProcessor.critical.section_("sinks")
self.config.AlertProcessor.soft.section_("sinks")
def setUp(self):
"""
_setUp_
"""
self.transitions = Transitions()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("changestate_t/jobs", "JobDump")
self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("job_summary", "WMStats")
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.couchServer = CouchServer(dburl = os.getenv("COUCHURL"))
self.config = Configuration()
self.config.component_("JobStateMachine")
self.config.JobStateMachine.couchurl = os.getenv("COUCHURL")
self.config.JobStateMachine.jobSummaryDBName = "job_summary"
return
def __init__(self, app, config, mount):
"""
:arg app: reference to application object; passed to all entities.
:arg config: reference to configuration; passed to all entities.
:arg str mount: API URL mount point; passed to all entities."""
RESTApi.__init__(self, app, config, mount)
cherrypy.log("ReqMgr entire configuration:\n%s" % Configuration.getInstance())
cherrypy.log("ReqMgr REST hub configuration subset:\n%s" % config)
# Makes raw format as default
# self.formats.insert(0, ('application/raw', RawFormat()))
self._add({"about": Info(app, IndividualCouchManager(config), config, mount),
"info": Info(app, IndividualCouchManager(config), config, mount),
"app_config": ReqMgrConfigData(app, IndividualCouchManager(config), config, mount),
"request": Request(app, IndividualCouchManager(config), config, mount),
"cmsswversions": CMSSWVersions(app, IndividualCouchManager(config), config, mount),
"wmagentconfig": WMAgentConfig(app, IndividualCouchManager(config), config, mount),
"permissions": PermissionsConfig(app, IndividualCouchManager(config), config, mount),
"campaignconfig": WMAgentConfig(app, IndividualCouchManager(config), config, mount),
"unifiedconfig": UnifiedConfig(app, IndividualCouchManager(config), config, mount),
"status": RequestStatus(app, IndividualCouchManager(config), config, mount),
"type": RequestType(app, IndividualCouchManager(config), config, mount),
"spec_template": RequestSpec(app, IndividualCouchManager(config), config, mount),
"workload_config": WorkloadConfig(app, IndividualCouchManager(config), config, mount),
"splitting": WorkloadSplitting(app, IndividualCouchManager(config), config, mount),
"wmstats_info": WMStatsInfo(app, IndividualCouchManager(config), config, mount),
"proc_status": ProcessMatrix(app, self, config, mount)
})
def __init__(self, app, config, mount):
"""
:arg app: reference to application object; passed to all entities.
:arg config: reference to configuration; passed to all entities.
:arg str mount: API URL mount point; passed to all entities."""
RESTApi.__init__(self, app, config, mount)
cherrypy.log("ReqMgr entire configuration:\n%s" % Configuration.getInstance())
cherrypy.log("ReqMgr REST hub configuration subset:\n%s" % config)
self.db_handler = ReqMgrCouch(config)
# Makes raw format as default
#self.formats.insert(0, ('application/raw', RawFormat()))
self._add({"about": Info(app, self, config, mount),
"info": Info(app, self, config, mount),
"request": Request(app, self, config, mount),
"group": Group(app, self, config, mount),
"team": Team(app, self, config, mount),
"software": Software(app, self, config, mount),
"status": RequestStatus(app, self, config, mount),
"type": RequestType(app, self, config, mount),
"spec_template": RequestSpec(self, app, config, mount),
"workload_config": WorkloadConfig(self, app, config, mount),
"splitting": WorkloadSplitting(self, app, config, mount),
"wmstats_info":WMStatsInfo(self, app, config, mount)
})
def createTier0Config():
"""
_createTier0Config_
Create a configuration object to hold the Tier0 configuration. Currently,
the configuration has two sections: Streams and Global.
"""
tier0Config = Configuration()
tier0Config.section_("Streams")
tier0Config.section_("Datasets")
tier0Config.section_("Global")
tier0Config.Global.ScramArches = {}
tier0Config.Global.Backfill = None
tier0Config.Global.ProcessingSite = "T2_CH_CERN_AI"
tier0Config.Global.BulkInjectNode = "T0_CH_CERN_Disk"
tier0Config.Global.ExpressInjectNode = "T0_CH_CERN_Disk"
tier0Config.Global.ExpressSubscribeNode = None
tier0Config.Global.DQMDataTier = "DQMIO"
tier0Config.Global.BaseRequestPriority = 150000
return tier0Config
def prepare_job_config(self, job):
""" Generates a CRAB config object from the Ganga job configuration. """
from WMCore.Configuration import Configuration
job_config = Configuration()
for section in job.backend.CRABConfig._schema.datadict.keys():
section_config = getattr(job.backend.CRABConfig, section)
ganga_section_config = Config.getConfig('CRABConfig_%s' % section)
task_section_config = job_config.section_(section)
for parameter_name, parameter_type in section_config._schema.allItems():
parameter_value = getattr(section_config, parameter_name)
if parameter_value not in (None, [None]):
# CRAB Config doesn't like Ganga sequence type instead of Lists
if parameter_type._meta['sequence']:
parameter_value = list(parameter_value)
task_section_config.__setattr__(parameter_name, parameter_value)
# Updating configuration in case of Ganga inline options specified
ganga_option = ganga_section_config[parameter_name]
if ganga_option:
# CRAB Config doesn't like Ganga sequence (or tuples) type instead of Lists
# Passing sequance with Ganga inline options makes it a tuple.
if parameter_type._meta['sequence']:
# Ugly but we need this because otherwise tuple of 1 element with a string would
# be transformed in a list of chars ( ('ab') --> ['a', 'b'] )
import json
ganga_option = json.loads(json.dumps(ganga_option))
if type(ganga_option) != list:
ganga_option = [ganga_option]
# loads makes strings 'utf' type, CRAB3 Server wants 'str' type
ganga_option = map(lambda x: str(x), ganga_option)
task_section_config.__setattr__(parameter_name, ganga_option)
# Some internal configuration
job_config.General.workArea = job.outputdir
return job_config
def createTier0Config():
"""
_createTier0Config_
Create a configuration object to hold the Tier0 configuration. Currently,
the configuration has two sections: Streams and Global.
"""
tier0Config = Configuration()
tier0Config.section_("Streams")
tier0Config.section_("Datasets")
tier0Config.section_("Global")
tier0Config.Global.InjectRuns = None
tier0Config.Global.InjectMinRun = None
tier0Config.Global.InjectMaxRun = None
tier0Config.Global.ScramArches = {}
tier0Config.Global.Backfill = None
tier0Config.Global.ProcessingSite = "T0_CH_CERN"
tier0Config.Global.StreamerPNN = "T0_CH_CERN_Disk"
tier0Config.Global.DQMDataTier = "DQMIO"
tier0Config.Global.BaseRequestPriority = 150000
tier0Config.Global.EnableUniqueWorkflowName = False
return tier0Config
def _wmAgentConfig(configFile):
config = Configuration()
config.section_("JobStateMachine")
#Waring setting couchDB to None will cause the ERROR:
# but that should be ignored, if you want to test couchDB
# set the real couchDB information here
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = os.getenv("COUCHDB")
config.section_("Agent")
# User specific parameter
config.Agent.hostName = "cmssrv52.fnal.gov"
# User specific parameter
config.Agent.contact = "*****@*****.**"
# User specific parameter
config.Agent.teamName = "DMWM"
# User specific parameter
config.Agent.agentName = "WMAgentCommissioning"
config.Agent.useMsgService = False
config.Agent.useTrigger = False
# BossAir setup
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
saveConfigurationFile(config, configFile)
def setup_test_server(module_name, class_name, app_name = None, authz_key_file=None, port=8888):
"""Helper function to set up a :class:`~.RESTMain` server from given
module and class. Creates a fake server configuration and instantiates
the server application from it.
:arg str module_name: module from which to import test class.
:arg str class_type: name of the server test class.
:arg str app_name: optional test application name, 'test' by default.
:returns: tuple with the server object and authz hmac signing key."""
if authz_key_file:
test_authz_key = authz_key_file
else:
test_authz_key = fake_authz_key_file()
cfg = Configuration()
main = cfg.section_('main')
main.application = app_name or 'test'
main.silent = True
main.index = 'top'
main.authz_defaults = { 'role': None, 'group': None, 'site': None }
main.section_('tools').section_('cms_auth').key_file = test_authz_key.name
app = cfg.section_(app_name or 'test')
app.admin = '*****@*****.**'
app.description = app.title = 'Test'
views = cfg.section_('views')
top = views.section_('top')
top.object = module_name + "." + class_name
server = RESTMain(cfg, os.getcwd())
server.validate_config()
server.setup_server()
server.install_application()
cherrypy.config.update({'server.socket_port': port})
cherrypy.config.update({'server.socket_host': '127.0.0.1'})
cherrypy.config.update({'request.show_tracebacks': True})
cherrypy.config.update({'environment': 'test_suite'})
for app in cherrypy.tree.apps.values():
if '/' in app.config:
app.config["/"]["request.show_tracebacks"] = True
return server, test_authz_key
def setup_dummy_server(module_name, class_name, app_name=None, authz_key_file=None, port=8888):
"""Helper function to set up a :class:`~.RESTMain` server from given
module and class. Creates a fake server configuration and instantiates
the server application from it.
:arg str module_name: module from which to import test class.
:arg str class_type: name of the server test class.
:arg str app_name: optional test application name, 'test' by default.
:returns: tuple with the server object and authz hmac signing key."""
if authz_key_file:
test_authz_key = authz_key_file
else:
test_authz_key = fake_authz_key_file()
cfg = Configuration()
main = cfg.section_("main")
main.application = app_name or "test"
main.silent = True
main.index = "top"
main.authz_defaults = {"role": None, "group": None, "site": None}
main.section_("tools").section_("cms_auth").key_file = test_authz_key.name
app = cfg.section_(app_name or "test")
app.admin = "*****@*****.**"
app.description = app.title = "Test"
views = cfg.section_("views")
top = views.section_("top")
top.object = module_name + "." + class_name
server = RESTMain(cfg, os.getcwd())
server.validate_config()
server.setup_server()
server.install_application()
cherrypy.config.update({"server.socket_port": port})
cherrypy.config.update({"server.socket_host": "127.0.0.1"})
cherrypy.config.update({"request.show_tracebacks": True})
cherrypy.config.update({"environment": "test_suite"})
for app in cherrypy.tree.apps.values():
if "/" in app.config:
app.config["/"]["request.show_tracebacks"] = True
return server, test_authz_key
def workqueueConfig(couchdb = localCouchDb):
"""
Returns an usable workqueue config
"""
config = Configuration()
config.section_("Agent")
config.Agent.hostName = gethostname()
config.component_("WorkQueueManager")
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.couchurl = couchdb
config.WorkQueueManager.dbname = workqueueDBName
config.WorkQueueManager.wmstatDBName = wmstatsDBName
config.WorkQueueManager.inboxDatabase = workqueueInboxDbName
config.WorkQueueManager.level = "GlobalQueue"
config.WorkQueueManager.queueParams = {'WMStatsCouchUrl': "%s/%s" % (couchdb, wmstatsDBName)}
config.WorkQueueManager.queueParams['QueueURL'] = '%s/%s' % (couchdb, workqueueDBName)
config.WorkQueueManager.reqMgrConfig = {}
config.WorkQueueManager.reqMgrConfig['endpoint'] = reqMgrEndpoint
return config
workqueueDBName = 'workqueue'
workqueueInboxDbName = 'workqueue_inbox'
wmstatDBName = 'wmstats'
HOST = "cmsweb-dev.cern.ch"
REQMGR = "https://cmsweb-dev.cern.ch/reqmgr/reqMgr"
COUCH = "https://cmsweb-dev.cern.ch/couchdb"
TEAMS = 'TestTeam'
WEBURL = "%s/%s" % (COUCH, workqueueDBName)
root = __file__.rsplit('/', 4)[0]
cache_dir = os.path.join(root, 'state', 'workqueue', 'cache')
os.environ['WMCORE_CACHE_DIR'] = cache_dir
# Nothing after this point should need to be changed.
config = Configuration()
config.section_("Agent")
config.Agent.hostName = HOST
config.Agent.teamName = TEAMS
config.component_("WorkQueueManager")
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.couchurl = COUCH
config.WorkQueueManager.dbname = workqueueDBName
config.WorkQueueManager.inboxDatabase = workqueueInboxDbName
config.WorkQueueManager.wmstatDBName = wmstatDBName
config.WorkQueueManager.level = "GlobalQueue"
config.WorkQueueManager.queueParams = {
'WMStatsCouchUrl': "%s/%s" % (COUCH, wmstatDBName)
}
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.assertTrue(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.assertTrue(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception as ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
self.assertTrue( "Parameter1" in section1Dict)
self.assertTrue( "Parameter2" in section1Dict)
self.assertTrue( "Parameter3" in section1Dict)
self.assertTrue( "Parameter4" in section1Dict)
self.assertEqual(section1Dict['Parameter1'],
config.Section1.Parameter1)
self.assertEqual(section1Dict['Parameter2'],
config.Section1.Parameter2)
self.assertEqual(section1Dict['Parameter3'],
config.Section1.Parameter3)
self.assertEqual(section1Dict['Parameter4'],
config.Section1.Parameter4)
# compound types
config.Section2.List = ["string", 123, 123.456, False]
config.Section2.Dictionary = { "string" : "string",
"int" : 123,
"float" : 123.456,
"bool" : False}
config.Section2.Tuple = ("string", 123, 123.456, False)
self.assertEqual(config.Section2.List,
["string", 123, 123.456, False])
self.assertEqual(config.Section2.Tuple,
("string", 123, 123.456, False))
class DummyObject:
pass
# unsupported parameter type
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadObject", DummyObject())
# unsupported data type in compound type
badList = [ DummyObject(), DummyObject()]
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadList", badList)
badDict = { "dict" : {}, "list": [DummyObject()], "tuple" : () }
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadDict", badDict)
goodDict = { "dict" : {}, "list": [], "tuple" : () }
config.Section2.GoodDict = goodDict
RUCIO_ACCT = "wmcore_output"
RULE_LIFETIME = 30 * 24 * 60 * 60 # 30 days
RULE_LIFETIME_RELVAL = 12 * 30 * 24 * 60 * 60 # 12 months
if BASE_URL == "https://cmsweb.cern.ch":
RUCIO_AUTH_URL="https://cms-rucio-auth.cern.ch"
RUCIO_URL="http://cms-rucio.cern.ch"
SEND_NOTIFICATION=True
ENABLE_DATA_PLACEMENT=True
else:
RUCIO_AUTH_URL="https://cms-rucio-auth-int.cern.ch"
RUCIO_URL="http://cms-rucio-int.cern.ch"
SEND_NOTIFICATION=False
ENABLE_DATA_PLACEMENT=False
config = Configuration()
main = config.section_("main")
srv = main.section_("server")
srv.thread_pool = 30
main.application = "ms-output"
main.port = 8245 # main application port it listens on
main.index = 'ui' # Configuration requires index attribute
# Security configuration
main.authz_defaults = {"role": None, "group": None, "site": None}
#set default logging (prevent duplicate)
main.log_screen = True
sec = main.section_("tools").section_("cms_auth")
sec.key_file = "%s/auth/wmcore-auth/header-auth-key" % ROOTDIR
Locale.py
Template for a locale configuration that contains all the stuff that agent operators want to customise
on a regular basis
Created by Dave Evans on 2011-05-20.
Copyright (c) 2011 Fermilab. All rights reserved.
"""
import sys
import os
import socket
from WMCore.Configuration import Configuration
config = Configuration()
config.section_('locale')
#
# Define the required sections
#
config.locale.section_("reqmgr")
config.locale.section_("gwq")
config.locale.section_("lwq")
config.locale.section_("agent")
config.locale.section_("crabserver")
config.locale.section_("couch")
config.locale.section_("mysql")
config.locale.section_("certificates")
# canned settings for each section
from WMCore.Configuration import Configuration
conf = Configuration()
main = conf.section_('main')
srv = main.section_('server')
srv.thread_pool = 5
main.application = 'crabserver'
main.port = 8270
main.index = 'data'
main.authz_defaults = {'role': None, 'group': None, 'site': None}
main.section_('tools').section_(
'cms_auth'
).key_file = "%s/auth/wmcore-auth/header-auth-key" % __file__.rsplit('/', 3)[0]
app = conf.section_('crabserver')
app.admin = '*****@*****.**'
app.description = 'CRABServer RESTFull API'
app.title = 'CRABRESTFull'
views = conf.section_('views')
data = views.section_('data')
data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI'
data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/'
data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
data.defaultBlacklist = ['T0_CH_CERN']
data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit(
'/', 3)[0]
data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit(
'/', 3)[0]
"""
DBS3 Migration Server Configuration
"""
from WMCore.Configuration import Configuration
import os, sys
ROOTDIR = os.path.normcase(os.path.abspath(__file__)).rsplit('/', 3)[0]
sys.path.append(os.path.join(ROOTDIR, 'auth/dbs'))
from DBSSecrets import dbs3_l3_i2
from DBSSecrets import dbs3_p2_i2
from DBSSecrets import dbs3_l1_i2
config = Configuration()
config.component_('web')
config.web.host = "127.0.0.1"
config.web.port = 8251
config.web.log_screen = True
config.web.thread_pool = 10
config.component_('dbsmigration')
config.dbsmigration.instances = ['prod/global', 'dev/global', 'int/global']
config.dbsmigration.section_('database')
db_instances = config.dbsmigration.database.section_('instances')
db_production_global = db_instances.section_('prod/global')
db_production_global.threads = 2
db_production_global.dbowner = dbs3_p2_i2['databaseOwner']
db_production_global.connectUrl = dbs3_p2_i2['connectUrl']['writer']
class WMDataMining(CherryPyPeriodicTask):
def __init__(self, rest, config):
CherryPyPeriodicTask.__init__(self, config)
def setConcurrentTasks(self, config):
"""
sets the list of functions which
"""
self.concurrentTasks = [{
'func': sayBye,
'duration': config.activeDuration
}, {
'func': Hello(),
'duration': config.archiveDuration
}]
if __name__ == '__main__':
import cherrypy
from WMCore.Configuration import Configuration
config = Configuration()
config.section_("wmmining")
config.wmmining.activeDuration = 5
config.wmmining.archiveDuration = 30
#helloTask = PeriodicWorker(sayHello, config.wmmining)
WMDataMining(None, config.wmmining)
cherrypy.quickstart()
import os
from WMCore.Configuration import Configuration
from CRABClient.UserUtilities import config, getUsernameFromCRIC
config = Configuration()
config.section_("General")
config.General.requestName = '2017_MuMu_C'
config.General.transferOutputs = True
config.General.transferLogs = True
config.section_("JobType")
config.JobType.allowUndistributedCMSSW = True
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = 'crab_PSet_2017_MuMu_C.py'
config.JobType.maxMemoryMB = 3000
config.JobType.maxJobRuntimeMin = 1800
config.JobType.numCores = 1
config.JobType.scriptExe = 'crab_script_2017_MuMu_C.sh'
config.JobType.inputFiles = [
'crab_script_2017_MuMu_C.py',
os.path.join(os.environ['CMSSW_BASE'],
'src/PhysicsTools/NanoAODTools/scripts/haddnano.py'),
]
config.JobType.outputFiles = [] #['hist.root']
config.JobType.sendPythonFolder = True
config.section_("Data")
config.Data.inputDataset = '/DoubleMuon/Run2017C-02Apr2020-v1/NANOAOD'
config.Data.inputDBS = 'global'
config.Data.splitting = 'FileBased'
if config.Data.splitting == 'FileBased':
config.Data.unitsPerJob = 1
import os
import glob
from WMCore.Configuration import Configuration
config = Configuration()
config.section_('General')
config.General.transferOutputs = True
config.General.transferLogs = True
config.General.requestName = 'ALCARECO_2017D_PixelTrees_2021Febr_v1-NoMT'
config.section_('JobType')
config.JobType.allowUndistributedCMSSW = True
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = '4pixelTree-ALCARECO-NoMT.py'
config.JobType.outputFiles = ['PixelTree.root']
config.JobType.disableAutomaticOutputCollection = True
config.JobType.maxJobRuntimeMin = 3000
config.JobType.maxMemoryMB = 4000
#config.JobType.numCores = 8
config.section_('Data')
#config.Data.inputDBS = 'phys03'
config.Data.inputDataset = '/SingleMuon/Run2017D-SiPixelCalSingleMuon-ForPixelALCARECO_UL2017-v1/ALCARECO'
#config.Data.runRange = '315257-316995'
config.Data.outLFNDirBase = '/store/user/tvami/PixelTrees/'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 1
config.Data.publication = True
config.Data.ignoreLocality = True
from WMCore.Configuration import Configuration
name = 'WWW'
steam_dir = 'xulyu'
config = Configuration()
config.section_("General")
config.General.requestName = 'newttW'
config.General.transferLogs = True
config.section_("JobType")
config.JobType.pluginName = 'Analysis'
config.JobType.inputFiles = ['Summer16_23Sep2016V3_MC_L1FastJet_AK4PFchs.txt','Summer16_23Sep2016V3_MC_L2Relative_AK4PFchs.txt','Summer16_23Sep2016V3_MC_L3Absolute_AK4PFchs.txt','Summer16_23Sep2016V3_MC_L1FastJet_AK8PFchs.txt','Summer16_23Sep2016V3_MC_L2Relative_AK8PFchs.txt','Summer16_23Sep2016V3_MC_L3Absolute_AK8PFchs.txt','Summer16_23Sep2016V3_MC_L1FastJet_AK8PFPuppi.txt','Summer16_23Sep2016V3_MC_L2Relative_AK8PFPuppi.txt','Summer16_23Sep2016V3_MC_L3Absolute_AK8PFPuppi.txt','Summer16_23Sep2016V3_MC_L1FastJet_AK4PFPuppi.txt','Summer16_23Sep2016V3_MC_L2Relative_AK4PFPuppi.txt','Summer16_23Sep2016V3_MC_L3Absolute_AK4PFPuppi.txt']
#config.JobType.inputFiles = ['PHYS14_25_V2_All_L1FastJet_AK4PFchs.txt','PHYS14_25_V2_All_L2Relative_AK4PFchs.txt','PHYS14_25_V2_All_L3Absolute_AK4PFchs.txt','PHYS14_25_V2_All_L1FastJet_AK8PFchs.txt','PHYS14_25_V2_All_L2Relative_AK8PFchs.txt','PHYS14_25_V2_All_L3Absolute_AK8PFchs.txt']
# Name of the CMSSW configuration file
#config.JobType.psetName = 'bkg_ana.py'
config.JobType.psetName = 'analysis.py'
#config.JobType.allowUndistributedCMSSW = True
config.JobType.allowUndistributedCMSSW = True
config.section_("Data")
#config.Data.inputDataset = '/WJetsToLNu_13TeV-madgraph-pythia8-tauola/Phys14DR-PU20bx25_PHYS14_25_V1-v1/MINIAODSIM'
config.Data.inputDataset = '/ttWJets_13TeV_madgraphMLM/RunIISummer16MiniAODv2-80X_mcRun2_asymptotic_2016_TrancheIV_v6-v1/MINIAODSIM'
config.Data.inputDBS = 'global'
#config.Data.inputDBS = 'phys03'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob =5
config.Data.totalUnits = -1
config.Data.publication = False
config.Data.outLFNDirBase = '/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STEAM/' + steam_dir + '/' + name + '/'
# This string is used to construct the output dataset name
config.JobType.sendExternalFolder = True
data)
except Exception as ex:
msg = "Error updating document %s in CouchDB: %s" % (docid,
str(ex))
self.logger.error(msg)
return
if __name__ == "__main__":
import os
import logging
from RESTInteractions import CRABRest
from WMCore.Configuration import Configuration
logging.basicConfig(level=logging.DEBUG)
config = Configuration()
config.section_("TaskWorker")
#will use X509_USER_PROXY var for this test
config.TaskWorker.cmscert = os.environ["X509_USER_PROXY"]
config.TaskWorker.cmskey = os.environ["X509_USER_PROXY"]
server_ = CRABRest('cmsweb-testbed.cern.ch', config.TaskWorker.cmscert,
config.TaskWorker.cmskey)
server_.setDbInstance('dev')
resubmitter = DagmanResubmitter(config, server_)
resubmitter.execute(
task={
'tm_taskname':
'141129_110306_crab3test-5:atanasi_crab_test_resubmit',
'user_proxy': os.environ["X509_USER_PROXY"],