def testD(self):
"""test documentation"""
config = Configuration()
config.section_("Section1")
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
config.Section1.Parameter5 = {"test1": "test2", "test3": 123}
config.Section1.document_("""This is Section1""")
config.Section1.document_("""This is Section1.Parameter1""",
"Parameter1")
config.Section1.document_("""This is Section1.Parameter2""",
"Parameter2")
config.Section1.document_(
"""This is Section1.Parameter3\n with multiline comments""",
"Parameter3")
try:
config.Section1.documentedString_()
except Exception, ex:
msg = "Error calling ConfigSection.documentedString_:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def __init__(self, **configDict):
"""
init jobCreator
"""
myThread = threading.currentThread()
self.transaction = myThread.transaction
#DAO factory for WMBS objects
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
# WMCore splitter factory for splitting up jobs.
self.splitterFactory = SplitterFactory()
config = Configuration()
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = configDict["couchURL"]
config.JobStateMachine.couch_retries = configDict["defaultRetries"]
config.JobStateMachine.couchDBName = configDict["couchDBName"]
self.config = config
#Variables
self.jobCacheDir = configDict['jobCacheDir']
self.defaultJobType = configDict['defaultJobType']
self.limit = configDict.get('fileLoadLimit', 500)
self.createWorkArea = CreateWorkArea()
self.changeState = ChangeState(self.config)
return
def testD(self):
"""test documentation"""
config = Configuration()
config.section_("Section1")
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
config.Section1.Parameter5 = {
"test1" : "test2", "test3" : 123
}
config.Section1.document_("""This is Section1""")
config.Section1.document_("""This is Section1.Parameter1""",
"Parameter1")
config.Section1.document_("""This is Section1.Parameter2""",
"Parameter2")
config.Section1.document_("""This is Section1.Parameter3\n with multiline comments""",
"Parameter3")
try:
config.Section1.documentedString_()
except Exception, ex:
msg = "Error calling ConfigSection.documentedString_:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def __init__(self, **configDict):
"""
init jobCreator
"""
myThread = threading.currentThread()
self.transaction = myThread.transaction
# DAO factory for WMBS objects
self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi)
# WMCore splitter factory for splitting up jobs.
self.splitterFactory = SplitterFactory()
config = Configuration()
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = configDict["couchURL"]
config.JobStateMachine.couch_retries = configDict["defaultRetries"]
config.JobStateMachine.couchDBName = configDict["couchDBName"]
self.config = config
# Variables
self.jobCacheDir = configDict["jobCacheDir"]
self.defaultJobType = configDict["defaultJobType"]
self.limit = configDict.get("fileLoadLimit", 500)
self.createWorkArea = CreateWorkArea()
self.changeState = ChangeState(self.config)
return
def setUp(self):
"Setup MSManager for testing"
config = Configuration()
data = config.section_('data')
data.reqmgr2Url = "http://localhost/reqmgr2"
data.verbose = True
data.interval = 600
data.quotaUsage = 0.8
data.quotaAccount = "DataOps"
data.enableStatusTransition = True
data.rucioAccount = "wma_test"
data.rucioUrl = "http://cmsrucio-int.cern.ch"
data.rucioAuthUrl = "https://cmsrucio-auth-int.cern.ch"
data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod"
data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader"
data.smtpServer = "localhost"
data.fromAddr = "*****@*****.**"
data.toAddr = ["*****@*****.**"]
data.warningTransferThreshold = 100. * (1000**4) # 100 TB (terabyte)
self.mgr = MSManager(data)
data.services = ['monitor']
self.mgr_monit = MSManager(data)
data.services = ['transferor']
data.limitRequestsPerCycle = 50
data.enableDataTransfer = True
self.mgr_trans = MSManager(data)
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("RetryManager")
config.RetryManager.logLevel = 'DEBUG'
config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager'
config.RetryManager.maxRetries = 10
config.RetryManager.pollInterval = 10
# These are the cooloff times for the RetryManager, the times it waits
# Before attempting resubmission
config.RetryManager.coolOffTime = {'create': 120, 'submit': 120, 'job': 120}
# Path to plugin directory
config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns'
#config.RetryManager.pluginName = ''
config.RetryManager.WMCoreBase = WMCore.WMInit.getWMBASE()
config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components')
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "retry_manager_t"
return config
def getConfig(self, dbs3UploadOnly = False):
"""
_getConfig_
This creates the actual config file used by the component.
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBS3Upload")
config.DBS3Upload.pollInterval = 10
config.DBS3Upload.logLevel = 'DEBUG'
#config.DBS3Upload.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/dev/global/DBSWriter"
#config.DBS3Upload.dbsUrl = "https://dbs3-dev01.cern.ch/dbs/prod/global/DBSWriter"
config.DBS3Upload.dbsUrl = self.dbsUrl
config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBS3Upload.nProcesses = 1
config.DBS3Upload.dbsWaitTime = 0.1
config.DBS3Upload.datasetType = "VALID"
config.DBS3Upload.dbs3UploadOnly = dbs3UploadOnly
return config
def getASOConfig(self):
"""
_getWMAgentConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("Agent")
config.Agent.contact = "*****@*****.**"
config.Agent.agentName = "testingagent"
config.Agent.hostName = "localhost"
config.Agent.teamName = "testingteam"
config.component_("AsyncTransfer")
config.AsyncTransfer.log_level = logging.DEBUG
config.AsyncTransfer.namespace = "AsyncStageOut.AsyncTransfer"
config.AsyncTransfer.componentDir = config.General.workDir
config.AsyncTransfer.pollInterval = 10
config.AsyncTransfer.pollViewsInterval = 10
config.AsyncTransfer.couch_instance = os.getenv('COUCHURL')
config.AsyncTransfer.files_database = 'asynctransfer2_t'
config.AsyncTransfer.statitics_database = 'asynctransfer_stat_t'
config.AsyncTransfer.requests_database = 'aso_request_database_t'
config.AsyncTransfer.data_source = os.getenv('COUCHURL')
config.AsyncTransfer.db_source = 'asynctrackerpoller_t_wmstats'
config.AsyncTransfer.pluginName = "CentralMonitoring"
config.AsyncTransfer.pluginDir = "AsyncStageOut.Plugins"
config.AsyncTransfer.max_files_per_transfer = 1000
config.AsyncTransfer.pool_size = 1
config.AsyncTransfer.max_retry = 0
# config.AsyncTransfer.credentialDir = credentialDir
# config.AsyncTransfer.UISetupScript = ui_script
# config.AsyncTransfer.transfer_script = 'ftscp'
# config.AsyncTransfer.serverDN = hostDN
# config.AsyncTransfer.pollStatInterval = 86400
# config.AsyncTransfer.expiration_days = 30
# config.AsyncTransfer.couch_statinstance = statCouchUrl
# config.AsyncTransfer.serviceCert = serviceCert
# config.AsyncTransfer.serviceKey = "/path/to/valid/host-key"
# config.AsyncTransfer.cleanEnvironment = True
config.AsyncTransfer.user_monitoring_db = 'user_monitoring_asynctransfer_t'
config.AsyncTransfer.couch_user_monitoring_instance = os.getenv('COUCHURL')
config.AsyncTransfer.analyticsPollingInterval = 60
# config.AsyncTransfer.filesCleaningPollingInterval = 14400
config.AsyncTransfer.summaries_expiration_days = 30
return config
def testA(self):
"""ctor"""
try:
config = Configuration()
except Exception as ex:
msg = "Failed to instantiate Configuration\n"
msg += str(ex)
self.fail(msg)
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def createConfig(self):
"""
Create a config object for testing
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("DBSInterface")
config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
config.DBSInterface.DBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSBlockMaxFiles = 10
config.DBSInterface.DBSBlockMaxSize = 9999999999
config.DBSInterface.DBSBlockMaxTime = 10000
config.DBSInterface.MaxFilesToCommit = 10
return config
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.failUnless(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.failUnless(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception, ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def setUp(self):
"Setup MSManager for testing"
config = Configuration()
data = config.section_('data')
data.reqmgr2Url = "http://localhost/reqmgr2"
data.verbose = True
data.interval = 600
data.quotaUsage = 0.8
data.quotaAccount = "DataOps"
data.enableStatusTransition = True
data.rucioAccount = "test"
data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod"
data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader"
self.mgr = MSManager(data)
data.services = ['monitor']
self.mgr_monit = MSManager(data)
data.services = ['transferor']
data.limitRequestsPerCycle = 50
data.enableDataTransfer = True
self.mgr_trans = MSManager(data)
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'DEBUG'
config.DBSUpload.DBSBlockMaxFiles = 500
config.DBSUpload.DBSBlockMaxTime = 600
config.DBSUpload.DBSBlockMaxSize = 999999999999
config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.nProcesses = 1
config.DBSUpload.dbsWaitTime = 1
return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") )
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print "****WARNING: the DBHOST environment variable will be deprecated soon***"
print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
# after this you can augment it with whatever you need.
return config
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.failUnless(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.failUnless(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception, ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'DEBUG'
config.DBSUpload.DBSBlockMaxFiles = 1
config.DBSUpload.DBSBlockMaxTime = 2
config.DBSUpload.DBSBlockMaxSize = 999999999999
config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
#config.DBSUpload.dbsUrl = 'https://localhost:1443/dbs/prod/global/DBSWriter'
config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.nProcesses = 1
config.DBSUpload.dbsWaitTime = 0.1
return config
def testC(self):
"""add components"""
config = Configuration()
config.component_("Component1")
config.component_("Component2")
config.component_("Component3")
comp1 = getattr(config, "Component1", None)
self.failUnless(comp1 != None)
comp2 = getattr(config, "Component2", None)
self.failUnless(comp2 != None)
def testE(self):
"""test save/load """
testValues = [
"string", 123, 123.456,
["list", 789, 10.1 ],
{ "dict1" : "value", "dict2" : 10.0 }
]
config = Configuration()
for x in range(0, 5):
config.section_("Section%s" % x)
config.component_("Component%s" % x)
sect = getattr(config, "Section%s" % x)
comp = getattr(config, "Component%s" % x)
sect.document_("This is Section%s" % x)
comp.document_("This is Component%s" % x)
for i in range(0, 5):
setattr(comp, "Parameter%s" % i, testValues[i])
setattr(sect, "Parameter%s" % i, testValues[i])
comp.document_("This is Parameter%s" % i,
"Parameter%s" %i)
sect.document_("This is Parameter%s" %i,
"Parameter%s" %i)
stringSave = str(config)
documentSave = config.documentedString_()
commentSave = config.commentedString_()
saveConfigurationFile(config, self.normalSave)
saveConfigurationFile(config, self.docSave, document = True)
saveConfigurationFile(config, self.commentSave, comment = True)
plainConfig = loadConfigurationFile(self.normalSave)
docConfig = loadConfigurationFile(self.docSave)
commentConfig = loadConfigurationFile(self.commentSave)
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") )
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print "****WARNING: the DBHOST environment variable will be deprecated soon***"
print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
# after this you can augment it with whatever you need.
return config
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["MockPlugin"]
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobAccountant_t/fwjrs',
"MergeSuccess.pkl")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return config
def getConfig(self):
"""
_getConfig_
Gets a basic config from default location
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
#Now the CoreDatabase information
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# BossAir and MockPlugin configuration
config.section_("BossAir")
config.BossAir.pluginNames = ['MockPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.multicoreTaskTypes = [
'MultiProcessing', 'MultiProduction'
]
config.BossAir.nCondorProcesses = 1
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh")
# JobSubmitter configuration
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'DEBUG'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
self.testDir, 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "jobsubmitter_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
#!/usr/bin/env python
"""
Defines default config values for HarvestingScheduler specific
parameters.
"""
__all__ = []
import os
import os.path
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("HarvestingScheduler")
config.HarvestingScheduler.namespace = "WMComponent.HarvestingScheduler.HarvestingScheduler"
config.HarvestingScheduler.componentDir = os.getcwd() + "/HarvestingScheduler"
config.HarvestingScheduler.workloadCache = os.getcwd(), +"/HarvestingScheduler/workloadCache"
config.HarvestingScheduler.targetSite = "srm-cms.cern.ch"
config.HarvestingScheduler.scramArch = "slc5_ia32_gcc434"
config.HarvestingScheduler.cmsPath = "/afs/cern.ch/cms/sw"
config.HarvestingScheduler.proxy = os.getenv("X509_USER_PROXY", "bad_proxy")
config.HarvestingScheduler.dqmGuiUrl = "https://cmsweb.cern.ch/dqm/dev"
config.HarvestingScheduler.couchurl = "http://127.0.0.1:5984"
config.HarvestingScheduler.couchDBName = "datasets_to_harvest"
config.HarvestingScheduler.doStageOut = True
config.HarvestingScheduler.doDqmUpload = True
config.HarvestingScheduler.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/"
config.HarvestingScheduler.dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
config.HarvestingScheduler.expiryTime = 3600 * 24 * 30 # One month
def createConfig(self):
"""
_createConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'ERROR'
config.DBSUpload.maxThreads = 1
config.DBSUpload.namespace = 'WMComponent.DBSUpload.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.workerThreads = 4
config.section_("DBSInterface")
config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
config.DBSInterface.DBSVersion = 'DBS_2_0_9'
config.DBSInterface.MaxFilesToCommit = 10
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# configure threshold of DBS upload queue size alert threshold
# reference: trac ticket #1628
config.DBSUpload.alertUploadQueueSize = 2000
return config
def createConfig(self):
"""
_createConfig_
This creates the actual config file used by the component
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "DBSUpload"
config.Agent.useHeartbeat = False
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = "ERROR"
config.DBSUpload.maxThreads = 1
config.DBSUpload.namespace = "WMComponent.DBSUpload.DBSUpload"
config.DBSUpload.componentDir = os.path.join(os.getcwd(), "Components")
config.DBSUpload.workerThreads = 4
config.section_("DBSInterface")
config.DBSInterface.globalDBSUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet"
config.DBSInterface.globalDBSVersion = "DBS_2_0_9"
config.DBSInterface.DBSUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
config.DBSInterface.DBSVersion = "DBS_2_0_9"
config.DBSInterface.DBSBlockMaxFiles = 10
config.DBSInterface.DBSBlockMaxSize = 9999999999
config.DBSInterface.DBSBlockMaxTime = 10000
config.DBSInterface.MaxFilesToCommit = 10
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# configure threshold of DBS upload queue size alert threshold
# reference: trac ticket #1628
config.DBSUpload.alertUploadQueueSize = 2000
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("ErrorHandler")
# The log level of the component.
config.ErrorHandler.logLevel = 'DEBUG'
# The namespace of the component
config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
# maximum number of threads we want to deal
# with messages per pool.
config.ErrorHandler.maxThreads = 30
# maximum number of retries we want for job
config.ErrorHandler.maxRetries = 5
# The poll interval at which to look for failed jobs
config.ErrorHandler.pollInterval = 60
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "errorhandler_t_jd"
config.section_('ACDC')
config.ACDC.couchurl = self.testInit.couchUrl
config.ACDC.database = "errorhandler_t"
return config
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["MockPlugin"]
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobAccountant_t/fwjrs',
"MergeSuccess.pkl")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError("You must set the DATABASE environment variable to run tests")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print("****WARNING: the DBHOST environment variable will be deprecated soon***")
print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
return config
#!/usr/bin/env python
"""
Defines default config values for JobStatusLite specific parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_('JobStatusLite')
config.JobStatusLite.namespace = 'WMComponent.JobStatusLite.JobStatusLite'
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.logLevel = 'INFO'
config.JobStatusLite.pollInterval = 180
config.JobStatusLite.queryInterval = 120
config.JobStatusLite.jobLoadLimit = 100
config.JobStatusLite.maxJobQuery = 100
config.JobStatusLite.taskLimit = 30
config.JobStatusLite.maxJobsCommit = 100
config.JobStatusLite.processes = 5
def getConfig(self):
"""
_getConfig_
Build a basic JobTracker config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(),
'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
'badtest.fnal.gov')
#config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh')
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# BossAir
config.component_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = "jobtracker_t"
return config
def getConfig(self):
"""
_getConfig_
This is the global test configuration object
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = 'DEBUG'
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components')
# JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter'
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'CondorGlobusPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'DEBUG'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
# JobAccountant
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobAccountant.logLevel = 'INFO'
# JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobArchiver.numberOfJobsToCluster = 1000
# Task Archiver
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testInit.generateWorkDir()
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.submitDir)
return config
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for DBSBuffer specific
parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("DBSBuffer")
#The log level of the component.
config.DBSBuffer.logLevel = 'INFO'
#The namespace of the buffer. Used to load the module as daemon
config.DBSBuffer.namespace = 'WMComponent.DBSBuffer.DBSBuffer'
# maximum number of threads we want to deal
# with messages per pool.
config.DBSBuffer.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSBuffer.jobSuccessHandler = \
'WMComponent.DBSBuffer.Handler.JobSuccess'
_StartComponent_
Start the component, reading its configuration from
the common configuration file, which is accessed by environment variable
"""
import os
import sys
import getopt
import logging
from WMCore.Agent.Configuration import Configuration
# Find and load the Configuration
config = Configuration()
####################
# General: General Settings Section
####################
config.section_("General")
config.General.workDir = '/tmp/TQ-test/'
######################
# Task Queue Component
######################
config.component_("TQComp")
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for CranWmbs specific
parameters.
"""
__all__ = []
__revision__ = "$Id: DefaultConfig.py,v 0.2 2009/09/30 01:26:34 hriahi Exp $"
__version__ = "$Revision: 0.2 $"
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("CrabJobCreator")
config.CrabJobCreator.logLevel = "DEBUG"
config.CrabJobCreator.componentName = "CrabJobCreator"
config.CrabJobCreator.componentDir = \
os.path.join(os.getenv("TESTDIR"), "CrabJobCreator")
# The maximum number of threads to process each message type
config.CrabJobCreator.maxThreads = 10
# CrabServer parameter
config.CrabJobCreator.wdir = '/data/Storage/logs'
config.CrabJobCreator.maxRetries = 3
config.CrabJobCreator.credentialType = "Proxy"
config.CrabJobCreator.ProxiesDir = "/tmp/del_proxies/"
config.CrabJobCreator.StorageName = "crab.pg.infn.it"
config.CrabJobCreator.storagePort = "2811"
config.CrabJobCreator.Protocol = "gridftp"
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for FeederManager specific
parameters.
"""
__all__ = []
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("FeederManager")
config.FeederManager.logLevel = "INFO"
config.FeederManager.componentName = "FeederManager"
config.FeederManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "FeederManager")
config.FeederManager.addDatasetWatchHandler = \
'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'
# The maximum number of threads to process each message type
config.FeederManager.maxThreads = 10
# The poll interval at which to look for new fileset/feeder association
config.FeederManager.pollInterval = 60
def getConfig(self):
"""
_createConfig_
General config file
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.General.WorkDir = os.getenv("TESTDIR", os.getcwd())
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984")
config.JobStateMachine.couchDBName = "jobarchiver_t_0"
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
#config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = self.testDir
config.JobArchiver.numberOfJobsToCluster = 1000
config.component_('WorkQueueManager')
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager"
config.WorkQueueManager.level = 'LocalQueue'
config.WorkQueueManager.logLevel = 'DEBUG'
config.WorkQueueManager.couchurl = 'https://None'
config.WorkQueueManager.dbname = 'whatever'
config.WorkQueueManager.inboxDatabase = 'whatever2'
config.WorkQueueManager.queueParams = {}
config.WorkQueueManager.queueParams["ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue"
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
return config
def getConfig(self):
"""
_createConfig_
General config file
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.General.WorkDir = os.getenv("TESTDIR", os.getcwd())
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL",
"cmssrv48.fnal.gov:5984")
config.JobStateMachine.couchDBName = "jobarchiver_t_0"
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
#config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = self.testDir
config.JobArchiver.numberOfJobsToCluster = 1000
config.component_('WorkQueueManager')
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager"
config.WorkQueueManager.level = 'LocalQueue'
config.WorkQueueManager.logLevel = 'DEBUG'
config.WorkQueueManager.couchurl = 'https://None'
config.WorkQueueManager.dbname = 'whatever'
config.WorkQueueManager.inboxDatabase = 'whatever2'
config.WorkQueueManager.queueParams = {}
config.WorkQueueManager.queueParams[
"ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue"
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
return config
#!/usr/bin/env python
#pylint: disable=E1101,E1103,C0103,R0902
"""
Defines default config values for WorkflowManager specific
parameters.
"""
__all__ = []
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("WorkflowManager")
config.WorkflowManager.logLevel = "INFO"
config.WorkflowManager.componentName = "WorkflowManager"
config.WorkflowManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "WorkflowManager")
# The maximum number of threads to process each message type
config.WorkflowManager.maxThreads = 10
# The poll interval at which to look for new filesets
config.WorkflowManager.pollInterval = 60
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for DBSUpload specific
parameters.
"""
__all__ = []
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.section_("CoreDatabase")
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = os.getenv('DIALECT', None)
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
#The log level of the component.
config.DBSUpload.logLevel = 'DEBUG'
# maximum number of threads we want to deal
# with messages per pool.
config.DBSUpload.maxThreads = 1
#
# JobSuccess Handler
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.failUnless(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.failUnless(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception as ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
self.failUnless( "Parameter1" in section1Dict)
self.failUnless( "Parameter2" in section1Dict)
self.failUnless( "Parameter3" in section1Dict)
self.failUnless( "Parameter4" in section1Dict)
self.assertEqual(section1Dict['Parameter1'],
config.Section1.Parameter1)
self.assertEqual(section1Dict['Parameter2'],
config.Section1.Parameter2)
self.assertEqual(section1Dict['Parameter3'],
config.Section1.Parameter3)
self.assertEqual(section1Dict['Parameter4'],
config.Section1.Parameter4)
# compound types
config.Section2.List = ["string", 123, 123.456, False]
config.Section2.Dictionary = { "string" : "string",
"int" : 123,
"float" : 123.456,
"bool" : False}
config.Section2.Tuple = ("string", 123, 123.456, False)
self.assertEqual(config.Section2.List,
["string", 123, 123.456, False])
self.assertEqual(config.Section2.Tuple,
("string", 123, 123.456, False))
class DummyObject:
pass
# unsupported parameter type
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadObject", DummyObject())
# unsupported data type in compound type
badList = [ DummyObject(), DummyObject()]
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadList", badList)
#!/usr/bin/env python
"""
Defines default config values for JobAccountant specific
parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("WorkQueueManager")
# set type of work queue
config.WorkQueueManager.level = "GlobalQueue"
#config.WorkQueueManager.level = "LocalQueue"
# In general can be left alone
config.WorkQueueManager.componentDir = config.General.WorkDir + "/WorkQueueManager"
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
#config.WorkQueueManager.logLevel = 'INFO'
#config.WorkQueueManager.pollInterval = 600
# RequestManager config
config.WorkQueueManager.reqMgrConfig = {}
# uncomment to override default reqMgr url
#config.WorkQueueManager.reqMgrConfig['endpoint'] = 'http://cmssrv49.fnal.gov:8585'
config.WorkQueueManager.reqMgrConfig['teamName'] = 'Dodgers'
# add parameters for global or local queue if default param is not what you want
config.WorkQueueManager.queueParams = {'LocationRefreshInterval': 10}
# uncomment to change CacheDir from default
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("ErrorHandler")
# The log level of the component.
config.ErrorHandler.logLevel = 'DEBUG'
# The namespace of the component
config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
# maximum number of threads we want to deal
# with messages per pool.
config.ErrorHandler.maxThreads = 30
# maximum number of retries we want for job
config.ErrorHandler.maxRetries = 5
# The poll interval at which to look for failed jobs
config.ErrorHandler.pollInterval = 60
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "errorhandler_t_jd"
config.section_('ACDC')
config.ACDC.couchurl = self.testInit.couchUrl
config.ACDC.database = "errorhandler_t"
return config
def getConfig(self):
"""
_getConfig_
Build a basic JobTracker config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
#config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh')
config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# BossAir
config.component_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = "jobtracker_t"
return config
Sample configuration for generating workflow.
"""
import os
try:
import cPickle as pickle
except ImportError:
import pickle
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.section_('General')
# output directory of generated files.
config.General.baseDir = '/tmp/PRODAGENT'
config.General.srcDir = os.path.join(config.General.baseDir, 'src/python/PA/Component')
config.General.testDir = os.path.join(config.General.baseDir, 'test/python/PA_t/Component_t')
# module prefix for python files.
config.General.pythonPrefix = 'PA.Component'
config.General.pythonTestPrefix = 'PA_t.Component_t'
config.General.handlers = []
config.General.synchronizers = []
config.General.plugins = []
# synchronizer is for trigger module.
synchronizer = {'ID' : 'JobPostProcess', \
#!/usr/bin/env python
#pylint: disable=E1101,E1103,C0103,R0902
"""
DefaultConfig.py
Sample configuration for generating workflow.
"""
import os
import cPickle
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.section_('General')
# output directory of generated files.
config.General.baseDir = '/tmp/PRODAGENT'
config.General.srcDir = os.path.join(config.General.baseDir,
'src/python/PA/Component')
config.General.testDir = os.path.join(config.General.baseDir,
'test/python/PA_t/Component_t')
# module prefix for python files.
config.General.pythonPrefix = 'PA.Component'
config.General.pythonTestPrefix = 'PA_t.Component_t'
config.General.handlers = []
config.General.synchronizers = []
config.General.plugins = []
# synchronizer is for trigger module.
synchronizer = {'ID' : 'JobPostProcess', \
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("RetryManager")
config.RetryManager.logLevel = 'DEBUG'
config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager'
config.RetryManager.pollInterval = 10
# These are the cooloff times for the RetryManager, the times it waits
# Before attempting resubmission
config.RetryManager.section_("DefaultRetryAlgo")
config.RetryManager.DefaultRetryAlgo.section_("default")
config.RetryManager.DefaultRetryAlgo.default.coolOffTime = {
'create': 120,
'submit': 120,
'job': 120
}
# Path to plugin directory
config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns'
config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE()
config.RetryManager.componentDir = os.path.join(
os.getcwd(), 'Components')
# ErrorHandler
# Not essential, but useful for ProcessingAlgo
config.component_("ErrorHandler")
config.ErrorHandler.maxRetries = 5
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "retry_manager_t"
return config
def getConfig(self):
"""
_getConfig_
Gets a basic config from default location
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
#Now the CoreDatabase information
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# BossAir and MockPlugin configuration
config.section_("BossAir")
config.BossAir.pluginNames = ['MockPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction']
config.BossAir.nCondorProcesses = 1
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
"submit.sh")
# JobSubmitter configuration
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'DEBUG'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.submitScript = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(self.testDir, 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "jobsubmitter_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
def getConfig(self):
"""
_getConfig_
Build a basic BossAir config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
config.Agent.useHeartbeat = False
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh'
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.pollInterval = 1
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 1
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
# JobStatusLite
config.component_('JobStatusLite')
config.JobStatusLite.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400}
config.JobStatusLite.pollInterval = 1
return config
def getConfiguration(self,
configurationFile=None,
connectUrl=None,
socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(
os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print "****WARNING: the DBHOST environment variable will be deprecated soon***"
print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (
config.JobStateMachine.couchurl,
config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
return config
def getConfig(self):
"""
_getConfig_
Creates a common config.
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "DashboardReporter"
config.Agent.useHeartbeat = False
config.section_("DashboardReporter")
config.DashboardReporter.dashboardHost = "cmssrv52.fnal.gov"
config.DashboardReporter.dashboardPort = 8884
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = "dashboardreporter_t"
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
return config
#!/usr/bin/env python
"""
Defines default config values for JobStatusLite specific parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_('JobStatusLite')
config.JobStatusLite.namespace = 'WMComponent.JobStatusLite.JobStatusLite'
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.logLevel = 'INFO'
config.JobStatusLite.pollInterval = 180
config.JobStatusLite.queryInterval = 120
config.JobStatusLite.jobLoadLimit = 100
config.JobStatusLite.maxJobQuery = 100
config.JobStatusLite.taskLimit = 30
config.JobStatusLite.maxJobsCommit = 100
config.JobStatusLite.processes = 5
def getConfig(self):
"""
_getConfig_
Build a basic BossAir config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
config.Agent.useHeartbeat = False
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.pollInterval = 1
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 1
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
# JobStatusLite
config.component_('JobStatusLite')
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400}
config.JobStatusLite.pollInterval = 1
return config
