def getConfig(self):
"""
_getConfig_
Build a basic JobTracker config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(),
'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
'badtest.fnal.gov')
#config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh')
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# BossAir
config.component_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = "jobtracker_t"
return config
def testD(self):
"""test documentation"""
config = Configuration()
config.section_("Section1")
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
config.Section1.Parameter5 = {"test1": "test2", "test3": 123}
config.Section1.document_("""This is Section1""")
config.Section1.document_("""This is Section1.Parameter1""",
"Parameter1")
config.Section1.document_("""This is Section1.Parameter2""",
"Parameter2")
config.Section1.document_(
"""This is Section1.Parameter3\n with multiline comments""",
"Parameter3")
try:
config.Section1.documentedString_()
except Exception, ex:
msg = "Error calling ConfigSection.documentedString_:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def __init__(self, **configDict):
"""
init jobCreator
"""
myThread = threading.currentThread()
self.transaction = myThread.transaction
#DAO factory for WMBS objects
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
# WMCore splitter factory for splitting up jobs.
self.splitterFactory = SplitterFactory()
config = Configuration()
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = configDict["couchURL"]
config.JobStateMachine.couch_retries = configDict["defaultRetries"]
config.JobStateMachine.couchDBName = configDict["couchDBName"]
self.config = config
#Variables
self.jobCacheDir = configDict['jobCacheDir']
self.defaultJobType = configDict['defaultJobType']
self.limit = configDict.get('fileLoadLimit', 500)
self.createWorkArea = CreateWorkArea()
self.changeState = ChangeState(self.config)
return
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.failUnless(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.failUnless(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception, ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def setUp(self):
"Setup MSManager for testing"
config = Configuration()
data = config.section_('data')
data.reqmgr2Url = "http://localhost/reqmgr2"
data.verbose = True
data.interval = 600
data.quotaUsage = 0.8
data.quotaAccount = "DataOps"
data.enableStatusTransition = True
data.rucioAccount = "wma_test"
data.rucioUrl = "http://cmsrucio-int.cern.ch"
data.rucioAuthUrl = "https://cmsrucio-auth-int.cern.ch"
data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod"
data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader"
data.smtpServer = "localhost"
data.fromAddr = "*****@*****.**"
data.toAddr = ["*****@*****.**"]
data.warningTransferThreshold = 100. * (1000**4) # 100 TB (terabyte)
self.mgr = MSManager(data)
data.services = ['monitor']
self.mgr_monit = MSManager(data)
data.services = ['transferor']
data.limitRequestsPerCycle = 50
data.enableDataTransfer = True
self.mgr_trans = MSManager(data)
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["MockPlugin"]
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobAccountant_t/fwjrs',
"MergeSuccess.pkl")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError("You must set the DATABASE environment variable to run tests")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print("****WARNING: the DBHOST environment variable will be deprecated soon***")
print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
return config
def testA(self):
"""ctor"""
try:
config = Configuration()
except Exception as ex:
msg = "Failed to instantiate Configuration\n"
msg += str(ex)
self.fail(msg)
def getConfig(self):
"""
_getConfig_
Build a basic BossAir config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
config.Agent.useHeartbeat = False
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh'
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.pollInterval = 1
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 1
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
# JobStatusLite
config.component_('JobStatusLite')
config.JobStatusLite.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400}
config.JobStatusLite.pollInterval = 1
return config
def getConfig(self):
"""
_getConfig_
Gets a basic config from default location
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
#Now the CoreDatabase information
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# BossAir and MockPlugin configuration
config.section_("BossAir")
config.BossAir.pluginNames = ['MockPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.multicoreTaskTypes = [
'MultiProcessing', 'MultiProduction'
]
config.BossAir.nCondorProcesses = 1
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh")
# JobSubmitter configuration
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'DEBUG'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
self.testDir, 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "jobsubmitter_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
def testC(self):
"""add components"""
config = Configuration()
config.component_("Component1")
config.component_("Component2")
config.component_("Component3")
comp1 = getattr(config, "Component1", None)
self.failUnless(comp1 != None)
comp2 = getattr(config, "Component2", None)
self.failUnless(comp2 != None)
def createConfig(self):
"""
_createConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'ERROR'
config.DBSUpload.maxThreads = 1
config.DBSUpload.namespace = 'WMComponent.DBSUpload.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.workerThreads = 4
config.section_("DBSInterface")
config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
config.DBSInterface.DBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSBlockMaxFiles = 10
config.DBSInterface.DBSBlockMaxSize = 9999999999
config.DBSInterface.DBSBlockMaxTime = 10000
config.DBSInterface.MaxFilesToCommit = 10
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# configure threshold of DBS upload queue size alert threshold
# reference: trac ticket #1628
config.DBSUpload.alertUploadQueueSize = 2000
return config
def getConfig(self):
"""
_createConfig_
General config file
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.General.WorkDir = os.getenv("TESTDIR", os.getcwd())
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL",
"cmssrv48.fnal.gov:5984")
config.JobStateMachine.couchDBName = "jobarchiver_t_0"
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
#config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = self.testDir
config.JobArchiver.numberOfJobsToCluster = 1000
config.component_('WorkQueueManager')
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager"
config.WorkQueueManager.level = 'LocalQueue'
config.WorkQueueManager.logLevel = 'DEBUG'
config.WorkQueueManager.couchurl = 'https://None'
config.WorkQueueManager.dbname = 'whatever'
config.WorkQueueManager.inboxDatabase = 'whatever2'
config.WorkQueueManager.queueParams = {}
config.WorkQueueManager.queueParams[
"ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue"
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("RetryManager")
config.RetryManager.logLevel = 'DEBUG'
config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager'
config.RetryManager.pollInterval = 10
# These are the cooloff times for the RetryManager, the times it waits
# Before attempting resubmission
config.RetryManager.section_("DefaultRetryAlgo")
config.RetryManager.DefaultRetryAlgo.section_("default")
config.RetryManager.DefaultRetryAlgo.default.coolOffTime = {
'create': 120,
'submit': 120,
'job': 120
}
# Path to plugin directory
config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns'
config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE()
config.RetryManager.componentDir = os.path.join(
os.getcwd(), 'Components')
# ErrorHandler
# Not essential, but useful for ProcessingAlgo
config.component_("ErrorHandler")
config.ErrorHandler.maxRetries = 5
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "retry_manager_t"
return config
def testE(self):
"""test save/load """
testValues = [
"string", 123, 123.456,
["list", 789, 10.1 ],
{ "dict1" : "value", "dict2" : 10.0 }
]
config = Configuration()
for x in range(0, 5):
config.section_("Section%s" % x)
config.component_("Component%s" % x)
sect = getattr(config, "Section%s" % x)
comp = getattr(config, "Component%s" % x)
sect.document_("This is Section%s" % x)
comp.document_("This is Component%s" % x)
for i in range(0, 5):
setattr(comp, "Parameter%s" % i, testValues[i])
setattr(sect, "Parameter%s" % i, testValues[i])
comp.document_("This is Parameter%s" % i,
"Parameter%s" %i)
sect.document_("This is Parameter%s" %i,
"Parameter%s" %i)
stringSave = str(config)
documentSave = config.documentedString_()
commentSave = config.commentedString_()
saveConfigurationFile(config, self.normalSave)
saveConfigurationFile(config, self.docSave, document = True)
saveConfigurationFile(config, self.commentSave, comment = True)
plainConfig = loadConfigurationFile(self.normalSave)
docConfig = loadConfigurationFile(self.docSave)
commentConfig = loadConfigurationFile(self.commentSave)
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return
def setUp(self):
"Setup MSManager for testing"
config = Configuration()
data = config.section_('data')
data.reqmgr2Url = "http://localhost/reqmgr2"
data.verbose = True
data.interval = 600
data.quotaUsage = 0.8
data.quotaAccount = "DataOps"
data.enableStatusTransition = True
data.rucioAccount = "test"
data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod"
data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader"
self.mgr = MSManager(data)
data.services = ['monitor']
self.mgr_monit = MSManager(data)
data.services = ['transferor']
data.limitRequestsPerCycle = 50
data.enableDataTransfer = True
self.mgr_trans = MSManager(data)
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("ErrorHandler")
# The log level of the component.
config.ErrorHandler.logLevel = 'DEBUG'
# The namespace of the component
config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
# maximum number of threads we want to deal
# with messages per pool.
config.ErrorHandler.maxThreads = 30
# maximum number of retries we want for job
config.ErrorHandler.maxRetries = 5
# The poll interval at which to look for failed jobs
config.ErrorHandler.pollInterval = 60
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "errorhandler_t_jd"
config.section_('ACDC')
config.ACDC.couchurl = self.testInit.couchUrl
config.ACDC.database = "errorhandler_t"
return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") )
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print "****WARNING: the DBHOST environment variable will be deprecated soon***"
print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
# after this you can augment it with whatever you need.
return config
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'DEBUG'
config.DBSUpload.DBSBlockMaxFiles = 500
config.DBSUpload.DBSBlockMaxTime = 600
config.DBSUpload.DBSBlockMaxSize = 999999999999
config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.nProcesses = 1
config.DBSUpload.dbsWaitTime = 1
return config
#!/usr/bin/env python
"""
Defines default config values for JobStatusLite specific parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_('JobStatusLite')
config.JobStatusLite.namespace = 'WMComponent.JobStatusLite.JobStatusLite'
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.logLevel = 'INFO'
config.JobStatusLite.pollInterval = 180
config.JobStatusLite.queryInterval = 120
config.JobStatusLite.jobLoadLimit = 100
config.JobStatusLite.maxJobQuery = 100
config.JobStatusLite.taskLimit = 30
config.JobStatusLite.maxJobsCommit = 100
config.JobStatusLite.processes = 5
def getConfig(self):
"""
_getConfig_
This is the global test configuration object
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = 'DEBUG'
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components')
# JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter'
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'CondorGlobusPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'DEBUG'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
# JobAccountant
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobAccountant.logLevel = 'INFO'
# JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobArchiver.numberOfJobsToCluster = 1000
# Task Archiver
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testInit.generateWorkDir()
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.submitDir)
return config
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.failUnless(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.failUnless(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception as ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
self.failUnless( "Parameter1" in section1Dict)
self.failUnless( "Parameter2" in section1Dict)
self.failUnless( "Parameter3" in section1Dict)
self.failUnless( "Parameter4" in section1Dict)
self.assertEqual(section1Dict['Parameter1'],
config.Section1.Parameter1)
self.assertEqual(section1Dict['Parameter2'],
config.Section1.Parameter2)
self.assertEqual(section1Dict['Parameter3'],
config.Section1.Parameter3)
self.assertEqual(section1Dict['Parameter4'],
config.Section1.Parameter4)
# compound types
config.Section2.List = ["string", 123, 123.456, False]
config.Section2.Dictionary = { "string" : "string",
"int" : 123,
"float" : 123.456,
"bool" : False}
config.Section2.Tuple = ("string", 123, 123.456, False)
self.assertEqual(config.Section2.List,
["string", 123, 123.456, False])
self.assertEqual(config.Section2.Tuple,
("string", 123, 123.456, False))
class DummyObject:
pass
# unsupported parameter type
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadObject", DummyObject())
# unsupported data type in compound type
badList = [ DummyObject(), DummyObject()]
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadList", badList)
