def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("RetryManager")
config.RetryManager.logLevel = 'DEBUG'
config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager'
config.RetryManager.maxRetries = 10
config.RetryManager.pollInterval = 10
# These are the cooloff times for the RetryManager, the times it waits
# Before attempting resubmission
config.RetryManager.coolOffTime = {'create': 120, 'submit': 120, 'job': 120}
# Path to plugin directory
config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns'
#config.RetryManager.pluginName = ''
config.RetryManager.WMCoreBase = WMCore.WMInit.getWMBASE()
config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components')
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "retry_manager_t"
return config
def getConfig(self, dbs3UploadOnly = False):
"""
_getConfig_
This creates the actual config file used by the component.
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBS3Upload")
config.DBS3Upload.pollInterval = 10
config.DBS3Upload.logLevel = 'DEBUG'
#config.DBS3Upload.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/dev/global/DBSWriter"
#config.DBS3Upload.dbsUrl = "https://dbs3-dev01.cern.ch/dbs/prod/global/DBSWriter"
config.DBS3Upload.dbsUrl = self.dbsUrl
config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBS3Upload.nProcesses = 1
config.DBS3Upload.dbsWaitTime = 0.1
config.DBS3Upload.datasetType = "VALID"
config.DBS3Upload.dbs3UploadOnly = dbs3UploadOnly
return config
def getConfig(self):
"""
_getConfig_
Creates a common config.
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "DashboardReporter"
config.Agent.useHeartbeat = False
config.section_("DashboardReporter")
config.DashboardReporter.dashboardHost = "cmssrv52.fnal.gov"
config.DashboardReporter.dashboardPort = 8884
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = "dashboardreporter_t"
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
return config
def testC(self):
"""add components"""
config = Configuration()
config.component_("Component1")
config.component_("Component2")
config.component_("Component3")
comp1 = getattr(config, "Component1", None)
self.failUnless(comp1 != None)
comp2 = getattr(config, "Component2", None)
self.failUnless(comp2 != None)
def createConfig(self):
"""
_createConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'ERROR'
config.DBSUpload.maxThreads = 1
config.DBSUpload.namespace = 'WMComponent.DBSUpload.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.workerThreads = 4
config.section_("DBSInterface")
config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
config.DBSInterface.DBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSBlockMaxFiles = 10
config.DBSInterface.DBSBlockMaxSize = 9999999999
config.DBSInterface.DBSBlockMaxTime = 10000
config.DBSInterface.MaxFilesToCommit = 10
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# configure threshold of DBS upload queue size alert threshold
# reference: trac ticket #1628
config.DBSUpload.alertUploadQueueSize = 2000
return config
def getASOConfig(self):
"""
_getWMAgentConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("Agent")
config.Agent.contact = "*****@*****.**"
config.Agent.agentName = "testingagent"
config.Agent.hostName = "localhost"
config.Agent.teamName = "testingteam"
config.component_("AsyncTransfer")
config.AsyncTransfer.log_level = logging.DEBUG
config.AsyncTransfer.namespace = "AsyncStageOut.AsyncTransfer"
config.AsyncTransfer.componentDir = config.General.workDir
config.AsyncTransfer.pollInterval = 10
config.AsyncTransfer.pollViewsInterval = 10
config.AsyncTransfer.couch_instance = os.getenv('COUCHURL')
config.AsyncTransfer.files_database = 'asynctransfer2_t'
config.AsyncTransfer.statitics_database = 'asynctransfer_stat_t'
config.AsyncTransfer.requests_database = 'aso_request_database_t'
config.AsyncTransfer.data_source = os.getenv('COUCHURL')
config.AsyncTransfer.db_source = 'asynctrackerpoller_t_wmstats'
config.AsyncTransfer.pluginName = "CentralMonitoring"
config.AsyncTransfer.pluginDir = "AsyncStageOut.Plugins"
config.AsyncTransfer.max_files_per_transfer = 1000
config.AsyncTransfer.pool_size = 1
config.AsyncTransfer.max_retry = 0
# config.AsyncTransfer.credentialDir = credentialDir
# config.AsyncTransfer.UISetupScript = ui_script
# config.AsyncTransfer.transfer_script = 'ftscp'
# config.AsyncTransfer.serverDN = hostDN
# config.AsyncTransfer.pollStatInterval = 86400
# config.AsyncTransfer.expiration_days = 30
# config.AsyncTransfer.couch_statinstance = statCouchUrl
# config.AsyncTransfer.serviceCert = serviceCert
# config.AsyncTransfer.serviceKey = "/path/to/valid/host-key"
# config.AsyncTransfer.cleanEnvironment = True
config.AsyncTransfer.user_monitoring_db = 'user_monitoring_asynctransfer_t'
config.AsyncTransfer.couch_user_monitoring_instance = os.getenv('COUCHURL')
config.AsyncTransfer.analyticsPollingInterval = 60
# config.AsyncTransfer.filesCleaningPollingInterval = 14400
config.AsyncTransfer.summaries_expiration_days = 30
return config
def createConfig(self):
"""
_createConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'ERROR'
config.DBSUpload.maxThreads = 1
config.DBSUpload.namespace = 'WMComponent.DBSUpload.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.workerThreads = 4
config.section_("DBSInterface")
config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
config.DBSInterface.DBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSBlockMaxFiles = 10
config.DBSInterface.DBSBlockMaxSize = 9999999999
config.DBSInterface.DBSBlockMaxTime = 10000
config.DBSInterface.MaxFilesToCommit = 10
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# configure threshold of DBS upload queue size alert threshold
# reference: trac ticket #1628
config.DBSUpload.alertUploadQueueSize = 2000
return config
def getConfig(self):
"""
_createConfig_
General config file
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.General.WorkDir = os.getenv("TESTDIR", os.getcwd())
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL",
"cmssrv48.fnal.gov:5984")
config.JobStateMachine.couchDBName = "jobarchiver_t_0"
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
#config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = self.testDir
config.JobArchiver.numberOfJobsToCluster = 1000
config.component_('WorkQueueManager')
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager"
config.WorkQueueManager.level = 'LocalQueue'
config.WorkQueueManager.logLevel = 'DEBUG'
config.WorkQueueManager.couchurl = 'https://None'
config.WorkQueueManager.dbname = 'whatever'
config.WorkQueueManager.inboxDatabase = 'whatever2'
config.WorkQueueManager.queueParams = {}
config.WorkQueueManager.queueParams[
"ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue"
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
return config
def getConfig(self):
"""
_getConfig_
Build a basic BossAir config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
config.Agent.useHeartbeat = False
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.pollInterval = 1
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 1
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
# JobStatusLite
config.component_('JobStatusLite')
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400}
config.JobStatusLite.pollInterval = 1
return config
def getConfig(self):
"""
_getConfig_
Build a basic JobTracker config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(),
'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
'badtest.fnal.gov')
#config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh')
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# BossAir
config.component_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = "jobtracker_t"
return config
def getConfig(self):
"""
_getConfig_
Build a basic JobTracker config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
#config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh')
config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# BossAir
config.component_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = "jobtracker_t"
return config
def getConfig(self):
"""
_createConfig_
General config file
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.General.WorkDir = os.getenv("TESTDIR", os.getcwd())
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984")
config.JobStateMachine.couchDBName = "jobarchiver_t_0"
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
#config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = self.testDir
config.JobArchiver.numberOfJobsToCluster = 1000
config.component_('WorkQueueManager')
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager"
config.WorkQueueManager.level = 'LocalQueue'
config.WorkQueueManager.logLevel = 'DEBUG'
config.WorkQueueManager.couchurl = 'https://None'
config.WorkQueueManager.dbname = 'whatever'
config.WorkQueueManager.inboxDatabase = 'whatever2'
config.WorkQueueManager.queueParams = {}
config.WorkQueueManager.queueParams["ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue"
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
return config
def getConfig(self):
"""
_getConfig_
Build a basic BossAir config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
config.Agent.useHeartbeat = False
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh'
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.pollInterval = 1
config.JobSubmitter.pluginName = 'AirPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMInit.getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'INFO'
config.JobTracker.pollInterval = 1
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
# JobStatusLite
config.component_('JobStatusLite')
config.JobStatusLite.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400}
config.JobStatusLite.pollInterval = 1
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def getConfig(self):
"""
_getConfig_
Build a basic BossAir config
"""
config = Configuration()
config.section_("Agent")
config.Agent.agentName = "testAgent"
config.Agent.componentName = "test"
config.Agent.useHeartbeat = False
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("BossAir")
config.BossAir.pluginNames = ["TestPlugin", "CondorPlugin"]
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = "INFO"
config.JobSubmitter.pollInterval = 1
config.JobSubmitter.pluginName = "AirPlugin"
config.JobSubmitter.pluginDir = "JobSubmitter.Plugins"
config.JobSubmitter.submitDir = os.path.join(self.testDir, "submit")
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", "badtest.fnal.gov")
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/JobSubmitter_t", "submit.sh"
)
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), "Components")
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), "config.cfg")
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = "INFO"
config.JobTracker.pollInterval = 1
# JobStateMachine
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "bossair_t"
# JobStatusLite
config.component_("JobStatusLite")
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), "Components")
config.JobStatusLite.stateTimeouts = {"Pending": 10, "Running": 86400}
config.JobStatusLite.pollInterval = 1
return config
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'DEBUG'
config.DBSUpload.DBSBlockMaxFiles = 1
config.DBSUpload.DBSBlockMaxTime = 2
config.DBSUpload.DBSBlockMaxSize = 999999999999
config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
#config.DBSUpload.dbsUrl = 'https://localhost:1443/dbs/prod/global/DBSWriter'
config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.nProcesses = 1
config.DBSUpload.dbsWaitTime = 0.1
return config
def testE(self):
"""test save/load """
testValues = [
"string", 123, 123.456,
["list", 789, 10.1 ],
{ "dict1" : "value", "dict2" : 10.0 }
]
config = Configuration()
for x in range(0, 5):
config.section_("Section%s" % x)
config.component_("Component%s" % x)
sect = getattr(config, "Section%s" % x)
comp = getattr(config, "Component%s" % x)
sect.document_("This is Section%s" % x)
comp.document_("This is Component%s" % x)
for i in range(0, 5):
setattr(comp, "Parameter%s" % i, testValues[i])
setattr(sect, "Parameter%s" % i, testValues[i])
comp.document_("This is Parameter%s" % i,
"Parameter%s" %i)
sect.document_("This is Parameter%s" %i,
"Parameter%s" %i)
stringSave = str(config)
documentSave = config.documentedString_()
commentSave = config.commentedString_()
saveConfigurationFile(config, self.normalSave)
saveConfigurationFile(config, self.docSave, document = True)
saveConfigurationFile(config, self.commentSave, comment = True)
plainConfig = loadConfigurationFile(self.normalSave)
docConfig = loadConfigurationFile(self.docSave)
commentConfig = loadConfigurationFile(self.commentSave)
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError("You must set the DATABASE environment variable to run tests")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print("****WARNING: the DBHOST environment variable will be deprecated soon***")
print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("ErrorHandler")
# The log level of the component.
config.ErrorHandler.logLevel = 'DEBUG'
# The namespace of the component
config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
# maximum number of threads we want to deal
# with messages per pool.
config.ErrorHandler.maxThreads = 30
# maximum number of retries we want for job
config.ErrorHandler.maxRetries = 5
# The poll interval at which to look for failed jobs
config.ErrorHandler.pollInterval = 60
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "errorhandler_t_jd"
config.section_('ACDC')
config.ACDC.couchurl = self.testInit.couchUrl
config.ACDC.database = "errorhandler_t"
return config
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'DEBUG'
config.DBSUpload.DBSBlockMaxFiles = 500
config.DBSUpload.DBSBlockMaxTime = 600
config.DBSUpload.DBSBlockMaxSize = 999999999999
config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.nProcesses = 1
config.DBSUpload.dbsWaitTime = 1
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("ErrorHandler")
# The log level of the component.
config.ErrorHandler.logLevel = 'DEBUG'
# The namespace of the component
config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
# maximum number of threads we want to deal
# with messages per pool.
config.ErrorHandler.maxThreads = 30
# maximum number of retries we want for job
config.ErrorHandler.maxRetries = 5
# The poll interval at which to look for failed jobs
config.ErrorHandler.pollInterval = 60
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "errorhandler_t_jd"
config.section_('ACDC')
config.ACDC.couchurl = self.testInit.couchUrl
config.ACDC.database = "errorhandler_t"
return config
def getConfig(self):
"""
_getConfig_
Gets a basic config from default location
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
#Now the CoreDatabase information
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# BossAir and MockPlugin configuration
config.section_("BossAir")
config.BossAir.pluginNames = ['MockPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.multicoreTaskTypes = [
'MultiProcessing', 'MultiProduction'
]
config.BossAir.nCondorProcesses = 1
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh")
# JobSubmitter configuration
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'DEBUG'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
self.testDir, 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "jobsubmitter_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
def getConfig(self):
"""
_getConfig_
Gets a basic config from default location
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
#Now the CoreDatabase information
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# BossAir and MockPlugin configuration
config.section_("BossAir")
config.BossAir.pluginNames = ['MockPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction']
config.BossAir.nCondorProcesses = 1
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
"submit.sh")
# JobSubmitter configuration
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'DEBUG'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.submitScript = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(self.testDir, 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "jobsubmitter_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
def getConfig(self):
"""
_getConfig_
Creates a common config.
"""
myThread = threading.currentThread()
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = self.componentName
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
#The log level of the component.
#config.JobCreator.logLevel = 'SQLDEBUG'
config.JobCreator.logLevel = 'INFO'
# maximum number of threads we want to deal
# with messages per pool.
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
#config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 4
config.JobCreator.componentDir = self.testDir
config.JobCreator.useWorkQueue = True
config.JobCreator.WorkQueueParams = {'emulateDBSReader': True}
# We now call the JobMaker from here
config.component_('JobMaker')
config.JobMaker.logLevel = 'INFO'
config.JobMaker.namespace = 'WMCore.WMSpec.Makers.JobMaker'
config.JobMaker.maxThreads = 1
config.JobMaker.makeJobsHandler = 'WMCore.WMSpec.Makers.Handlers.MakeJobs'
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
config.JobStateMachine.couchDBName = self.couchdbname
return config
def getConfig(self):
"""
_getConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("RetryManager")
config.RetryManager.logLevel = 'DEBUG'
config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager'
config.RetryManager.pollInterval = 10
# These are the cooloff times for the RetryManager, the times it waits
# Before attempting resubmission
config.RetryManager.section_("DefaultRetryAlgo")
config.RetryManager.DefaultRetryAlgo.section_("default")
config.RetryManager.DefaultRetryAlgo.default.coolOffTime = {
'create': 120,
'submit': 120,
'job': 120
}
# Path to plugin directory
config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns'
config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE()
config.RetryManager.componentDir = os.path.join(
os.getcwd(), 'Components')
# ErrorHandler
# Not essential, but useful for ProcessingAlgo
config.component_("ErrorHandler")
config.ErrorHandler.maxRetries = 5
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "retry_manager_t"
return config
def getWMAgentConfig(self):
"""
_getWMAgentConfig_
"""
config = Configuration()
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("AsyncStageoutTracker")
# The log level of the component.
config.AsyncStageoutTracker.logLevel = 'DEBUG'
# The namespace of the component
config.AsyncStageoutTracker.namespace = 'WMComponent.AsyncStageoutTracker.AsyncStageoutTracker'
# maximum number of threads we want to deal
# with messages per pool.
config.AsyncStageoutTracker.maxThreads = 1
# maximum number of retries we want for job
config.AsyncStageoutTracker.maxRetries = 5
# The poll interval at which to look for failed jobs
config.AsyncStageoutTracker.pollInterval = 60
config.AsyncStageoutTracker.couchurl = "http://127.0.0.1:5984"
config.AsyncStageoutTracker.couchDBName = "user_monitoring_asynctransfer_t"
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = "asynctrackerpoller_t_jd"
config.JobStateMachine.jobSummaryDBName= 'asynctrackerpoller_t_wmstats'
config.component_('JobAccountant')
config.section_('ACDC')
config.ACDC.couchurl = self.testInit.couchUrl
config.ACDC.database = "asynctrackerpoller_t"
return config
#!/usr/bin/env python
"""
Defines default config values for JobAccountant specific
parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("WorkQueueManager")
# set type of work queue
config.WorkQueueManager.level = "GlobalQueue"
#config.WorkQueueManager.level = "LocalQueue"
# In general can be left alone
config.WorkQueueManager.componentDir = config.General.WorkDir + "/WorkQueueManager"
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
#config.WorkQueueManager.logLevel = 'INFO'
#config.WorkQueueManager.pollInterval = 600
# RequestManager config
config.WorkQueueManager.reqMgrConfig = {}
# uncomment to override default reqMgr url
#config.WorkQueueManager.reqMgrConfig['endpoint'] = 'http://cmssrv49.fnal.gov:8585'
config.WorkQueueManager.reqMgrConfig['teamName'] = 'Dodgers'
#!/usr/bin/env python
"""
Defines default config values for HarvestingScheduler specific
parameters.
"""
__all__ = []
import os
import os.path
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("HarvestingScheduler")
config.HarvestingScheduler.namespace = "WMComponent.HarvestingScheduler.HarvestingScheduler"
config.HarvestingScheduler.componentDir = os.getcwd() + "/HarvestingScheduler"
config.HarvestingScheduler.workloadCache = os.getcwd(), +"/HarvestingScheduler/workloadCache"
config.HarvestingScheduler.targetSite = "srm-cms.cern.ch"
config.HarvestingScheduler.scramArch = "slc5_ia32_gcc434"
config.HarvestingScheduler.cmsPath = "/afs/cern.ch/cms/sw"
config.HarvestingScheduler.proxy = os.getenv("X509_USER_PROXY", "bad_proxy")
config.HarvestingScheduler.dqmGuiUrl = "https://cmsweb.cern.ch/dqm/dev"
config.HarvestingScheduler.couchurl = "http://127.0.0.1:5984"
config.HarvestingScheduler.couchDBName = "datasets_to_harvest"
config.HarvestingScheduler.doStageOut = True
config.HarvestingScheduler.doDqmUpload = True
config.HarvestingScheduler.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/"
config.HarvestingScheduler.dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
config.HarvestingScheduler.expiryTime = 3600 * 24 * 30 # One month
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for CranWmbs specific
parameters.
"""
__all__ = []
__revision__ = "$Id: DefaultConfig.py,v 0.2 2009/09/30 01:26:34 hriahi Exp $"
__version__ = "$Revision: 0.2 $"
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("CrabJobCreator")
config.CrabJobCreator.logLevel = "DEBUG"
config.CrabJobCreator.componentName = "CrabJobCreator"
config.CrabJobCreator.componentDir = \
os.path.join(os.getenv("TESTDIR"), "CrabJobCreator")
# The maximum number of threads to process each message type
config.CrabJobCreator.maxThreads = 10
# CrabServer parameter
config.CrabJobCreator.wdir = '/data/Storage/logs'
config.CrabJobCreator.maxRetries = 3
config.CrabJobCreator.credentialType = "Proxy"
config.CrabJobCreator.ProxiesDir = "/tmp/del_proxies/"
config.CrabJobCreator.StorageName = "crab.pg.infn.it"
config.CrabJobCreator.storagePort = "2811"
config.CrabJobCreator.Protocol = "gridftp"
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.section_("CoreDatabase")
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = os.getenv('DIALECT', None)
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
#The log level of the component.
config.DBSUpload.logLevel = 'DEBUG'
# maximum number of threads we want to deal
# with messages per pool.
config.DBSUpload.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSUpload.bufferSuccessHandler = \
'WMComponent.DBSUpload.Handler.BufferSuccess'
config.DBSUpload.newWorkflowHandler = \
'WMComponent.DBSUpload.Handler.NewWorkflowHandler'
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for DBSBuffer specific
parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("DBSBuffer")
#The log level of the component.
config.DBSBuffer.logLevel = 'INFO'
#The namespace of the buffer. Used to load the module as daemon
config.DBSBuffer.namespace = 'WMComponent.DBSBuffer.DBSBuffer'
# maximum number of threads we want to deal
# with messages per pool.
config.DBSBuffer.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSBuffer.jobSuccessHandler = \
'WMComponent.DBSBuffer.Handler.JobSuccess'
def getConfig(
self, configPath=os.path.join(WMCore.WMInit.getWMBASE(), "src/python/WMComponent/JobSubmitter/DefaultConfig.py")
):
"""
_getConfig_
Gets a basic config from default location
"""
myThread = threading.currentThread()
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = "testAgent"
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("BossAir")
config.BossAir.pluginNames = ["TestPlugin", "CondorPlugin"]
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = "INFO"
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = "CondorGlobusPlugin"
config.JobSubmitter.pluginDir = "JobSubmitter.Plugins"
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", "badtest.fnal.gov")
config.JobSubmitter.submitScript = os.path.join(
WMCore.WMBase.getTestBase(), "WMComponent_t/JobSubmitter_t", "submit.sh"
)
config.JobSubmitter.componentDir = os.path.join(self.testDir, "Components")
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
config.JobSubmitter.inputFile = os.path.join(
WMCore.WMBase.getTestBase(), "WMComponent_t/JobSubmitter_t", "FrameworkJobReport-4540.xml"
)
config.JobSubmitter.deleteJDLFiles = False
# JobStateMachine
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "jobsubmitter_t"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
def getConfig(self):
"""
_getConfig_
This is the global test configuration object
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = 'DEBUG'
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components')
# JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter'
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'CondorGlobusPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'DEBUG'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
# JobAccountant
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobAccountant.logLevel = 'INFO'
# JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobArchiver.numberOfJobsToCluster = 1000
# Task Archiver
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testInit.generateWorkDir()
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.submitDir)
return config
config = Configuration()
####################
# General: General Settings Section
####################
config.section_("General")
config.General.workDir = '/tmp/TQ-test/'
######################
# Task Queue Component
######################
config.component_("TQComp")
config.TQComp.namespace = "TQComp.TQComp"
#The log level of the component.
config.TQComp.logLevel = 'DEBUG'
# maximum number of threads we want to deal
# with messages per pool
config.TQComp.maxThreads = 5
# Component dir (twice due to some bug in wmcoreD)
config.TQComp.componentDir = '/tmp/TQ-test/TQComp'
config.TQComp.ComponentDir = '/tmp/TQ-test/TQComp'
# Where to download specs and sandboxes from
config.TQComp.downloadBaseUrl = 'http://glitece.ciemat.es:20030'
#!/usr/bin/env python
"""
Defines default config values for JobStatusLite specific parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_('JobStatusLite')
config.JobStatusLite.namespace = 'WMComponent.JobStatusLite.JobStatusLite'
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.logLevel = 'INFO'
config.JobStatusLite.pollInterval = 180
config.JobStatusLite.queryInterval = 120
config.JobStatusLite.jobLoadLimit = 100
config.JobStatusLite.maxJobQuery = 100
config.JobStatusLite.taskLimit = 30
config.JobStatusLite.maxJobsCommit = 100
config.JobStatusLite.processes = 5
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.section_("CoreDatabase")
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = os.getenv('DIALECT', None)
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
#The log level of the component.
config.DBSUpload.logLevel = 'DEBUG'
# maximum number of threads we want to deal
# with messages per pool.
config.DBSUpload.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSUpload.bufferSuccessHandler = \
'WMComponent.DBSUpload.Handler.BufferSuccess'
config.DBSUpload.newWorkflowHandler = \
'WMComponent.DBSUpload.Handler.NewWorkflowHandler'
the common configuration file, which is accessed by environment variable
"""
import os
import sys
import getopt
import logging
from WMCore.Agent.Configuration import Configuration
# //
# // Find and load the Configuration
# //
config = Configuration()
config.component_("PilotManagerComponent")
# The log level of the component.
config.PilotManagerComponent.logLevel = "DEBUG"
# maximum number of threads we want to deal
# with messages per pool.
config.PilotManagerComponent.maxThreads = 30
# emulation mode
config.PilotManagerComponent.emulationMode = True
# maximum number of retries we want for job
config.PilotManagerComponent.maxRetries = 10
# component dir
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for WorkflowManager specific
parameters.
"""
__all__ = []
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("WorkflowManager")
config.WorkflowManager.logLevel = "INFO"
config.WorkflowManager.componentName = "WorkflowManager"
config.WorkflowManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "WorkflowManager")
# The maximum number of threads to process each message type
config.WorkflowManager.maxThreads = 10
# The poll interval at which to look for new filesets
config.WorkflowManager.pollInterval = 60
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for FeederManager specific
parameters.
"""
__all__ = []
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("FeederManager")
config.FeederManager.logLevel = "INFO"
config.FeederManager.componentName = "FeederManager"
config.FeederManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "FeederManager")
config.FeederManager.addDatasetWatchHandler = \
'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'
# The maximum number of threads to process each message type
config.FeederManager.maxThreads = 10
# The poll interval at which to look for new fileset/feeder association
config.FeederManager.pollInterval = 60
#!/usr/bin/env python
#pylint: disable=E1101,E1103,C0103,R0902
"""
Defines default config values for WorkflowManager specific
parameters.
"""
__all__ = []
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("WorkflowManager")
config.WorkflowManager.logLevel = "INFO"
config.WorkflowManager.componentName = "WorkflowManager"
config.WorkflowManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "WorkflowManager")
# The maximum number of threads to process each message type
config.WorkflowManager.maxThreads = 10
# The poll interval at which to look for new filesets
config.WorkflowManager.pollInterval = 60
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for FeederManager specific
parameters.
"""
__all__ = []
import os
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("FeederManager")
config.FeederManager.logLevel = "INFO"
config.FeederManager.componentName = "FeederManager"
config.FeederManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "FeederManager")
config.FeederManager.addDatasetWatchHandler = \
'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'
# The maximum number of threads to process each message type
config.FeederManager.maxThreads = 10
# The poll interval at which to look for new fileset/feeder association
config.FeederManager.pollInterval = 60
#!/usr/bin/env python
"""
Defines default config values for JobStatusLite specific parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_('JobStatusLite')
config.JobStatusLite.namespace = 'WMComponent.JobStatusLite.JobStatusLite'
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.logLevel = 'INFO'
config.JobStatusLite.pollInterval = 180
config.JobStatusLite.queryInterval = 120
config.JobStatusLite.jobLoadLimit = 100
config.JobStatusLite.maxJobQuery = 100
config.JobStatusLite.taskLimit = 30
config.JobStatusLite.maxJobsCommit = 100
config.JobStatusLite.processes = 5
#!/usr/bin/env python
"""
Defines default config values for JobAccountant specific
parameters.
"""
__all__ = []
from WMCore.Agent.Configuration import Configuration
config = Configuration()
config.component_("WorkQueueManager")
# set type of work queue
config.WorkQueueManager.level = "GlobalQueue"
#config.WorkQueueManager.level = "LocalQueue"
# In general can be left alone
config.WorkQueueManager.componentDir = config.General.WorkDir + "/WorkQueueManager"
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
#config.WorkQueueManager.logLevel = 'INFO'
#config.WorkQueueManager.pollInterval = 600
# RequestManager config
config.WorkQueueManager.reqMgrConfig = {}
# uncomment to override default reqMgr url
#config.WorkQueueManager.reqMgrConfig['endpoint'] = 'http://cmssrv49.fnal.gov:8585'
config.WorkQueueManager.reqMgrConfig['teamName'] = 'Dodgers'
# add parameters for global or local queue if default param is not what you want
config.WorkQueueManager.queueParams = {'LocationRefreshInterval': 10}
# uncomment to change CacheDir from default