def testD(self): """test documentation""" config = Configuration() config.section_("Section1") config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 config.Section1.Parameter5 = {"test1": "test2", "test3": 123} config.Section1.document_("""This is Section1""") config.Section1.document_("""This is Section1.Parameter1""", "Parameter1") config.Section1.document_("""This is Section1.Parameter2""", "Parameter2") config.Section1.document_( """This is Section1.Parameter3\n with multiline comments""", "Parameter3") try: config.Section1.documentedString_() except Exception, ex: msg = "Error calling ConfigSection.documentedString_:\n" msg += "%s\n" % str(ex) self.fail(msg)
def __init__(self, **configDict): """ init jobCreator """ myThread = threading.currentThread() self.transaction = myThread.transaction #DAO factory for WMBS objects self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) # WMCore splitter factory for splitting up jobs. self.splitterFactory = SplitterFactory() config = Configuration() config.section_("JobStateMachine") config.JobStateMachine.couchurl = configDict["couchURL"] config.JobStateMachine.couch_retries = configDict["defaultRetries"] config.JobStateMachine.couchDBName = configDict["couchDBName"] self.config = config #Variables self.jobCacheDir = configDict['jobCacheDir'] self.defaultJobType = configDict['defaultJobType'] self.limit = configDict.get('fileLoadLimit', 500) self.createWorkArea = CreateWorkArea() self.changeState = ChangeState(self.config) return
def testD(self): """test documentation""" config = Configuration() config.section_("Section1") config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 config.Section1.Parameter5 = { "test1" : "test2", "test3" : 123 } config.Section1.document_("""This is Section1""") config.Section1.document_("""This is Section1.Parameter1""", "Parameter1") config.Section1.document_("""This is Section1.Parameter2""", "Parameter2") config.Section1.document_("""This is Section1.Parameter3\n with multiline comments""", "Parameter3") try: config.Section1.documentedString_() except Exception, ex: msg = "Error calling ConfigSection.documentedString_:\n" msg += "%s\n" % str(ex) self.fail(msg)
def __init__(self, **configDict): """ init jobCreator """ myThread = threading.currentThread() self.transaction = myThread.transaction # DAO factory for WMBS objects self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) # WMCore splitter factory for splitting up jobs. self.splitterFactory = SplitterFactory() config = Configuration() config.section_("JobStateMachine") config.JobStateMachine.couchurl = configDict["couchURL"] config.JobStateMachine.couch_retries = configDict["defaultRetries"] config.JobStateMachine.couchDBName = configDict["couchDBName"] self.config = config # Variables self.jobCacheDir = configDict["jobCacheDir"] self.defaultJobType = configDict["defaultJobType"] self.limit = configDict.get("fileLoadLimit", 500) self.createWorkArea = CreateWorkArea() self.changeState = ChangeState(self.config) return
def setUp(self): "Setup MSManager for testing" config = Configuration() data = config.section_('data') data.reqmgr2Url = "http://localhost/reqmgr2" data.verbose = True data.interval = 600 data.quotaUsage = 0.8 data.quotaAccount = "DataOps" data.enableStatusTransition = True data.rucioAccount = "wma_test" data.rucioUrl = "http://cmsrucio-int.cern.ch" data.rucioAuthUrl = "https://cmsrucio-auth-int.cern.ch" data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod" data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader" data.smtpServer = "localhost" data.fromAddr = "*****@*****.**" data.toAddr = ["*****@*****.**"] data.warningTransferThreshold = 100. * (1000**4) # 100 TB (terabyte) self.mgr = MSManager(data) data.services = ['monitor'] self.mgr_monit = MSManager(data) data.services = ['transferor'] data.limitRequestsPerCycle = 50 data.enableDataTransfer = True self.mgr_trans = MSManager(data)
def getConfig(self): """ _getConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("RetryManager") config.RetryManager.logLevel = 'DEBUG' config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager' config.RetryManager.maxRetries = 10 config.RetryManager.pollInterval = 10 # These are the cooloff times for the RetryManager, the times it waits # Before attempting resubmission config.RetryManager.coolOffTime = {'create': 120, 'submit': 120, 'job': 120} # Path to plugin directory config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns' #config.RetryManager.pluginName = '' config.RetryManager.WMCoreBase = WMCore.WMInit.getWMBASE() config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components') # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "retry_manager_t" return config
def getConfig(self, dbs3UploadOnly = False): """ _getConfig_ This creates the actual config file used by the component. """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBS3Upload") config.DBS3Upload.pollInterval = 10 config.DBS3Upload.logLevel = 'DEBUG' #config.DBS3Upload.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/dev/global/DBSWriter" #config.DBS3Upload.dbsUrl = "https://dbs3-dev01.cern.ch/dbs/prod/global/DBSWriter" config.DBS3Upload.dbsUrl = self.dbsUrl config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBS3Upload.nProcesses = 1 config.DBS3Upload.dbsWaitTime = 0.1 config.DBS3Upload.datasetType = "VALID" config.DBS3Upload.dbs3UploadOnly = dbs3UploadOnly return config
def getASOConfig(self): """ _getWMAgentConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("Agent") config.Agent.contact = "*****@*****.**" config.Agent.agentName = "testingagent" config.Agent.hostName = "localhost" config.Agent.teamName = "testingteam" config.component_("AsyncTransfer") config.AsyncTransfer.log_level = logging.DEBUG config.AsyncTransfer.namespace = "AsyncStageOut.AsyncTransfer" config.AsyncTransfer.componentDir = config.General.workDir config.AsyncTransfer.pollInterval = 10 config.AsyncTransfer.pollViewsInterval = 10 config.AsyncTransfer.couch_instance = os.getenv('COUCHURL') config.AsyncTransfer.files_database = 'asynctransfer2_t' config.AsyncTransfer.statitics_database = 'asynctransfer_stat_t' config.AsyncTransfer.requests_database = 'aso_request_database_t' config.AsyncTransfer.data_source = os.getenv('COUCHURL') config.AsyncTransfer.db_source = 'asynctrackerpoller_t_wmstats' config.AsyncTransfer.pluginName = "CentralMonitoring" config.AsyncTransfer.pluginDir = "AsyncStageOut.Plugins" config.AsyncTransfer.max_files_per_transfer = 1000 config.AsyncTransfer.pool_size = 1 config.AsyncTransfer.max_retry = 0 # config.AsyncTransfer.credentialDir = credentialDir # config.AsyncTransfer.UISetupScript = ui_script # config.AsyncTransfer.transfer_script = 'ftscp' # config.AsyncTransfer.serverDN = hostDN # config.AsyncTransfer.pollStatInterval = 86400 # config.AsyncTransfer.expiration_days = 30 # config.AsyncTransfer.couch_statinstance = statCouchUrl # config.AsyncTransfer.serviceCert = serviceCert # config.AsyncTransfer.serviceKey = "/path/to/valid/host-key" # config.AsyncTransfer.cleanEnvironment = True config.AsyncTransfer.user_monitoring_db = 'user_monitoring_asynctransfer_t' config.AsyncTransfer.couch_user_monitoring_instance = os.getenv('COUCHURL') config.AsyncTransfer.analyticsPollingInterval = 60 # config.AsyncTransfer.filesCleaningPollingInterval = 14400 config.AsyncTransfer.summaries_expiration_days = 30 return config
def testA(self): """ctor""" try: config = Configuration() except Exception as ex: msg = "Failed to instantiate Configuration\n" msg += str(ex) self.fail(msg)
def getConfig(self): """ _getConfig_ """ config = Configuration() config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = 'wmagent_jobdump' return config
def createConfig(self): """ Create a config object for testing """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("DBSInterface") config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet' config.DBSInterface.globalDBSVersion = 'DBS_2_0_9' config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet' config.DBSInterface.DBSVersion = 'DBS_2_0_9' config.DBSInterface.DBSBlockMaxFiles = 10 config.DBSInterface.DBSBlockMaxSize = 9999999999 config.DBSInterface.DBSBlockMaxTime = 10000 config.DBSInterface.MaxFilesToCommit = 10 return config
def testB(self): """add settings""" config = Configuration() config.section_("Section1") section1 = getattr(config, "Section1", None) self.failUnless(section1 != None) config.section_("Section2") section2 = getattr(config, "Section2", None) self.failUnless(section2 != None) self.assertRaises(AttributeError, getattr, config, "Section3") # basic types config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 self.assertEqual(config.Section1.Parameter1, True) self.assertEqual(config.Section1.Parameter2, "string") self.assertEqual(config.Section1.Parameter3, 123) self.assertEqual(config.Section1.Parameter4, 123.456) # dictionary format: try: section1Dict = config.Section1.dictionary_() except Exception, ex: msg = "Error converting section to dictionary:\n" msg += "%s\n" % str(ex) self.fail(msg)
def setUp(self): "Setup MSManager for testing" config = Configuration() data = config.section_('data') data.reqmgr2Url = "http://localhost/reqmgr2" data.verbose = True data.interval = 600 data.quotaUsage = 0.8 data.quotaAccount = "DataOps" data.enableStatusTransition = True data.rucioAccount = "test" data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod" data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader" self.mgr = MSManager(data) data.services = ['monitor'] self.mgr_monit = MSManager(data) data.services = ['transferor'] data.limitRequestsPerCycle = 50 data.enableDataTransfer = True self.mgr_trans = MSManager(data)
def getConfig(self): """ _getConfig_ This creates the actual config file used by the component """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = 'DEBUG' config.DBSUpload.DBSBlockMaxFiles = 500 config.DBSUpload.DBSBlockMaxTime = 600 config.DBSUpload.DBSBlockMaxSize = 999999999999 config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter' config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBSUpload.nProcesses = 1 config.DBSUpload.dbsWaitTime = 1 return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None): """ Loads (if available) your configuration file and augments it with the standard settings used in multiple tests. """ if configurationFile != None: config = loadConfigurationFile(configurationFile) else: config = Configuration() # some general settings that would come from the general default # config file config.Agent.contact = "*****@*****.**" config.Agent.teamName = "Lakers" config.Agent.agentName = "Lebron James" config.section_("General") # If you need a testDir, call testInit.generateWorkDir # config.General.workDir = os.getenv("TESTDIR") config.section_("CoreDatabase") if connectUrl: config.CoreDatabase.connectUrl = connectUrl config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl) config.CoreDatabase.socket = socket or os.getenv("DBSOCK") else: if (os.getenv('DATABASE') == None): raise RuntimeError, \ "You must set the DATABASE environment variable to run tests" config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") ) config.CoreDatabase.socket = os.getenv("DBSOCK") if os.getenv("DBHOST"): print "****WARNING: the DBHOST environment variable will be deprecated soon***" print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****" # after this you can augment it with whatever you need. return config
def getConfig(self): """ _getConfig_ This creates the actual config file used by the component """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = 'DEBUG' config.DBSUpload.DBSBlockMaxFiles = 1 config.DBSUpload.DBSBlockMaxTime = 2 config.DBSUpload.DBSBlockMaxSize = 999999999999 config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter' #config.DBSUpload.dbsUrl = 'https://localhost:1443/dbs/prod/global/DBSWriter' config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBSUpload.nProcesses = 1 config.DBSUpload.dbsWaitTime = 0.1 return config
def testC(self): """add components""" config = Configuration() config.component_("Component1") config.component_("Component2") config.component_("Component3") comp1 = getattr(config, "Component1", None) self.failUnless(comp1 != None) comp2 = getattr(config, "Component2", None) self.failUnless(comp2 != None)
def testE(self): """test save/load """ testValues = [ "string", 123, 123.456, ["list", 789, 10.1 ], { "dict1" : "value", "dict2" : 10.0 } ] config = Configuration() for x in range(0, 5): config.section_("Section%s" % x) config.component_("Component%s" % x) sect = getattr(config, "Section%s" % x) comp = getattr(config, "Component%s" % x) sect.document_("This is Section%s" % x) comp.document_("This is Component%s" % x) for i in range(0, 5): setattr(comp, "Parameter%s" % i, testValues[i]) setattr(sect, "Parameter%s" % i, testValues[i]) comp.document_("This is Parameter%s" % i, "Parameter%s" %i) sect.document_("This is Parameter%s" %i, "Parameter%s" %i) stringSave = str(config) documentSave = config.documentedString_() commentSave = config.commentedString_() saveConfigurationFile(config, self.normalSave) saveConfigurationFile(config, self.docSave, document = True) saveConfigurationFile(config, self.commentSave, comment = True) plainConfig = loadConfigurationFile(self.normalSave) docConfig = loadConfigurationFile(self.docSave) commentConfig = loadConfigurationFile(self.commentSave)
def createConfig(self): """ _createConfig_ Create a config and save it to the temp dir. Set the WMAGENT_CONFIG environment variable so the config gets picked up. """ config = Configuration() config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "resource_control_t" config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") configHandle = open(os.path.join(self.tempDir, "config.py"), "w") configHandle.write(str(config)) configHandle.close() os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py") return
def createConfig(self): """ _createConfig_ Create a config and save it to the temp dir. Set the WMAGENT_CONFIG environment variable so the config gets picked up. """ config = Configuration() config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "resource_control_t" config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "bossair_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' config.JobStateMachine.summaryStatsDBName = 'stat_summary_t' config.section_("BossAir") config.BossAir.pluginDir = "WMCore.BossAir.Plugins" config.BossAir.pluginNames = ["MockPlugin"] config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobAccountant_t/fwjrs', "MergeSuccess.pkl") configHandle = open(os.path.join(self.tempDir, "config.py"), "w") configHandle.write(str(config)) configHandle.close() os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py") return config
def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.multicoreTaskTypes = [ 'MultiProcessing', 'MultiProduction' ] config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config
#!/usr/bin/env python """ Defines default config values for HarvestingScheduler specific parameters. """ __all__ = [] import os import os.path from WMCore.Agent.Configuration import Configuration config = Configuration() config.component_("HarvestingScheduler") config.HarvestingScheduler.namespace = "WMComponent.HarvestingScheduler.HarvestingScheduler" config.HarvestingScheduler.componentDir = os.getcwd() + "/HarvestingScheduler" config.HarvestingScheduler.workloadCache = os.getcwd(), +"/HarvestingScheduler/workloadCache" config.HarvestingScheduler.targetSite = "srm-cms.cern.ch" config.HarvestingScheduler.scramArch = "slc5_ia32_gcc434" config.HarvestingScheduler.cmsPath = "/afs/cern.ch/cms/sw" config.HarvestingScheduler.proxy = os.getenv("X509_USER_PROXY", "bad_proxy") config.HarvestingScheduler.dqmGuiUrl = "https://cmsweb.cern.ch/dqm/dev" config.HarvestingScheduler.couchurl = "http://127.0.0.1:5984" config.HarvestingScheduler.couchDBName = "datasets_to_harvest" config.HarvestingScheduler.doStageOut = True config.HarvestingScheduler.doDqmUpload = True config.HarvestingScheduler.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/" config.HarvestingScheduler.dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" config.HarvestingScheduler.expiryTime = 3600 * 24 * 30 # One month
def createConfig(self): """ _createConfig_ This creates the actual config file used by the component """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = 'ERROR' config.DBSUpload.maxThreads = 1 config.DBSUpload.namespace = 'WMComponent.DBSUpload.DBSUpload' config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBSUpload.workerThreads = 4 config.section_("DBSInterface") config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet' config.DBSInterface.globalDBSVersion = 'DBS_2_0_9' config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet' config.DBSInterface.DBSVersion = 'DBS_2_0_9' config.DBSInterface.MaxFilesToCommit = 10 # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" # configure threshold of DBS upload queue size alert threshold # reference: trac ticket #1628 config.DBSUpload.alertUploadQueueSize = 2000 return config
def createConfig(self): """ _createConfig_ This creates the actual config file used by the component """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "DBSUpload" config.Agent.useHeartbeat = False # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = "ERROR" config.DBSUpload.maxThreads = 1 config.DBSUpload.namespace = "WMComponent.DBSUpload.DBSUpload" config.DBSUpload.componentDir = os.path.join(os.getcwd(), "Components") config.DBSUpload.workerThreads = 4 config.section_("DBSInterface") config.DBSInterface.globalDBSUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet" config.DBSInterface.globalDBSVersion = "DBS_2_0_9" config.DBSInterface.DBSUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" config.DBSInterface.DBSVersion = "DBS_2_0_9" config.DBSInterface.DBSBlockMaxFiles = 10 config.DBSInterface.DBSBlockMaxSize = 9999999999 config.DBSInterface.DBSBlockMaxTime = 10000 config.DBSInterface.MaxFilesToCommit = 10 # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" # configure threshold of DBS upload queue size alert threshold # reference: trac ticket #1628 config.DBSUpload.alertUploadQueueSize = 2000 return config
def getConfig(self): """ _getConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("ErrorHandler") # The log level of the component. config.ErrorHandler.logLevel = 'DEBUG' # The namespace of the component config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler' # maximum number of threads we want to deal # with messages per pool. config.ErrorHandler.maxThreads = 30 # maximum number of retries we want for job config.ErrorHandler.maxRetries = 5 # The poll interval at which to look for failed jobs config.ErrorHandler.pollInterval = 60 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "errorhandler_t_jd" config.section_('ACDC') config.ACDC.couchurl = self.testInit.couchUrl config.ACDC.database = "errorhandler_t" return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None): """ Loads (if available) your configuration file and augments it with the standard settings used in multiple tests. """ if configurationFile != None: config = loadConfigurationFile(configurationFile) else: config = Configuration() # some general settings that would come from the general default # config file config.Agent.contact = "*****@*****.**" config.Agent.teamName = "Lakers" config.Agent.agentName = "Lebron James" config.Agent.hostName = "testhost.laker.world" config.section_("General") # If you need a testDir, call testInit.generateWorkDir # config.General.workDir = os.getenv("TESTDIR") config.section_("CoreDatabase") if connectUrl: config.CoreDatabase.connectUrl = connectUrl config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl) config.CoreDatabase.socket = socket or os.getenv("DBSOCK") else: if os.getenv('DATABASE') == None: raise RuntimeError("You must set the DATABASE environment variable to run tests") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE")) config.CoreDatabase.socket = os.getenv("DBSOCK") if os.getenv("DBHOST"): print("****WARNING: the DBHOST environment variable will be deprecated soon***") print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****") # after this you can augment it with whatever you need. couchurl = os.getenv("COUCHURL") config.section_("ACDC") config.ACDC.couchurl = couchurl config.ACDC.database = "wmagent_acdc_t" config.component_("JobStateMachine") config.JobStateMachine.couchurl = couchurl config.JobStateMachine.couchDBName = "wmagent_job_test" config.JobStateMachine.jobSummaryDBName = "job_summary" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.getcwd() config.JobAccountant.logLevel = 'SQLDEBUG' config.component_("TaskArchiver") config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) config.TaskArchiver.ReqMgrSeviceURL = "request manager service url" config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2" return config
#!/usr/bin/env python """ Defines default config values for JobStatusLite specific parameters. """ __all__ = [] from WMCore.Agent.Configuration import Configuration config = Configuration() config.component_('JobStatusLite') config.JobStatusLite.namespace = 'WMComponent.JobStatusLite.JobStatusLite' config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components') config.JobStatusLite.logLevel = 'INFO' config.JobStatusLite.pollInterval = 180 config.JobStatusLite.queryInterval = 120 config.JobStatusLite.jobLoadLimit = 100 config.JobStatusLite.maxJobQuery = 100 config.JobStatusLite.taskLimit = 30 config.JobStatusLite.maxJobsCommit = 100 config.JobStatusLite.processes = 5
def getConfig(self): """ _getConfig_ Build a basic JobTracker config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') #config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh') config.JobSubmitter.submitScript = os.path.join( WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # BossAir config.component_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "jobtracker_t" return config
def getConfig(self): """ _getConfig_ This is the global test configuration object """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobCreator config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' config.JobCreator.logLevel = 'DEBUG' config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromResourceControl = True config.JobCreator.pollInterval = 10 config.JobCreator.jobCacheDir = self.testDir config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 2 config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components') # JobSubmitter config.component_("JobSubmitter") config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter' config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'CondorGlobusPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join(getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'DEBUG' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 # JobAccountant config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components') config.JobAccountant.logLevel = 'INFO' # JobArchiver config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components') config.JobArchiver.numberOfJobsToCluster = 1000 # Task Archiver config.component_("TaskArchiver") config.TaskArchiver.componentDir = self.testInit.generateWorkDir() config.TaskArchiver.WorkQueueParams = {} config.TaskArchiver.pollInterval = 60 config.TaskArchiver.logLevel = 'INFO' config.TaskArchiver.timeOut = 0 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'mnorman:[email protected]:5984') config.JobStateMachine.couchDBName = "mnorman_test" # Needed, because this is a test os.makedirs(config.JobSubmitter.submitDir) return config
#!/usr/bin/env python #pylint: disable-msg=E1101,E1103,C0103,R0902 """ Defines default config values for DBSBuffer specific parameters. """ __all__ = [] from WMCore.Agent.Configuration import Configuration config = Configuration() config.component_("DBSBuffer") #The log level of the component. config.DBSBuffer.logLevel = 'INFO' #The namespace of the buffer. Used to load the module as daemon config.DBSBuffer.namespace = 'WMComponent.DBSBuffer.DBSBuffer' # maximum number of threads we want to deal # with messages per pool. config.DBSBuffer.maxThreads = 1 # # JobSuccess Handler # config.DBSBuffer.jobSuccessHandler = \ 'WMComponent.DBSBuffer.Handler.JobSuccess'
_StartComponent_ Start the component, reading its configuration from the common configuration file, which is accessed by environment variable """ import os import sys import getopt import logging from WMCore.Agent.Configuration import Configuration # Find and load the Configuration config = Configuration() #################### # General: General Settings Section #################### config.section_("General") config.General.workDir = '/tmp/TQ-test/' ###################### # Task Queue Component ###################### config.component_("TQComp")
#!/usr/bin/env python #pylint: disable-msg=E1101,E1103,C0103,R0902 """ Defines default config values for CranWmbs specific parameters. """ __all__ = [] __revision__ = "$Id: DefaultConfig.py,v 0.2 2009/09/30 01:26:34 hriahi Exp $" __version__ = "$Revision: 0.2 $" import os from WMCore.Agent.Configuration import Configuration config = Configuration() config.component_("CrabJobCreator") config.CrabJobCreator.logLevel = "DEBUG" config.CrabJobCreator.componentName = "CrabJobCreator" config.CrabJobCreator.componentDir = \ os.path.join(os.getenv("TESTDIR"), "CrabJobCreator") # The maximum number of threads to process each message type config.CrabJobCreator.maxThreads = 10 # CrabServer parameter config.CrabJobCreator.wdir = '/data/Storage/logs' config.CrabJobCreator.maxRetries = 3 config.CrabJobCreator.credentialType = "Proxy" config.CrabJobCreator.ProxiesDir = "/tmp/del_proxies/" config.CrabJobCreator.StorageName = "crab.pg.infn.it" config.CrabJobCreator.storagePort = "2811" config.CrabJobCreator.Protocol = "gridftp"
#!/usr/bin/env python #pylint: disable-msg=E1101,E1103,C0103,R0902 """ Defines default config values for FeederManager specific parameters. """ __all__ = [] import os from WMCore.Agent.Configuration import Configuration config = Configuration() config.component_("FeederManager") config.FeederManager.logLevel = "INFO" config.FeederManager.componentName = "FeederManager" config.FeederManager.componentDir = \ os.path.join(os.getenv("TESTDIR"), "FeederManager") config.FeederManager.addDatasetWatchHandler = \ 'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch' # The maximum number of threads to process each message type config.FeederManager.maxThreads = 10 # The poll interval at which to look for new fileset/feeder association config.FeederManager.pollInterval = 60
def getConfig(self): """ _createConfig_ General config file """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.General.WorkDir = os.getenv("TESTDIR", os.getcwd()) #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984") config.JobStateMachine.couchDBName = "jobarchiver_t_0" config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' #config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = self.testDir config.JobArchiver.numberOfJobsToCluster = 1000 config.component_('WorkQueueManager') config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager" config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager" config.WorkQueueManager.level = 'LocalQueue' config.WorkQueueManager.logLevel = 'DEBUG' config.WorkQueueManager.couchurl = 'https://None' config.WorkQueueManager.dbname = 'whatever' config.WorkQueueManager.inboxDatabase = 'whatever2' config.WorkQueueManager.queueParams = {} config.WorkQueueManager.queueParams["ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue" # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" return config
def getConfig(self): """ _createConfig_ General config file """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.General.WorkDir = os.getenv("TESTDIR", os.getcwd()) #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984") config.JobStateMachine.couchDBName = "jobarchiver_t_0" config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' #config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = self.testDir config.JobArchiver.numberOfJobsToCluster = 1000 config.component_('WorkQueueManager') config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager" config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager" config.WorkQueueManager.level = 'LocalQueue' config.WorkQueueManager.logLevel = 'DEBUG' config.WorkQueueManager.couchurl = 'https://None' config.WorkQueueManager.dbname = 'whatever' config.WorkQueueManager.inboxDatabase = 'whatever2' config.WorkQueueManager.queueParams = {} config.WorkQueueManager.queueParams[ "ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue" # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" return config
#!/usr/bin/env python #pylint: disable=E1101,E1103,C0103,R0902 """ Defines default config values for WorkflowManager specific parameters. """ __all__ = [] import os from WMCore.Agent.Configuration import Configuration config = Configuration() config.component_("WorkflowManager") config.WorkflowManager.logLevel = "INFO" config.WorkflowManager.componentName = "WorkflowManager" config.WorkflowManager.componentDir = \ os.path.join(os.getenv("TESTDIR"), "WorkflowManager") # The maximum number of threads to process each message type config.WorkflowManager.maxThreads = 10 # The poll interval at which to look for new filesets config.WorkflowManager.pollInterval = 60
#!/usr/bin/env python #pylint: disable-msg=E1101,E1103,C0103,R0902 """ Defines default config values for DBSUpload specific parameters. """ __all__ = [] import os from WMCore.Agent.Configuration import Configuration config = Configuration() config.section_("CoreDatabase") if (os.getenv('DATABASE') == None): raise RuntimeError, \ "You must set the DATABASE environment variable to run tests" config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = os.getenv('DIALECT', None) config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") #The log level of the component. config.DBSUpload.logLevel = 'DEBUG' # maximum number of threads we want to deal # with messages per pool. config.DBSUpload.maxThreads = 1 # # JobSuccess Handler
def testB(self): """add settings""" config = Configuration() config.section_("Section1") section1 = getattr(config, "Section1", None) self.failUnless(section1 != None) config.section_("Section2") section2 = getattr(config, "Section2", None) self.failUnless(section2 != None) self.assertRaises(AttributeError, getattr, config, "Section3") # basic types config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 self.assertEqual(config.Section1.Parameter1, True) self.assertEqual(config.Section1.Parameter2, "string") self.assertEqual(config.Section1.Parameter3, 123) self.assertEqual(config.Section1.Parameter4, 123.456) # dictionary format: try: section1Dict = config.Section1.dictionary_() except Exception as ex: msg = "Error converting section to dictionary:\n" msg += "%s\n" % str(ex) self.fail(msg) self.failUnless( "Parameter1" in section1Dict) self.failUnless( "Parameter2" in section1Dict) self.failUnless( "Parameter3" in section1Dict) self.failUnless( "Parameter4" in section1Dict) self.assertEqual(section1Dict['Parameter1'], config.Section1.Parameter1) self.assertEqual(section1Dict['Parameter2'], config.Section1.Parameter2) self.assertEqual(section1Dict['Parameter3'], config.Section1.Parameter3) self.assertEqual(section1Dict['Parameter4'], config.Section1.Parameter4) # compound types config.Section2.List = ["string", 123, 123.456, False] config.Section2.Dictionary = { "string" : "string", "int" : 123, "float" : 123.456, "bool" : False} config.Section2.Tuple = ("string", 123, 123.456, False) self.assertEqual(config.Section2.List, ["string", 123, 123.456, False]) self.assertEqual(config.Section2.Tuple, ("string", 123, 123.456, False)) class DummyObject: pass # unsupported parameter type self.assertRaises( RuntimeError, setattr, config.Section2, "BadObject", DummyObject()) # unsupported data type in compound type badList = [ DummyObject(), DummyObject()] self.assertRaises( RuntimeError, setattr, config.Section2, "BadList", badList)
#!/usr/bin/env python """ Defines default config values for JobAccountant specific parameters. """ __all__ = [] from WMCore.Agent.Configuration import Configuration config = Configuration() config.component_("WorkQueueManager") # set type of work queue config.WorkQueueManager.level = "GlobalQueue" #config.WorkQueueManager.level = "LocalQueue" # In general can be left alone config.WorkQueueManager.componentDir = config.General.WorkDir + "/WorkQueueManager" config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager" #config.WorkQueueManager.logLevel = 'INFO' #config.WorkQueueManager.pollInterval = 600 # RequestManager config config.WorkQueueManager.reqMgrConfig = {} # uncomment to override default reqMgr url #config.WorkQueueManager.reqMgrConfig['endpoint'] = 'http://cmssrv49.fnal.gov:8585' config.WorkQueueManager.reqMgrConfig['teamName'] = 'Dodgers' # add parameters for global or local queue if default param is not what you want config.WorkQueueManager.queueParams = {'LocationRefreshInterval': 10} # uncomment to change CacheDir from default
def getConfig(self): """ _getConfig_ Build a basic JobTracker config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') #config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh') config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # BossAir config.component_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "jobtracker_t" return config
Sample configuration for generating workflow. """ import os try: import cPickle as pickle except ImportError: import pickle from WMCore.Agent.Configuration import Configuration config = Configuration() config.section_('General') # output directory of generated files. config.General.baseDir = '/tmp/PRODAGENT' config.General.srcDir = os.path.join(config.General.baseDir, 'src/python/PA/Component') config.General.testDir = os.path.join(config.General.baseDir, 'test/python/PA_t/Component_t') # module prefix for python files. config.General.pythonPrefix = 'PA.Component' config.General.pythonTestPrefix = 'PA_t.Component_t' config.General.handlers = [] config.General.synchronizers = [] config.General.plugins = [] # synchronizer is for trigger module. synchronizer = {'ID' : 'JobPostProcess', \
#!/usr/bin/env python #pylint: disable=E1101,E1103,C0103,R0902 """ DefaultConfig.py Sample configuration for generating workflow. """ import os import cPickle from WMCore.Agent.Configuration import Configuration config = Configuration() config.section_('General') # output directory of generated files. config.General.baseDir = '/tmp/PRODAGENT' config.General.srcDir = os.path.join(config.General.baseDir, 'src/python/PA/Component') config.General.testDir = os.path.join(config.General.baseDir, 'test/python/PA_t/Component_t') # module prefix for python files. config.General.pythonPrefix = 'PA.Component' config.General.pythonTestPrefix = 'PA_t.Component_t' config.General.handlers = [] config.General.synchronizers = [] config.General.plugins = [] # synchronizer is for trigger module. synchronizer = {'ID' : 'JobPostProcess', \
def getConfig(self): """ _getConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("RetryManager") config.RetryManager.logLevel = 'DEBUG' config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager' config.RetryManager.pollInterval = 10 # These are the cooloff times for the RetryManager, the times it waits # Before attempting resubmission config.RetryManager.section_("DefaultRetryAlgo") config.RetryManager.DefaultRetryAlgo.section_("default") config.RetryManager.DefaultRetryAlgo.default.coolOffTime = { 'create': 120, 'submit': 120, 'job': 120 } # Path to plugin directory config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns' config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE() config.RetryManager.componentDir = os.path.join( os.getcwd(), 'Components') # ErrorHandler # Not essential, but useful for ProcessingAlgo config.component_("ErrorHandler") config.ErrorHandler.maxRetries = 5 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "retry_manager_t" return config
def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction'] config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config
def getConfig(self): """ _getConfig_ Build a basic BossAir config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' config.Agent.useHeartbeat = False config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh' config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.pollInterval = 1 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join( WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 1 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "bossair_t" # JobStatusLite config.component_('JobStatusLite') config.JobStatusLite.componentDir = os.path.join( os.getcwd(), 'Components') config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400} config.JobStatusLite.pollInterval = 1 return config
def getConfiguration(self, configurationFile=None, connectUrl=None, socket=None): """ Loads (if available) your configuration file and augments it with the standard settings used in multiple tests. """ if configurationFile != None: config = loadConfigurationFile(configurationFile) else: config = Configuration() # some general settings that would come from the general default # config file config.Agent.contact = "*****@*****.**" config.Agent.teamName = "Lakers" config.Agent.agentName = "Lebron James" config.Agent.hostName = "testhost.laker.world" config.section_("General") # If you need a testDir, call testInit.generateWorkDir # config.General.workDir = os.getenv("TESTDIR") config.section_("CoreDatabase") if connectUrl: config.CoreDatabase.connectUrl = connectUrl config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl) config.CoreDatabase.socket = socket or os.getenv("DBSOCK") else: if (os.getenv('DATABASE') == None): raise RuntimeError, \ "You must set the DATABASE environment variable to run tests" config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE")) config.CoreDatabase.socket = os.getenv("DBSOCK") if os.getenv("DBHOST"): print "****WARNING: the DBHOST environment variable will be deprecated soon***" print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****" # after this you can augment it with whatever you need. couchurl = os.getenv("COUCHURL") config.section_("ACDC") config.ACDC.couchurl = couchurl config.ACDC.database = "wmagent_acdc_t" config.component_("JobStateMachine") config.JobStateMachine.couchurl = couchurl config.JobStateMachine.couchDBName = "wmagent_job_test" config.JobStateMachine.jobSummaryDBName = "job_summary" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.getcwd() config.JobAccountant.logLevel = 'SQLDEBUG' config.component_("TaskArchiver") config.TaskArchiver.localWMStatsURL = "%s/%s" % ( config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) config.TaskArchiver.ReqMgrSeviceURL = "request manager service url" return config
def getConfig(self): """ _getConfig_ Creates a common config. """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "DashboardReporter" config.Agent.useHeartbeat = False config.section_("DashboardReporter") config.DashboardReporter.dashboardHost = "cmssrv52.fnal.gov" config.DashboardReporter.dashboardPort = 8884 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "dashboardreporter_t" # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" return config
def getConfig(self): """ _getConfig_ Build a basic BossAir config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' config.Agent.useHeartbeat = False config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.pollInterval = 1 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 1 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "bossair_t" # JobStatusLite config.component_('JobStatusLite') config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components') config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400} config.JobStatusLite.pollInterval = 1 return config