def testB(self): """add settings""" config = Configuration() config.section_("Section1") section1 = getattr(config, "Section1", None) self.failUnless(section1 != None) config.section_("Section2") section2 = getattr(config, "Section2", None) self.failUnless(section2 != None) self.assertRaises(AttributeError, getattr, config, "Section3") # basic types config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 self.assertEqual(config.Section1.Parameter1, True) self.assertEqual(config.Section1.Parameter2, "string") self.assertEqual(config.Section1.Parameter3, 123) self.assertEqual(config.Section1.Parameter4, 123.456) # dictionary format: try: section1Dict = config.Section1.dictionary_() except Exception, ex: msg = "Error converting section to dictionary:\n" msg += "%s\n" % str(ex) self.fail(msg)
def testD(self): """test documentation""" config = Configuration() config.section_("Section1") config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 config.Section1.Parameter5 = {"test1": "test2", "test3": 123} config.Section1.document_("""This is Section1""") config.Section1.document_("""This is Section1.Parameter1""", "Parameter1") config.Section1.document_("""This is Section1.Parameter2""", "Parameter2") config.Section1.document_( """This is Section1.Parameter3\n with multiline comments""", "Parameter3") try: config.Section1.documentedString_() except Exception, ex: msg = "Error calling ConfigSection.documentedString_:\n" msg += "%s\n" % str(ex) self.fail(msg)
def __init__(self, **configDict): """ init jobCreator """ myThread = threading.currentThread() self.transaction = myThread.transaction # DAO factory for WMBS objects self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) # WMCore splitter factory for splitting up jobs. self.splitterFactory = SplitterFactory() config = Configuration() config.section_("JobStateMachine") config.JobStateMachine.couchurl = configDict["couchURL"] config.JobStateMachine.couch_retries = configDict["defaultRetries"] config.JobStateMachine.couchDBName = configDict["couchDBName"] self.config = config # Variables self.jobCacheDir = configDict["jobCacheDir"] self.defaultJobType = configDict["defaultJobType"] self.limit = configDict.get("fileLoadLimit", 500) self.createWorkArea = CreateWorkArea() self.changeState = ChangeState(self.config) return
def createConfig(self): """ _createConfig_ Create a config and save it to the temp dir. Set the WMAGENT_CONFIG environment variable so the config gets picked up. """ config = Configuration() config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "resource_control_t" config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "bossair_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' config.JobStateMachine.summaryStatsDBName = 'stat_summary_t' config.section_("BossAir") config.BossAir.pluginDir = "WMCore.BossAir.Plugins" config.BossAir.pluginNames = ["MockPlugin"] config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobAccountant_t/fwjrs', "MergeSuccess.pkl") configHandle = open(os.path.join(self.tempDir, "config.py"), "w") configHandle.write(str(config)) configHandle.close() os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py") return config
def testD(self): """test documentation""" config = Configuration() config.section_("Section1") config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 config.Section1.Parameter5 = { "test1" : "test2", "test3" : 123 } config.Section1.document_("""This is Section1""") config.Section1.document_("""This is Section1.Parameter1""", "Parameter1") config.Section1.document_("""This is Section1.Parameter2""", "Parameter2") config.Section1.document_("""This is Section1.Parameter3\n with multiline comments""", "Parameter3") try: config.Section1.documentedString_() except Exception, ex: msg = "Error calling ConfigSection.documentedString_:\n" msg += "%s\n" % str(ex) self.fail(msg)
def getConfig(self): """ _getConfig_ Build a basic JobTracker config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') #config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh') config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # BossAir config.component_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "jobtracker_t" return config
def __init__(self, **configDict): """ init jobCreator """ myThread = threading.currentThread() self.transaction = myThread.transaction #DAO factory for WMBS objects self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) # WMCore splitter factory for splitting up jobs. self.splitterFactory = SplitterFactory() config = Configuration() config.section_("JobStateMachine") config.JobStateMachine.couchurl = configDict["couchURL"] config.JobStateMachine.couch_retries = configDict["defaultRetries"] config.JobStateMachine.couchDBName = configDict["couchDBName"] self.config = config #Variables self.jobCacheDir = configDict['jobCacheDir'] self.defaultJobType = configDict['defaultJobType'] self.limit = configDict.get('fileLoadLimit', 500) self.createWorkArea = CreateWorkArea() self.changeState = ChangeState(self.config) return
def getConfig(self): """ _getConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("RetryManager") config.RetryManager.logLevel = 'DEBUG' config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager' config.RetryManager.maxRetries = 10 config.RetryManager.pollInterval = 10 # These are the cooloff times for the RetryManager, the times it waits # Before attempting resubmission config.RetryManager.coolOffTime = {'create': 120, 'submit': 120, 'job': 120} # Path to plugin directory config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns' #config.RetryManager.pluginName = '' config.RetryManager.WMCoreBase = WMCore.WMInit.getWMBASE() config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components') # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "retry_manager_t" return config
def getConfig(self): """ _getConfig_ Build a basic JobTracker config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') #config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh') config.JobSubmitter.submitScript = os.path.join( WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # BossAir config.component_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "jobtracker_t" return config
def createConfig(self): """ _createConfig_ This creates the actual config file used by the component """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = 'ERROR' config.DBSUpload.maxThreads = 1 config.DBSUpload.namespace = 'WMComponent.DBSUpload.DBSUpload' config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBSUpload.workerThreads = 4 config.section_("DBSInterface") config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet' config.DBSInterface.globalDBSVersion = 'DBS_2_0_9' config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet' config.DBSInterface.DBSVersion = 'DBS_2_0_9' config.DBSInterface.DBSBlockMaxFiles = 10 config.DBSInterface.DBSBlockMaxSize = 9999999999 config.DBSInterface.DBSBlockMaxTime = 10000 config.DBSInterface.MaxFilesToCommit = 10 # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" # configure threshold of DBS upload queue size alert threshold # reference: trac ticket #1628 config.DBSUpload.alertUploadQueueSize = 2000 return config
def getConfig(self): """ _getConfig_ Creates a common config. """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "DashboardReporter" config.Agent.useHeartbeat = False config.section_("DashboardReporter") config.DashboardReporter.dashboardHost = "cmssrv52.fnal.gov" config.DashboardReporter.dashboardPort = 8884 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = "dashboardreporter_t" # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" return config
def setUp(self): "Setup MSManager for testing" config = Configuration() data = config.section_('data') data.reqmgr2Url = "http://localhost/reqmgr2" data.verbose = True data.interval = 600 data.quotaUsage = 0.8 data.quotaAccount = "DataOps" data.enableStatusTransition = True data.rucioAccount = "wma_test" data.rucioUrl = "http://cmsrucio-int.cern.ch" data.rucioAuthUrl = "https://cmsrucio-auth-int.cern.ch" data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod" data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader" data.smtpServer = "localhost" data.fromAddr = "*****@*****.**" data.toAddr = ["*****@*****.**"] data.warningTransferThreshold = 100. * (1000**4) # 100 TB (terabyte) self.mgr = MSManager(data) data.services = ['monitor'] self.mgr_monit = MSManager(data) data.services = ['transferor'] data.limitRequestsPerCycle = 50 data.enableDataTransfer = True self.mgr_trans = MSManager(data)
def getConfig(self): """ _getConfig_ """ config = Configuration() config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = 'wmagent_jobdump' return config
def getConfig(self): """ _getConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("RetryManager") config.RetryManager.logLevel = 'DEBUG' config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager' config.RetryManager.pollInterval = 10 # These are the cooloff times for the RetryManager, the times it waits # Before attempting resubmission config.RetryManager.section_("DefaultRetryAlgo") config.RetryManager.DefaultRetryAlgo.section_("default") config.RetryManager.DefaultRetryAlgo.default.coolOffTime = { 'create': 120, 'submit': 120, 'job': 120 } # Path to plugin directory config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns' config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE() config.RetryManager.componentDir = os.path.join( os.getcwd(), 'Components') # ErrorHandler # Not essential, but useful for ProcessingAlgo config.component_("ErrorHandler") config.ErrorHandler.maxRetries = 5 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "retry_manager_t" return config
def testE(self): """test save/load """ testValues = [ "string", 123, 123.456, ["list", 789, 10.1 ], { "dict1" : "value", "dict2" : 10.0 } ] config = Configuration() for x in range(0, 5): config.section_("Section%s" % x) config.component_("Component%s" % x) sect = getattr(config, "Section%s" % x) comp = getattr(config, "Component%s" % x) sect.document_("This is Section%s" % x) comp.document_("This is Component%s" % x) for i in range(0, 5): setattr(comp, "Parameter%s" % i, testValues[i]) setattr(sect, "Parameter%s" % i, testValues[i]) comp.document_("This is Parameter%s" % i, "Parameter%s" %i) sect.document_("This is Parameter%s" %i, "Parameter%s" %i) stringSave = str(config) documentSave = config.documentedString_() commentSave = config.commentedString_() saveConfigurationFile(config, self.normalSave) saveConfigurationFile(config, self.docSave, document = True) saveConfigurationFile(config, self.commentSave, comment = True) plainConfig = loadConfigurationFile(self.normalSave) docConfig = loadConfigurationFile(self.docSave) commentConfig = loadConfigurationFile(self.commentSave)
def getConfig(self): """ _createConfig_ General config file """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.General.WorkDir = os.getenv("TESTDIR", os.getcwd()) #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984") config.JobStateMachine.couchDBName = "jobarchiver_t_0" config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' #config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = self.testDir config.JobArchiver.numberOfJobsToCluster = 1000 config.component_('WorkQueueManager') config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager" config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager" config.WorkQueueManager.level = 'LocalQueue' config.WorkQueueManager.logLevel = 'DEBUG' config.WorkQueueManager.couchurl = 'https://None' config.WorkQueueManager.dbname = 'whatever' config.WorkQueueManager.inboxDatabase = 'whatever2' config.WorkQueueManager.queueParams = {} config.WorkQueueManager.queueParams[ "ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue" # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" return config
def getConfig(self): """ _getConfig_ Build a basic BossAir config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' config.Agent.useHeartbeat = False config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.pollInterval = 1 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join(WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 1 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "bossair_t" # JobStatusLite config.component_('JobStatusLite') config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components') config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400} config.JobStatusLite.pollInterval = 1 return config
def createConfig(self): """ Create a config object for testing """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("DBSInterface") config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet' config.DBSInterface.globalDBSVersion = 'DBS_2_0_9' config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet' config.DBSInterface.DBSVersion = 'DBS_2_0_9' config.DBSInterface.DBSBlockMaxFiles = 10 config.DBSInterface.DBSBlockMaxSize = 9999999999 config.DBSInterface.DBSBlockMaxTime = 10000 config.DBSInterface.MaxFilesToCommit = 10 return config
def getConfig(self, dbs3UploadOnly = False): """ _getConfig_ This creates the actual config file used by the component. """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBS3Upload") config.DBS3Upload.pollInterval = 10 config.DBS3Upload.logLevel = 'DEBUG' #config.DBS3Upload.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/dev/global/DBSWriter" #config.DBS3Upload.dbsUrl = "https://dbs3-dev01.cern.ch/dbs/prod/global/DBSWriter" config.DBS3Upload.dbsUrl = self.dbsUrl config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBS3Upload.nProcesses = 1 config.DBS3Upload.dbsWaitTime = 0.1 config.DBS3Upload.datasetType = "VALID" config.DBS3Upload.dbs3UploadOnly = dbs3UploadOnly return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None): """ Loads (if available) your configuration file and augments it with the standard settings used in multiple tests. """ if configurationFile != None: config = loadConfigurationFile(configurationFile) else: config = Configuration() # some general settings that would come from the general default # config file config.Agent.contact = "*****@*****.**" config.Agent.teamName = "Lakers" config.Agent.agentName = "Lebron James" config.Agent.hostName = "testhost.laker.world" config.section_("General") # If you need a testDir, call testInit.generateWorkDir # config.General.workDir = os.getenv("TESTDIR") config.section_("CoreDatabase") if connectUrl: config.CoreDatabase.connectUrl = connectUrl config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl) config.CoreDatabase.socket = socket or os.getenv("DBSOCK") else: if os.getenv('DATABASE') == None: raise RuntimeError("You must set the DATABASE environment variable to run tests") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE")) config.CoreDatabase.socket = os.getenv("DBSOCK") if os.getenv("DBHOST"): print("****WARNING: the DBHOST environment variable will be deprecated soon***") print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****") # after this you can augment it with whatever you need. couchurl = os.getenv("COUCHURL") config.section_("ACDC") config.ACDC.couchurl = couchurl config.ACDC.database = "wmagent_acdc_t" config.component_("JobStateMachine") config.JobStateMachine.couchurl = couchurl config.JobStateMachine.couchDBName = "wmagent_job_test" config.JobStateMachine.jobSummaryDBName = "job_summary" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.getcwd() config.JobAccountant.logLevel = 'SQLDEBUG' config.component_("TaskArchiver") config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) config.TaskArchiver.ReqMgrSeviceURL = "request manager service url" config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2" return config
def getConfig(self): """ _createConfig_ General config file """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.General.WorkDir = os.getenv("TESTDIR", os.getcwd()) #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984") config.JobStateMachine.couchDBName = "jobarchiver_t_0" config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' #config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = self.testDir config.JobArchiver.numberOfJobsToCluster = 1000 config.component_('WorkQueueManager') config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager" config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager" config.WorkQueueManager.level = 'LocalQueue' config.WorkQueueManager.logLevel = 'DEBUG' config.WorkQueueManager.couchurl = 'https://None' config.WorkQueueManager.dbname = 'whatever' config.WorkQueueManager.inboxDatabase = 'whatever2' config.WorkQueueManager.queueParams = {} config.WorkQueueManager.queueParams["ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue" # addition for Alerts messaging framework, work (alerts) and control # channel addresses to which the component will be sending alerts # these are destination addresses where AlertProcessor:Receiver listens config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" return config
def getConfig(self): """ _getConfig_ Build a basic BossAir config """ config = Configuration() config.section_("Agent") config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' config.Agent.useHeartbeat = False config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh' config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.pollInterval = 1 config.JobSubmitter.pluginName = 'AirPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join( WMCore.WMInit.getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg') # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'INFO' config.JobTracker.pollInterval = 1 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "bossair_t" # JobStatusLite config.component_('JobStatusLite') config.JobStatusLite.componentDir = os.path.join( os.getcwd(), 'Components') config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400} config.JobStatusLite.pollInterval = 1 return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None): """ Loads (if available) your configuration file and augments it with the standard settings used in multiple tests. """ if configurationFile != None: config = loadConfigurationFile(configurationFile) else: config = Configuration() # some general settings that would come from the general default # config file config.Agent.contact = "*****@*****.**" config.Agent.teamName = "Lakers" config.Agent.agentName = "Lebron James" config.section_("General") # If you need a testDir, call testInit.generateWorkDir # config.General.workDir = os.getenv("TESTDIR") config.section_("CoreDatabase") if connectUrl: config.CoreDatabase.connectUrl = connectUrl config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl) config.CoreDatabase.socket = socket or os.getenv("DBSOCK") else: if (os.getenv('DATABASE') == None): raise RuntimeError, \ "You must set the DATABASE environment variable to run tests" config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") ) config.CoreDatabase.socket = os.getenv("DBSOCK") if os.getenv("DBHOST"): print "****WARNING: the DBHOST environment variable will be deprecated soon***" print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****" # after this you can augment it with whatever you need. return config
def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction'] config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config
def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.multicoreTaskTypes = [ 'MultiProcessing', 'MultiProduction' ] config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config
def getConfig(self): """ _getConfig_ Creates a common config. """ myThread = threading.currentThread() config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = self.componentName #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' #The log level of the component. #config.JobCreator.logLevel = 'SQLDEBUG' config.JobCreator.logLevel = 'INFO' # maximum number of threads we want to deal # with messages per pool. config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromResourceControl = True config.JobCreator.pollInterval = 10 #config.JobCreator.jobCacheDir = self.testDir config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 4 config.JobCreator.componentDir = self.testDir config.JobCreator.useWorkQueue = True config.JobCreator.WorkQueueParams = {'emulateDBSReader': True} # We now call the JobMaker from here config.component_('JobMaker') config.JobMaker.logLevel = 'INFO' config.JobMaker.namespace = 'WMCore.WMSpec.Makers.JobMaker' config.JobMaker.maxThreads = 1 config.JobMaker.makeJobsHandler = 'WMCore.WMSpec.Makers.Handlers.MakeJobs' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984') config.JobStateMachine.couchDBName = self.couchdbname return config
def getConfig(self): """ _getConfig_ Build a basic BossAir config """ config = Configuration() config.section_("Agent") config.Agent.agentName = "testAgent" config.Agent.componentName = "test" config.Agent.useHeartbeat = False config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("BossAir") config.BossAir.pluginNames = ["TestPlugin", "CondorPlugin"] config.BossAir.pluginDir = "WMCore.BossAir.Plugins" config.component_("JobSubmitter") config.JobSubmitter.logLevel = "INFO" config.JobSubmitter.pollInterval = 1 config.JobSubmitter.pluginName = "AirPlugin" config.JobSubmitter.pluginDir = "JobSubmitter.Plugins" config.JobSubmitter.submitDir = os.path.join(self.testDir, "submit") config.JobSubmitter.submitNode = os.getenv("HOSTNAME", "badtest.fnal.gov") config.JobSubmitter.submitScript = os.path.join( WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/JobSubmitter_t", "submit.sh" ) config.JobSubmitter.componentDir = os.path.join(os.getcwd(), "Components") config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), "config.cfg") # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = "INFO" config.JobTracker.pollInterval = 1 # JobStateMachine config.component_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = "bossair_t" # JobStatusLite config.component_("JobStatusLite") config.JobStatusLite.componentDir = os.path.join(os.getcwd(), "Components") config.JobStatusLite.stateTimeouts = {"Pending": 10, "Running": 86400} config.JobStatusLite.pollInterval = 1 return config
def getASOConfig(self): """ _getWMAgentConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("Agent") config.Agent.contact = "*****@*****.**" config.Agent.agentName = "testingagent" config.Agent.hostName = "localhost" config.Agent.teamName = "testingteam" config.component_("AsyncTransfer") config.AsyncTransfer.log_level = logging.DEBUG config.AsyncTransfer.namespace = "AsyncStageOut.AsyncTransfer" config.AsyncTransfer.componentDir = config.General.workDir config.AsyncTransfer.pollInterval = 10 config.AsyncTransfer.pollViewsInterval = 10 config.AsyncTransfer.couch_instance = os.getenv('COUCHURL') config.AsyncTransfer.files_database = 'asynctransfer2_t' config.AsyncTransfer.statitics_database = 'asynctransfer_stat_t' config.AsyncTransfer.requests_database = 'aso_request_database_t' config.AsyncTransfer.data_source = os.getenv('COUCHURL') config.AsyncTransfer.db_source = 'asynctrackerpoller_t_wmstats' config.AsyncTransfer.pluginName = "CentralMonitoring" config.AsyncTransfer.pluginDir = "AsyncStageOut.Plugins" config.AsyncTransfer.max_files_per_transfer = 1000 config.AsyncTransfer.pool_size = 1 config.AsyncTransfer.max_retry = 0 # config.AsyncTransfer.credentialDir = credentialDir # config.AsyncTransfer.UISetupScript = ui_script # config.AsyncTransfer.transfer_script = 'ftscp' # config.AsyncTransfer.serverDN = hostDN # config.AsyncTransfer.pollStatInterval = 86400 # config.AsyncTransfer.expiration_days = 30 # config.AsyncTransfer.couch_statinstance = statCouchUrl # config.AsyncTransfer.serviceCert = serviceCert # config.AsyncTransfer.serviceKey = "/path/to/valid/host-key" # config.AsyncTransfer.cleanEnvironment = True config.AsyncTransfer.user_monitoring_db = 'user_monitoring_asynctransfer_t' config.AsyncTransfer.couch_user_monitoring_instance = os.getenv('COUCHURL') config.AsyncTransfer.analyticsPollingInterval = 60 # config.AsyncTransfer.filesCleaningPollingInterval = 14400 config.AsyncTransfer.summaries_expiration_days = 30 return config
def setUp(self): "Setup MSManager for testing" config = Configuration() data = config.section_('data') data.reqmgr2Url = "http://localhost/reqmgr2" data.verbose = True data.interval = 600 data.quotaUsage = 0.8 data.quotaAccount = "DataOps" data.enableStatusTransition = True data.rucioAccount = "test" data.phedexUrl = "https://cmsweb.cern.ch/phedex/datasvc/json/prod" data.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader" self.mgr = MSManager(data) data.services = ['monitor'] self.mgr_monit = MSManager(data) data.services = ['transferor'] data.limitRequestsPerCycle = 50 data.enableDataTransfer = True self.mgr_trans = MSManager(data)
def getWMAgentConfig(self): """ _getWMAgentConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("AsyncStageoutTracker") # The log level of the component. config.AsyncStageoutTracker.logLevel = 'DEBUG' # The namespace of the component config.AsyncStageoutTracker.namespace = 'WMComponent.AsyncStageoutTracker.AsyncStageoutTracker' # maximum number of threads we want to deal # with messages per pool. config.AsyncStageoutTracker.maxThreads = 1 # maximum number of retries we want for job config.AsyncStageoutTracker.maxRetries = 5 # The poll interval at which to look for failed jobs config.AsyncStageoutTracker.pollInterval = 60 config.AsyncStageoutTracker.couchurl = "http://127.0.0.1:5984" config.AsyncStageoutTracker.couchDBName = "user_monitoring_asynctransfer_t" # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "asynctrackerpoller_t_jd" config.JobStateMachine.jobSummaryDBName= 'asynctrackerpoller_t_wmstats' config.component_('JobAccountant') config.section_('ACDC') config.ACDC.couchurl = self.testInit.couchUrl config.ACDC.database = "asynctrackerpoller_t" return config
def getConfig(self): """ _getConfig_ This creates the actual config file used by the component """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = 'DEBUG' config.DBSUpload.DBSBlockMaxFiles = 1 config.DBSUpload.DBSBlockMaxTime = 2 config.DBSUpload.DBSBlockMaxSize = 999999999999 config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter' #config.DBSUpload.dbsUrl = 'https://localhost:1443/dbs/prod/global/DBSWriter' config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBSUpload.nProcesses = 1 config.DBSUpload.dbsWaitTime = 0.1 return config
def createConfig(self): """ _createConfig_ Create a config and save it to the temp dir. Set the WMAGENT_CONFIG environment variable so the config gets picked up. """ config = Configuration() config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "resource_control_t" config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") configHandle = open(os.path.join(self.tempDir, "config.py"), "w") configHandle.write(str(config)) configHandle.close() os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py") return
def getConfig(self): """ _getConfig_ """ config = Configuration() # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("ErrorHandler") # The log level of the component. config.ErrorHandler.logLevel = 'DEBUG' # The namespace of the component config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler' # maximum number of threads we want to deal # with messages per pool. config.ErrorHandler.maxThreads = 30 # maximum number of retries we want for job config.ErrorHandler.maxRetries = 5 # The poll interval at which to look for failed jobs config.ErrorHandler.pollInterval = 60 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = "errorhandler_t_jd" config.section_('ACDC') config.ACDC.couchurl = self.testInit.couchUrl config.ACDC.database = "errorhandler_t" return config
def getConfig(self): """ _getConfig_ This creates the actual config file used by the component """ config = Configuration() #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = 'DEBUG' config.DBSUpload.DBSBlockMaxFiles = 500 config.DBSUpload.DBSBlockMaxTime = 600 config.DBSUpload.DBSBlockMaxSize = 999999999999 config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter' config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBSUpload.nProcesses = 1 config.DBSUpload.dbsWaitTime = 1 return config
def getConfig(self): """ _getConfig_ This is the global test configuration object """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobCreator config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' config.JobCreator.logLevel = 'DEBUG' config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromResourceControl = True config.JobCreator.pollInterval = 10 config.JobCreator.jobCacheDir = self.testDir config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 2 config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components') # JobSubmitter config.component_("JobSubmitter") config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter' config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'CondorGlobusPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join(getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'DEBUG' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 # JobAccountant config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components') config.JobAccountant.logLevel = 'INFO' # JobArchiver config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components') config.JobArchiver.numberOfJobsToCluster = 1000 # Task Archiver config.component_("TaskArchiver") config.TaskArchiver.componentDir = self.testInit.generateWorkDir() config.TaskArchiver.WorkQueueParams = {} config.TaskArchiver.pollInterval = 60 config.TaskArchiver.logLevel = 'INFO' config.TaskArchiver.timeOut = 0 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'mnorman:[email protected]:5984') config.JobStateMachine.couchDBName = "mnorman_test" # Needed, because this is a test os.makedirs(config.JobSubmitter.submitDir) return config
""" import os try: import cPickle as pickle except ImportError: import pickle from WMCore.Agent.Configuration import Configuration config = Configuration() config.section_('General') # output directory of generated files. config.General.baseDir = '/tmp/PRODAGENT' config.General.srcDir = os.path.join(config.General.baseDir, 'src/python/PA/Component') config.General.testDir = os.path.join(config.General.baseDir, 'test/python/PA_t/Component_t') # module prefix for python files. config.General.pythonPrefix = 'PA.Component' config.General.pythonTestPrefix = 'PA_t.Component_t' config.General.handlers = [] config.General.synchronizers = [] config.General.plugins = [] # synchronizer is for trigger module. synchronizer = {'ID' : 'JobPostProcess', \ 'action' : 'PA.Core.Trigger.PrepareCleanup'}
def testB(self): """add settings""" config = Configuration() config.section_("Section1") section1 = getattr(config, "Section1", None) self.failUnless(section1 != None) config.section_("Section2") section2 = getattr(config, "Section2", None) self.failUnless(section2 != None) self.assertRaises(AttributeError, getattr, config, "Section3") # basic types config.Section1.Parameter1 = True config.Section1.Parameter2 = "string" config.Section1.Parameter3 = 123 config.Section1.Parameter4 = 123.456 self.assertEqual(config.Section1.Parameter1, True) self.assertEqual(config.Section1.Parameter2, "string") self.assertEqual(config.Section1.Parameter3, 123) self.assertEqual(config.Section1.Parameter4, 123.456) # dictionary format: try: section1Dict = config.Section1.dictionary_() except Exception as ex: msg = "Error converting section to dictionary:\n" msg += "%s\n" % str(ex) self.fail(msg) self.failUnless( "Parameter1" in section1Dict) self.failUnless( "Parameter2" in section1Dict) self.failUnless( "Parameter3" in section1Dict) self.failUnless( "Parameter4" in section1Dict) self.assertEqual(section1Dict['Parameter1'], config.Section1.Parameter1) self.assertEqual(section1Dict['Parameter2'], config.Section1.Parameter2) self.assertEqual(section1Dict['Parameter3'], config.Section1.Parameter3) self.assertEqual(section1Dict['Parameter4'], config.Section1.Parameter4) # compound types config.Section2.List = ["string", 123, 123.456, False] config.Section2.Dictionary = { "string" : "string", "int" : 123, "float" : 123.456, "bool" : False} config.Section2.Tuple = ("string", 123, 123.456, False) self.assertEqual(config.Section2.List, ["string", 123, 123.456, False]) self.assertEqual(config.Section2.Tuple, ("string", 123, 123.456, False)) class DummyObject: pass # unsupported parameter type self.assertRaises( RuntimeError, setattr, config.Section2, "BadObject", DummyObject()) # unsupported data type in compound type badList = [ DummyObject(), DummyObject()] self.assertRaises( RuntimeError, setattr, config.Section2, "BadList", badList)
""" Defines default config values for DBSUpload specific parameters. """ __all__ = [] import os from WMCore.Agent.Configuration import Configuration config = Configuration() config.section_("CoreDatabase") if (os.getenv('DATABASE') == None): raise RuntimeError, \ "You must set the DATABASE environment variable to run tests" config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = os.getenv('DIALECT', None) config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") #The log level of the component. config.DBSUpload.logLevel = 'DEBUG' # maximum number of threads we want to deal # with messages per pool. config.DBSUpload.maxThreads = 1
#!/usr/bin/env python #pylint: disable=E1101,E1103,C0103,R0902 """ DefaultConfig.py Sample configuration for generating workflow. """ import os import cPickle from WMCore.Agent.Configuration import Configuration config = Configuration() config.section_('General') # output directory of generated files. config.General.baseDir = '/tmp/PRODAGENT' config.General.srcDir = os.path.join(config.General.baseDir, 'src/python/PA/Component') config.General.testDir = os.path.join(config.General.baseDir, 'test/python/PA_t/Component_t') # module prefix for python files. config.General.pythonPrefix = 'PA.Component' config.General.pythonTestPrefix = 'PA_t.Component_t' config.General.handlers = [] config.General.synchronizers = [] config.General.plugins = [] # synchronizer is for trigger module. synchronizer = {'ID' : 'JobPostProcess', \
import os import sys import getopt import logging from WMCore.Agent.Configuration import Configuration # Find and load the Configuration config = Configuration() #################### # General: General Settings Section #################### config.section_("General") config.General.workDir = '/tmp/TQ-test/' ###################### # Task Queue Component ###################### config.component_("TQComp") config.TQComp.namespace = "TQComp.TQComp" #The log level of the component. config.TQComp.logLevel = 'DEBUG' # maximum number of threads we want to deal # with messages per pool
def getConfig( self, configPath=os.path.join(WMCore.WMInit.getWMBASE(), "src/python/WMComponent/JobSubmitter/DefaultConfig.py") ): """ _getConfig_ Gets a basic config from default location """ myThread = threading.currentThread() config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = "testAgent" config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("BossAir") config.BossAir.pluginNames = ["TestPlugin", "CondorPlugin"] config.BossAir.pluginDir = "WMCore.BossAir.Plugins" config.component_("JobSubmitter") config.JobSubmitter.logLevel = "INFO" config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = "CondorGlobusPlugin" config.JobSubmitter.pluginDir = "JobSubmitter.Plugins" config.JobSubmitter.submitNode = os.getenv("HOSTNAME", "badtest.fnal.gov") config.JobSubmitter.submitScript = os.path.join( WMCore.WMBase.getTestBase(), "WMComponent_t/JobSubmitter_t", "submit.sh" ) config.JobSubmitter.componentDir = os.path.join(self.testDir, "Components") config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 config.JobSubmitter.inputFile = os.path.join( WMCore.WMBase.getTestBase(), "WMComponent_t/JobSubmitter_t", "FrameworkJobReport-4540.xml" ) config.JobSubmitter.deleteJDLFiles = False # JobStateMachine config.component_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = "jobsubmitter_t" # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config
#!/usr/bin/env python #pylint: disable-msg=E1101,E1103,C0103,R0902 """ Defines default config values for DBSUpload specific parameters. """ __all__ = [] import os from WMCore.Agent.Configuration import Configuration config = Configuration() config.section_("CoreDatabase") if (os.getenv('DATABASE') == None): raise RuntimeError, \ "You must set the DATABASE environment variable to run tests" config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = os.getenv('DIALECT', None) config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBSUpload") #The log level of the component. config.DBSUpload.logLevel = 'DEBUG' # maximum number of threads we want to deal # with messages per pool. config.DBSUpload.maxThreads = 1 # # JobSuccess Handler