Exemplo n.º 1
0
    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.component_("RetryManager")
        config.RetryManager.logLevel     = 'DEBUG'
        config.RetryManager.namespace    = 'WMComponent.RetryManager.RetryManager'
        config.RetryManager.maxRetries   = 10
        config.RetryManager.pollInterval = 10
        # These are the cooloff times for the RetryManager, the times it waits
        # Before attempting resubmission
        config.RetryManager.coolOffTime  = {'create': 120, 'submit': 120, 'job': 120}
        # Path to plugin directory
        config.RetryManager.pluginPath   = 'WMComponent.RetryManager.PlugIns'
        #config.RetryManager.pluginName   = ''
        config.RetryManager.WMCoreBase   = WMCore.WMInit.getWMBASE()
        config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components')


        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName     = "retry_manager_t"

        return config
Exemplo n.º 2
0
    def getConfig(self, dbs3UploadOnly = False):
        """
        _getConfig_

        This creates the actual config file used by the component.
        """
        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat  = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.component_("DBS3Upload")
        config.DBS3Upload.pollInterval     = 10
        config.DBS3Upload.logLevel         = 'DEBUG'
        #config.DBS3Upload.dbsUrl           = "https://cmsweb-testbed.cern.ch/dbs/dev/global/DBSWriter"
        #config.DBS3Upload.dbsUrl           = "https://dbs3-dev01.cern.ch/dbs/prod/global/DBSWriter"
        config.DBS3Upload.dbsUrl           = self.dbsUrl
        config.DBS3Upload.namespace        = 'WMComponent.DBS3Buffer.DBSUpload'
        config.DBS3Upload.componentDir     = os.path.join(os.getcwd(), 'Components')
        config.DBS3Upload.nProcesses       = 1
        config.DBS3Upload.dbsWaitTime      = 0.1
        config.DBS3Upload.datasetType      = "VALID"
        config.DBS3Upload.dbs3UploadOnly   = dbs3UploadOnly
        return config
Exemplo n.º 3
0
    def getConfig(self):
        """
        _getConfig_

        Creates a common config.
        """

        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName   = "DashboardReporter"
        config.Agent.useHeartbeat    = False

        config.section_("DashboardReporter")
        config.DashboardReporter.dashboardHost = "cmssrv52.fnal.gov"
        config.DashboardReporter.dashboardPort = 8884

        #JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
        config.JobStateMachine.couchDBName     = "dashboardreporter_t"
        
        # addition for Alerts messaging framework, work (alerts) and control
        # channel addresses to which the component will be sending alerts
        # these are destination addresses where AlertProcessor:Receiver listens
        config.section_("Alert")
        config.Alert.address = "tcp://127.0.0.1:5557"
        config.Alert.controlAddr = "tcp://127.0.0.1:5559"        

        return config
Exemplo n.º 4
0
    def testC(self):
        """add components"""

        config = Configuration()
        config.component_("Component1")
        config.component_("Component2")
        config.component_("Component3")

        comp1 = getattr(config, "Component1", None)
        self.failUnless(comp1 != None)
        comp2 = getattr(config, "Component2", None)
        self.failUnless(comp2 != None)
Exemplo n.º 5
0
    def createConfig(self):
        """
        _createConfig_

        This creates the actual config file used by the component

        """
        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat    = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")


        config.component_("DBSUpload")
        config.DBSUpload.pollInterval  = 10
        config.DBSUpload.logLevel      = 'ERROR'
        config.DBSUpload.maxThreads    = 1
        config.DBSUpload.namespace     = 'WMComponent.DBSUpload.DBSUpload'
        config.DBSUpload.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.DBSUpload.workerThreads = 4

        config.section_("DBSInterface")
        config.DBSInterface.globalDBSUrl     = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
        config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
        config.DBSInterface.DBSUrl           = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
        config.DBSInterface.DBSVersion       = 'DBS_2_0_9'
        config.DBSInterface.DBSBlockMaxFiles = 10
        config.DBSInterface.DBSBlockMaxSize  = 9999999999
        config.DBSInterface.DBSBlockMaxTime  = 10000
        config.DBSInterface.MaxFilesToCommit = 10

        # addition for Alerts messaging framework, work (alerts) and control
        # channel addresses to which the component will be sending alerts
        # these are destination addresses where AlertProcessor:Receiver listens
        config.section_("Alert")
        config.Alert.address = "tcp://127.0.0.1:5557"
        config.Alert.controlAddr = "tcp://127.0.0.1:5559"
        # configure threshold of DBS upload queue size alert threshold
        # reference: trac ticket #1628
        config.DBSUpload.alertUploadQueueSize = 2000

        return config
Exemplo n.º 6
0
     def getASOConfig(self):
         """
         _getWMAgentConfig_
 
         """
         config = Configuration()
 
         # First the general stuff
         config.section_("General")
         config.General.workDir = os.getenv("TESTDIR", self.testDir)
         config.section_("CoreDatabase")
         config.CoreDatabase.connectUrl = os.getenv("DATABASE")
         config.CoreDatabase.socket     = os.getenv("DBSOCK")
 
         config.section_("Agent")
         config.Agent.contact = "*****@*****.**"
         config.Agent.agentName = "testingagent"
         config.Agent.hostName = "localhost"
         config.Agent.teamName = "testingteam"
         
         config.component_("AsyncTransfer")
         config.AsyncTransfer.log_level = logging.DEBUG
         config.AsyncTransfer.namespace = "AsyncStageOut.AsyncTransfer"
         config.AsyncTransfer.componentDir  = config.General.workDir
         config.AsyncTransfer.pollInterval = 10
         config.AsyncTransfer.pollViewsInterval = 10
         config.AsyncTransfer.couch_instance = os.getenv('COUCHURL')
         config.AsyncTransfer.files_database = 'asynctransfer2_t'
         config.AsyncTransfer.statitics_database = 'asynctransfer_stat_t'
         config.AsyncTransfer.requests_database = 'aso_request_database_t'
         config.AsyncTransfer.data_source = os.getenv('COUCHURL')
         config.AsyncTransfer.db_source = 'asynctrackerpoller_t_wmstats'
         config.AsyncTransfer.pluginName = "CentralMonitoring"
         config.AsyncTransfer.pluginDir = "AsyncStageOut.Plugins"
         config.AsyncTransfer.max_files_per_transfer = 1000
         config.AsyncTransfer.pool_size = 1
         config.AsyncTransfer.max_retry = 0
 #        config.AsyncTransfer.credentialDir = credentialDir
 #        config.AsyncTransfer.UISetupScript = ui_script
 #        config.AsyncTransfer.transfer_script = 'ftscp'
 #        config.AsyncTransfer.serverDN = hostDN
 #        config.AsyncTransfer.pollStatInterval = 86400
 #        config.AsyncTransfer.expiration_days = 30
 #        config.AsyncTransfer.couch_statinstance = statCouchUrl
 #        config.AsyncTransfer.serviceCert = serviceCert
 #        config.AsyncTransfer.serviceKey = "/path/to/valid/host-key"
 #        config.AsyncTransfer.cleanEnvironment = True
         config.AsyncTransfer.user_monitoring_db = 'user_monitoring_asynctransfer_t'
         config.AsyncTransfer.couch_user_monitoring_instance = os.getenv('COUCHURL')
         config.AsyncTransfer.analyticsPollingInterval = 60
 #        config.AsyncTransfer.filesCleaningPollingInterval = 14400
         config.AsyncTransfer.summaries_expiration_days = 30
         return config  
Exemplo n.º 7
0
    def createConfig(self):
        """
        _createConfig_

        This creates the actual config file used by the component

        """
        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat    = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")


        config.component_("DBSUpload")
        config.DBSUpload.pollInterval  = 10
        config.DBSUpload.logLevel      = 'ERROR'
        config.DBSUpload.maxThreads    = 1
        config.DBSUpload.namespace     = 'WMComponent.DBSUpload.DBSUpload'
        config.DBSUpload.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.DBSUpload.workerThreads = 4

        config.section_("DBSInterface")
        config.DBSInterface.globalDBSUrl     = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
        config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
        config.DBSInterface.DBSUrl           = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
        config.DBSInterface.DBSVersion       = 'DBS_2_0_9'
        config.DBSInterface.DBSBlockMaxFiles = 10
        config.DBSInterface.DBSBlockMaxSize  = 9999999999
        config.DBSInterface.DBSBlockMaxTime  = 10000
        config.DBSInterface.MaxFilesToCommit = 10

        # addition for Alerts messaging framework, work (alerts) and control
        # channel addresses to which the component will be sending alerts
        # these are destination addresses where AlertProcessor:Receiver listens
        config.section_("Alert")
        config.Alert.address = "tcp://127.0.0.1:5557"
        config.Alert.controlAddr = "tcp://127.0.0.1:5559"        
        # configure threshold of DBS upload queue size alert threshold
        # reference: trac ticket #1628
        config.DBSUpload.alertUploadQueueSize = 2000
        
        return config
Exemplo n.º 8
0
    def getConfig(self):
        """
        _createConfig_

        General config file
        """
        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())
        config.General.WorkDir = os.getenv("TESTDIR", os.getcwd())

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.section_("JobStateMachine")
        config.JobStateMachine.couchurl = os.getenv("COUCHURL",
                                                    "cmssrv48.fnal.gov:5984")
        config.JobStateMachine.couchDBName = "jobarchiver_t_0"

        config.component_("JobArchiver")
        config.JobArchiver.pollInterval = 60
        config.JobArchiver.logLevel = 'INFO'
        #config.JobArchiver.logDir                = os.path.join(self.testDir, 'logs')
        config.JobArchiver.componentDir = self.testDir
        config.JobArchiver.numberOfJobsToCluster = 1000

        config.component_('WorkQueueManager')
        config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
        config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager"
        config.WorkQueueManager.level = 'LocalQueue'
        config.WorkQueueManager.logLevel = 'DEBUG'
        config.WorkQueueManager.couchurl = 'https://None'
        config.WorkQueueManager.dbname = 'whatever'
        config.WorkQueueManager.inboxDatabase = 'whatever2'
        config.WorkQueueManager.queueParams = {}
        config.WorkQueueManager.queueParams[
            "ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue"

        # addition for Alerts messaging framework, work (alerts) and control
        # channel addresses to which the component will be sending alerts
        # these are destination addresses where AlertProcessor:Receiver listens
        config.section_("Alert")
        config.Alert.address = "tcp://127.0.0.1:5557"
        config.Alert.controlAddr = "tcp://127.0.0.1:5559"

        return config
Exemplo n.º 9
0
    def getConfig(self):
        """
        _getConfig_

        Build a basic BossAir config
        """

        config = Configuration()

        config.section_("Agent")
        config.Agent.agentName  = 'testAgent'
        config.Agent.componentName = 'test'
        config.Agent.useHeartbeat = False

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")


        config.section_("BossAir")
        config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
        config.BossAir.pluginDir   = 'WMCore.BossAir.Plugins'

        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel      = 'INFO'
        config.JobSubmitter.pollInterval  = 1
        config.JobSubmitter.pluginName    = 'AirPlugin'
        config.JobSubmitter.pluginDir     = 'JobSubmitter.Plugins'
        config.JobSubmitter.submitDir     = os.path.join(self.testDir, 'submit')
        config.JobSubmitter.submitNode    = os.getenv("HOSTNAME", 'badtest.fnal.gov')
        config.JobSubmitter.submitScript  = os.path.join(WMCore.WMInit.getWMBASE(),
                                                         'test/python/WMComponent_t/JobSubmitter_t',
                                                         'submit.sh')
        config.JobSubmitter.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200
        config.JobSubmitter.gLiteConf     = os.path.join(os.getcwd(), 'config.cfg')



        # JobTracker
        config.component_("JobTracker")
        config.JobTracker.logLevel      = 'INFO'
        config.JobTracker.pollInterval  = 1


        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL')
        config.JobStateMachine.couchDBName     = "bossair_t"


        # JobStatusLite
        config.component_('JobStatusLite')
        config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
        config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400}
        config.JobStatusLite.pollInterval = 1


        return config
Exemplo n.º 10
0
    def getConfig(self):
        """
        _getConfig_

        Build a basic JobTracker config
        """

        config = Configuration()

        config.section_("Agent")
        config.Agent.agentName = 'testAgent'

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        # JobTracker
        config.component_("JobTracker")
        config.JobTracker.logLevel = 'INFO'
        config.JobTracker.pollInterval = 10
        config.JobTracker.trackerName = 'CondorTracker'
        config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
        config.JobTracker.componentDir = os.path.join(os.getcwd(),
                                                      'Components')
        config.JobTracker.runTimeLimit = 7776000  #Jobs expire after 90 days
        config.JobTracker.idleTimeLimit = 7776000
        config.JobTracker.heldTimeLimit = 7776000
        config.JobTracker.unknTimeLimit = 7776000

        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel = 'INFO'
        config.JobSubmitter.maxThreads = 1
        config.JobSubmitter.pollInterval = 10
        config.JobSubmitter.pluginName = 'AirPlugin'
        config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
        config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
        config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
                                                   'badtest.fnal.gov')
        #config.JobSubmitter.submitScript  = os.path.join(os.getcwd(), 'submit.sh')
        config.JobSubmitter.submitScript = os.path.join(
            WMCore.WMInit.getWMBASE(),
            'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
        config.JobSubmitter.componentDir = os.path.join(
            os.getcwd(), 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200
        config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')

        # BossAir
        config.component_("BossAir")
        config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
        config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'

        #JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl = os.getenv('COUCHURL',
                                                    'cmssrv52.fnal.gov:5984')
        config.JobStateMachine.couchDBName = "jobtracker_t"

        return config
Exemplo n.º 11
0
    def getConfig(self):
        """
        _getConfig_

        Build a basic JobTracker config
        """
                       
        config = Configuration()

        config.section_("Agent")
        config.Agent.agentName  = 'testAgent'

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        # JobTracker
        config.component_("JobTracker")
        config.JobTracker.logLevel      = 'INFO'
        config.JobTracker.pollInterval  = 10
        config.JobTracker.trackerName   = 'CondorTracker'
        config.JobTracker.pluginDir     = 'WMComponent.JobTracker.Plugins'
        config.JobTracker.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.JobTracker.runTimeLimit  = 7776000 #Jobs expire after 90 days
        config.JobTracker.idleTimeLimit = 7776000
        config.JobTracker.heldTimeLimit = 7776000
        config.JobTracker.unknTimeLimit = 7776000


        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel      = 'INFO'
        config.JobSubmitter.maxThreads    = 1
        config.JobSubmitter.pollInterval  = 10
        config.JobSubmitter.pluginName    = 'AirPlugin'
        config.JobSubmitter.pluginDir     = 'JobSubmitter.Plugins'
        config.JobSubmitter.submitDir     = os.path.join(self.testDir, 'submit')
        config.JobSubmitter.submitNode    = os.getenv("HOSTNAME", 'badtest.fnal.gov')
        #config.JobSubmitter.submitScript  = os.path.join(os.getcwd(), 'submit.sh')
        config.JobSubmitter.submitScript  = os.path.join(WMCore.WMInit.getWMBASE(),
                                                         'test/python/WMComponent_t/JobSubmitter_t',
                                                         'submit.sh')
        config.JobSubmitter.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200
        config.JobSubmitter.gLiteConf     = os.path.join(os.getcwd(), 'config.cfg')



        # BossAir
        config.component_("BossAir")
        config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
        config.BossAir.pluginDir   = 'WMCore.BossAir.Plugins'


        #JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
        config.JobStateMachine.couchDBName     = "jobtracker_t"
        
        return config
Exemplo n.º 12
0
    def getConfig(self):
        """
        _createConfig_

        General config file
        """
        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())
        config.General.WorkDir = os.getenv("TESTDIR", os.getcwd())

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.section_("JobStateMachine")
        config.JobStateMachine.couchurl    = os.getenv("COUCHURL", "cmssrv48.fnal.gov:5984")
        config.JobStateMachine.couchDBName = "jobarchiver_t_0"

        config.component_("JobArchiver")
        config.JobArchiver.pollInterval          = 60
        config.JobArchiver.logLevel              = 'INFO'
        #config.JobArchiver.logDir                = os.path.join(self.testDir, 'logs')
        config.JobArchiver.componentDir          = self.testDir
        config.JobArchiver.numberOfJobsToCluster = 1000

        config.component_('WorkQueueManager')
	config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
	config.WorkQueueManager.componentDir = config.General.workDir + "/WorkQueueManager"
	config.WorkQueueManager.level = 'LocalQueue'
        config.WorkQueueManager.logLevel = 'DEBUG'
        config.WorkQueueManager.couchurl = 'https://None'
        config.WorkQueueManager.dbname = 'whatever'
        config.WorkQueueManager.inboxDatabase = 'whatever2'
        config.WorkQueueManager.queueParams = {}
        config.WorkQueueManager.queueParams["ParentQueueCouchUrl"] = "https://cmsweb.cern.ch/couchdb/workqueue"
        
        # addition for Alerts messaging framework, work (alerts) and control
        # channel addresses to which the component will be sending alerts
        # these are destination addresses where AlertProcessor:Receiver listens
        config.section_("Alert")
        config.Alert.address = "tcp://127.0.0.1:5557"
        config.Alert.controlAddr = "tcp://127.0.0.1:5559"

        return config        
Exemplo n.º 13
0
    def getConfig(self):
        """
        _getConfig_

        Build a basic BossAir config
        """

        config = Configuration()

        config.section_("Agent")
        config.Agent.agentName = 'testAgent'
        config.Agent.componentName = 'test'
        config.Agent.useHeartbeat = False

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.section_("BossAir")
        config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
        config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
        config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh'

        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel = 'INFO'
        config.JobSubmitter.pollInterval = 1
        config.JobSubmitter.pluginName = 'AirPlugin'
        config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
        config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
        config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
                                                   'badtest.fnal.gov')
        config.JobSubmitter.submitScript = os.path.join(
            WMCore.WMInit.getWMBASE(),
            'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh')
        config.JobSubmitter.componentDir = os.path.join(
            os.getcwd(), 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200
        config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), 'config.cfg')

        # JobTracker
        config.component_("JobTracker")
        config.JobTracker.logLevel = 'INFO'
        config.JobTracker.pollInterval = 1

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl = os.getenv('COUCHURL')
        config.JobStateMachine.couchDBName = "bossair_t"

        # JobStatusLite
        config.component_('JobStatusLite')
        config.JobStatusLite.componentDir = os.path.join(
            os.getcwd(), 'Components')
        config.JobStatusLite.stateTimeouts = {'Pending': 10, 'Running': 86400}
        config.JobStatusLite.pollInterval = 1

        return config
Exemplo n.º 14
0
    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName = 'wmagent_jobdump'

        return config
Exemplo n.º 15
0
    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName     = 'wmagent_jobdump'

        return config
Exemplo n.º 16
0
    def getConfig(self):
        """
        _getConfig_

        Build a basic BossAir config
        """

        config = Configuration()

        config.section_("Agent")
        config.Agent.agentName = "testAgent"
        config.Agent.componentName = "test"
        config.Agent.useHeartbeat = False

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.section_("BossAir")
        config.BossAir.pluginNames = ["TestPlugin", "CondorPlugin"]
        config.BossAir.pluginDir = "WMCore.BossAir.Plugins"

        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel = "INFO"
        config.JobSubmitter.pollInterval = 1
        config.JobSubmitter.pluginName = "AirPlugin"
        config.JobSubmitter.pluginDir = "JobSubmitter.Plugins"
        config.JobSubmitter.submitDir = os.path.join(self.testDir, "submit")
        config.JobSubmitter.submitNode = os.getenv("HOSTNAME", "badtest.fnal.gov")
        config.JobSubmitter.submitScript = os.path.join(
            WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/JobSubmitter_t", "submit.sh"
        )
        config.JobSubmitter.componentDir = os.path.join(os.getcwd(), "Components")
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200
        config.JobSubmitter.gLiteConf = os.path.join(os.getcwd(), "config.cfg")

        # JobTracker
        config.component_("JobTracker")
        config.JobTracker.logLevel = "INFO"
        config.JobTracker.pollInterval = 1

        # JobStateMachine
        config.component_("JobStateMachine")
        config.JobStateMachine.couchurl = os.getenv("COUCHURL")
        config.JobStateMachine.couchDBName = "bossair_t"

        # JobStatusLite
        config.component_("JobStatusLite")
        config.JobStatusLite.componentDir = os.path.join(os.getcwd(), "Components")
        config.JobStatusLite.stateTimeouts = {"Pending": 10, "Running": 86400}
        config.JobStatusLite.pollInterval = 1

        return config
Exemplo n.º 17
0
    def getConfig(self):
        """
        _getConfig_

        This creates the actual config file used by the component

        """


        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat  = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")


        config.component_("DBSUpload")
        config.DBSUpload.pollInterval     = 10
        config.DBSUpload.logLevel         = 'DEBUG'
        config.DBSUpload.DBSBlockMaxFiles = 1
        config.DBSUpload.DBSBlockMaxTime  = 2
        config.DBSUpload.DBSBlockMaxSize  = 999999999999
        config.DBSUpload.dbsUrl           = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
        #config.DBSUpload.dbsUrl           = 'https://localhost:1443/dbs/prod/global/DBSWriter'
        config.DBSUpload.namespace        = 'WMComponent.DBS3Buffer.DBSUpload'
        config.DBSUpload.componentDir     = os.path.join(os.getcwd(), 'Components')
        config.DBSUpload.nProcesses       = 1
        config.DBSUpload.dbsWaitTime      = 0.1


        return config
Exemplo n.º 18
0
    def testE(self):
        """test save/load """

        testValues = [
            "string", 123, 123.456,
            ["list", 789, 10.1 ],
            { "dict1" : "value", "dict2" : 10.0 }
            ]

        config = Configuration()
        for x in range(0, 5):
            config.section_("Section%s" % x)
            config.component_("Component%s" % x)
            sect = getattr(config, "Section%s" % x)
            comp = getattr(config, "Component%s" % x)
            sect.document_("This is Section%s" % x)
            comp.document_("This is Component%s" % x)

            for i in range(0, 5):
                setattr(comp, "Parameter%s" % i, testValues[i])
                setattr(sect, "Parameter%s" % i, testValues[i])
                comp.document_("This is Parameter%s" % i,
                               "Parameter%s" %i)
                sect.document_("This is Parameter%s" %i,
                               "Parameter%s" %i)

        stringSave = str(config)
        documentSave = config.documentedString_()
        commentSave = config.commentedString_()


        saveConfigurationFile(config, self.normalSave)
        saveConfigurationFile(config, self.docSave, document = True)
        saveConfigurationFile(config, self.commentSave, comment = True)

        plainConfig = loadConfigurationFile(self.normalSave)

        docConfig = loadConfigurationFile(self.docSave)


        commentConfig = loadConfigurationFile(self.commentSave)
Exemplo n.º 19
0
    def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
        """
        Loads (if available) your configuration file and augments
        it with the standard settings used in multiple tests.
        """
        if configurationFile != None:
            config = loadConfigurationFile(configurationFile)
        else:
            config = Configuration()

        # some general settings that would come from the general default
        # config file
        config.Agent.contact = "*****@*****.**"
        config.Agent.teamName = "Lakers"
        config.Agent.agentName = "Lebron James"
        config.Agent.hostName = "testhost.laker.world"

        config.section_("General")
        # If you need a testDir, call testInit.generateWorkDir
        # config.General.workDir = os.getenv("TESTDIR")

        config.section_("CoreDatabase")
        if connectUrl:
            config.CoreDatabase.connectUrl = connectUrl
            config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
            config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
        else:
            if os.getenv('DATABASE') == None:
                raise RuntimeError("You must set the DATABASE environment variable to run tests")
            config.CoreDatabase.connectUrl = os.getenv("DATABASE")
            config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
            config.CoreDatabase.socket = os.getenv("DBSOCK")
            if os.getenv("DBHOST"):
                print("****WARNING: the DBHOST environment variable will be deprecated soon***")
                print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
            # after this you can augment it with whatever you need.

        couchurl = os.getenv("COUCHURL")
        config.section_("ACDC")
        config.ACDC.couchurl = couchurl
        config.ACDC.database = "wmagent_acdc_t"

        config.component_("JobStateMachine")
        config.JobStateMachine.couchurl = couchurl
        config.JobStateMachine.couchDBName = "wmagent_job_test"
        config.JobStateMachine.jobSummaryDBName = "job_summary"
        config.JobStateMachine.summaryStatsDBName = "stat_summary_test"

        config.component_("JobAccountant")
        config.JobAccountant.pollInterval = 60
        config.JobAccountant.componentDir = os.getcwd()
        config.JobAccountant.logLevel = 'SQLDEBUG'

        config.component_("TaskArchiver")
        config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
        config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
        config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"

        return config
Exemplo n.º 20
0
    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.component_("ErrorHandler")
        # The log level of the component. 
        config.ErrorHandler.logLevel = 'DEBUG'
        # The namespace of the component
        config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
        # maximum number of threads we want to deal
        # with messages per pool.
        config.ErrorHandler.maxThreads = 30
        # maximum number of retries we want for job
        config.ErrorHandler.maxRetries = 5
        # The poll interval at which to look for failed jobs
        config.ErrorHandler.pollInterval = 60

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName     = "errorhandler_t_jd"


        config.section_('ACDC')
        config.ACDC.couchurl = self.testInit.couchUrl
        config.ACDC.database = "errorhandler_t"

        return config
Exemplo n.º 21
0
    def getConfig(self):
        """
        _getConfig_

        This creates the actual config file used by the component

        """

        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.component_("DBSUpload")
        config.DBSUpload.pollInterval = 10
        config.DBSUpload.logLevel = 'DEBUG'
        config.DBSUpload.DBSBlockMaxFiles = 500
        config.DBSUpload.DBSBlockMaxTime = 600
        config.DBSUpload.DBSBlockMaxSize = 999999999999
        config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
        config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
        config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
        config.DBSUpload.nProcesses = 1
        config.DBSUpload.dbsWaitTime = 1

        return config
Exemplo n.º 22
0
    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.component_("ErrorHandler")
        # The log level of the component.
        config.ErrorHandler.logLevel = 'DEBUG'
        # The namespace of the component
        config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
        # maximum number of threads we want to deal
        # with messages per pool.
        config.ErrorHandler.maxThreads = 30
        # maximum number of retries we want for job
        config.ErrorHandler.maxRetries = 5
        # The poll interval at which to look for failed jobs
        config.ErrorHandler.pollInterval = 60

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName = "errorhandler_t_jd"

        config.section_('ACDC')
        config.ACDC.couchurl = self.testInit.couchUrl
        config.ACDC.database = "errorhandler_t"

        return config
Exemplo n.º 23
0
    def getConfig(self):
        """
        _getConfig_

        Gets a basic config from default location
        """

        config = Configuration()

        config.component_("Agent")
        config.Agent.WMSpecDirectory = self.testDir
        config.Agent.agentName = 'testAgent'
        config.Agent.componentName = self.componentName
        config.Agent.useHeartbeat = False

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)

        #Now the CoreDatabase information
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        # BossAir and MockPlugin configuration
        config.section_("BossAir")
        config.BossAir.pluginNames = ['MockPlugin']
        config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
        config.BossAir.multicoreTaskTypes = [
            'MultiProcessing', 'MultiProduction'
        ]
        config.BossAir.nCondorProcesses = 1
        config.BossAir.section_("MockPlugin")
        config.BossAir.MockPlugin.fakeReport = os.path.join(
            getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh")
        # JobSubmitter configuration
        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel = 'DEBUG'
        config.JobSubmitter.maxThreads = 1
        config.JobSubmitter.pollInterval = 10
        config.JobSubmitter.submitScript = os.path.join(
            getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
        config.JobSubmitter.componentDir = os.path.join(
            self.testDir, 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200

        #JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl = os.getenv('COUCHURL')
        config.JobStateMachine.couchDBName = "jobsubmitter_t"
        config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'

        # Needed, because this is a test
        os.makedirs(config.JobSubmitter.componentDir)

        return config
Exemplo n.º 24
0
    def getConfig(self):
        """
        _getConfig_

        Gets a basic config from default location
        """

        config = Configuration()

        config.component_("Agent")
        config.Agent.WMSpecDirectory = self.testDir
        config.Agent.agentName       = 'testAgent'
        config.Agent.componentName   = self.componentName
        config.Agent.useHeartbeat    = False


        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)

        #Now the CoreDatabase information
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        # BossAir and MockPlugin configuration
        config.section_("BossAir")
        config.BossAir.pluginNames = ['MockPlugin']
        config.BossAir.pluginDir   = 'WMCore.BossAir.Plugins'
        config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction']
        config.BossAir.nCondorProcesses = 1
        config.BossAir.section_("MockPlugin")
        config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
                                                         'WMComponent_t/JobSubmitter_t',
                                                         "submit.sh")
        # JobSubmitter configuration
        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel      = 'DEBUG'
        config.JobSubmitter.maxThreads    = 1
        config.JobSubmitter.pollInterval  = 10
        config.JobSubmitter.submitScript  = os.path.join(getTestBase(),
                                                         'WMComponent_t/JobSubmitter_t',
                                                         'submit.sh')
        config.JobSubmitter.componentDir  = os.path.join(self.testDir, 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200

        #JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL')
        config.JobStateMachine.couchDBName     = "jobsubmitter_t"
        config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'

        # Needed, because this is a test
        os.makedirs(config.JobSubmitter.componentDir)

        return config
Exemplo n.º 25
0
    def getConfig(self):
        """
        _getConfig_

        Creates a common config.
        """


        myThread = threading.currentThread()

        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName   = self.componentName

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.component_("JobCreator")
        config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
        #The log level of the component.
        #config.JobCreator.logLevel = 'SQLDEBUG'
        config.JobCreator.logLevel = 'INFO'

        # maximum number of threads we want to deal
        # with messages per pool.
        config.JobCreator.maxThreads                = 1
        config.JobCreator.UpdateFromResourceControl = True
        config.JobCreator.pollInterval              = 10
        #config.JobCreator.jobCacheDir               = self.testDir
        config.JobCreator.defaultJobType            = 'processing' #Type of jobs that we run, used for resource control
        config.JobCreator.workerThreads             = 4
        config.JobCreator.componentDir              = self.testDir
        config.JobCreator.useWorkQueue              = True
        config.JobCreator.WorkQueueParams           = {'emulateDBSReader': True}

        # We now call the JobMaker from here
        config.component_('JobMaker')
        config.JobMaker.logLevel        = 'INFO'
        config.JobMaker.namespace       = 'WMCore.WMSpec.Makers.JobMaker'
        config.JobMaker.maxThreads      = 1
        config.JobMaker.makeJobsHandler = 'WMCore.WMSpec.Makers.Handlers.MakeJobs'

        #JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', 'cmssrv52.fnal.gov:5984')
        config.JobStateMachine.couchDBName     = self.couchdbname

        return config
Exemplo n.º 26
0
    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.component_("RetryManager")
        config.RetryManager.logLevel = 'DEBUG'
        config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager'
        config.RetryManager.pollInterval = 10
        # These are the cooloff times for the RetryManager, the times it waits
        # Before attempting resubmission
        config.RetryManager.section_("DefaultRetryAlgo")
        config.RetryManager.DefaultRetryAlgo.section_("default")
        config.RetryManager.DefaultRetryAlgo.default.coolOffTime = {
            'create': 120,
            'submit': 120,
            'job': 120
        }
        # Path to plugin directory
        config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns'
        config.RetryManager.WMCoreBase = WMCore.WMBase.getWMBASE()
        config.RetryManager.componentDir = os.path.join(
            os.getcwd(), 'Components')

        # ErrorHandler
        # Not essential, but useful for ProcessingAlgo
        config.component_("ErrorHandler")
        config.ErrorHandler.maxRetries = 5

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName = "retry_manager_t"

        return config
Exemplo n.º 27
0
     def getWMAgentConfig(self):
         """
         _getWMAgentConfig_
 
         """
         config = Configuration()
 
         # First the general stuff
         config.section_("General")
         config.General.workDir = os.getenv("TESTDIR", self.testDir)
         config.section_("CoreDatabase")
         config.CoreDatabase.connectUrl = os.getenv("DATABASE")
         config.CoreDatabase.socket     = os.getenv("DBSOCK")
 
         config.component_("AsyncStageoutTracker")
         # The log level of the component. 
         config.AsyncStageoutTracker.logLevel = 'DEBUG'
         # The namespace of the component
         config.AsyncStageoutTracker.namespace = 'WMComponent.AsyncStageoutTracker.AsyncStageoutTracker'
         # maximum number of threads we want to deal
         # with messages per pool.
         config.AsyncStageoutTracker.maxThreads = 1
         # maximum number of retries we want for job
         config.AsyncStageoutTracker.maxRetries = 5
         # The poll interval at which to look for failed jobs
         config.AsyncStageoutTracker.pollInterval = 60
         config.AsyncStageoutTracker.couchurl     = "http://127.0.0.1:5984"
         config.AsyncStageoutTracker.couchDBName  = "user_monitoring_asynctransfer_t" 
 
         # JobStateMachine
         config.component_('JobStateMachine')
         config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
         config.JobStateMachine.couchDBName     = "asynctrackerpoller_t_jd"
         config.JobStateMachine.jobSummaryDBName= 'asynctrackerpoller_t_wmstats'
 
         config.component_('JobAccountant')
         
         config.section_('ACDC')
         config.ACDC.couchurl = self.testInit.couchUrl
         config.ACDC.database = "asynctrackerpoller_t"
 
         return config  
Exemplo n.º 28
0
#!/usr/bin/env python

"""
Defines default config values for JobAccountant specific
parameters.
"""
__all__ = []



from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("WorkQueueManager")

# set type of work queue
config.WorkQueueManager.level = "GlobalQueue"
#config.WorkQueueManager.level = "LocalQueue"

# In general can be left alone
config.WorkQueueManager.componentDir = config.General.WorkDir + "/WorkQueueManager"
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
#config.WorkQueueManager.logLevel = 'INFO'
#config.WorkQueueManager.pollInterval = 600


# RequestManager config
config.WorkQueueManager.reqMgrConfig = {}
# uncomment to override default reqMgr url
#config.WorkQueueManager.reqMgrConfig['endpoint'] = 'http://cmssrv49.fnal.gov:8585'
config.WorkQueueManager.reqMgrConfig['teamName'] = 'Dodgers'
Exemplo n.º 29
0
#!/usr/bin/env python

"""
Defines default config values for HarvestingScheduler specific
parameters.
"""
__all__ = []


import os
import os.path

from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("HarvestingScheduler")
config.HarvestingScheduler.namespace = "WMComponent.HarvestingScheduler.HarvestingScheduler"
config.HarvestingScheduler.componentDir = os.getcwd() + "/HarvestingScheduler"
config.HarvestingScheduler.workloadCache = os.getcwd(), +"/HarvestingScheduler/workloadCache"
config.HarvestingScheduler.targetSite = "srm-cms.cern.ch"
config.HarvestingScheduler.scramArch = "slc5_ia32_gcc434"
config.HarvestingScheduler.cmsPath = "/afs/cern.ch/cms/sw"
config.HarvestingScheduler.proxy = os.getenv("X509_USER_PROXY", "bad_proxy")
config.HarvestingScheduler.dqmGuiUrl = "https://cmsweb.cern.ch/dqm/dev"
config.HarvestingScheduler.couchurl = "http://127.0.0.1:5984"
config.HarvestingScheduler.couchDBName = "datasets_to_harvest"
config.HarvestingScheduler.doStageOut = True
config.HarvestingScheduler.doDqmUpload = True
config.HarvestingScheduler.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/"
config.HarvestingScheduler.dbsUrl = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
config.HarvestingScheduler.expiryTime = 3600 * 24 * 30  # One month
Exemplo n.º 30
0
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for CranWmbs specific
parameters.
"""
__all__ = []
__revision__ = "$Id: DefaultConfig.py,v 0.2 2009/09/30 01:26:34 hriahi Exp $"
__version__ = "$Revision: 0.2 $"

import os
from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("CrabJobCreator")
config.CrabJobCreator.logLevel = "DEBUG"
config.CrabJobCreator.componentName = "CrabJobCreator"
config.CrabJobCreator.componentDir = \
    os.path.join(os.getenv("TESTDIR"), "CrabJobCreator")

# The maximum number of threads to process each message type
config.CrabJobCreator.maxThreads = 10

# CrabServer parameter
config.CrabJobCreator.wdir = '/data/Storage/logs' 
config.CrabJobCreator.maxRetries = 3
config.CrabJobCreator.credentialType = "Proxy"
config.CrabJobCreator.ProxiesDir = "/tmp/del_proxies/"
config.CrabJobCreator.StorageName = "crab.pg.infn.it"
config.CrabJobCreator.storagePort = "2811"
config.CrabJobCreator.Protocol = "gridftp"
Exemplo n.º 31
0
import os

from WMCore.Agent.Configuration import Configuration

config = Configuration()

config.section_("CoreDatabase")
if (os.getenv('DATABASE') == None):
    raise RuntimeError, \
          "You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = os.getenv('DIALECT', None)
config.CoreDatabase.socket = os.getenv("DBSOCK")

config.component_("DBSUpload")
#The log level of the component.
config.DBSUpload.logLevel = 'DEBUG'

# maximum number of threads we want to deal
# with messages per pool.
config.DBSUpload.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSUpload.bufferSuccessHandler = \
    'WMComponent.DBSUpload.Handler.BufferSuccess'

config.DBSUpload.newWorkflowHandler = \
    'WMComponent.DBSUpload.Handler.NewWorkflowHandler'
Exemplo n.º 32
0
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for DBSBuffer specific
parameters.
"""
__all__ = []




from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("DBSBuffer")
#The log level of the component. 
config.DBSBuffer.logLevel = 'INFO'
#The namespace of the buffer. Used to load the module as daemon
config.DBSBuffer.namespace = 'WMComponent.DBSBuffer.DBSBuffer'
# maximum number of threads we want to deal
# with messages per pool.
config.DBSBuffer.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSBuffer.jobSuccessHandler = \
    'WMComponent.DBSBuffer.Handler.JobSuccess'

Exemplo n.º 33
0
    def getConfig(
        self, configPath=os.path.join(WMCore.WMInit.getWMBASE(), "src/python/WMComponent/JobSubmitter/DefaultConfig.py")
    ):
        """
        _getConfig_

        Gets a basic config from default location
        """

        myThread = threading.currentThread()

        config = Configuration()

        config.component_("Agent")
        config.Agent.WMSpecDirectory = self.testDir
        config.Agent.agentName = "testAgent"
        config.Agent.componentName = self.componentName
        config.Agent.useHeartbeat = False

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)

        # Now the CoreDatabase information
        # This should be the dialect, dburl, etc

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.section_("BossAir")
        config.BossAir.pluginNames = ["TestPlugin", "CondorPlugin"]
        config.BossAir.pluginDir = "WMCore.BossAir.Plugins"

        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel = "INFO"
        config.JobSubmitter.maxThreads = 1
        config.JobSubmitter.pollInterval = 10
        config.JobSubmitter.pluginName = "CondorGlobusPlugin"
        config.JobSubmitter.pluginDir = "JobSubmitter.Plugins"
        config.JobSubmitter.submitNode = os.getenv("HOSTNAME", "badtest.fnal.gov")
        config.JobSubmitter.submitScript = os.path.join(
            WMCore.WMBase.getTestBase(), "WMComponent_t/JobSubmitter_t", "submit.sh"
        )
        config.JobSubmitter.componentDir = os.path.join(self.testDir, "Components")
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200
        config.JobSubmitter.inputFile = os.path.join(
            WMCore.WMBase.getTestBase(), "WMComponent_t/JobSubmitter_t", "FrameworkJobReport-4540.xml"
        )
        config.JobSubmitter.deleteJDLFiles = False

        # JobStateMachine
        config.component_("JobStateMachine")
        config.JobStateMachine.couchurl = os.getenv("COUCHURL")
        config.JobStateMachine.couchDBName = "jobsubmitter_t"

        # Needed, because this is a test
        os.makedirs(config.JobSubmitter.componentDir)

        return config
Exemplo n.º 34
0
    def getConfig(self):
        """
        _getConfig_

        This is the global test configuration object
        """



        config = Configuration()

        config.component_("Agent")
        config.Agent.WMSpecDirectory = self.testDir
        config.Agent.agentName       = 'testAgent'
        config.Agent.componentName   = 'test'


        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)

        # Now the CoreDatabase information
        # This should be the dialect, dburl, etc

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")



        # JobCreator
        config.component_("JobCreator")
        config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
        config.JobCreator.logLevel  = 'DEBUG'
        config.JobCreator.maxThreads                = 1
        config.JobCreator.UpdateFromResourceControl = True
        config.JobCreator.pollInterval              = 10
        config.JobCreator.jobCacheDir               = self.testDir
        config.JobCreator.defaultJobType            = 'processing' #Type of jobs that we run, used for resource control
        config.JobCreator.workerThreads             = 2
        config.JobCreator.componentDir              = os.path.join(os.getcwd(), 'Components')



        # JobSubmitter
        config.component_("JobSubmitter")
        config.JobSubmitter.namespace     = 'WMComponent.JobSubmitter.JobSubmitter'
        config.JobSubmitter.logLevel      = 'INFO'
        config.JobSubmitter.maxThreads    = 1
        config.JobSubmitter.pollInterval  = 10
        config.JobSubmitter.pluginName    = 'CondorGlobusPlugin'
        config.JobSubmitter.pluginDir     = 'JobSubmitter.Plugins'
        config.JobSubmitter.submitDir     = os.path.join(self.testDir, 'submit')
        config.JobSubmitter.submitNode    = os.getenv("HOSTNAME", 'badtest.fnal.gov')
        config.JobSubmitter.submitScript  = os.path.join(getWMBASE(),
                                                         'test/python/WMComponent_t/JobSubmitter_t',
                                                         'submit.sh')
        config.JobSubmitter.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200




        # JobTracker
        config.component_("JobTracker")
        config.JobTracker.logLevel      = 'DEBUG'
        config.JobTracker.pollInterval  = 10
        config.JobTracker.trackerName   = 'CondorTracker'
        config.JobTracker.pluginDir     = 'WMComponent.JobTracker.Plugins'
        config.JobTracker.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.JobTracker.runTimeLimit  = 7776000 #Jobs expire after 90 days
        config.JobTracker.idleTimeLimit = 7776000
        config.JobTracker.heldTimeLimit = 7776000
        config.JobTracker.unknTimeLimit = 7776000



        # JobAccountant
        config.component_("JobAccountant")
        config.JobAccountant.pollInterval = 60
        config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
        config.JobAccountant.logLevel     = 'INFO'



        # JobArchiver
        config.component_("JobArchiver")
        config.JobArchiver.pollInterval          = 60
        config.JobArchiver.logLevel              = 'INFO'
        config.JobArchiver.logDir                = os.path.join(self.testDir, 'logs')
        config.JobArchiver.componentDir          = os.path.join(os.getcwd(), 'Components')
        config.JobArchiver.numberOfJobsToCluster = 1000



        # Task Archiver
        config.component_("TaskArchiver")
        config.TaskArchiver.componentDir    = self.testInit.generateWorkDir()
        config.TaskArchiver.WorkQueueParams = {}
        config.TaskArchiver.pollInterval    = 60
        config.TaskArchiver.logLevel        = 'INFO'
        config.TaskArchiver.timeOut         = 0



        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL',
                                                           'mnorman:[email protected]:5984')
        config.JobStateMachine.couchDBName     = "mnorman_test"


        # Needed, because this is a test
        os.makedirs(config.JobSubmitter.submitDir)


        return config
Exemplo n.º 35
0
config = Configuration()



####################
# General: General Settings Section
####################
config.section_("General")
config.General.workDir = '/tmp/TQ-test/'



######################
# Task Queue Component 
######################
config.component_("TQComp")
config.TQComp.namespace = "TQComp.TQComp"

#The log level of the component.
config.TQComp.logLevel = 'DEBUG'

# maximum number of threads we want to deal
# with messages per pool
config.TQComp.maxThreads = 5

# Component dir (twice due to some bug in wmcoreD)
config.TQComp.componentDir = '/tmp/TQ-test/TQComp'
config.TQComp.ComponentDir = '/tmp/TQ-test/TQComp'

# Where to download specs and sandboxes from
config.TQComp.downloadBaseUrl = 'http://glitece.ciemat.es:20030'
Exemplo n.º 36
0
#!/usr/bin/env python

"""
Defines default config values for JobStatusLite specific parameters.
"""
__all__ = []



from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_('JobStatusLite')
config.JobStatusLite.namespace       = 'WMComponent.JobStatusLite.JobStatusLite'
config.JobStatusLite.componentDir    = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.logLevel        = 'INFO'
config.JobStatusLite.pollInterval    = 180
config.JobStatusLite.queryInterval   = 120
config.JobStatusLite.jobLoadLimit    = 100
config.JobStatusLite.maxJobQuery     = 100
config.JobStatusLite.taskLimit       = 30
config.JobStatusLite.maxJobsCommit   = 100
config.JobStatusLite.processes       = 5
Exemplo n.º 37
0
from WMCore.Agent.Configuration import Configuration


config = Configuration()

config.section_("CoreDatabase")
if (os.getenv('DATABASE') == None):
    raise RuntimeError, \
          "You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = os.getenv('DIALECT', None)
config.CoreDatabase.socket = os.getenv("DBSOCK")


config.component_("DBSUpload")
#The log level of the component.
config.DBSUpload.logLevel = 'DEBUG'

# maximum number of threads we want to deal
# with messages per pool.
config.DBSUpload.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSUpload.bufferSuccessHandler = \
    'WMComponent.DBSUpload.Handler.BufferSuccess'

config.DBSUpload.newWorkflowHandler = \
    'WMComponent.DBSUpload.Handler.NewWorkflowHandler'
Exemplo n.º 38
0
the common configuration file, which is accessed by environment variable

"""

import os
import sys
import getopt
import logging

from WMCore.Agent.Configuration import Configuration

#  //
# // Find and load the Configuration
# //
config = Configuration()
config.component_("PilotManagerComponent")

# The log level of the component.
config.PilotManagerComponent.logLevel = "DEBUG"

# maximum number of threads we want to deal
# with messages per pool.
config.PilotManagerComponent.maxThreads = 30

# emulation mode
config.PilotManagerComponent.emulationMode = True

# maximum number of retries we want for job
config.PilotManagerComponent.maxRetries = 10

# component dir
Exemplo n.º 39
0
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for WorkflowManager specific
parameters.
"""
__all__ = []



import os

from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("WorkflowManager")
config.WorkflowManager.logLevel = "INFO"
config.WorkflowManager.componentName = "WorkflowManager"
config.WorkflowManager.componentDir = \
    os.path.join(os.getenv("TESTDIR"), "WorkflowManager")

# The maximum number of threads to process each message type
config.WorkflowManager.maxThreads = 10

# The poll interval at which to look for new filesets
config.WorkflowManager.pollInterval = 60
Exemplo n.º 40
0
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for FeederManager specific
parameters.
"""
__all__ = []



import os

from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("FeederManager")
config.FeederManager.logLevel = "INFO"
config.FeederManager.componentName = "FeederManager"
config.FeederManager.componentDir = \
    os.path.join(os.getenv("TESTDIR"), "FeederManager")
config.FeederManager.addDatasetWatchHandler = \
    'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'

# The maximum number of threads to process each message type
config.FeederManager.maxThreads = 10

# The poll interval at which to look for new fileset/feeder association
config.FeederManager.pollInterval = 60
Exemplo n.º 41
0
#!/usr/bin/env python
#pylint: disable=E1101,E1103,C0103,R0902
"""
Defines default config values for WorkflowManager specific
parameters.
"""
__all__ = []

import os

from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("WorkflowManager")
config.WorkflowManager.logLevel = "INFO"
config.WorkflowManager.componentName = "WorkflowManager"
config.WorkflowManager.componentDir = \
    os.path.join(os.getenv("TESTDIR"), "WorkflowManager")

# The maximum number of threads to process each message type
config.WorkflowManager.maxThreads = 10

# The poll interval at which to look for new filesets
config.WorkflowManager.pollInterval = 60
Exemplo n.º 42
0
#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for FeederManager specific
parameters.
"""
__all__ = []

import os

from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("FeederManager")
config.FeederManager.logLevel = "INFO"
config.FeederManager.componentName = "FeederManager"
config.FeederManager.componentDir = \
    os.path.join(os.getenv("TESTDIR"), "FeederManager")
config.FeederManager.addDatasetWatchHandler = \
    'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'

# The maximum number of threads to process each message type
config.FeederManager.maxThreads = 10

# The poll interval at which to look for new fileset/feeder association
config.FeederManager.pollInterval = 60
Exemplo n.º 43
0
#!/usr/bin/env python
"""
Defines default config values for JobStatusLite specific parameters.
"""
__all__ = []

from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_('JobStatusLite')
config.JobStatusLite.namespace = 'WMComponent.JobStatusLite.JobStatusLite'
config.JobStatusLite.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobStatusLite.logLevel = 'INFO'
config.JobStatusLite.pollInterval = 180
config.JobStatusLite.queryInterval = 120
config.JobStatusLite.jobLoadLimit = 100
config.JobStatusLite.maxJobQuery = 100
config.JobStatusLite.taskLimit = 30
config.JobStatusLite.maxJobsCommit = 100
config.JobStatusLite.processes = 5
Exemplo n.º 44
0
#!/usr/bin/env python
"""
Defines default config values for JobAccountant specific
parameters.
"""
__all__ = []

from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("WorkQueueManager")

# set type of work queue
config.WorkQueueManager.level = "GlobalQueue"
#config.WorkQueueManager.level = "LocalQueue"

# In general can be left alone
config.WorkQueueManager.componentDir = config.General.WorkDir + "/WorkQueueManager"
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
#config.WorkQueueManager.logLevel = 'INFO'
#config.WorkQueueManager.pollInterval = 600

# RequestManager config
config.WorkQueueManager.reqMgrConfig = {}
# uncomment to override default reqMgr url
#config.WorkQueueManager.reqMgrConfig['endpoint'] = 'http://cmssrv49.fnal.gov:8585'
config.WorkQueueManager.reqMgrConfig['teamName'] = 'Dodgers'

# add parameters for global or local queue if default param is not what you want
config.WorkQueueManager.queueParams = {'LocationRefreshInterval': 10}
# uncomment to change CacheDir from default