def setUp(self): """ If we dont have a wmspec file create one """ WorkQueueTestCase.setUp(self) self.cacheDir = tempfile.mkdtemp() self.specGenerator = WMSpecGenerator(self.cacheDir) self.specs = self.createReRecoSpec(1, "file") # setup rucio parameters for global/local queue self.queueParams = {} self.queueParams['log_reporter'] = "lq_profile_test" self.queueParams['rucioAccount'] = "wma_test" self.queueParams['rucioAuthUrl'] = "http://cmsrucio-int.cern.ch" self.queueParams['rucioUrl'] = "https://cmsrucio-auth-int.cern.ch" # Create queues self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" % (self.couchURL, self.logDBName), **self.queueParams)
def __init__(self, resources): """ Initialise class members """ BaseWorkerThread.__init__(self) self.resources = resources self.wq = localQueue()
def setupLocalQueue(self): """Create a local queue""" localQ = localQueue(DbName=self.localQDB, InboxDbName=self.localQInboxDB, QueueURL=self.localQCouchUrl, Teams=["The A-Team", "some other bloke"], ParentQueueCouchUrl=self.globalQCouchUrl, CacheDir=self.testInit.testDir) return localQ
def setupLocalQueue(self): """Create a local queue""" localQ = localQueue(DbName = self.localQDB, InboxDbName = self.localQInboxDB, QueueURL = self.localQCouchUrl, Teams = ["The A-Team", "some other bloke"], ParentQueueCouchUrl = self.globalQCouchUrl, CacheDir = self.testInit.testDir) return localQ
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec( specName, "file", assignKwargs={'SiteWhitelist': ["T2_XX_SiteA"]}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True, **self.queueParams) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox', **self.queueParams) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) numWorks = localQ.pullWork({'T2_XX_SiteA': 10}) self.assertTrue(numWorks > 0) # replicate from GQ to LQ manually localQ.backend.pullFromParent(continuous=False) # wait until replication is done time.sleep(2) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') # overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) availableWF = localApi.getAvailableWorkflows() self.assertEqual(availableWF, set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except Exception as ex: self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file", assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox' ) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) numWorks = localQ.pullWork({'T2_XX_SiteA': 10}) self.assertTrue(numWorks > 0) # replicate from GQ to LQ manually localQ.backend.pullFromParent(continuous=False) # wait until replication is done time.sleep(2) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') # overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) availableWF = localApi.getAvailableWorkflows() self.assertEqual(availableWF, set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except Exception as ex: self.fail('No exception should be raised.: %s' % str(ex))
def __init__(self, config): """ Initialise class members """ BaseWorkerThread.__init__(self) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) self.config = config self.jobCacheDir = self.config.JobCreator.jobCacheDir if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False: # Get workqueue setup from config unless overridden if hasattr(self.config.TaskArchiver, 'WorkQueueParams'): self.workQueue = localQueue( **self.config.TaskArchiver.WorkQueueParams) else: from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig self.workQueue = queueFromConfig(self.config) else: self.workQueue = None self.timeout = getattr(self.config.TaskArchiver, "timeOut", None) self.useReqMgrForCompletionCheck = getattr( self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True) if not self.useReqMgrForCompletionCheck: #sets the local monitor summary couch db self.requestLocalCouchDB = RequestDBWriter( self.config.AnalyticsDataCollector.localT0RequestDBURL, couchapp=self.config.AnalyticsDataCollector.RequestCouchApp) self.centralCouchDBWriter = self.requestLocalCouchDB else: self.centralCouchDBWriter = RequestDBWriter( self.config.AnalyticsDataCollector.centralRequestDBURL) self.reqmgr2Svc = ReqMgr( self.config.TaskArchiver.ReqMgr2ServiceURL) #TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only) self.reqmgrSvc = RequestManager( {'endpoint': self.config.TaskArchiver.ReqMgrServiceURL}) #Load the cleanout state ID and save it stateIDDAO = self.daoFactory(classname="Jobs.GetStateID") self.stateID = stateIDDAO.execute("cleanout") return
def __init__(self, config): """ Initialise class members """ BaseWorkerThread.__init__(self) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.config = config self.jobCacheDir = self.config.JobCreator.jobCacheDir if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False: # Get workqueue setup from config unless overridden if hasattr(self.config.TaskArchiver, 'WorkQueueParams'): self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams) else: from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig self.workQueue = queueFromConfig(self.config) else: self.workQueue = None self.timeout = getattr(self.config.TaskArchiver, "timeOut", 0) self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3) # Set up optional histograms self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", []) self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10) self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0) # Start a couch server for getting job info # from the FWJRs for committal to archive try: self.dbname = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL', self.config.JobStateMachine.couchDBName) self.couchdb = CouchServer(self.config.JobStateMachine.couchurl) self.summarydb = getattr(self.config.TaskArchiver, "summaryDBName", self.dbname) self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname) self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname) self.workdatabase = self.couchdb.connectDatabase(self.dbname) logging.debug("Using url %s" % self.config.JobStateMachine.couchurl) logging.debug("Writing to %s" % self.dbname) self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False) except Exception, ex: msg = "Error in connecting to couch.\n" msg += str(ex) logging.error(msg) self.jobsdatabase = None self.fwjrdatabase = None if getattr(self.config.TaskArchiver, 'requireCouch', False): raise TaskArchiverPollerException(msg)
def queueFromConfig(config): """Create a queue from the config object""" config = queueConfigFromConfigObject(config) if config.WorkQueueManager.level == 'GlobalQueue': from WMCore.WorkQueue.WorkQueue import globalQueue return globalQueue(**config.WorkQueueManager.queueParams) elif config.WorkQueueManager.level == 'LocalQueue': from WMCore.WorkQueue.WorkQueue import localQueue return localQueue(**config.WorkQueueManager.queueParams) else: from WMCore.WorkQueue.WorkQueue import WorkQueue return WorkQueue(**config.WorkQueueManager.queueParams)
def __init__(self, config): """ Initialise class members """ BaseWorkerThread.__init__(self) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.dbsDaoFactory = DAOFactory( package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi ) self.config = config self.jobCacheDir = self.config.JobCreator.jobCacheDir if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False: # Get workqueue setup from config unless overridden if hasattr(self.config.TaskArchiver, "WorkQueueParams"): self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams) else: from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig self.workQueue = queueFromConfig(self.config) else: self.workQueue = None self.timeout = getattr(self.config.TaskArchiver, "timeOut", None) self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, "useReqMgrForCompletionCheck", True) if not self.useReqMgrForCompletionCheck: # sets the local monitor summary couch db self.requestLocalCouchDB = RequestDBWriter( self.config.AnalyticsDataCollector.localT0RequestDBURL, couchapp=self.config.AnalyticsDataCollector.RequestCouchApp, ) self.centralCouchDBWriter = self.requestLocalCouchDB else: self.centralCouchDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL) self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL) # TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only) self.reqmgrSvc = RequestManager({"endpoint": self.config.TaskArchiver.ReqMgrServiceURL}) # Load the cleanout state ID and save it stateIDDAO = self.daoFactory(classname="Jobs.GetStateID") self.stateID = stateIDDAO.execute("cleanout") return
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox') # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) self.assertTrue( localQ.pullWork({'T2_XX_SiteA': 10}, continuousReplication=False) > 0) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') #overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except: self.fail('No exception should be raised.')
def setUp(self): """ If we dont have a wmspec file create one """ EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=True) WorkQueueTestCase.setUp(self) self.cacheDir = tempfile.mkdtemp() self.specGenerator = WMSpecGenerator(self.cacheDir) self.specs = self.createReRecoSpec(1, "file") # Create queues self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, \ NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" %(self.couchURL, self.logDBName), log_reporter='lq_profile_test')
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName = 'workqueue_t', QueueURL = self.testInit.couchUrl) localQ = localQueue(DbName = 'local_workqueue_t', QueueURL = self.testInit.couchUrl, CacheDir = self.testInit.testDir, ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName = 'workqueue_t_inbox' ) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce' : True, 'group' : True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0) localQ.processInboundWork(continuous = False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') #overwrite default - can't test with stale view localApi.defaultOptions = {'reduce' : True, 'group' : True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except: self.fail('No exception should be raised.')
def setUp(self): """ If we dont have a wmspec file create one """ EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True, requestMgr = True) WorkQueueTestCase.setUp(self) self.cacheDir = tempfile.mkdtemp() self.specGenerator = WMSpecGenerator(self.cacheDir) self.specs = self.createReRecoSpec(1, "file") # Create queues self.localQueue = localQueue(DbName = self.queueDB, InboxDbName = self.queueInboxDB, NegotiationTimeout = 0, QueueURL = 'global.example.com', CacheDir = self.cacheDir)
def setUp(self): """ If we dont have a wmspec file create one """ EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True, requestMgr = False) #set up WMAgent config file for couchdb self.configFile = EmulatorSetup.setupWMAgentConfig() WorkQueueTestCase.setUp(self) # Basic production Spec self.spec = monteCarloWorkload('testProduction', mcArgs) getFirstTask(self.spec).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(self.spec).addProduction(totalevents = 10000) self.spec.setSpecUrl(os.path.join(self.workDir, 'testworkflow.spec')) self.spec.save(self.spec.specUrl()) # Sample Tier1 ReReco spec self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs) self.processingSpec.setSpecUrl(os.path.join(self.workDir, 'testProcessing.spec')) self.processingSpec.save(self.processingSpec.specUrl()) # Sample Tier1 ReReco spec self.parentProcSpec = rerecoWorkload('testParentProcessing', parentProcArgs) self.parentProcSpec.setSpecUrl(os.path.join(self.workDir, 'testParentProcessing.spec')) self.parentProcSpec.save(self.parentProcSpec.specUrl()) # ReReco spec with blacklist self.blacklistSpec = rerecoWorkload('blacklistSpec', rerecoArgs) self.blacklistSpec.setSpecUrl(os.path.join(self.workDir, 'testBlacklist.spec')) getFirstTask(self.blacklistSpec).data.constraints.sites.blacklist = ['T2_XX_SiteA'] self.blacklistSpec.save(self.blacklistSpec.specUrl()) # ReReco spec with whitelist self.whitelistSpec = rerecoWorkload('whitelistlistSpec', rerecoArgs) self.whitelistSpec.setSpecUrl(os.path.join(self.workDir, 'testWhitelist.spec')) getFirstTask(self.whitelistSpec).data.constraints.sites.whitelist = ['T2_XX_SiteB'] self.whitelistSpec.save(self.whitelistSpec.specUrl()) # setup Mock DBS and PhEDEx inputDataset = getFirstTask(self.processingSpec).inputDataset() self.dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) # Create queues globalCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB) self.globalQueue = globalQueue(DbName = self.globalQDB, InboxDbName = self.globalQInboxDB, QueueURL = globalCouchUrl) # self.midQueue = WorkQueue(SplitByBlock = False, # mid-level queue # PopulateFilesets = False, # ParentQueue = self.globalQueue, # CacheDir = None) # ignore mid queue as it causes database duplication's # copy jobStateMachine couchDB configuration here since we don't want/need to pass whole configuration jobCouchConfig = Configuration() jobCouchConfig.section_("JobStateMachine") jobCouchConfig.JobStateMachine.couchurl = os.environ["COUCHURL"] jobCouchConfig.JobStateMachine.couchDBName = "testcouchdb" # copy bossAir configuration here since we don't want/need to pass whole configuration bossAirConfig = Configuration() bossAirConfig.section_("BossAir") bossAirConfig.BossAir.pluginDir = "WMCore.BossAir.Plugins" bossAirConfig.BossAir.pluginNames = ["CondorPlugin"] bossAirConfig.section_("Agent") bossAirConfig.Agent.agentName = "TestAgent" self.localQueue = localQueue(DbName = self.localQDB, InboxDbName = self.localQInboxDB, ParentQueueCouchUrl = globalCouchUrl, JobDumpConfig = jobCouchConfig, BossAirConfig = bossAirConfig, CacheDir = self.workDir) self.localQueue2 = localQueue(DbName = self.localQDB2, InboxDbName = self.localQInboxDB2, ParentQueueCouchUrl = globalCouchUrl, JobDumpConfig = jobCouchConfig, BossAirConfig = bossAirConfig, CacheDir = self.workDir) # configuration for the Alerts messaging framework, work (alerts) and # control channel addresses to which alerts # these are destination addresses where AlertProcessor:Receiver listens config = Configuration() config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" # standalone queue for unit tests self.queue = WorkQueue(JobDumpConfig = jobCouchConfig, BossAirConfig = bossAirConfig, DbName = self.queueDB, InboxDbName = self.queueInboxDB, CacheDir = self.workDir, config = config) # create relevant sites in wmbs rc = ResourceControl() for site, se in self.queue.SiteDB.mapping.items(): rc.insertSite(site, 100, se, cmsName = site) daofactory = DAOFactory(package = "WMCore.WMBS", logger = threading.currentThread().logger, dbinterface = threading.currentThread().dbi) addLocation = daofactory(classname = "Locations.New") addLocation.execute(siteName = site, seName = se)
def __init__(self, config): """ Initialise class members """ BaseWorkerThread.__init__(self) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.config = config self.jobCacheDir = self.config.JobCreator.jobCacheDir if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False: # Get workqueue setup from config unless overridden if hasattr(self.config.TaskArchiver, 'WorkQueueParams'): self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams) else: from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig self.workQueue = queueFromConfig(self.config) else: self.workQueue = None self.maxProcessSize = getattr(self.config.TaskArchiver, 'maxProcessSize', 250) self.timeout = getattr(self.config.TaskArchiver, "timeOut", None) self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3) self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True) self.uploadPublishInfo = getattr(self.config.TaskArchiver, 'uploadPublishInfo', False) self.uploadPublishDir = getattr(self.config.TaskArchiver, 'uploadPublishDir', None) self.userFileCacheURL = getattr(self.config.TaskArchiver, 'userFileCacheURL', None) # Set up optional histograms self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", []) self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10) self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0) if not self.useReqMgrForCompletionCheck: #sets the local monitor summary couch db self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL); # Start a couch server for getting job info # from the FWJRs for committal to archive try: workDBName = getattr(self.config.TaskArchiver, 'workloadSummaryCouchDBName', 'workloadsummary') workDBurl = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL') jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url'] jobDBName = self.config.JobStateMachine.couchDBName self.jobCouchdb = CouchServer(jobDBurl) self.workCouchdb = CouchServer(workDBurl) self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName) self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName) self.workdatabase = self.workCouchdb.connectDatabase(workDBName) logging.debug("Using url %s/%s for job" % (jobDBurl, jobDBName)) logging.debug("Writing to %s/%s for workloadSummary" % (sanitizeURL(workDBurl)['url'], workDBName)) self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False) except Exception, ex: msg = "Error in connecting to couch.\n" msg += str(ex) logging.error(msg) self.jobsdatabase = None self.fwjrdatabase = None if getattr(self.config.TaskArchiver, 'requireCouch', False): raise TaskArchiverPollerException(msg)
def __init__(self, config): """ Initialise class members """ BaseWorkerThread.__init__(self) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.config = config self.jobCacheDir = self.config.JobCreator.jobCacheDir if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False: # Get workqueue setup from config unless overridden if hasattr(self.config.TaskArchiver, 'WorkQueueParams'): self.workQueue = localQueue( **self.config.TaskArchiver.WorkQueueParams) else: from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig self.workQueue = queueFromConfig(self.config) else: self.workQueue = None self.maxProcessSize = getattr(self.config.TaskArchiver, 'maxProcessSize', 250) self.timeout = getattr(self.config.TaskArchiver, "timeOut", None) self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3) self.useReqMgrForCompletionCheck = getattr( self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True) self.uploadPublishInfo = getattr(self.config.TaskArchiver, 'uploadPublishInfo', False) self.uploadPublishDir = getattr(self.config.TaskArchiver, 'uploadPublishDir', None) self.userFileCacheURL = getattr(self.config.TaskArchiver, 'userFileCacheURL', None) # Set up optional histograms self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", []) self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10) self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0) if not self.useReqMgrForCompletionCheck: #sets the local monitor summary couch db self.wmstatsCouchDB = WMStatsWriter( self.config.TaskArchiver.localWMStatsURL) self.centralCouchDBWriter = self.wmstatsCouchDB else: self.centralCouchDBWriter = WMStatsWriter( self.config.TaskArchiver.centralWMStatsURL) # Start a couch server for getting job info # from the FWJRs for committal to archive try: workDBName = getattr(self.config.TaskArchiver, 'workloadSummaryCouchDBName', 'workloadsummary') workDBurl = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL') jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url'] jobDBName = self.config.JobStateMachine.couchDBName self.jobCouchdb = CouchServer(jobDBurl) self.workCouchdb = CouchServer(workDBurl) self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName) self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName) self.workdatabase = self.workCouchdb.connectDatabase(workDBName) logging.debug("Using url %s/%s for job" % (jobDBurl, jobDBName)) logging.debug("Writing to %s/%s for workloadSummary" % (sanitizeURL(workDBurl)['url'], workDBName)) self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False) except Exception, ex: msg = "Error in connecting to couch.\n" msg += str(ex) logging.error(msg) self.jobsdatabase = None self.fwjrdatabase = None if getattr(self.config.TaskArchiver, 'requireCouch', False): raise TaskArchiverPollerException(msg)