def setUp(self):
"""
If we dont have a wmspec file create one
"""
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# setup rucio parameters for global/local queue
self.queueParams = {}
self.queueParams['log_reporter'] = "lq_profile_test"
self.queueParams['rucioAccount'] = "wma_test"
self.queueParams['rucioAuthUrl'] = "http://cmsrucio-int.cern.ch"
self.queueParams['rucioUrl'] = "https://cmsrucio-auth-int.cern.ch"
# Create queues
self.localQueue = localQueue(DbName=self.queueDB,
InboxDbName=self.queueInboxDB,
NegotiationTimeout=0,
QueueURL='global.example.com',
CacheDir=self.cacheDir,
central_logdb_url="%s/%s" %
(self.couchURL, self.logDBName),
**self.queueParams)
def __init__(self, resources):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.resources = resources
self.wq = localQueue()
def __init__(self, resources):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.resources = resources
self.wq = localQueue()
def setupLocalQueue(self):
"""Create a local queue"""
localQ = localQueue(DbName=self.localQDB,
InboxDbName=self.localQInboxDB,
QueueURL=self.localQCouchUrl,
Teams=["The A-Team", "some other bloke"],
ParentQueueCouchUrl=self.globalQCouchUrl,
CacheDir=self.testInit.testDir)
return localQ
def setupLocalQueue(self):
"""Create a local queue"""
localQ = localQueue(DbName = self.localQDB,
InboxDbName = self.localQInboxDB,
QueueURL = self.localQCouchUrl,
Teams = ["The A-Team", "some other bloke"],
ParentQueueCouchUrl = self.globalQCouchUrl,
CacheDir = self.testInit.testDir)
return localQ
def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(
specName, "file", assignKwargs={'SiteWhitelist': ["T2_XX_SiteA"]})
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True,
**self.queueParams)
localQ = localQueue(DbName='local_workqueue_t',
QueueURL=self.testInit.couchUrl,
CacheDir=self.testInit.testDir,
ParentQueueCouchUrl='%s/workqueue_t' %
self.testInit.couchUrl,
ParentQueueInboxCouchDBName='workqueue_t_inbox',
**self.queueParams)
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce': True, 'group': True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
numWorks = localQ.pullWork({'T2_XX_SiteA': 10})
self.assertTrue(numWorks > 0)
# replicate from GQ to LQ manually
localQ.backend.pullFromParent(continuous=False)
# wait until replication is done
time.sleep(2)
localQ.processInboundWork(continuous=False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
# overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce': True, 'group': True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
availableWF = localApi.getAvailableWorkflows()
self.assertEqual(availableWF, set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except Exception as ex:
self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file",
assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]})
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True)
localQ = localQueue(DbName='local_workqueue_t',
QueueURL=self.testInit.couchUrl,
CacheDir=self.testInit.testDir,
ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl,
ParentQueueInboxCouchDBName='workqueue_t_inbox'
)
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce': True, 'group': True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
numWorks = localQ.pullWork({'T2_XX_SiteA': 10})
self.assertTrue(numWorks > 0)
# replicate from GQ to LQ manually
localQ.backend.pullFromParent(continuous=False)
# wait until replication is done
time.sleep(2)
localQ.processInboundWork(continuous=False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
# overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce': True, 'group': True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
availableWF = localApi.getAvailableWorkflows()
self.assertEqual(availableWF, set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except Exception as ex:
self.fail('No exception should be raised.: %s' % str(ex))
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(
**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.requestLocalCouchDB = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralCouchDBWriter = self.requestLocalCouchDB
else:
self.centralCouchDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.centralRequestDBURL)
self.reqmgr2Svc = ReqMgr(
self.config.TaskArchiver.ReqMgr2ServiceURL)
#TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
self.reqmgrSvc = RequestManager(
{'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
#Load the cleanout state ID and save it
stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
self.stateID = stateIDDAO.execute("cleanout")
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", 0)
self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)
# Set up optional histograms
self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", [])
self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10)
self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0)
# Start a couch server for getting job info
# from the FWJRs for committal to archive
try:
self.dbname = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL',
self.config.JobStateMachine.couchDBName)
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self.summarydb = getattr(self.config.TaskArchiver, "summaryDBName", self.dbname)
self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname)
self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname)
self.workdatabase = self.couchdb.connectDatabase(self.dbname)
logging.debug("Using url %s" % self.config.JobStateMachine.couchurl)
logging.debug("Writing to %s" % self.dbname)
self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False)
except Exception, ex:
msg = "Error in connecting to couch.\n"
msg += str(ex)
logging.error(msg)
self.jobsdatabase = None
self.fwjrdatabase = None
if getattr(self.config.TaskArchiver, 'requireCouch', False):
raise TaskArchiverPollerException(msg)
def queueFromConfig(config):
"""Create a queue from the config object"""
config = queueConfigFromConfigObject(config)
if config.WorkQueueManager.level == 'GlobalQueue':
from WMCore.WorkQueue.WorkQueue import globalQueue
return globalQueue(**config.WorkQueueManager.queueParams)
elif config.WorkQueueManager.level == 'LocalQueue':
from WMCore.WorkQueue.WorkQueue import localQueue
return localQueue(**config.WorkQueueManager.queueParams)
else:
from WMCore.WorkQueue.WorkQueue import WorkQueue
return WorkQueue(**config.WorkQueueManager.queueParams)
def queueFromConfig(config):
"""Create a queue from the config object"""
config = queueConfigFromConfigObject(config)
if config.WorkQueueManager.level == 'GlobalQueue':
from WMCore.WorkQueue.WorkQueue import globalQueue
return globalQueue(**config.WorkQueueManager.queueParams)
elif config.WorkQueueManager.level == 'LocalQueue':
from WMCore.WorkQueue.WorkQueue import localQueue
return localQueue(**config.WorkQueueManager.queueParams)
else:
from WMCore.WorkQueue.WorkQueue import WorkQueue
return WorkQueue(**config.WorkQueueManager.queueParams)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(
package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi
)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, "WorkQueueParams"):
self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, "useReqMgrForCompletionCheck", True)
if not self.useReqMgrForCompletionCheck:
# sets the local monitor summary couch db
self.requestLocalCouchDB = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp,
)
self.centralCouchDBWriter = self.requestLocalCouchDB
else:
self.centralCouchDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL)
self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
# TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
self.reqmgrSvc = RequestManager({"endpoint": self.config.TaskArchiver.ReqMgrServiceURL})
# Load the cleanout state ID and save it
stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
self.stateID = stateIDDAO.execute("cleanout")
return
def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file")
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl)
localQ = localQueue(DbName='local_workqueue_t',
QueueURL=self.testInit.couchUrl,
CacheDir=self.testInit.testDir,
ParentQueueCouchUrl='%s/workqueue_t' %
self.testInit.couchUrl,
ParentQueueInboxCouchDBName='workqueue_t_inbox')
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
#overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce': True, 'group': True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
self.assertTrue(
localQ.pullWork({'T2_XX_SiteA': 10}, continuousReplication=False) >
0)
localQ.processInboundWork(continuous=False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
#overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce': True, 'group': True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
self.assertEqual(localApi.getAvailableWorkflows(),
set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except:
self.fail('No exception should be raised.')
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, \
NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" %(self.couchURL, self.logDBName), log_reporter='lq_profile_test')
def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file")
globalQ = globalQueue(DbName = 'workqueue_t',
QueueURL = self.testInit.couchUrl)
localQ = localQueue(DbName = 'local_workqueue_t',
QueueURL = self.testInit.couchUrl,
CacheDir = self.testInit.testDir,
ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl,
ParentQueueInboxCouchDBName = 'workqueue_t_inbox'
)
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
#overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce' : True, 'group' : True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0)
localQ.processInboundWork(continuous = False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
#overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce' : True, 'group' : True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except:
self.fail('No exception should be raised.')
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex=False,
dbs=False,
siteDB=True,
requestMgr=True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, \
NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" %(self.couchURL, self.logDBName), log_reporter='lq_profile_test')
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
NegotiationTimeout = 0,
QueueURL = 'global.example.com',
CacheDir = self.cacheDir)
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
NegotiationTimeout = 0,
QueueURL = 'global.example.com',
CacheDir = self.cacheDir)
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
#set up WMAgent config file for couchdb
self.configFile = EmulatorSetup.setupWMAgentConfig()
WorkQueueTestCase.setUp(self)
# Basic production Spec
self.spec = monteCarloWorkload('testProduction', mcArgs)
getFirstTask(self.spec).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(self.spec).addProduction(totalevents = 10000)
self.spec.setSpecUrl(os.path.join(self.workDir, 'testworkflow.spec'))
self.spec.save(self.spec.specUrl())
# Sample Tier1 ReReco spec
self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
self.processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessing.spec'))
self.processingSpec.save(self.processingSpec.specUrl())
# Sample Tier1 ReReco spec
self.parentProcSpec = rerecoWorkload('testParentProcessing', parentProcArgs)
self.parentProcSpec.setSpecUrl(os.path.join(self.workDir,
'testParentProcessing.spec'))
self.parentProcSpec.save(self.parentProcSpec.specUrl())
# ReReco spec with blacklist
self.blacklistSpec = rerecoWorkload('blacklistSpec', rerecoArgs)
self.blacklistSpec.setSpecUrl(os.path.join(self.workDir,
'testBlacklist.spec'))
getFirstTask(self.blacklistSpec).data.constraints.sites.blacklist = ['T2_XX_SiteA']
self.blacklistSpec.save(self.blacklistSpec.specUrl())
# ReReco spec with whitelist
self.whitelistSpec = rerecoWorkload('whitelistlistSpec', rerecoArgs)
self.whitelistSpec.setSpecUrl(os.path.join(self.workDir,
'testWhitelist.spec'))
getFirstTask(self.whitelistSpec).data.constraints.sites.whitelist = ['T2_XX_SiteB']
self.whitelistSpec.save(self.whitelistSpec.specUrl())
# setup Mock DBS and PhEDEx
inputDataset = getFirstTask(self.processingSpec).inputDataset()
self.dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
# Create queues
globalCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.globalQueue = globalQueue(DbName = self.globalQDB,
InboxDbName = self.globalQInboxDB,
QueueURL = globalCouchUrl)
# self.midQueue = WorkQueue(SplitByBlock = False, # mid-level queue
# PopulateFilesets = False,
# ParentQueue = self.globalQueue,
# CacheDir = None)
# ignore mid queue as it causes database duplication's
# copy jobStateMachine couchDB configuration here since we don't want/need to pass whole configuration
jobCouchConfig = Configuration()
jobCouchConfig.section_("JobStateMachine")
jobCouchConfig.JobStateMachine.couchurl = os.environ["COUCHURL"]
jobCouchConfig.JobStateMachine.couchDBName = "testcouchdb"
# copy bossAir configuration here since we don't want/need to pass whole configuration
bossAirConfig = Configuration()
bossAirConfig.section_("BossAir")
bossAirConfig.BossAir.pluginDir = "WMCore.BossAir.Plugins"
bossAirConfig.BossAir.pluginNames = ["CondorPlugin"]
bossAirConfig.section_("Agent")
bossAirConfig.Agent.agentName = "TestAgent"
self.localQueue = localQueue(DbName = self.localQDB,
InboxDbName = self.localQInboxDB,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
self.localQueue2 = localQueue(DbName = self.localQDB2,
InboxDbName = self.localQInboxDB2,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
# configuration for the Alerts messaging framework, work (alerts) and
# control channel addresses to which alerts
# these are destination addresses where AlertProcessor:Receiver listens
config = Configuration()
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# standalone queue for unit tests
self.queue = WorkQueue(JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
CacheDir = self.workDir,
config = config)
# create relevant sites in wmbs
rc = ResourceControl()
for site, se in self.queue.SiteDB.mapping.items():
rc.insertSite(site, 100, se, cmsName = site)
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = threading.currentThread().logger,
dbinterface = threading.currentThread().dbi)
addLocation = daofactory(classname = "Locations.New")
addLocation.execute(siteName = site, seName = se)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.maxProcessSize = getattr(self.config.TaskArchiver, 'maxProcessSize', 250)
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)
self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.uploadPublishInfo = getattr(self.config.TaskArchiver, 'uploadPublishInfo', False)
self.uploadPublishDir = getattr(self.config.TaskArchiver, 'uploadPublishDir', None)
self.userFileCacheURL = getattr(self.config.TaskArchiver, 'userFileCacheURL', None)
# Set up optional histograms
self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", [])
self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10)
self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL);
# Start a couch server for getting job info
# from the FWJRs for committal to archive
try:
workDBName = getattr(self.config.TaskArchiver, 'workloadSummaryCouchDBName',
'workloadsummary')
workDBurl = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL')
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.workCouchdb = CouchServer(workDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
self.workdatabase = self.workCouchdb.connectDatabase(workDBName)
logging.debug("Using url %s/%s for job" % (jobDBurl, jobDBName))
logging.debug("Writing to %s/%s for workloadSummary" % (sanitizeURL(workDBurl)['url'], workDBName))
self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False)
except Exception, ex:
msg = "Error in connecting to couch.\n"
msg += str(ex)
logging.error(msg)
self.jobsdatabase = None
self.fwjrdatabase = None
if getattr(self.config.TaskArchiver, 'requireCouch', False):
raise TaskArchiverPollerException(msg)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(
**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.maxProcessSize = getattr(self.config.TaskArchiver,
'maxProcessSize', 250)
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.uploadPublishInfo = getattr(self.config.TaskArchiver,
'uploadPublishInfo', False)
self.uploadPublishDir = getattr(self.config.TaskArchiver,
'uploadPublishDir', None)
self.userFileCacheURL = getattr(self.config.TaskArchiver,
'userFileCacheURL', None)
# Set up optional histograms
self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys",
[])
self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins",
10)
self.histogramLimit = getattr(self.config.TaskArchiver,
"histogramLimit", 5.0)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.wmstatsCouchDB = WMStatsWriter(
self.config.TaskArchiver.localWMStatsURL)
self.centralCouchDBWriter = self.wmstatsCouchDB
else:
self.centralCouchDBWriter = WMStatsWriter(
self.config.TaskArchiver.centralWMStatsURL)
# Start a couch server for getting job info
# from the FWJRs for committal to archive
try:
workDBName = getattr(self.config.TaskArchiver,
'workloadSummaryCouchDBName',
'workloadsummary')
workDBurl = getattr(self.config.TaskArchiver,
'workloadSummaryCouchURL')
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.workCouchdb = CouchServer(workDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" %
jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" %
jobDBName)
self.workdatabase = self.workCouchdb.connectDatabase(workDBName)
logging.debug("Using url %s/%s for job" % (jobDBurl, jobDBName))
logging.debug("Writing to %s/%s for workloadSummary" %
(sanitizeURL(workDBurl)['url'], workDBName))
self.requireCouch = getattr(self.config.TaskArchiver,
'requireCouch', False)
except Exception, ex:
msg = "Error in connecting to couch.\n"
msg += str(ex)
logging.error(msg)
self.jobsdatabase = None
self.fwjrdatabase = None
if getattr(self.config.TaskArchiver, 'requireCouch', False):
raise TaskArchiverPollerException(msg)
