def preInitialization(self):
print("WorkQueueManager.preInitialization")
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.WorkQueueManager.pollInterval
### Global queue special functions
if self.config.WorkQueueManager.level == 'GlobalQueue':
# Get work from ReqMgr, report back & delete finished requests
myThread.workerThreadManager.addWorker(
WorkQueueManagerReqMgrPoller(
queueFromConfig(self.config),
getattr(self.config.WorkQueueManager,
'reqMgrConfig', {})
),
pollInterval)
### local queue special function
elif self.config.WorkQueueManager.level == 'LocalQueue':
# pull work from parent queue
myThread.workerThreadManager.addWorker(
WorkQueueManagerWorkPoller(queueFromConfig(self.config),
self.config),
pollInterval)
# inject acquired work into wmbs
myThread.workerThreadManager.addWorker(
WorkQueueManagerWMBSFileFeeder(queueFromConfig(self.config),
self.config),
pollInterval)
### general functions
# Data location updates
myThread.workerThreadManager.addWorker(
WorkQueueManagerLocationPoller(queueFromConfig(self.config),
self.config),
pollInterval)
# Clean finished work & apply end policies
myThread.workerThreadManager.addWorker(
WorkQueueManagerCleaner(queueFromConfig(self.config),
self.config),
pollInterval)
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.changeState = ChangeState(self.config)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.loadAction = self.daoFactory(classname="Jobs.LoadFromIDWithWorkflow")
# Variables
self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
"numberOfJobsToCluster", 1000)
# initialize the alert framework (if available)
self.initAlerts(compName="JobArchiver")
try:
self.uploadPublishDir = getattr(self.config.JobArchiver, 'uploadPublishDir',
os.path.join(config.JobArchiver.componentDir, 'publishDir'))
self.logDir = getattr(config.JobArchiver, 'logDir',
os.path.join(config.JobArchiver.componentDir, 'logDir'))
if not os.path.isdir(self.logDir):
os.makedirs(self.logDir)
if not os.path.isdir(self.uploadPublishDir):
os.makedirs(self.uploadPublishDir)
except Exception as ex:
msg = "Unhandled exception while setting up logDir and/or uploadPublishDir!\n"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg=msg)
try:
logging.debug("Directory: %s", self.logDir)
logging.debug("Config: %s", config)
except:
pass
raise JobArchiverPollerException(msg)
try:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
except Exception as ex:
msg = "Could not load workQueue"
msg += str(ex)
logging.error(msg)
# raise JobArchiverPollerException(msg)
self.uploadPublishInfo = getattr(self.config.JobArchiver, 'uploadPublishInfo', False)
self.userFileCacheURL = getattr(self.config.JobArchiver, 'userFileCacheURL', None)
self.handleWorkflowInjection = getattr(self.config.JobArchiver,
'handleInjected', True)
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.changeState = ChangeState(self.config)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.loadAction = self.daoFactory(classname = "Jobs.LoadFromIDWithWorkflow")
# Variables
self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
"numberOfJobsToCluster", 1000)
# initialize the alert framework (if available)
self.initAlerts(compName = "JobArchiver")
try:
self.uploadPublishDir = getattr(self.config.JobArchiver, 'uploadPublishDir',
os.path.join(config.JobArchiver.componentDir, 'publishDir'))
self.logDir = getattr(config.JobArchiver, 'logDir',
os.path.join(config.JobArchiver.componentDir, 'logDir'))
if not os.path.isdir(self.logDir):
os.makedirs(self.logDir)
if not os.path.isdir(self.uploadPublishDir):
os.makedirs(self.uploadPublishDir)
except Exception as ex:
msg = "Unhandled exception while setting up logDir and/or uploadPublishDir!\n"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg = msg)
try:
logging.debug("Directory: %s" % self.logDir)
logging.debug("Config: %s" % config)
except:
pass
raise JobArchiverPollerException(msg)
try:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
except Exception as ex:
msg = "Could not load workQueue"
msg += str(ex)
logging.error(msg)
#raise JobArchiverPollerException(msg)
self.uploadPublishInfo = getattr(self.config.JobArchiver, 'uploadPublishInfo', False)
self.userFileCacheURL = getattr(self.config.JobArchiver, 'userFileCacheURL', None)
self.handleWorkflowInjection = getattr(self.config.JobArchiver,
'handleInjected', True)
return
def preInitialization(self):
print("WorkQueueManager.preInitialization")
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.WorkQueueManager.pollInterval
dataLocationInterval = getattr(self.config.WorkQueueManager,
"dataLocationInterval", 1 * 60 * 60)
### Global queue special functions
if self.config.WorkQueueManager.level == 'GlobalQueue':
# Get work from ReqMgr, report back & delete finished requests
myThread.workerThreadManager.addWorker(
WorkQueueManagerReqMgrPoller(
queueFromConfig(self.config),
getattr(self.config.WorkQueueManager, 'reqMgrConfig', {})),
pollInterval)
### local queue special function
elif self.config.WorkQueueManager.level == 'LocalQueue':
# pull work from parent queue
myThread.workerThreadManager.addWorker(
WorkQueueManagerWorkPoller(queueFromConfig(self.config),
self.config), pollInterval)
# inject acquired work into wmbs
myThread.workerThreadManager.addWorker(
WorkQueueManagerWMBSFileFeeder(queueFromConfig(self.config),
self.config), pollInterval)
### general functions
# Data location updates
myThread.workerThreadManager.addWorker(
WorkQueueManagerLocationPoller(queueFromConfig(self.config),
self.config), dataLocationInterval)
# Clean finished work & apply end policies
myThread.workerThreadManager.addWorker(
WorkQueueManagerCleaner(queueFromConfig(self.config), self.config),
pollInterval)
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(
**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.requestLocalCouchDB = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralCouchDBWriter = self.requestLocalCouchDB
else:
self.centralCouchDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.centralRequestDBURL)
self.reqmgr2Svc = ReqMgr(
self.config.TaskArchiver.ReqMgr2ServiceURL)
#TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
self.reqmgrSvc = RequestManager(
{'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
#Load the cleanout state ID and save it
stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
self.stateID = stateIDDAO.execute("cleanout")
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", 0)
self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)
# Set up optional histograms
self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", [])
self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10)
self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0)
# Start a couch server for getting job info
# from the FWJRs for committal to archive
try:
self.dbname = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL',
self.config.JobStateMachine.couchDBName)
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self.summarydb = getattr(self.config.TaskArchiver, "summaryDBName", self.dbname)
self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname)
self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname)
self.workdatabase = self.couchdb.connectDatabase(self.dbname)
logging.debug("Using url %s" % self.config.JobStateMachine.couchurl)
logging.debug("Writing to %s" % self.dbname)
self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False)
except Exception, ex:
msg = "Error in connecting to couch.\n"
msg += str(ex)
logging.error(msg)
self.jobsdatabase = None
self.fwjrdatabase = None
if getattr(self.config.TaskArchiver, 'requireCouch', False):
raise TaskArchiverPollerException(msg)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(
package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi
)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, "WorkQueueParams"):
self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, "useReqMgrForCompletionCheck", True)
if not self.useReqMgrForCompletionCheck:
# sets the local monitor summary couch db
self.requestLocalCouchDB = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp,
)
self.centralCouchDBWriter = self.requestLocalCouchDB
else:
self.centralCouchDBWriter = RequestDBWriter(self.config.AnalyticsDataCollector.centralRequestDBURL)
self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
# TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
self.reqmgrSvc = RequestManager({"endpoint": self.config.TaskArchiver.ReqMgrServiceURL})
# Load the cleanout state ID and save it
stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
self.stateID = stateIDDAO.execute("cleanout")
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.changeState = ChangeState(self.config)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.loadAction = self.daoFactory(
classname="Jobs.LoadFromIDWithWorkflow")
# Variables
self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
"numberOfJobsToCluster", 1000)
self.numberOfJobsToArchive = getattr(self.config.JobArchiver,
"numberOfJobsToArchive", 10000)
# initialize the alert framework (if available)
self.initAlerts(compName="JobArchiver")
try:
self.logDir = getattr(
config.JobArchiver, 'logDir',
os.path.join(config.JobArchiver.componentDir, 'logDir'))
if not os.path.isdir(self.logDir):
os.makedirs(self.logDir)
except Exception as ex:
msg = "Unhandled exception while setting up logDir!\n"
msg += str(ex)
logging.exception(msg)
raise JobArchiverPollerException(msg)
self.tier0Mode = hasattr(config, "Tier0Feeder")
try:
if not self.tier0Mode:
self.workQueue = queueFromConfig(self.config)
except Exception as ex:
msg = "Could not load workQueue"
msg += str(ex)
logging.error(msg)
# raise JobArchiverPollerException(msg)
return
def getAcquiredAvailableWork():
"""
Get jobs from already split elements that are not yet
in WMBS (i.e. no jobs created)
"""
workStatistics = {}
queueConfig = queueConfigFromConfigObject(workqueueConfig(couchdb = externalCouchDb))
workqueue = queueFromConfig(queueConfig)
acquiredAvailableElements = workqueue.backend.getElements(status = 'Acquired')
acquiredAvailableElements.extend(workqueue.backend.getElements(status = 'Available'))
for element in acquiredAvailableElements:
workflow = element['ParentQueueId']
if workflow not in workStatistics:
workStatistics[workflow] = 0
workStatistics[workflow] += element['Jobs']
return workStatistics
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.changeState = ChangeState(self.config)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.loadAction = self.daoFactory(classname="Jobs.LoadFromIDWithWorkflow")
# Variables
self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
"numberOfJobsToCluster", 1000)
self.numberOfJobsToArchive = getattr(self.config.JobArchiver,
"numberOfJobsToArchive", 10000)
try:
self.logDir = getattr(config.JobArchiver, 'logDir',
os.path.join(config.JobArchiver.componentDir, 'logDir'))
if not os.path.isdir(self.logDir):
os.makedirs(self.logDir)
except Exception as ex:
msg = "Unhandled exception while setting up logDir!\n"
msg += str(ex)
logging.exception(msg)
raise JobArchiverPollerException(msg)
self.tier0Mode = hasattr(config, "Tier0Feeder")
try:
if not self.tier0Mode:
self.workQueue = queueFromConfig(self.config)
except Exception as ex:
msg = "Could not load workQueue"
msg += str(ex)
logging.error(msg)
# raise JobArchiverPollerException(msg)
return
def getAssignedApprovedWork():
"""
Split the un-split. Use a local couch for it.
"""
workStatistics = {}
wmstatsReader = WMStatsReader(wmstatsEndpoint)
unAssignedRequests = wmstatsReader.workflowsByStatus(['assignment-approved'], stale = False)
queueConfig = queueConfigFromConfigObject(workqueueConfig())
workqueue = queueFromConfig(queueConfig)
for requestName in unAssignedRequests:
if 'TEST' in requestName:
continue
workqueue.queueWork('%s/reqmgr_workload_cache/%s/spec' % (externalCouchDb, requestName), requestName, 'notreallyateam')
for requestName in unAssignedRequests:
workStatistics[requestName] = 0
workElements = workqueue.backend.getElementsForWorkflow(requestName)
for element in workElements:
jobs = element['Jobs']
workStatistics[requestName] += jobs
return workStatistics
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(
**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.maxProcessSize = getattr(self.config.TaskArchiver,
'maxProcessSize', 250)
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.uploadPublishInfo = getattr(self.config.TaskArchiver,
'uploadPublishInfo', False)
self.uploadPublishDir = getattr(self.config.TaskArchiver,
'uploadPublishDir', None)
self.userFileCacheURL = getattr(self.config.TaskArchiver,
'userFileCacheURL', None)
# Set up optional histograms
self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys",
[])
self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins",
10)
self.histogramLimit = getattr(self.config.TaskArchiver,
"histogramLimit", 5.0)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.wmstatsCouchDB = WMStatsWriter(
self.config.TaskArchiver.localWMStatsURL)
self.centralCouchDBWriter = self.wmstatsCouchDB
else:
self.centralCouchDBWriter = WMStatsWriter(
self.config.TaskArchiver.centralWMStatsURL)
# Start a couch server for getting job info
# from the FWJRs for committal to archive
try:
workDBName = getattr(self.config.TaskArchiver,
'workloadSummaryCouchDBName',
'workloadsummary')
workDBurl = getattr(self.config.TaskArchiver,
'workloadSummaryCouchURL')
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.workCouchdb = CouchServer(workDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" %
jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" %
jobDBName)
self.workdatabase = self.workCouchdb.connectDatabase(workDBName)
logging.debug("Using url %s/%s for job" % (jobDBurl, jobDBName))
logging.debug("Writing to %s/%s for workloadSummary" %
(sanitizeURL(workDBurl)['url'], workDBName))
self.requireCouch = getattr(self.config.TaskArchiver,
'requireCouch', False)
except Exception, ex:
msg = "Error in connecting to couch.\n"
msg += str(ex)
logging.error(msg)
self.jobsdatabase = None
self.fwjrdatabase = None
if getattr(self.config.TaskArchiver, 'requireCouch', False):
raise TaskArchiverPollerException(msg)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.maxProcessSize = getattr(self.config.TaskArchiver, 'maxProcessSize', 250)
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)
self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.uploadPublishInfo = getattr(self.config.TaskArchiver, 'uploadPublishInfo', False)
self.uploadPublishDir = getattr(self.config.TaskArchiver, 'uploadPublishDir', None)
self.userFileCacheURL = getattr(self.config.TaskArchiver, 'userFileCacheURL', None)
# Set up optional histograms
self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", [])
self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10)
self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL);
# Start a couch server for getting job info
# from the FWJRs for committal to archive
try:
workDBName = getattr(self.config.TaskArchiver, 'workloadSummaryCouchDBName',
'workloadsummary')
workDBurl = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL')
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.workCouchdb = CouchServer(workDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
self.workdatabase = self.workCouchdb.connectDatabase(workDBName)
logging.debug("Using url %s/%s for job" % (jobDBurl, jobDBName))
logging.debug("Writing to %s/%s for workloadSummary" % (sanitizeURL(workDBurl)['url'], workDBName))
self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False)
except Exception, ex:
msg = "Error in connecting to couch.\n"
msg += str(ex)
logging.error(msg)
self.jobsdatabase = None
self.fwjrdatabase = None
if getattr(self.config.TaskArchiver, 'requireCouch', False):
raise TaskArchiverPollerException(msg)
