def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.config = config
self.condorAPI = PyCondorAPI()
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI()
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.validSpeedDrainConfigKeys = [
'CondorPriority', 'NoJobRetries', 'EnableAllSites'
]
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.queue = queue
self.config = config
self.condorAPI = PyCondorAPI()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.listSubsWithoutJobs = self.daoFactory(
classname="Subscriptions.GetSubsWithoutJobGroup")
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI(config)
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.previousConfig = {}
self.validSpeedDrainConfigKeys = [
'CondorPriority', 'NoJobRetries', 'EnableAllSites'
]
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
self.emailAlert = EmailAlert(config.EmailAlert.dictionary_())
self.condorStates = ("Running", "Idle")
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self):
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
queries = [["1", "idle"], ["2", "running"]]
for query in queries:
jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], [])
# if there is an error, report it instead of the length of an empty list
if jobs is None:
results[query[1]] = "unknown (schedd query error)"
else:
results[query[1]] = len(jobs)
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.config = config
self.condorAPI = PyCondorAPI()
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI(config)
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.validSpeedDrainConfigKeys = ['CondorPriority', 'NoJobRetries', 'EnableAllSites']
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.queue = queue
self.config = config
self.condorAPI = PyCondorAPI()
self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi)
self.listSubsWithoutJobs = self.daoFactory(classname="Subscriptions.GetSubsWithoutJobGroup")
def __init__(self):
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
class WorkQueueManagerWorkPoller(BaseWorkerThread):
"""
Polls for Work
"""
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.queue = queue
self.config = config
self.condorAPI = PyCondorAPI()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.listSubsWithoutJobs = self.daoFactory(
classname="Subscriptions.GetSubsWithoutJobGroup")
def setup(self, parameters):
"""
Called at startup - introduce random delay
to avoid workers all starting at once
"""
t = random.randrange(self.idleTime)
self.logger.info('Sleeping for %d seconds before 1st loop', t)
time.sleep(t)
@timeFunction
def algorithm(self, parameters):
"""
Pull in work
"""
self.logger.info("Starting WorkQueueManagerWorkPoller thread ...")
try:
self.pullWork()
except Exception as ex:
self.queue.logger.error("Error in work pull loop: %s", str(ex))
try:
# process if we get work or not - we may have to split old work
# i.e. if transient errors were seen during splitting
self.processWork()
except Exception as ex:
self.queue.logger.error("Error in new work split loop: %s",
str(ex))
return
def passRetrieveCondition(self):
"""
_passRetrieveCondition_
Return true if the component can proceed with fetching work.
False if the component should skip pulling work this cycle.
For now, it only checks whether the agent is in drain mode or
MAX_JOBS_PER_OWNER is reached or if the condor schedd is overloaded.
"""
passCond = "OK"
myThread = threading.currentThread()
if isDrainMode(self.config):
passCond = "agent is in drain mode"
elif availableScheddSlots(myThread.dbi) <= 0:
passCond = "schedd slot is maxed: MAX_JOBS_PER_OWNER"
elif self.condorAPI.isScheddOverloaded():
passCond = "schedd is overloaded"
else:
subscriptions = self.listSubsWithoutJobs.execute()
if subscriptions:
passCond = "JobCreator hasn't created jobs for subscriptions %s" % subscriptions
return passCond
def pullWork(self):
"""Get work from parent"""
self.queue.logger.info("Pulling work from %s",
self.queue.parent_queue.queueUrl)
myThread = threading.currentThread()
try:
cond = self.passRetrieveCondition()
if cond == "OK":
work = self.queue.pullWork()
self.queue.logger.info("Obtained %s unit(s) of work", work)
myThread.logdbClient.delete("LocalWorkQueue_pullWork",
"warning",
this_thread=True)
else:
self.queue.logger.warning("No work will be pulled, reason: %s",
cond)
myThread.logdbClient.post("LocalWorkQueue_pullWork", cond,
"warning")
except IOError as ex:
self.queue.logger.exception(
"Error opening connection to work queue: %s", str(ex))
except Exception as ex:
self.queue.logger.exception(
"Unable to pull work from parent Error: %s", str(ex))
def processWork(self):
"""Process new work"""
self.queue.logger.info("Splitting new work")
try:
self.queue.processInboundWork()
except Exception as ex:
self.queue.logger.exception('Error during split: %s', str(ex))
self.logger.info('Splitting finished')
return
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
results['local_wq_status'] = self.checkLocalWQStatus(dbname="workqueue")
results['local_wqinbox_status'] = self.checkLocalWQStatus(dbname="workqueue_inbox")
results['global_wq_status'] = self.checkGlobalWQStatus()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
queries = [["1", "idle"], ["2", "running"]]
for query in queries:
jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], [])
# if there is an error, report it instead of the length of an empty list
if jobs is None:
results[query[1]] = "unknown (schedd query error)"
else:
results[query[1]] = len(jobs)
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def checkLocalWQStatus(self, dbname):
"""
Query local WorkQueue workqueue/workqueue_inbox database to see whether
there are any active elements in this agent.
"""
results = {}
for st in ('Available', 'Negotiating', 'Acquired', 'Running'):
if dbname == "workqueue":
elements = self.localBackend.getElements(status=st, returnIdOnly=True)
else:
elements = self.localBackend.getInboxElements(status=st, returnIdOnly=True)
results[st] = len(elements)
return results
def checkGlobalWQStatus(self):
"""
Query Global WorkQueue workqueue database to see whether there are
any active elements set to this agent.
"""
results = {}
for st in ("Acquired", "Running"):
elements = self.globalBackend.getElements(status=st, returnIdOnly=True,
ChildQueueUrl=self.thisAgentUrl)
results[st] = len(elements)
return results
class WorkQueueManagerWorkPoller(BaseWorkerThread):
"""
Polls for Work
"""
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.config = config
self.condorAPI = PyCondorAPI()
def setup(self, parameters):
"""
Called at startup - introduce random delay
to avoid workers all starting at once
"""
t = random.randrange(self.idleTime)
self.logger.info('Sleeping for %d seconds before 1st loop' % t)
time.sleep(t)
def algorithm(self, parameters):
"""
Pull in work
"""
try:
self.pullWork()
except Exception as ex:
self.queue.logger.error("Error in work pull loop: %s" % str(ex))
try:
# process if we get work or not - we may have to split old work
# i.e. if transient errors were seen during splitting
self.processWork()
except Exception as ex:
self.queue.logger.error("Error in new work split loop: %s" %
str(ex))
return
def passRetrieveCondition(self):
"""
_passRetrieveCondition_
Return true if the component can proceed with fetching work.
False if the component should skip pulling work this cycle.
For now, it only checks whether the agent is in drain mode or
if the condor schedd is overloaded.
"""
passCond = True
if isDrainMode(self.config):
passCond = False
elif self.condorAPI.isScheddOverloaded():
passCond = False
return passCond
def pullWork(self):
"""Get work from parent"""
self.queue.logger.info("Pulling work from %s" %
self.queue.parent_queue.queueUrl)
work = 0
myThread = threading.currentThread()
try:
if self.passRetrieveCondition():
work = self.queue.pullWork()
myThread.logdbClient.delete("LocalWorkQueue_pullWork",
"warning",
this_thread=True)
else:
msg = "Workqueue didn't pass the retrieve condition: NOT pulling work"
self.queue.logger.warning(msg)
myThread.logdbClient.post("LocalWorkQueue_pullWork", msg,
"warning")
except IOError as ex:
self.queue.logger.error(
"Error opening connection to work queue: %s \n%s" %
(str(ex), traceback.format_exc()))
except Exception as ex:
self.queue.logger.error(
"Unable to pull work from parent Error: %s\n%s" %
(str(ex), traceback.format_exc()))
self.queue.logger.info("Obtained %s unit(s) of work" % work)
return work
def processWork(self):
"""Process new work"""
self.queue.logger.info("Splitting new work")
try:
self.queue.processInboundWork()
except Exception as ex:
self.queue.logger.exception('Error during split: %s' % str(ex))
self.logger.info('Splitting finished')
return
class WorkQueueManagerWorkPoller(BaseWorkerThread):
"""
Polls for Work
"""
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.queue = queue
self.config = config
self.condorAPI = PyCondorAPI()
self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi)
self.listSubsWithoutJobs = self.daoFactory(classname="Subscriptions.GetSubsWithoutJobGroup")
def setup(self, parameters):
"""
Called at startup - introduce random delay
to avoid workers all starting at once
"""
t = random.randrange(self.idleTime)
self.logger.info('Sleeping for %d seconds before 1st loop' % t)
time.sleep(t)
@timeFunction
def algorithm(self, parameters):
"""
Pull in work
"""
try:
self.pullWork()
except Exception as ex:
self.queue.logger.error("Error in work pull loop: %s" % str(ex))
try:
# process if we get work or not - we may have to split old work
# i.e. if transient errors were seen during splitting
self.processWork()
except Exception as ex:
self.queue.logger.error("Error in new work split loop: %s" % str(ex))
return
def passRetrieveCondition(self):
"""
_passRetrieveCondition_
Return true if the component can proceed with fetching work.
False if the component should skip pulling work this cycle.
For now, it only checks whether the agent is in drain mode or
MAX_JOBS_PER_OWNER is reached or if the condor schedd is overloaded.
"""
passCond = "OK"
myThread = threading.currentThread()
if isDrainMode(self.config):
passCond = "No work will be pulled: Agent is in drain"
elif availableScheddSlots(myThread.dbi) <= 0:
passCond = "No work will be pulled: schedd slot is maxed: MAX_JOBS_PER_OWNER"
elif self.condorAPI.isScheddOverloaded():
passCond = "No work will be pulled: schedd is overloaded"
else:
subscriptions = self.listSubsWithoutJobs.execute()
if subscriptions:
passCond = "No work will be pulled: "
passCond += "JobCreator hasn't created jobs for subscriptions %s" % subscriptions
return passCond
def pullWork(self):
"""Get work from parent"""
self.queue.logger.info("Pulling work from %s" % self.queue.parent_queue.queueUrl)
work = 0
myThread = threading.currentThread()
try:
cond = self.passRetrieveCondition()
if cond == "OK":
work = self.queue.pullWork()
myThread.logdbClient.delete("LocalWorkQueue_pullWork", "warning", this_thread=True)
else:
self.queue.logger.warning(cond)
myThread.logdbClient.post("LocalWorkQueue_pullWork", cond, "warning")
except IOError as ex:
self.queue.logger.error("Error opening connection to work queue: %s \n%s" %
(str(ex), traceback.format_exc()))
except Exception as ex:
self.queue.logger.error("Unable to pull work from parent Error: %s\n%s"
% (str(ex), traceback.format_exc()))
self.queue.logger.info("Obtained %s unit(s) of work" % work)
return work
def processWork(self):
"""Process new work"""
self.queue.logger.info("Splitting new work")
try:
self.queue.processInboundWork()
except Exception as ex:
self.queue.logger.exception('Error during split: %s' % str(ex))
self.logger.info('Splitting finished')
return
class DrainStatusPoller(BaseWorkerThread):
"""
Collects information related to the agent drain status
"""
# class variable that contains drain statistics
drainStats = {}
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI()
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.validSpeedDrainConfigKeys = [
'CondorPriority', 'NoJobRetries', 'EnableAllSites'
]
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
@timeFunction
def algorithm(self, parameters):
"""
Update drainStats if agent is in drain mode
"""
logging.info("Running agent drain algorithm...")
self.agentConfig = self.reqAuxDB.getWMAgentConfig(
self.config.Agent.hostName)
if isDrainMode(self.config):
# check to see if the agent hit any speed drain thresholds
thresholdsHit = self.checkSpeedDrainThresholds()
if thresholdsHit:
logging.info("Updating agent configuration for speed drain...")
self.updateAgentSpeedDrainConfig(thresholdsHit)
try:
DrainStatusPoller.drainStats = self.drainAPI.collectDrainInfo()
logging.info("Finished collecting agent drain status.")
logging.info("Drain stats: " +
str(DrainStatusPoller.drainStats))
except Exception as ex:
msg = "Error occurred, will retry later:\n"
msg += str(ex)
logging.exception(msg)
else:
logging.info(
"Agent not in drain mode. Resetting flags and skipping drain check..."
)
self.resetAgentSpeedDrainConfig()
@classmethod
def getDrainInfo(cls):
"""
Return drainStats class variable
"""
return cls.drainStats
def updateAgentSpeedDrainConfig(self, thresholdsHit):
"""
Takes a list of speed drain configuration keys and updates the agent configuration
"""
updateConfig = False
condorPriorityFlag = False
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
if 'CondorPriority' in thresholdsHit:
logging.info(
"Bumping condor job priority to 999999 for Production/Processing pending jobs."
)
self.condorAPI.editCondorJobs(
"JobStatus=?=1 && (CMS_JobType =?= \"Production\" || CMS_JobType =?= \"Processing\")",
"JobPrio", "999999")
condorPriorityFlag = True
if condorPriorityFlag != speedDrainConfig['CondorPriority']['Enabled']:
# CondorPriority setting is irreversible so the flag only indicates weather
# priority is increased or not. It is not checked by other components
logging.info("Enabling CondorPriority flag.")
speedDrainConfig['CondorPriority']['Enabled'] = condorPriorityFlag
updateConfig = True
if 'NoJobRetries' in thresholdsHit:
logging.info(
"Enabling NoJobRetries flag: Error Handler won't retry the jobs"
)
# ErrorHandler will pick this up and set max retries to 0
speedDrainConfig['NoJobRetries']['Enabled'] = True
updateConfig = True
if 'EnableAllSites' in thresholdsHit:
logging.info(
"Enabling EnableAllSites flag: Updating agent to submit to all sites."
)
# setting this value to True makes JobSubmitterPoller ignore site status
speedDrainConfig['EnableAllSites']['Enabled'] = True
updateConfig = True
# update the aux db speed drain config with any changes
if updateConfig:
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainMode", True)
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainConfig",
speedDrainConfig)
return
def resetAgentSpeedDrainConfig(self):
"""
resetting SpeedDrainMode to False and SpeedDrainiConfig Enabled to False
"""
if self.agentConfig.get("SpeedDrainMode"):
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainMode", False)
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
for key, v in speedDrainConfig.items():
if key in self.validSpeedDrainConfigKeys and v['Enabled']:
speedDrainConfig[key]['Enabled'] = False
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainConfig",
speedDrainConfig)
return
def checkSpeedDrainThresholds(self):
"""
Check the current number of jobs in Condor and create a list of agent configuration parameters
that need updated for speed draining
"""
enableKeys = []
# get the current speed drain status
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
# get condor jobs
jobs = self.condorAPI.getCondorJobs("", [])
if jobs is None:
logging.warning(
"There was an error querying the schedd. Not checking speed drain thresholds."
)
return []
# loop through the speed drain configuration and make a list of what thresholds have been hit
for k, v in speedDrainConfig.items():
# make sure keys in the speed drain config are valid
if k in self.validSpeedDrainConfigKeys and isinstance(
v['Threshold'], int) and isinstance(v['Enabled'], bool):
# we always want to apply the condor priority change if the threshold is hit
if not v['Enabled'] or k == 'CondorPriority':
logging.info("Checking speed drain threshold for %s. ", k)
if len(jobs) < v['Threshold']:
logging.info(
"Agent will update speed drain configuration for %s. ",
k)
enableKeys.append(k)
else:
logging.warning(
"Speed drain configuration error for %s. Please check aux db contents.",
k)
return enableKeys
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
self.condorStates = ("Running", "Idle")
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
results['local_wq_status'] = self.checkLocalWQStatus(
dbname="workqueue")
results['local_wqinbox_status'] = self.checkLocalWQStatus(
dbname="workqueue_inbox")
results['global_wq_status'] = self.checkGlobalWQStatus()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
jobs = self.condorAPI.getCondorJobsSummary()
for state in self.condorStates:
# if there is an error, report it instead of the length of an empty list
if not jobs:
results[state.lower()] = None
else:
results[state.lower()] = int(jobs[0].get(state))
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(
status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def checkLocalWQStatus(self, dbname):
"""
Query local WorkQueue workqueue/workqueue_inbox database to see whether
there are any active elements in this agent.
"""
results = {}
for st in ('Available', 'Negotiating', 'Acquired', 'Running'):
if dbname == "workqueue":
elements = self.localBackend.getElements(status=st,
returnIdOnly=True)
else:
elements = self.localBackend.getInboxElements(
status=st, returnIdOnly=True)
results[st] = len(elements)
return results
def checkGlobalWQStatus(self):
"""
Query Global WorkQueue workqueue database to see whether there are
any active elements set to this agent.
"""
results = {}
for st in ("Acquired", "Running"):
elements = self.globalBackend.getElements(
status=st, returnIdOnly=True, ChildQueueUrl=self.thisAgentUrl)
results[st] = len(elements)
return results
class DrainStatusPoller(BaseWorkerThread):
"""
Collects information related to the agent drain status
"""
# class variable that contains drain statistics
drainStats = {}
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI(config)
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.validSpeedDrainConfigKeys = ['CondorPriority', 'NoJobRetries', 'EnableAllSites']
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
@timeFunction
def algorithm(self, parameters):
"""
Update drainStats if agent is in drain mode
"""
logging.info("Running agent drain algorithm...")
self.agentConfig = self.reqAuxDB.getWMAgentConfig(self.config.Agent.hostName)
if not self.agentConfig:
logging.error("Failed to fetch agent configuration from the auxiliary DB")
return
if isDrainMode(self.config):
# check to see if the agent hit any speed drain thresholds
thresholdsHit = self.checkSpeedDrainThresholds()
if thresholdsHit:
logging.info("Updating agent configuration for speed drain...")
self.updateAgentSpeedDrainConfig(thresholdsHit)
# now collect drain statistics
try:
DrainStatusPoller.drainStats = self.drainAPI.collectDrainInfo()
logging.info("Finished collecting agent drain status.")
logging.info("Drain stats: " + str(DrainStatusPoller.drainStats))
except Exception as ex:
msg = "Error occurred, will retry later:\n"
msg += str(ex)
logging.exception(msg)
else:
logging.info("Agent not in drain mode. Resetting flags and skipping drain check...")
self.resetAgentSpeedDrainConfig()
@classmethod
def getDrainInfo(cls):
"""
Return drainStats class variable
"""
return cls.drainStats
def updateAgentSpeedDrainConfig(self, thresholdsHit):
"""
Takes a list of speed drain configuration keys and updates the agent configuration
"""
updateConfig = False
condorPriorityFlag = False
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
if 'CondorPriority' in thresholdsHit:
logging.info("Bumping condor job priority to 999999 for Production/Processing pending jobs.")
self.condorAPI.editCondorJobs(
"JobStatus=?=1 && (CMS_JobType =?= \"Production\" || CMS_JobType =?= \"Processing\")",
"JobPrio", "999999")
condorPriorityFlag = True
if condorPriorityFlag != speedDrainConfig['CondorPriority']['Enabled']:
# CondorPriority setting is irreversible so the flag only indicates weather
# priority is increased or not. It is not checked by other components
logging.info("Enabling CondorPriority flag.")
speedDrainConfig['CondorPriority']['Enabled'] = condorPriorityFlag
updateConfig = True
if 'NoJobRetries' in thresholdsHit:
logging.info("Enabling NoJobRetries flag: Error Handler won't retry the jobs")
# ErrorHandler will pick this up and set max retries to 0
speedDrainConfig['NoJobRetries']['Enabled'] = True
updateConfig = True
if 'EnableAllSites' in thresholdsHit:
logging.info("Enabling EnableAllSites flag: Updating agent to submit to all sites.")
# setting this value to True makes JobSubmitterPoller ignore site status
speedDrainConfig['EnableAllSites']['Enabled'] = True
updateConfig = True
# update the aux db speed drain config with any changes
if updateConfig:
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainMode", True)
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainConfig", speedDrainConfig)
return
def resetAgentSpeedDrainConfig(self):
"""
resetting SpeedDrainMode to False and SpeedDrainiConfig Enabled to False
"""
if self.agentConfig.get("SpeedDrainMode"):
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainMode", False)
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
for key, v in speedDrainConfig.items():
if key in self.validSpeedDrainConfigKeys and v['Enabled']:
speedDrainConfig[key]['Enabled'] = False
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainConfig", speedDrainConfig)
return
def checkSpeedDrainThresholds(self):
"""
Check the current number of jobs in Condor and create a list of agent configuration parameters
that need updated for speed draining
"""
enableKeys = []
# get the current speed drain status
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
# get condor jobs
jobs = self.condorAPI.getCondorJobs("", [])
if jobs is None:
logging.warning("There was an error querying the schedd. Not checking speed drain thresholds.")
return []
# loop through the speed drain configuration and make a list of what thresholds have been hit
for k, v in speedDrainConfig.items():
# make sure keys in the speed drain config are valid
if k in self.validSpeedDrainConfigKeys and isinstance(v['Threshold'], int) and isinstance(v['Enabled'], bool):
# we always want to apply the condor priority change if the threshold is hit
if not v['Enabled'] or k == 'CondorPriority':
logging.info("Checking speed drain threshold for %s. ", k)
if len(jobs) < v['Threshold']:
logging.info("Agent will update speed drain configuration for %s. ", k)
enableKeys.append(k)
else:
logging.warning("Speed drain configuration error for %s. Please check aux db contents.", k)
return enableKeys
class DrainStatusPoller(BaseWorkerThread):
"""
Collects information related to the agent drain status
"""
# class variable that contains drain statistics
drainStats = {}
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI(config)
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.previousConfig = {}
self.validSpeedDrainConfigKeys = [
'CondorPriority', 'NoJobRetries', 'EnableAllSites'
]
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
self.emailAlert = EmailAlert(config.EmailAlert.dictionary_())
self.condorStates = ("Running", "Idle")
@timeFunction
def algorithm(self, parameters):
"""
Update drainStats if agent is in drain mode
"""
logging.info("Running agent drain algorithm...")
if self.agentConfig:
# make a copy of the previous agent aux db configuration to compare against later
self.previousConfig = copy.deepcopy(self.agentConfig)
# grab a new copy of the agent aux db configuration
self.agentConfig = self.reqAuxDB.getWMAgentConfig(
self.config.Agent.hostName)
if not self.agentConfig:
logging.error(
"Failed to fetch agent configuration from the auxiliary DB")
return
try:
# see if the agent is in drain mode
if self.agentConfig["UserDrainMode"] or self.agentConfig[
"AgentDrainMode"]:
# check to see if the agent hit any speed drain thresholds
thresholdsHit = self.checkSpeedDrainThresholds()
if thresholdsHit:
logging.info(
"Updating agent configuration for speed drain...")
self.updateAgentSpeedDrainConfig(thresholdsHit)
# now collect drain statistics
DrainStatusPoller.drainStats = self.drainAPI.collectDrainInfo()
logging.info("Finished collecting agent drain status.")
logging.info("Drain stats: %s",
str(DrainStatusPoller.drainStats))
else:
logging.info(
"Agent not in drain mode. Resetting flags and skipping drain check..."
)
self.resetAgentSpeedDrainConfig()
# finally, check for any changes in drain status
self.checkDrainStatusChanges()
except Exception as ex:
msg = "Error occurred, will retry later:\n"
msg += str(ex)
logging.exception(msg)
@classmethod
def getDrainInfo(cls):
"""
Return drainStats class variable
"""
return cls.drainStats
def checkDrainStatusChanges(self):
"""
Check to see if any drain statuses have changed in the auxiliary db
If yes, send email notification and update local drain thread variables
"""
message = ""
drainStatusKeys = ['UserDrainMode', 'AgentDrainMode', 'SpeedDrainMode']
if not self.previousConfig:
return
for key in drainStatusKeys:
if self.previousConfig[key] != self.agentConfig[key]:
message += "Agent had a drain status transition to %s = %s\n" % (
str(key), str(self.agentConfig[key]))
if message:
self.emailAlert.send(
"DrainMode status change on " +
getattr(self.config.Agent, "hostName"), message)
logging.info("Drain mode status change: %s", message)
return
def updateAgentSpeedDrainConfig(self, thresholdsHit):
"""
Takes a list of speed drain configuration keys and updates the agent configuration
"""
updateConfig = False
condorPriorityFlag = False
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
if 'CondorPriority' in thresholdsHit:
logging.info(
"Bumping condor job priority to 999999 for Production/Processing pending jobs."
)
self.condorAPI.editCondorJobs(
"JobStatus=?=1 && (CMS_JobType =?= \"Production\" || CMS_JobType =?= \"Processing\")",
"JobPrio", "999999")
condorPriorityFlag = True
if condorPriorityFlag != speedDrainConfig['CondorPriority']['Enabled']:
# CondorPriority setting is irreversible so the flag only indicates weather
# priority is increased or not. It is not checked by other components
logging.info("Enabling CondorPriority flag.")
speedDrainConfig['CondorPriority']['Enabled'] = condorPriorityFlag
updateConfig = True
if 'NoJobRetries' in thresholdsHit:
logging.info(
"Enabling NoJobRetries flag: Error Handler won't retry the jobs"
)
# ErrorHandler will pick this up and set max retries to 0
speedDrainConfig['NoJobRetries']['Enabled'] = True
updateConfig = True
if 'EnableAllSites' in thresholdsHit:
logging.info(
"Enabling EnableAllSites flag: Updating agent to submit to all sites."
)
# setting this value to True makes JobSubmitterPoller ignore site status
speedDrainConfig['EnableAllSites']['Enabled'] = True
updateConfig = True
# update the aux db speed drain config with any changes
if updateConfig:
self.agentConfig['SpeedDrainMode'] = True
self.reqAuxDB.updateWMAgentConfig(self.config.Agent.hostName,
self.agentConfig)
return
def resetAgentSpeedDrainConfig(self):
"""
resetting SpeedDrainMode to False and SpeedDrainConfig Enabled to False
"""
if self.agentConfig.get("SpeedDrainMode"):
self.agentConfig['SpeedDrainMode'] = False
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
for key, v in viewitems(speedDrainConfig):
if key in self.validSpeedDrainConfigKeys and v['Enabled']:
speedDrainConfig[key]['Enabled'] = False
self.reqAuxDB.updateWMAgentConfig(self.config.Agent.hostName,
self.agentConfig)
return
def checkSpeedDrainThresholds(self):
"""
Check the current number of jobs in Condor and create a list of agent configuration parameters
that need updated for speed draining
"""
enableKeys = []
# first, update our summary of condor jobs
totalJobs = self.getTotalCondorJobs()
if totalJobs is None:
msg = "Cannot check speed drain because there was an error fetching job summary from HTCondor."
msg += " Will retry again in the next cycle."
logging.warning(msg)
return []
# get the current speed drain status
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
# loop through the speed drain configuration and make a list of what thresholds have been hit
for k, v in viewitems(speedDrainConfig):
# make sure keys in the speed drain config are valid
if k in self.validSpeedDrainConfigKeys and isinstance(
v['Threshold'], int) and isinstance(v['Enabled'], bool):
# we always want to apply the condor priority change if the threshold is hit
if not v['Enabled'] or k == 'CondorPriority':
logging.info("Checking speed drain threshold for %s. ", k)
if totalJobs < v['Threshold']:
logging.info(
"Agent will update speed drain configuration for %s. ",
k)
enableKeys.append(k)
else:
logging.warning(
"Speed drain configuration error for %s. Please check aux db contents.",
k)
return enableKeys
def getTotalCondorJobs(self):
"""
Retrieve a summary of the jobs in condor and return an absolute number
of the jobs in Idle and Running states.
:return: returns an integer with the total number of jobs, or None if it failed.
"""
jobs = self.condorAPI.getCondorJobsSummary()
if not jobs:
return None
results = 0
if jobs:
for state in self.condorStates:
results += int(jobs[0].get(state))
return results
