class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self):
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
queries = [["1", "idle"], ["2", "running"]]
for query in queries:
jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], [])
# if there is an error, report it instead of the length of an empty list
if jobs is None:
results[query[1]] = "unknown (schedd query error)"
else:
results[query[1]] = len(jobs)
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
class DrainStatusPoller(BaseWorkerThread):
"""
Collects information related to the agent drain status
"""
# class variable that contains drain statistics
drainStats = {}
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI()
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.validSpeedDrainConfigKeys = [
'CondorPriority', 'NoJobRetries', 'EnableAllSites'
]
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
@timeFunction
def algorithm(self, parameters):
"""
Update drainStats if agent is in drain mode
"""
logging.info("Running agent drain algorithm...")
self.agentConfig = self.reqAuxDB.getWMAgentConfig(
self.config.Agent.hostName)
if isDrainMode(self.config):
# check to see if the agent hit any speed drain thresholds
thresholdsHit = self.checkSpeedDrainThresholds()
if thresholdsHit:
logging.info("Updating agent configuration for speed drain...")
self.updateAgentSpeedDrainConfig(thresholdsHit)
try:
DrainStatusPoller.drainStats = self.drainAPI.collectDrainInfo()
logging.info("Finished collecting agent drain status.")
logging.info("Drain stats: " +
str(DrainStatusPoller.drainStats))
except Exception as ex:
msg = "Error occurred, will retry later:\n"
msg += str(ex)
logging.exception(msg)
else:
logging.info(
"Agent not in drain mode. Resetting flags and skipping drain check..."
)
self.resetAgentSpeedDrainConfig()
@classmethod
def getDrainInfo(cls):
"""
Return drainStats class variable
"""
return cls.drainStats
def updateAgentSpeedDrainConfig(self, thresholdsHit):
"""
Takes a list of speed drain configuration keys and updates the agent configuration
"""
updateConfig = False
condorPriorityFlag = False
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
if 'CondorPriority' in thresholdsHit:
logging.info(
"Bumping condor job priority to 999999 for Production/Processing pending jobs."
)
self.condorAPI.editCondorJobs(
"JobStatus=?=1 && (CMS_JobType =?= \"Production\" || CMS_JobType =?= \"Processing\")",
"JobPrio", "999999")
condorPriorityFlag = True
if condorPriorityFlag != speedDrainConfig['CondorPriority']['Enabled']:
# CondorPriority setting is irreversible so the flag only indicates weather
# priority is increased or not. It is not checked by other components
logging.info("Enabling CondorPriority flag.")
speedDrainConfig['CondorPriority']['Enabled'] = condorPriorityFlag
updateConfig = True
if 'NoJobRetries' in thresholdsHit:
logging.info(
"Enabling NoJobRetries flag: Error Handler won't retry the jobs"
)
# ErrorHandler will pick this up and set max retries to 0
speedDrainConfig['NoJobRetries']['Enabled'] = True
updateConfig = True
if 'EnableAllSites' in thresholdsHit:
logging.info(
"Enabling EnableAllSites flag: Updating agent to submit to all sites."
)
# setting this value to True makes JobSubmitterPoller ignore site status
speedDrainConfig['EnableAllSites']['Enabled'] = True
updateConfig = True
# update the aux db speed drain config with any changes
if updateConfig:
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainMode", True)
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainConfig",
speedDrainConfig)
return
def resetAgentSpeedDrainConfig(self):
"""
resetting SpeedDrainMode to False and SpeedDrainiConfig Enabled to False
"""
if self.agentConfig.get("SpeedDrainMode"):
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainMode", False)
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
for key, v in speedDrainConfig.items():
if key in self.validSpeedDrainConfigKeys and v['Enabled']:
speedDrainConfig[key]['Enabled'] = False
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName,
"SpeedDrainConfig",
speedDrainConfig)
return
def checkSpeedDrainThresholds(self):
"""
Check the current number of jobs in Condor and create a list of agent configuration parameters
that need updated for speed draining
"""
enableKeys = []
# get the current speed drain status
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
# get condor jobs
jobs = self.condorAPI.getCondorJobs("", [])
if jobs is None:
logging.warning(
"There was an error querying the schedd. Not checking speed drain thresholds."
)
return []
# loop through the speed drain configuration and make a list of what thresholds have been hit
for k, v in speedDrainConfig.items():
# make sure keys in the speed drain config are valid
if k in self.validSpeedDrainConfigKeys and isinstance(
v['Threshold'], int) and isinstance(v['Enabled'], bool):
# we always want to apply the condor priority change if the threshold is hit
if not v['Enabled'] or k == 'CondorPriority':
logging.info("Checking speed drain threshold for %s. ", k)
if len(jobs) < v['Threshold']:
logging.info(
"Agent will update speed drain configuration for %s. ",
k)
enableKeys.append(k)
else:
logging.warning(
"Speed drain configuration error for %s. Please check aux db contents.",
k)
return enableKeys
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
results['local_wq_status'] = self.checkLocalWQStatus(
dbname="workqueue")
results['local_wqinbox_status'] = self.checkLocalWQStatus(
dbname="workqueue_inbox")
results['global_wq_status'] = self.checkGlobalWQStatus()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
queries = [["1", "idle"], ["2", "running"]]
for query in queries:
jobs = self.condorAPI.getCondorJobs("JobStatus==" + query[0], [])
# if there is an error, report it instead of the length of an empty list
if jobs is None:
results[query[1]] = "unknown (schedd query error)"
else:
results[query[1]] = len(jobs)
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(
status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def checkLocalWQStatus(self, dbname):
"""
Query local WorkQueue workqueue/workqueue_inbox database to see whether
there are any active elements in this agent.
"""
results = {}
for st in ('Available', 'Negotiating', 'Acquired', 'Running'):
if dbname == "workqueue":
elements = self.localBackend.getElements(status=st,
returnIdOnly=True)
else:
elements = self.localBackend.getInboxElements(
status=st, returnIdOnly=True)
results[st] = len(elements)
return results
def checkGlobalWQStatus(self):
"""
Query Global WorkQueue workqueue database to see whether there are
any active elements set to this agent.
"""
results = {}
for st in ("Acquired", "Running"):
elements = self.globalBackend.getElements(
status=st, returnIdOnly=True, ChildQueueUrl=self.thisAgentUrl)
results[st] = len(elements)
return results
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
results['local_wq_status'] = self.checkLocalWQStatus(dbname="workqueue")
results['local_wqinbox_status'] = self.checkLocalWQStatus(dbname="workqueue_inbox")
results['global_wq_status'] = self.checkGlobalWQStatus()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
queries = [["1", "idle"], ["2", "running"]]
for query in queries:
jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], [])
# if there is an error, report it instead of the length of an empty list
if jobs is None:
results[query[1]] = "unknown (schedd query error)"
else:
results[query[1]] = len(jobs)
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def checkLocalWQStatus(self, dbname):
"""
Query local WorkQueue workqueue/workqueue_inbox database to see whether
there are any active elements in this agent.
"""
results = {}
for st in ('Available', 'Negotiating', 'Acquired', 'Running'):
if dbname == "workqueue":
elements = self.localBackend.getElements(status=st, returnIdOnly=True)
else:
elements = self.localBackend.getInboxElements(status=st, returnIdOnly=True)
results[st] = len(elements)
return results
def checkGlobalWQStatus(self):
"""
Query Global WorkQueue workqueue database to see whether there are
any active elements set to this agent.
"""
results = {}
for st in ("Acquired", "Running"):
elements = self.globalBackend.getElements(status=st, returnIdOnly=True,
ChildQueueUrl=self.thisAgentUrl)
results[st] = len(elements)
return results
class DrainStatusPoller(BaseWorkerThread):
"""
Collects information related to the agent drain status
"""
# class variable that contains drain statistics
drainStats = {}
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI(config)
self.condorAPI = PyCondorAPI()
self.agentConfig = {}
self.validSpeedDrainConfigKeys = ['CondorPriority', 'NoJobRetries', 'EnableAllSites']
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
@timeFunction
def algorithm(self, parameters):
"""
Update drainStats if agent is in drain mode
"""
logging.info("Running agent drain algorithm...")
self.agentConfig = self.reqAuxDB.getWMAgentConfig(self.config.Agent.hostName)
if not self.agentConfig:
logging.error("Failed to fetch agent configuration from the auxiliary DB")
return
if isDrainMode(self.config):
# check to see if the agent hit any speed drain thresholds
thresholdsHit = self.checkSpeedDrainThresholds()
if thresholdsHit:
logging.info("Updating agent configuration for speed drain...")
self.updateAgentSpeedDrainConfig(thresholdsHit)
# now collect drain statistics
try:
DrainStatusPoller.drainStats = self.drainAPI.collectDrainInfo()
logging.info("Finished collecting agent drain status.")
logging.info("Drain stats: " + str(DrainStatusPoller.drainStats))
except Exception as ex:
msg = "Error occurred, will retry later:\n"
msg += str(ex)
logging.exception(msg)
else:
logging.info("Agent not in drain mode. Resetting flags and skipping drain check...")
self.resetAgentSpeedDrainConfig()
@classmethod
def getDrainInfo(cls):
"""
Return drainStats class variable
"""
return cls.drainStats
def updateAgentSpeedDrainConfig(self, thresholdsHit):
"""
Takes a list of speed drain configuration keys and updates the agent configuration
"""
updateConfig = False
condorPriorityFlag = False
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
if 'CondorPriority' in thresholdsHit:
logging.info("Bumping condor job priority to 999999 for Production/Processing pending jobs.")
self.condorAPI.editCondorJobs(
"JobStatus=?=1 && (CMS_JobType =?= \"Production\" || CMS_JobType =?= \"Processing\")",
"JobPrio", "999999")
condorPriorityFlag = True
if condorPriorityFlag != speedDrainConfig['CondorPriority']['Enabled']:
# CondorPriority setting is irreversible so the flag only indicates weather
# priority is increased or not. It is not checked by other components
logging.info("Enabling CondorPriority flag.")
speedDrainConfig['CondorPriority']['Enabled'] = condorPriorityFlag
updateConfig = True
if 'NoJobRetries' in thresholdsHit:
logging.info("Enabling NoJobRetries flag: Error Handler won't retry the jobs")
# ErrorHandler will pick this up and set max retries to 0
speedDrainConfig['NoJobRetries']['Enabled'] = True
updateConfig = True
if 'EnableAllSites' in thresholdsHit:
logging.info("Enabling EnableAllSites flag: Updating agent to submit to all sites.")
# setting this value to True makes JobSubmitterPoller ignore site status
speedDrainConfig['EnableAllSites']['Enabled'] = True
updateConfig = True
# update the aux db speed drain config with any changes
if updateConfig:
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainMode", True)
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainConfig", speedDrainConfig)
return
def resetAgentSpeedDrainConfig(self):
"""
resetting SpeedDrainMode to False and SpeedDrainiConfig Enabled to False
"""
if self.agentConfig.get("SpeedDrainMode"):
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainMode", False)
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
for key, v in speedDrainConfig.items():
if key in self.validSpeedDrainConfigKeys and v['Enabled']:
speedDrainConfig[key]['Enabled'] = False
self.reqAuxDB.updateAgentConfig(self.config.Agent.hostName, "SpeedDrainConfig", speedDrainConfig)
return
def checkSpeedDrainThresholds(self):
"""
Check the current number of jobs in Condor and create a list of agent configuration parameters
that need updated for speed draining
"""
enableKeys = []
# get the current speed drain status
speedDrainConfig = self.agentConfig.get("SpeedDrainConfig")
# get condor jobs
jobs = self.condorAPI.getCondorJobs("", [])
if jobs is None:
logging.warning("There was an error querying the schedd. Not checking speed drain thresholds.")
return []
# loop through the speed drain configuration and make a list of what thresholds have been hit
for k, v in speedDrainConfig.items():
# make sure keys in the speed drain config are valid
if k in self.validSpeedDrainConfigKeys and isinstance(v['Threshold'], int) and isinstance(v['Enabled'], bool):
# we always want to apply the condor priority change if the threshold is hit
if not v['Enabled'] or k == 'CondorPriority':
logging.info("Checking speed drain threshold for %s. ", k)
if len(jobs) < v['Threshold']:
logging.info("Agent will update speed drain configuration for %s. ", k)
enableKeys.append(k)
else:
logging.warning("Speed drain configuration error for %s. Please check aux db contents.", k)
return enableKeys