class DrainStatusAPI(object): """ Provides methods for querying dbs and condor for drain statistics """ def __init__(self): self.dbsUtil = DBSBufferUtil() self.condorAPI = PyCondorAPI() def collectDrainInfo(self): """ Call methods to check the drain status """ results = {} results['workflows_completed'] = self.checkWorkflows() # if workflows are completed, collect additional drain statistics if results['workflows_completed']: results['upload_status'] = self.checkFileUploadStatus() results['condor_status'] = self.checkCondorStates() return results def checkWorkflows(self): """ Check to see if all workflows have a 'completed' status """ results = self.dbsUtil.isAllWorkflowCompleted() return results def checkCondorStates(self): """ Check idle and running jobs in Condor """ results = {} queries = [["1", "idle"], ["2", "running"]] for query in queries: jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], []) # if there is an error, report it instead of the length of an empty list if jobs is None: results[query[1]] = "unknown (schedd query error)" else: results[query[1]] = len(jobs) return results def checkFileUploadStatus(self): """ Check file upload status: Blocks open in DBS Files not uploaded in DBS Files not uploaded to Phedex """ results = {} results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks() results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED") results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded() return results
class DrainStatusAPI(object): """ Provides methods for querying dbs and condor for drain statistics """ def __init__(self, config): # queue url used in WorkQueueManager self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984" self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl) self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl) self.dbsUtil = DBSBufferUtil() self.condorAPI = PyCondorAPI() self.condorStates = ("Running", "Idle") def collectDrainInfo(self): """ Call methods to check the drain status """ results = {} results['workflows_completed'] = self.checkWorkflows() # if workflows are completed, collect additional drain statistics if results['workflows_completed']: results['upload_status'] = self.checkFileUploadStatus() results['condor_status'] = self.checkCondorStates() results['local_wq_status'] = self.checkLocalWQStatus( dbname="workqueue") results['local_wqinbox_status'] = self.checkLocalWQStatus( dbname="workqueue_inbox") results['global_wq_status'] = self.checkGlobalWQStatus() return results def checkWorkflows(self): """ Check to see if all workflows have a 'completed' status """ results = self.dbsUtil.isAllWorkflowCompleted() return results def checkCondorStates(self): """ Check idle and running jobs in Condor """ results = {} jobs = self.condorAPI.getCondorJobsSummary() for state in self.condorStates: # if there is an error, report it instead of the length of an empty list if not jobs: results[state.lower()] = None else: results[state.lower()] = int(jobs[0].get(state)) return results def checkFileUploadStatus(self): """ Check file upload status: Blocks open in DBS Files not uploaded in DBS Files not uploaded to Phedex """ results = {} results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks() results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus( status="NOTUPLOADED") results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded() return results def checkLocalWQStatus(self, dbname): """ Query local WorkQueue workqueue/workqueue_inbox database to see whether there are any active elements in this agent. """ results = {} for st in ('Available', 'Negotiating', 'Acquired', 'Running'): if dbname == "workqueue": elements = self.localBackend.getElements(status=st, returnIdOnly=True) else: elements = self.localBackend.getInboxElements( status=st, returnIdOnly=True) results[st] = len(elements) return results def checkGlobalWQStatus(self): """ Query Global WorkQueue workqueue database to see whether there are any active elements set to this agent. """ results = {} for st in ("Acquired", "Running"): elements = self.globalBackend.getElements( status=st, returnIdOnly=True, ChildQueueUrl=self.thisAgentUrl) results[st] = len(elements) return results
class DrainStatusAPI(object): """ Provides methods for querying dbs and condor for drain statistics """ def __init__(self, config): # queue url used in WorkQueueManager self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984" self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl) self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl) self.dbsUtil = DBSBufferUtil() self.condorAPI = PyCondorAPI() def collectDrainInfo(self): """ Call methods to check the drain status """ results = {} results['workflows_completed'] = self.checkWorkflows() # if workflows are completed, collect additional drain statistics if results['workflows_completed']: results['upload_status'] = self.checkFileUploadStatus() results['condor_status'] = self.checkCondorStates() results['local_wq_status'] = self.checkLocalWQStatus(dbname="workqueue") results['local_wqinbox_status'] = self.checkLocalWQStatus(dbname="workqueue_inbox") results['global_wq_status'] = self.checkGlobalWQStatus() return results def checkWorkflows(self): """ Check to see if all workflows have a 'completed' status """ results = self.dbsUtil.isAllWorkflowCompleted() return results def checkCondorStates(self): """ Check idle and running jobs in Condor """ results = {} queries = [["1", "idle"], ["2", "running"]] for query in queries: jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], []) # if there is an error, report it instead of the length of an empty list if jobs is None: results[query[1]] = "unknown (schedd query error)" else: results[query[1]] = len(jobs) return results def checkFileUploadStatus(self): """ Check file upload status: Blocks open in DBS Files not uploaded in DBS Files not uploaded to Phedex """ results = {} results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks() results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED") results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded() return results def checkLocalWQStatus(self, dbname): """ Query local WorkQueue workqueue/workqueue_inbox database to see whether there are any active elements in this agent. """ results = {} for st in ('Available', 'Negotiating', 'Acquired', 'Running'): if dbname == "workqueue": elements = self.localBackend.getElements(status=st, returnIdOnly=True) else: elements = self.localBackend.getInboxElements(status=st, returnIdOnly=True) results[st] = len(elements) return results def checkGlobalWQStatus(self): """ Query Global WorkQueue workqueue database to see whether there are any active elements set to this agent. """ results = {} for st in ("Acquired", "Running"): elements = self.globalBackend.getElements(status=st, returnIdOnly=True, ChildQueueUrl=self.thisAgentUrl) results[st] = len(elements) return results