def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
#fwjrInfoFromCouch = self.localCouchDB.getEventSummaryByWorkflow()
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSite()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = self.localQueue.getAnalyticsData()
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s) ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(finishedTasks), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
self.agentInfo, uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralWMStatsCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
DataUploadTime.setInfo(self, uploadTime, "ok")
except Exception, ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(self, False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = self.localQueue.getAnalyticsData()
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s) ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(finishedTasks), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
self.agentInfo, uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralRequestCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
DataUploadTime.setInfo(self, uploadTime, "ok")
except Exception as ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(self, False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
def collectAgentInfo(self):
#TODO: agent info (need to include job Slots for the sites)
# always checks couch first
source = self.config.JobStateMachine.jobSummaryDBName
target = self.config.AnalyticsDataCollector.centralWMStatsURL
couchInfo = self.localCouchServer.recoverReplicationErrors(source, target)
logging.info("getting couchdb replication status: %s" % couchInfo)
agentInfo = self.wmagentDB.getComponentStatus(self.config)
agentInfo.update(self.agentInfo)
if isDrainMode(self.config):
logging.info("Agent is in DrainMode")
agentInfo['drain_mode'] = True
agentInfo['status'] = "warning"
else:
agentInfo['drain_mode'] = False
if (couchInfo['status'] != 'ok'):
agentInfo['down_components'].append("CouchServer")
agentInfo['status'] = couchInfo['status']
couchInfo['name'] = "CouchServer"
agentInfo['down_component_detail'].append(couchInfo)
# Disk space warning
diskUseList = diskUse()
diskUseThreshold = float(self.config.AnalyticsDataCollector.diskUseThreshold)
agentInfo['disk_warning'] = []
for disk in diskUseList:
if float(disk['percent'].strip('%')) >= diskUseThreshold:
agentInfo['disk_warning'].append(disk)
# Couch process warning
couchProc = numberCouchProcess()
couchProcessThreshold = float(self.config.AnalyticsDataCollector.couchProcessThreshold)
if couchProc >= couchProcessThreshold:
agentInfo['couch_process_warning'] = couchProc
else:
agentInfo['couch_process_warning'] = 0
# This adds the last time and message when data was updated to agentInfo
lastDataUpload = DataUploadTime.getInfo(self)
if lastDataUpload['data_last_update']!=0:
agentInfo['data_last_update'] = lastDataUpload['data_last_update']
if lastDataUpload['data_error']!="":
agentInfo['data_error'] = lastDataUpload['data_error']
# Change status if there is data_error, couch process maxed out or disk full problems.
if agentInfo['status'] == 'ok':
if agentInfo['disk_warning'] != []:
agentInfo['status'] = "warning"
if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
if (agentInfo.has_key('data_error') and agentInfo['data_error'] != 'ok') or \
(agentInfo.has_key('couch_process_warning') and agentInfo['couch_process_warning'] != 0):
agentInfo['status'] = "error"
return agentInfo
def collectAgentInfo(self):
agentInfo = self.wmagentDB.getComponentStatus(self.config)
agentInfo.update(self.agentInfo)
if isDrainMode(self.config):
logging.info("Agent is in DrainMode")
agentInfo['drain_mode'] = True
agentInfo['status'] = "warning"
else:
agentInfo['drain_mode'] = False
couchInfo = self.collectCouchDBInfo()
if (couchInfo['status'] != 'ok'):
agentInfo['down_components'].append("CouchServer")
agentInfo['status'] = couchInfo['status']
couchInfo['name'] = "CouchServer"
agentInfo['down_component_detail'].append(couchInfo)
# Disk space warning
diskUseList = diskUse()
diskUseThreshold = float(
self.config.AnalyticsDataCollector.diskUseThreshold)
agentInfo['disk_warning'] = []
for disk in diskUseList:
if float(disk['percent'].strip('%')) >= diskUseThreshold and disk[
'mounted'] not in self.config.AnalyticsDataCollector.ignoreDisk:
agentInfo['disk_warning'].append(disk)
# Couch process warning
couchProc = numberCouchProcess()
couchProcessThreshold = float(
self.config.AnalyticsDataCollector.couchProcessThreshold)
if couchProc >= couchProcessThreshold:
agentInfo['couch_process_warning'] = couchProc
else:
agentInfo['couch_process_warning'] = 0
# This adds the last time and message when data was updated to agentInfo
lastDataUpload = DataUploadTime.getInfo(self)
if lastDataUpload['data_last_update'] != 0:
agentInfo['data_last_update'] = lastDataUpload['data_last_update']
if lastDataUpload['data_error'] != "":
agentInfo['data_error'] = lastDataUpload['data_error']
# Change status if there is data_error, couch process maxed out or disk full problems.
if agentInfo['status'] == 'ok':
if agentInfo['disk_warning'] != []:
agentInfo['status'] = "warning"
if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
if ('data_error' in agentInfo and agentInfo['data_error'] != 'ok') or \
('couch_process_warning' in agentInfo and agentInfo['couch_process_warning'] != 0):
agentInfo['status'] = "error"
return agentInfo
def collectAgentInfo(self):
"""
Monitors the general health of the agent, as:
1. status of the agent processes
2. status of the agent threads based on the database info
3. couchdb active tasks and its replications
4. check the disk usage
5. check the number of couch processes
:return: a dict with all the info collected
"""
logging.info("Getting agent info ...")
agentInfo = self.wmagentDB.getComponentStatus(self.config)
agentInfo.update(self.agentInfo)
agentInfo['disk_warning'] = listDiskUsageOverThreshold(self.config, updateDB=True)
if isDrainMode(self.config):
logging.info("Agent is in DrainMode")
agentInfo['drain_mode'] = True
agentInfo['drain_stats'] = DrainStatusPoller.getDrainInfo()
else:
agentInfo['drain_mode'] = False
couchInfo = self.collectCouchDBInfo()
if couchInfo['status'] != 'ok':
agentInfo['down_components'].append(couchInfo['name'])
agentInfo['status'] = couchInfo['status']
agentInfo['down_component_detail'].append(couchInfo)
# Couch process warning
couchProc = numberCouchProcess()
logging.info("CouchDB is running with %d processes", couchProc)
couchProcessThreshold = self.config.AnalyticsDataCollector.couchProcessThreshold
if couchProc >= couchProcessThreshold:
agentInfo['couch_process_warning'] = couchProc
else:
agentInfo['couch_process_warning'] = 0
# This adds the last time and message when data was updated to agentInfo
lastDataUpload = DataUploadTime.getInfo()
if lastDataUpload['data_last_update']:
agentInfo['data_last_update'] = lastDataUpload['data_last_update']
if lastDataUpload['data_error']:
agentInfo['data_error'] = lastDataUpload['data_error']
# Change status if there is data_error, couch process maxed out or disk full problems.
if agentInfo['status'] == 'ok' and (agentInfo['drain_mode'] or agentInfo['disk_warning']):
agentInfo['status'] = "warning"
if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
if agentInfo.get('data_error', 'ok') != 'ok' or agentInfo.get('couch_process_warning', 0):
agentInfo['status'] = "error"
logging.info("List of agent components down: %s", agentInfo['down_components'])
return agentInfo
def collectAgentInfo(self):
"""
Monitors the general health of the agent, as:
1. status of the agent processes
2. status of the agent threads based on the database info
3. couchdb active tasks and its replications
4. check the disk usage
5. check the number of couch processes
:return: a dict with all the info collected
"""
logging.info("Getting agent info ...")
agentInfo = self.wmagentDB.getComponentStatus(self.config)
agentInfo.update(self.agentInfo)
agentInfo['disk_warning'] = listDiskUsageOverThreshold(self.config, updateDB=True)
if isDrainMode(self.config):
logging.info("Agent is in DrainMode")
agentInfo['drain_mode'] = True
agentInfo['drain_stats'] = DrainStatusPoller.getDrainInfo()
else:
agentInfo['drain_mode'] = False
couchInfo = self.collectCouchDBInfo()
if couchInfo['status'] != 'ok':
agentInfo['down_components'].append(couchInfo['name'])
agentInfo['status'] = couchInfo['status']
agentInfo['down_component_detail'].append(couchInfo)
# Couch process warning
couchProc = numberCouchProcess()
logging.info("CouchDB is running with %d processes", couchProc)
couchProcessThreshold = self.config.AnalyticsDataCollector.couchProcessThreshold
if couchProc >= couchProcessThreshold:
agentInfo['couch_process_warning'] = couchProc
else:
agentInfo['couch_process_warning'] = 0
# This adds the last time and message when data was updated to agentInfo
lastDataUpload = DataUploadTime.getInfo()
if lastDataUpload['data_last_update']:
agentInfo['data_last_update'] = lastDataUpload['data_last_update']
if lastDataUpload['data_error']:
agentInfo['data_error'] = lastDataUpload['data_error']
# Change status if there is data_error, couch process maxed out or disk full problems.
if agentInfo['status'] == 'ok' and (agentInfo['drain_mode'] or agentInfo['disk_warning']):
agentInfo['status'] = "warning"
if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
if agentInfo.get('data_error', 'ok') != 'ok' or agentInfo.get('couch_process_warning', 0):
agentInfo['status'] = "error"
logging.info("List of agent components down: %s", agentInfo['down_components'])
return agentInfo
def collectAgentInfo(self):
agentInfo = self.wmagentDB.getComponentStatus(self.config)
agentInfo.update(self.agentInfo)
if isDrainMode(self.config):
logging.info("Agent is in DrainMode")
agentInfo['drain_mode'] = True
agentInfo['status'] = "warning"
else:
agentInfo['drain_mode'] = False
couchInfo = self.collectCouchDBInfo()
if (couchInfo['status'] != 'ok'):
agentInfo['down_components'].append("CouchServer")
agentInfo['status'] = couchInfo['status']
couchInfo['name'] = "CouchServer"
agentInfo['down_component_detail'].append(couchInfo)
# Disk space warning
diskUseList = diskUse()
diskUseThreshold = float(self.config.AnalyticsDataCollector.diskUseThreshold)
agentInfo['disk_warning'] = []
for disk in diskUseList:
if float(disk['percent'].strip('%')) >= diskUseThreshold and disk['mounted'] not in self.config.AnalyticsDataCollector.ignoreDisk:
agentInfo['disk_warning'].append(disk)
# Couch process warning
couchProc = numberCouchProcess()
couchProcessThreshold = float(self.config.AnalyticsDataCollector.couchProcessThreshold)
if couchProc >= couchProcessThreshold:
agentInfo['couch_process_warning'] = couchProc
else:
agentInfo['couch_process_warning'] = 0
# This adds the last time and message when data was updated to agentInfo
lastDataUpload = DataUploadTime.getInfo(self)
if lastDataUpload['data_last_update']!=0:
agentInfo['data_last_update'] = lastDataUpload['data_last_update']
if lastDataUpload['data_error']!="":
agentInfo['data_error'] = lastDataUpload['data_error']
# Change status if there is data_error, couch process maxed out or disk full problems.
if agentInfo['status'] == 'ok':
if agentInfo['disk_warning'] != []:
agentInfo['status'] = "warning"
if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
if ('data_error' in agentInfo and agentInfo['data_error'] != 'ok') or \
('couch_process_warning' in agentInfo and agentInfo['couch_process_warning'] != 0):
agentInfo['status'] = "error"
return agentInfo
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
skippedInfoFromCouch = self.localCouchDB.getSkippedFilesSummaryByWorkflow()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
logging.info("Getting DBS PhEDEx upload status ...")
completedWfs = self.dbsBufferUtil.getPhEDExDBSStatusForCompletedWorkflows(summary=True)
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = {}
if not hasattr(self.config, "Tier0Feeder"):
localQInfo = self.localQueue.getAnalyticsData()
else:
logging.debug("Tier-0 instance, not checking WorkQueue")
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
WorkflowsWithSkippedFile(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s)
Completed workflows(%s).. ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(skippedInfoFromCouch),
len(batchJobInfo), len(finishedTasks), len(localQInfo), len(completedWfs)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
tempCombinedData2 = combineAnalyticsData(tempCombinedData, localQInfo)
combinedRequests = combineAnalyticsData(tempCombinedData2, completedWfs)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
skippedInfoFromCouch, self.agentInfo,
uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralRequestCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
DataUploadTime.setInfo(uploadTime, "ok")
except Exception as ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
def collectAgentInfo(self):
"""
Monitors the general health of the agent, as:
1. status of the agent processes
2. status of the agent threads based on the database info
3. couchdb active tasks and its replications
4. check the disk usage
5. check the number of couch processes
6. check proxy and certificate validity
:return: a dict with all the info collected
"""
logging.info("Getting agent info ...")
agentInfo = self.wmagentDB.getComponentStatus(self.config)
agentInfo.update(self.agentInfo)
if isDrainMode(self.config):
logging.info("Agent is in DrainMode")
agentInfo['drain_mode'] = True
else:
agentInfo['drain_mode'] = False
couchInfo = self.collectCouchDBInfo()
if couchInfo['status'] != 'ok':
agentInfo['down_components'].append(couchInfo['name'])
agentInfo['status'] = couchInfo['status']
agentInfo['down_component_detail'].append(couchInfo)
# Disk space warning
diskUseList = diskUse()
diskUseThreshold = float(self.config.AnalyticsDataCollector.diskUseThreshold)
agentInfo['disk_warning'] = []
for disk in diskUseList:
if float(disk['percent'].strip('%')) >= diskUseThreshold and \
disk['mounted'] not in self.config.AnalyticsDataCollector.ignoreDisk:
agentInfo['disk_warning'].append(disk)
# Couch process warning
couchProc = numberCouchProcess()
logging.info("CouchDB is running with %d processes", couchProc)
couchProcessThreshold = self.config.AnalyticsDataCollector.couchProcessThreshold
if couchProc >= couchProcessThreshold:
agentInfo['couch_process_warning'] = couchProc
else:
agentInfo['couch_process_warning'] = 0
# This adds the last time and message when data was updated to agentInfo
lastDataUpload = DataUploadTime.getInfo()
if lastDataUpload['data_last_update']:
agentInfo['data_last_update'] = lastDataUpload['data_last_update']
if lastDataUpload['data_error']:
agentInfo['data_error'] = lastDataUpload['data_error']
# Change status if there is data_error, couch process maxed out or disk full problems.
if agentInfo['status'] == 'ok' and (agentInfo['drain_mode'] or agentInfo['disk_warning']):
agentInfo['status'] = "warning"
if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
if agentInfo.get('data_error', 'ok') != 'ok' or agentInfo.get('couch_process_warning', 0):
agentInfo['status'] = "error"
if agentInfo['down_components']:
logging.info("List of agent components down: %s" % agentInfo['down_components'])
# Check agent proxy and certificate validity
collectProxyInfo(agentInfo)
return agentInfo
def collectAgentInfo(self):
#TODO: agent info (need to include job Slots for the sites)
# always checks couch first
source = self.config.JobStateMachine.jobSummaryDBName
target = self.config.AnalyticsDataCollector.centralWMStatsURL
couchInfo = self.localCouchServer.recoverReplicationErrors(
source, target, filter="WMStats/repfilter")
logging.info("getting couchdb replication status: %s" % couchInfo)
agentInfo = self.wmagentDB.getComponentStatus(self.config)
agentInfo.update(self.agentInfo)
if isDrainMode(self.config):
logging.info("Agent is in DrainMode")
agentInfo['drain_mode'] = True
agentInfo['status'] = "warning"
else:
agentInfo['drain_mode'] = False
if (couchInfo['status'] != 'ok'):
agentInfo['down_components'].append("CouchServer")
agentInfo['status'] = couchInfo['status']
couchInfo['name'] = "CouchServer"
agentInfo['down_component_detail'].append(couchInfo)
# Disk space warning
diskUseList = diskUse()
diskUseThreshold = float(
self.config.AnalyticsDataCollector.diskUseThreshold)
agentInfo['disk_warning'] = []
for disk in diskUseList:
if float(disk['percent'].strip('%')) >= diskUseThreshold and disk[
'mounted'] not in self.config.AnalyticsDataCollector.ignoreDisk:
agentInfo['disk_warning'].append(disk)
# Couch process warning
couchProc = numberCouchProcess()
couchProcessThreshold = float(
self.config.AnalyticsDataCollector.couchProcessThreshold)
if couchProc >= couchProcessThreshold:
agentInfo['couch_process_warning'] = couchProc
else:
agentInfo['couch_process_warning'] = 0
# This adds the last time and message when data was updated to agentInfo
lastDataUpload = DataUploadTime.getInfo(self)
if lastDataUpload['data_last_update'] != 0:
agentInfo['data_last_update'] = lastDataUpload['data_last_update']
if lastDataUpload['data_error'] != "":
agentInfo['data_error'] = lastDataUpload['data_error']
# Change status if there is data_error, couch process maxed out or disk full problems.
if agentInfo['status'] == 'ok':
if agentInfo['disk_warning'] != []:
agentInfo['status'] = "warning"
if agentInfo['status'] == 'ok' or agentInfo['status'] == 'warning':
if (agentInfo.has_key('data_error') and agentInfo['data_error'] != 'ok') or \
(agentInfo.has_key('couch_process_warning') and agentInfo['couch_process_warning'] != 0):
agentInfo['status'] = "error"
return agentInfo