def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
#fwjrInfoFromCouch = self.localCouchDB.getEventSummaryByWorkflow()
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSite()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = self.localQueue.getAnalyticsData()
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s) ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(finishedTasks), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
self.agentInfo, uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralWMStatsCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
DataUploadTime.setInfo(self, uploadTime, "ok")
except Exception, ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(self, False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = self.localQueue.getAnalyticsData()
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s) ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(finishedTasks), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
self.agentInfo, uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralRequestCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
DataUploadTime.setInfo(self, uploadTime, "ok")
except Exception as ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(self, False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
skippedInfoFromCouch = self.localCouchDB.getSkippedFilesSummaryByWorkflow()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
logging.info("Getting DBS PhEDEx upload status ...")
completedWfs = self.dbsBufferUtil.getPhEDExDBSStatusForCompletedWorkflows(summary=True)
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = {}
if not hasattr(self.config, "Tier0Feeder"):
localQInfo = self.localQueue.getAnalyticsData()
else:
logging.debug("Tier-0 instance, not checking WorkQueue")
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
WorkflowsWithSkippedFile(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s)
Completed workflows(%s).. ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(skippedInfoFromCouch),
len(batchJobInfo), len(finishedTasks), len(localQInfo), len(completedWfs)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
tempCombinedData2 = combineAnalyticsData(tempCombinedData, localQInfo)
combinedRequests = combineAnalyticsData(tempCombinedData2, completedWfs)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
skippedInfoFromCouch, self.agentInfo,
uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralRequestCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
DataUploadTime.setInfo(uploadTime, "ok")
except Exception as ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
