def algorithm(self, parameters): """ get information from wmbs, workqueue and local couch """ try: #jobs per request info logging.info("Getting Job Couch Data ...") jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite() #fwjr per request info logging.info("Getting FWJRJob Couch Data ...") fwjrInfoFromCouch = self.localCouchDB.getEventSummaryByWorkflow() logging.info("Getting Batch Job Data ...") batchJobInfo = self.wmagentDB.getBatchJobInfo() # get the data from local workqueue: # request name, input dataset, inWMBS, inQueue logging.info("Getting Local Queue Data ...") localQInfo = self.localQueue.getAnalyticsData() # combine all the data from 3 sources logging.info("""Combining data from Job Couch(%s), FWJR(%s), Batch Job(%s), Local Queue(%s) ...""" % (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(localQInfo))) tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo) combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo) #set the uploadTime - should be the same for all docs uploadTime = int(time.time()) logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests)) requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, self.agentInfo, uploadTime, self.summaryLevel) if self.plugin != None: self.plugin(requestDocs, self.localSummaryCouchDB, self.centralWMStatsCouchDB) self.localSummaryCouchDB.uploadData(requestDocs) logging.info("Request data upload success\n %s request \n uploading agent data" % len(requestDocs)) #TODO: agent info (need to include job Slots for the sites) agentInfo = self.wmagentDB.getHeartBeatWarning() agentInfo.update(self.agentInfo) agentDocs = convertToAgentCouchDoc(agentInfo, self.config.ACDC, uploadTime) self.localSummaryCouchDB.updateAgentInfo(agentDocs) logging.info("Agent data upload success\n %s request" % len(agentDocs)) except Exception, ex: logging.error("Error occured, will retry later:") logging.error(str(ex)) logging.error("Traceback: \n%s" % traceback.format_exc())
def uploadAgentInfoToCentralWMStats(self, agentInfo, uploadTime): # direct data upload to the remote to prevent data conflict when agent is cleaned up and redeployed agentDocs = convertToAgentCouchDoc(agentInfo, self.config.ACDC, uploadTime) self.centralWMStatsCouchDB.updateAgentInfo(agentDocs)
def uploadAgentInfoToCentralWMStats(self, agentInfo, uploadTime): #direct data upload to the remote to prevent data conflict when agent is cleaned up and redeployed agentDocs = convertToAgentCouchDoc(agentInfo, self.config.ACDC, uploadTime) self.centralWMStatsCouchDB.updateAgentInfo(agentDocs)