def _updatePilotSummary(self, sitesData): result = RPCClient( "WorkloadManagement/WMSAdministrator").getPilotSummary() if not result['OK']: gLogger.error("Cannot get the pilot summary", result['Message']) return result summaryData = result['Value'] result = getCESiteMapping() if not result['OK']: return S_ERROR('Could not get CE site mapping') ceMapping = result['Value'] pilotSummary = {} for ce in summaryData: if ce not in ceMapping: continue ceData = summaryData[ce] siteName = ceMapping[ce] if siteName not in pilotSummary: pilotSummary[siteName] = {} for status in ceData: if status not in pilotSummary[siteName]: pilotSummary[siteName][status] = 0 pilotSummary[siteName][status] += ceData[status] for site in sitesData: if site in pilotSummary: sitesData[site]['pilotSummary'] = pilotSummary[site] return S_OK(sitesData)
def _updatePilotSummary( self, sitesData ): result = RPCClient( "WorkloadManagement/WMSAdministrator" ).getPilotSummary() if not result[ 'OK' ]: gLogger.error( "Cannot get the pilot summary", result['Message'] ) return result summaryData = result[ 'Value' ] result = getCESiteMapping() if not result['OK']: return S_ERROR( 'Could not get CE site mapping' ) ceMapping = result[ 'Value' ] pilotSummary = {} for ce in summaryData: if ce not in ceMapping: continue ceData = summaryData[ ce ] siteName = ceMapping[ ce ] if siteName not in pilotSummary: pilotSummary[ siteName ] = {} for status in ceData: if status not in pilotSummary[ siteName ]: pilotSummary[ siteName ][ status ] = 0 pilotSummary[ siteName ][ status ] += ceData[ status ] for site in sitesData: if site in pilotSummary: sitesData[ site ][ 'pilotSummary' ] = pilotSummary[ site ] return S_OK( sitesData )
def getARCPilotOutput( proxy, pilotRef ): tmp_dir = mkdtemp() myce = pilotRef.split(":")[1].strip("/") gridEnv = getGridEnv() mySite = getCESiteMapping()['Value'][myce] workDB = gConfig.getValue(cfgPath('Resources/Sites/LCG', mySite, 'CEs', myce, 'JobListFile')) myWorkDB = os.path.join("/opt/dirac/runit/WorkloadManagement/SiteDirector-RAL", workDB) cmd = [ 'arcget' ] cmd.extend( ['-k', '-c', myce, '-j', myWorkDB, '-D', tmp_dir, pilotRef] ) ret = executeGridCommand( proxy, cmd, gridEnv ) if not ret['OK']: shutil.rmtree( tmp_dir ) return ret status, output, error = ret['Value'] if 'Results stored at:' in output : tmp_dir = os.path.join( tmp_dir, os.listdir( tmp_dir )[0] ) result = S_OK() result['FileList'] = os.listdir( tmp_dir ) for filename in result['FileList']: tmpname = os.path.join( tmp_dir, filename ) if os.path.exists( tmpname ): myfile = file( tmpname, 'r' ) f = myfile.read() myfile.close() else : f = ' ' if ".out" in filename: filename = 'StdOut' if ".err" in filename: filename = 'StdErr' result[filename] = f shutil.rmtree( tmp_dir ) return result if 'Warning: Job not found in job list' in output: shutil.rmtree( tmp_dir ) message = "Pilot not yet visible in ARC dB" return S_ERROR( message ) return S_ERROR("Sorry - requested pilot output not yet available")
def getPilotSummaryWeb(self,selectDict,sortList,startItem,maxItems): """ Get summary of the pilot jobs status by CE/site in a standard structure """ stateNames = ['Submitted','Ready','Scheduled','Waiting','Running','Done','Aborted'] allStateNames = stateNames + ['Done_Empty','Aborted_Hour'] paramNames = ['Site','CE']+allStateNames resultDict = {} last_update = None if selectDict.has_key('LastUpdateTime'): last_update = selectDict['LastUpdateTime'] del selectDict['LastUpdateTime'] site_select = [] if selectDict.has_key('GridSite'): site_select = selectDict['GridSite'] if type(site_select) != type([]): site_select = [site_select] del selectDict['GridSite'] status_select = [] if selectDict.has_key('Status'): status_select = selectDict['Status'] if type(status_select) != type([]): status_select = [status_select] del selectDict['Status'] expand_site = '' if selectDict.has_key('ExpandSite'): expand_site = selectDict['ExpandSite'] site_select = [expand_site] del selectDict['ExpandSite'] start = time.time() # Get all the data from the database with various selections result = self.getCounters('PilotAgents', ['GridSite','DestinationSite','Status'], selectDict,newer=last_update,timeStamp='LastUpdateTime') if not result['OK']: return result last_update = Time.dateTime() - Time.hour selectDict['Status'] = 'Aborted' resultHour = self.getCounters('PilotAgents', ['GridSite','DestinationSite','Status'], selectDict,newer=last_update,timeStamp='LastUpdateTime') if not resultHour['OK']: return resultHour last_update = Time.dateTime() - Time.day selectDict['Status'] = ['Aborted','Done'] resultDay = self.getCounters('PilotAgents', ['GridSite','DestinationSite','Status'], selectDict,newer=last_update,timeStamp='LastUpdateTime') if not resultDay['OK']: return resultDay selectDict['CurrentJobID'] = 0 selectDict['Status'] = 'Done' resultDayEmpty = self.getCounters('PilotAgents', ['GridSite','DestinationSite','Status'], selectDict,newer=last_update,timeStamp='LastUpdateTime') if not resultDayEmpty['OK']: return resultDayEmpty ceMap = {} resMap = getCESiteMapping() if resMap['OK']: ceMap = resMap['Value'] # Sort out different counters resultDict = {} resultDict['Unknown']={} for attDict,count in result['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ce != "Multiple" and ceMap.has_key(ce): site = ceMap[ce] if not resultDict.has_key(site): resultDict[site] = {} if not resultDict[site].has_key(ce): resultDict[site][ce] = {} for p in allStateNames: resultDict[site][ce][p] = 0 resultDict[site][ce][state] = count for attDict,count in resultDay['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce): site = ceMap[ce] if state == "Done": resultDict[site][ce]["Done"] = count if state == "Aborted": resultDict[site][ce]["Aborted"] = count for attDict,count in resultDayEmpty['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce): site = ceMap[ce] if state == "Done": resultDict[site][ce]["Done_Empty"] = count for attDict,count in resultHour['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce): site = ceMap[ce] if state == "Aborted": resultDict[site][ce]["Aborted_Hour"] = count records = [] siteSumDict = {} for site in resultDict: sumDict = {} for state in allStateNames: if not sumDict.has_key(state): sumDict[state] = 0 sumDict['Total'] = 0 for ce in resultDict[site]: itemList = [site,ce] total = 0 for state in allStateNames: itemList.append(resultDict[site][ce][state]) sumDict[state] += resultDict[site][ce][state] if state == "Done": done = resultDict[site][ce][state] if state == "Done_Empty": empty = resultDict[site][ce][state] if state == "Aborted": aborted = resultDict[site][ce][state] if state == "Aborted_Hour": aborted_hour = resultDict[site][ce][state] if state != "Aborted_Hour" and state != "Done_Empty": total += resultDict[site][ce][state] sumDict['Total'] += total # Add the total number of pilots seen in the last day itemList.append(total) # Add pilot submission efficiency evaluation if (done-empty) > 0: eff = float(done)/float(done-empty) elif done == 0: eff = 0. elif empty == done: eff = 99. else: eff = 0. itemList.append('%.2f' % eff) # Add pilot job efficiency evaluation if total > 0: eff = float(total-aborted)/float(total)*100. else: eff = 100. itemList.append('%.2f' % eff) # Evaluate the quality status of the CE if total > 10: if eff < 25.: itemList.append('Bad') elif eff < 60.: itemList.append('Poor') elif eff < 85.: itemList.append('Fair') else: itemList.append('Good') else: itemList.append('Idle') if len(resultDict[site]) == 1 or expand_site: records.append(itemList) if len(resultDict[site]) > 1 and not expand_site: itemList = [site,'Multiple'] for state in allStateNames+['Total']: if sumDict.has_key(state): itemList.append(sumDict[state]) else: itemList.append(0) done = sumDict["Done"] empty = sumDict["Done_Empty"] aborted = sumDict["Aborted"] aborted_hour = sumDict["Aborted_Hour"] total = sumDict["Total"] # Add pilot submission efficiency evaluation if (done-empty) > 0: eff = float(done)/float(done-empty) elif done == 0: eff = 0. elif empty == done: eff = 99. else: eff = 0. itemList.append('%.2f' % eff) # Add pilot job efficiency evaluation if total > 0: eff = float(total-aborted)/float(total)*100. else: eff = 100. itemList.append('%.2f' % eff) # Evaluate the quality status of the Site if total > 10: if eff < 25.: itemList.append('Bad') elif eff < 60.: itemList.append('Poor') elif eff < 85.: itemList.append('Fair') else: itemList.append('Good') else: itemList.append('Idle') records.append(itemList) for state in allStateNames+['Total']: if not siteSumDict.has_key(state): siteSumDict[state] = sumDict[state] else: siteSumDict[state] += sumDict[state] # Perform site selection if site_select: new_records = [] for r in records: if r[0] in site_select: new_records.append(r) records = new_records # Perform status selection if status_select: new_records = [] for r in records: if r[14] in status_select: new_records.append(r) records = new_records # Get the Site Mask data client = RPCClient('WorkloadManagement/WMSAdministrator') result = client.getSiteMask() if result['OK']: siteMask = result['Value'] for r in records: if r[0] in siteMask: r.append('Yes') else: r.append('No') else: for r in records: r.append('Unknown') finalDict = {} finalDict['TotalRecords'] = len(records) finalDict['ParameterNames'] = paramNames+ \ ['Total','PilotsPerJob','PilotJobEff','Status','InMask'] # Return all the records if maxItems == 0 or the specified number otherwise if maxItems: finalDict['Records'] = records[startItem:startItem+maxItems] else: finalDict['Records'] = records done = siteSumDict["Done"] empty = siteSumDict["Done_Empty"] aborted = siteSumDict["Aborted"] aborted_hour = siteSumDict["Aborted_Hour"] total = siteSumDict["Total"] # Add pilot submission efficiency evaluation if (done-empty) > 0: eff = float(done)/float(done-empty) elif done == 0: eff = 0. elif empty == done: eff = 99. else: eff = 0. siteSumDict['PilotsPerJob'] = '%.2f' % eff # Add pilot job efficiency evaluation if total > 0: eff = float(total-aborted)/float(total)*100. else: eff = 100. siteSumDict['PilotJobEff'] = '%.2f' % eff # Evaluate the overall quality status if total > 100: if eff < 25.: siteSumDict['Status'] = 'Bad' elif eff < 60.: siteSumDict['Status'] = 'Poor' elif eff < 85.: siteSumDict['Status'] = 'Fair' else: siteSumDict['Status'] = 'Good' else: siteSumDict['Status'] = 'Idle' finalDict['Extras'] = siteSumDict return S_OK(finalDict)
def getPilotSummaryWeb(self, selectDict, sortList, startItem, maxItems): """ Get summary of the pilot jobs status by CE/site in a standard structure """ stateNames = [ 'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', 'Done', 'Aborted', 'Failed' ] allStateNames = stateNames + ['Done_Empty', 'Aborted_Hour'] paramNames = ['Site', 'CE'] + allStateNames last_update = None if 'LastUpdateTime' in selectDict: last_update = selectDict['LastUpdateTime'] del selectDict['LastUpdateTime'] site_select = [] if 'GridSite' in selectDict: site_select = selectDict['GridSite'] if not isinstance(site_select, type([])): site_select = [site_select] del selectDict['GridSite'] status_select = [] if 'Status' in selectDict: status_select = selectDict['Status'] if not isinstance(status_select, type([])): status_select = [status_select] del selectDict['Status'] expand_site = '' if 'ExpandSite' in selectDict: expand_site = selectDict['ExpandSite'] site_select = [expand_site] del selectDict['ExpandSite'] # Get all the data from the database with various selections result = self.getCounters('PilotAgents', ['GridSite', 'DestinationSite', 'Status'], selectDict, newer=last_update, timeStamp='LastUpdateTime') if not result['OK']: return result last_update = Time.dateTime() - Time.hour selectDict['Status'] = 'Aborted' resultHour = self.getCounters( 'PilotAgents', ['GridSite', 'DestinationSite', 'Status'], selectDict, newer=last_update, timeStamp='LastUpdateTime') if not resultHour['OK']: return resultHour last_update = Time.dateTime() - Time.day selectDict['Status'] = ['Aborted', 'Done'] resultDay = self.getCounters('PilotAgents', ['GridSite', 'DestinationSite', 'Status'], selectDict, newer=last_update, timeStamp='LastUpdateTime') if not resultDay['OK']: return resultDay selectDict['CurrentJobID'] = 0 selectDict['Status'] = 'Done' resultDayEmpty = self.getCounters( 'PilotAgents', ['GridSite', 'DestinationSite', 'Status'], selectDict, newer=last_update, timeStamp='LastUpdateTime') if not resultDayEmpty['OK']: return resultDayEmpty ceMap = {} resMap = getCESiteMapping() if resMap['OK']: ceMap = resMap['Value'] # Sort out different counters resultDict = {} resultDict['Unknown'] = {} for attDict, count in result['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ce != "Multiple" and ce in ceMap: site = ceMap[ce] if site not in resultDict: resultDict[site] = {} if ce not in resultDict[site]: resultDict[site][ce] = {} for p in allStateNames: resultDict[site][ce][p] = 0 resultDict[site][ce][state] = count for attDict, count in resultDay['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ce in ceMap: site = ceMap[ce] if state == "Done": resultDict[site][ce]["Done"] = count if state == "Aborted": resultDict[site][ce]["Aborted"] = count for attDict, count in resultDayEmpty['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ce in ceMap: site = ceMap[ce] if state == "Done": resultDict[site][ce]["Done_Empty"] = count for attDict, count in resultHour['Value']: site = attDict['GridSite'] ce = attDict['DestinationSite'] state = attDict['Status'] if site == 'Unknown' and ce != "Unknown" and ce in ceMap: site = ceMap[ce] if state == "Aborted": resultDict[site][ce]["Aborted_Hour"] = count records = [] siteSumDict = {} for site in resultDict: sumDict = {} for state in allStateNames: if state not in sumDict: sumDict[state] = 0 sumDict['Total'] = 0 for ce in resultDict[site]: itemList = [site, ce] total = 0 for state in allStateNames: itemList.append(resultDict[site][ce][state]) sumDict[state] += resultDict[site][ce][state] if state == "Done": done = resultDict[site][ce][state] if state == "Done_Empty": empty = resultDict[site][ce][state] if state == "Aborted": aborted = resultDict[site][ce][state] if state != "Aborted_Hour" and state != "Done_Empty": total += resultDict[site][ce][state] sumDict['Total'] += total # Add the total number of pilots seen in the last day itemList.append(total) # Add pilot submission efficiency evaluation if (done - empty) > 0: eff = float(done) / float(done - empty) elif done == 0: eff = 0. elif empty == done: eff = 99. else: eff = 0. itemList.append('%.2f' % eff) # Add pilot job efficiency evaluation if total > 0: eff = float(total - aborted) / float(total) * 100. else: eff = 100. itemList.append('%.2f' % eff) # Evaluate the quality status of the CE if total > 10: if eff < 25.: itemList.append('Bad') elif eff < 60.: itemList.append('Poor') elif eff < 85.: itemList.append('Fair') else: itemList.append('Good') else: itemList.append('Idle') if len(resultDict[site]) == 1 or expand_site: records.append(itemList) if len(resultDict[site]) > 1 and not expand_site: itemList = [site, 'Multiple'] for state in allStateNames + ['Total']: if state in sumDict: itemList.append(sumDict[state]) else: itemList.append(0) done = sumDict["Done"] empty = sumDict["Done_Empty"] aborted = sumDict["Aborted"] total = sumDict["Total"] # Add pilot submission efficiency evaluation if (done - empty) > 0: eff = float(done) / float(done - empty) elif done == 0: eff = 0. elif empty == done: eff = 99. else: eff = 0. itemList.append('%.2f' % eff) # Add pilot job efficiency evaluation if total > 0: eff = float(total - aborted) / float(total) * 100. else: eff = 100. itemList.append('%.2f' % eff) # Evaluate the quality status of the Site if total > 10: if eff < 25.: itemList.append('Bad') elif eff < 60.: itemList.append('Poor') elif eff < 85.: itemList.append('Fair') else: itemList.append('Good') else: itemList.append('Idle') records.append(itemList) for state in allStateNames + ['Total']: if state not in siteSumDict: siteSumDict[state] = sumDict[state] else: siteSumDict[state] += sumDict[state] # Perform site selection if site_select: new_records = [] for r in records: if r[0] in site_select: new_records.append(r) records = new_records # Perform status selection if status_select: new_records = [] for r in records: if r[14] in status_select: new_records.append(r) records = new_records # Get the Site Mask data result = SiteStatus().getUsableSites() if result['OK']: siteMask = result['Value'] for r in records: if r[0] in siteMask: r.append('Yes') else: r.append('No') else: for r in records: r.append('Unknown') finalDict = {} finalDict['TotalRecords'] = len(records) finalDict['ParameterNames'] = paramNames + \ ['Total', 'PilotsPerJob', 'PilotJobEff', 'Status', 'InMask'] # Return all the records if maxItems == 0 or the specified number otherwise if maxItems: finalDict['Records'] = records[startItem:startItem + maxItems] else: finalDict['Records'] = records done = siteSumDict["Done"] empty = siteSumDict["Done_Empty"] aborted = siteSumDict["Aborted"] total = siteSumDict["Total"] # Add pilot submission efficiency evaluation if (done - empty) > 0: eff = float(done) / float(done - empty) elif done == 0: eff = 0. elif empty == done: eff = 99. else: eff = 0. siteSumDict['PilotsPerJob'] = '%.2f' % eff # Add pilot job efficiency evaluation if total > 0: eff = float(total - aborted) / float(total) * 100. else: eff = 100. siteSumDict['PilotJobEff'] = '%.2f' % eff # Evaluate the overall quality status if total > 100: if eff < 25.: siteSumDict['Status'] = 'Bad' elif eff < 60.: siteSumDict['Status'] = 'Poor' elif eff < 85.: siteSumDict['Status'] = 'Fair' else: siteSumDict['Status'] = 'Good' else: siteSumDict['Status'] = 'Idle' finalDict['Extras'] = siteSumDict return S_OK(finalDict)