def _updatePilotSummary(self, sitesData):
result = RPCClient(
"WorkloadManagement/WMSAdministrator").getPilotSummary()
if not result['OK']:
gLogger.error("Cannot get the pilot summary", result['Message'])
return result
summaryData = result['Value']
result = getCESiteMapping()
if not result['OK']:
return S_ERROR('Could not get CE site mapping')
ceMapping = result['Value']
pilotSummary = {}
for ce in summaryData:
if ce not in ceMapping:
continue
ceData = summaryData[ce]
siteName = ceMapping[ce]
if siteName not in pilotSummary:
pilotSummary[siteName] = {}
for status in ceData:
if status not in pilotSummary[siteName]:
pilotSummary[siteName][status] = 0
pilotSummary[siteName][status] += ceData[status]
for site in sitesData:
if site in pilotSummary:
sitesData[site]['pilotSummary'] = pilotSummary[site]
return S_OK(sitesData)
def _updatePilotSummary( self, sitesData ):
result = RPCClient( "WorkloadManagement/WMSAdministrator" ).getPilotSummary()
if not result[ 'OK' ]:
gLogger.error( "Cannot get the pilot summary", result['Message'] )
return result
summaryData = result[ 'Value' ]
result = getCESiteMapping()
if not result['OK']:
return S_ERROR( 'Could not get CE site mapping' )
ceMapping = result[ 'Value' ]
pilotSummary = {}
for ce in summaryData:
if ce not in ceMapping:
continue
ceData = summaryData[ ce ]
siteName = ceMapping[ ce ]
if siteName not in pilotSummary:
pilotSummary[ siteName ] = {}
for status in ceData:
if status not in pilotSummary[ siteName ]:
pilotSummary[ siteName ][ status ] = 0
pilotSummary[ siteName ][ status ] += ceData[ status ]
for site in sitesData:
if site in pilotSummary:
sitesData[ site ][ 'pilotSummary' ] = pilotSummary[ site ]
return S_OK( sitesData )
def getARCPilotOutput( proxy, pilotRef ):
tmp_dir = mkdtemp()
myce = pilotRef.split(":")[1].strip("/")
gridEnv = getGridEnv()
mySite = getCESiteMapping()['Value'][myce]
workDB = gConfig.getValue(cfgPath('Resources/Sites/LCG', mySite, 'CEs', myce, 'JobListFile'))
myWorkDB = os.path.join("/opt/dirac/runit/WorkloadManagement/SiteDirector-RAL", workDB)
cmd = [ 'arcget' ]
cmd.extend( ['-k', '-c', myce, '-j', myWorkDB, '-D', tmp_dir, pilotRef] )
ret = executeGridCommand( proxy, cmd, gridEnv )
if not ret['OK']:
shutil.rmtree( tmp_dir )
return ret
status, output, error = ret['Value']
if 'Results stored at:' in output :
tmp_dir = os.path.join( tmp_dir, os.listdir( tmp_dir )[0] )
result = S_OK()
result['FileList'] = os.listdir( tmp_dir )
for filename in result['FileList']:
tmpname = os.path.join( tmp_dir, filename )
if os.path.exists( tmpname ):
myfile = file( tmpname, 'r' )
f = myfile.read()
myfile.close()
else :
f = ' '
if ".out" in filename:
filename = 'StdOut'
if ".err" in filename:
filename = 'StdErr'
result[filename] = f
shutil.rmtree( tmp_dir )
return result
if 'Warning: Job not found in job list' in output:
shutil.rmtree( tmp_dir )
message = "Pilot not yet visible in ARC dB"
return S_ERROR( message )
return S_ERROR("Sorry - requested pilot output not yet available")
def getPilotSummaryWeb(self,selectDict,sortList,startItem,maxItems):
""" Get summary of the pilot jobs status by CE/site in a standard structure
"""
stateNames = ['Submitted','Ready','Scheduled','Waiting','Running','Done','Aborted']
allStateNames = stateNames + ['Done_Empty','Aborted_Hour']
paramNames = ['Site','CE']+allStateNames
resultDict = {}
last_update = None
if selectDict.has_key('LastUpdateTime'):
last_update = selectDict['LastUpdateTime']
del selectDict['LastUpdateTime']
site_select = []
if selectDict.has_key('GridSite'):
site_select = selectDict['GridSite']
if type(site_select) != type([]):
site_select = [site_select]
del selectDict['GridSite']
status_select = []
if selectDict.has_key('Status'):
status_select = selectDict['Status']
if type(status_select) != type([]):
status_select = [status_select]
del selectDict['Status']
expand_site = ''
if selectDict.has_key('ExpandSite'):
expand_site = selectDict['ExpandSite']
site_select = [expand_site]
del selectDict['ExpandSite']
start = time.time()
# Get all the data from the database with various selections
result = self.getCounters('PilotAgents',
['GridSite','DestinationSite','Status'],
selectDict,newer=last_update,timeStamp='LastUpdateTime')
if not result['OK']:
return result
last_update = Time.dateTime() - Time.hour
selectDict['Status'] = 'Aborted'
resultHour = self.getCounters('PilotAgents',
['GridSite','DestinationSite','Status'],
selectDict,newer=last_update,timeStamp='LastUpdateTime')
if not resultHour['OK']:
return resultHour
last_update = Time.dateTime() - Time.day
selectDict['Status'] = ['Aborted','Done']
resultDay = self.getCounters('PilotAgents',
['GridSite','DestinationSite','Status'],
selectDict,newer=last_update,timeStamp='LastUpdateTime')
if not resultDay['OK']:
return resultDay
selectDict['CurrentJobID'] = 0
selectDict['Status'] = 'Done'
resultDayEmpty = self.getCounters('PilotAgents',
['GridSite','DestinationSite','Status'],
selectDict,newer=last_update,timeStamp='LastUpdateTime')
if not resultDayEmpty['OK']:
return resultDayEmpty
ceMap = {}
resMap = getCESiteMapping()
if resMap['OK']:
ceMap = resMap['Value']
# Sort out different counters
resultDict = {}
resultDict['Unknown']={}
for attDict,count in result['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce != "Multiple" and ceMap.has_key(ce):
site = ceMap[ce]
if not resultDict.has_key(site):
resultDict[site] = {}
if not resultDict[site].has_key(ce):
resultDict[site][ce] = {}
for p in allStateNames:
resultDict[site][ce][p] = 0
resultDict[site][ce][state] = count
for attDict,count in resultDay['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce):
site = ceMap[ce]
if state == "Done":
resultDict[site][ce]["Done"] = count
if state == "Aborted":
resultDict[site][ce]["Aborted"] = count
for attDict,count in resultDayEmpty['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce):
site = ceMap[ce]
if state == "Done":
resultDict[site][ce]["Done_Empty"] = count
for attDict,count in resultHour['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce):
site = ceMap[ce]
if state == "Aborted":
resultDict[site][ce]["Aborted_Hour"] = count
records = []
siteSumDict = {}
for site in resultDict:
sumDict = {}
for state in allStateNames:
if not sumDict.has_key(state):
sumDict[state] = 0
sumDict['Total'] = 0
for ce in resultDict[site]:
itemList = [site,ce]
total = 0
for state in allStateNames:
itemList.append(resultDict[site][ce][state])
sumDict[state] += resultDict[site][ce][state]
if state == "Done":
done = resultDict[site][ce][state]
if state == "Done_Empty":
empty = resultDict[site][ce][state]
if state == "Aborted":
aborted = resultDict[site][ce][state]
if state == "Aborted_Hour":
aborted_hour = resultDict[site][ce][state]
if state != "Aborted_Hour" and state != "Done_Empty":
total += resultDict[site][ce][state]
sumDict['Total'] += total
# Add the total number of pilots seen in the last day
itemList.append(total)
# Add pilot submission efficiency evaluation
if (done-empty) > 0:
eff = float(done)/float(done-empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
itemList.append('%.2f' % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = float(total-aborted)/float(total)*100.
else:
eff = 100.
itemList.append('%.2f' % eff)
# Evaluate the quality status of the CE
if total > 10:
if eff < 25.:
itemList.append('Bad')
elif eff < 60.:
itemList.append('Poor')
elif eff < 85.:
itemList.append('Fair')
else:
itemList.append('Good')
else:
itemList.append('Idle')
if len(resultDict[site]) == 1 or expand_site:
records.append(itemList)
if len(resultDict[site]) > 1 and not expand_site:
itemList = [site,'Multiple']
for state in allStateNames+['Total']:
if sumDict.has_key(state):
itemList.append(sumDict[state])
else:
itemList.append(0)
done = sumDict["Done"]
empty = sumDict["Done_Empty"]
aborted = sumDict["Aborted"]
aborted_hour = sumDict["Aborted_Hour"]
total = sumDict["Total"]
# Add pilot submission efficiency evaluation
if (done-empty) > 0:
eff = float(done)/float(done-empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
itemList.append('%.2f' % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = float(total-aborted)/float(total)*100.
else:
eff = 100.
itemList.append('%.2f' % eff)
# Evaluate the quality status of the Site
if total > 10:
if eff < 25.:
itemList.append('Bad')
elif eff < 60.:
itemList.append('Poor')
elif eff < 85.:
itemList.append('Fair')
else:
itemList.append('Good')
else:
itemList.append('Idle')
records.append(itemList)
for state in allStateNames+['Total']:
if not siteSumDict.has_key(state):
siteSumDict[state] = sumDict[state]
else:
siteSumDict[state] += sumDict[state]
# Perform site selection
if site_select:
new_records = []
for r in records:
if r[0] in site_select:
new_records.append(r)
records = new_records
# Perform status selection
if status_select:
new_records = []
for r in records:
if r[14] in status_select:
new_records.append(r)
records = new_records
# Get the Site Mask data
client = RPCClient('WorkloadManagement/WMSAdministrator')
result = client.getSiteMask()
if result['OK']:
siteMask = result['Value']
for r in records:
if r[0] in siteMask:
r.append('Yes')
else:
r.append('No')
else:
for r in records:
r.append('Unknown')
finalDict = {}
finalDict['TotalRecords'] = len(records)
finalDict['ParameterNames'] = paramNames+ \
['Total','PilotsPerJob','PilotJobEff','Status','InMask']
# Return all the records if maxItems == 0 or the specified number otherwise
if maxItems:
finalDict['Records'] = records[startItem:startItem+maxItems]
else:
finalDict['Records'] = records
done = siteSumDict["Done"]
empty = siteSumDict["Done_Empty"]
aborted = siteSumDict["Aborted"]
aborted_hour = siteSumDict["Aborted_Hour"]
total = siteSumDict["Total"]
# Add pilot submission efficiency evaluation
if (done-empty) > 0:
eff = float(done)/float(done-empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
siteSumDict['PilotsPerJob'] = '%.2f' % eff
# Add pilot job efficiency evaluation
if total > 0:
eff = float(total-aborted)/float(total)*100.
else:
eff = 100.
siteSumDict['PilotJobEff'] = '%.2f' % eff
# Evaluate the overall quality status
if total > 100:
if eff < 25.:
siteSumDict['Status'] = 'Bad'
elif eff < 60.:
siteSumDict['Status'] = 'Poor'
elif eff < 85.:
siteSumDict['Status'] = 'Fair'
else:
siteSumDict['Status'] = 'Good'
else:
siteSumDict['Status'] = 'Idle'
finalDict['Extras'] = siteSumDict
return S_OK(finalDict)
def getPilotSummaryWeb(self, selectDict, sortList, startItem, maxItems):
""" Get summary of the pilot jobs status by CE/site in a standard structure
"""
stateNames = [
'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', 'Done',
'Aborted', 'Failed'
]
allStateNames = stateNames + ['Done_Empty', 'Aborted_Hour']
paramNames = ['Site', 'CE'] + allStateNames
last_update = None
if 'LastUpdateTime' in selectDict:
last_update = selectDict['LastUpdateTime']
del selectDict['LastUpdateTime']
site_select = []
if 'GridSite' in selectDict:
site_select = selectDict['GridSite']
if not isinstance(site_select, type([])):
site_select = [site_select]
del selectDict['GridSite']
status_select = []
if 'Status' in selectDict:
status_select = selectDict['Status']
if not isinstance(status_select, type([])):
status_select = [status_select]
del selectDict['Status']
expand_site = ''
if 'ExpandSite' in selectDict:
expand_site = selectDict['ExpandSite']
site_select = [expand_site]
del selectDict['ExpandSite']
# Get all the data from the database with various selections
result = self.getCounters('PilotAgents',
['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not result['OK']:
return result
last_update = Time.dateTime() - Time.hour
selectDict['Status'] = 'Aborted'
resultHour = self.getCounters(
'PilotAgents', ['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not resultHour['OK']:
return resultHour
last_update = Time.dateTime() - Time.day
selectDict['Status'] = ['Aborted', 'Done']
resultDay = self.getCounters('PilotAgents',
['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not resultDay['OK']:
return resultDay
selectDict['CurrentJobID'] = 0
selectDict['Status'] = 'Done'
resultDayEmpty = self.getCounters(
'PilotAgents', ['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not resultDayEmpty['OK']:
return resultDayEmpty
ceMap = {}
resMap = getCESiteMapping()
if resMap['OK']:
ceMap = resMap['Value']
# Sort out different counters
resultDict = {}
resultDict['Unknown'] = {}
for attDict, count in result['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce != "Multiple" and ce in ceMap:
site = ceMap[ce]
if site not in resultDict:
resultDict[site] = {}
if ce not in resultDict[site]:
resultDict[site][ce] = {}
for p in allStateNames:
resultDict[site][ce][p] = 0
resultDict[site][ce][state] = count
for attDict, count in resultDay['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == "Done":
resultDict[site][ce]["Done"] = count
if state == "Aborted":
resultDict[site][ce]["Aborted"] = count
for attDict, count in resultDayEmpty['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == "Done":
resultDict[site][ce]["Done_Empty"] = count
for attDict, count in resultHour['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == "Aborted":
resultDict[site][ce]["Aborted_Hour"] = count
records = []
siteSumDict = {}
for site in resultDict:
sumDict = {}
for state in allStateNames:
if state not in sumDict:
sumDict[state] = 0
sumDict['Total'] = 0
for ce in resultDict[site]:
itemList = [site, ce]
total = 0
for state in allStateNames:
itemList.append(resultDict[site][ce][state])
sumDict[state] += resultDict[site][ce][state]
if state == "Done":
done = resultDict[site][ce][state]
if state == "Done_Empty":
empty = resultDict[site][ce][state]
if state == "Aborted":
aborted = resultDict[site][ce][state]
if state != "Aborted_Hour" and state != "Done_Empty":
total += resultDict[site][ce][state]
sumDict['Total'] += total
# Add the total number of pilots seen in the last day
itemList.append(total)
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = float(done) / float(done - empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
itemList.append('%.2f' % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = float(total - aborted) / float(total) * 100.
else:
eff = 100.
itemList.append('%.2f' % eff)
# Evaluate the quality status of the CE
if total > 10:
if eff < 25.:
itemList.append('Bad')
elif eff < 60.:
itemList.append('Poor')
elif eff < 85.:
itemList.append('Fair')
else:
itemList.append('Good')
else:
itemList.append('Idle')
if len(resultDict[site]) == 1 or expand_site:
records.append(itemList)
if len(resultDict[site]) > 1 and not expand_site:
itemList = [site, 'Multiple']
for state in allStateNames + ['Total']:
if state in sumDict:
itemList.append(sumDict[state])
else:
itemList.append(0)
done = sumDict["Done"]
empty = sumDict["Done_Empty"]
aborted = sumDict["Aborted"]
total = sumDict["Total"]
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = float(done) / float(done - empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
itemList.append('%.2f' % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = float(total - aborted) / float(total) * 100.
else:
eff = 100.
itemList.append('%.2f' % eff)
# Evaluate the quality status of the Site
if total > 10:
if eff < 25.:
itemList.append('Bad')
elif eff < 60.:
itemList.append('Poor')
elif eff < 85.:
itemList.append('Fair')
else:
itemList.append('Good')
else:
itemList.append('Idle')
records.append(itemList)
for state in allStateNames + ['Total']:
if state not in siteSumDict:
siteSumDict[state] = sumDict[state]
else:
siteSumDict[state] += sumDict[state]
# Perform site selection
if site_select:
new_records = []
for r in records:
if r[0] in site_select:
new_records.append(r)
records = new_records
# Perform status selection
if status_select:
new_records = []
for r in records:
if r[14] in status_select:
new_records.append(r)
records = new_records
# Get the Site Mask data
result = SiteStatus().getUsableSites()
if result['OK']:
siteMask = result['Value']
for r in records:
if r[0] in siteMask:
r.append('Yes')
else:
r.append('No')
else:
for r in records:
r.append('Unknown')
finalDict = {}
finalDict['TotalRecords'] = len(records)
finalDict['ParameterNames'] = paramNames + \
['Total', 'PilotsPerJob', 'PilotJobEff', 'Status', 'InMask']
# Return all the records if maxItems == 0 or the specified number otherwise
if maxItems:
finalDict['Records'] = records[startItem:startItem + maxItems]
else:
finalDict['Records'] = records
done = siteSumDict["Done"]
empty = siteSumDict["Done_Empty"]
aborted = siteSumDict["Aborted"]
total = siteSumDict["Total"]
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = float(done) / float(done - empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
siteSumDict['PilotsPerJob'] = '%.2f' % eff
# Add pilot job efficiency evaluation
if total > 0:
eff = float(total - aborted) / float(total) * 100.
else:
eff = 100.
siteSumDict['PilotJobEff'] = '%.2f' % eff
# Evaluate the overall quality status
if total > 100:
if eff < 25.:
siteSumDict['Status'] = 'Bad'
elif eff < 60.:
siteSumDict['Status'] = 'Poor'
elif eff < 85.:
siteSumDict['Status'] = 'Fair'
else:
siteSumDict['Status'] = 'Good'
else:
siteSumDict['Status'] = 'Idle'
finalDict['Extras'] = siteSumDict
return S_OK(finalDict)