def doQuery(self, jobtype='production', hours=12, region=None):
"""
<ul>
<li>jobtype - production,analysis, test
<li>hours - production,analysis, test
<li>VO - virtual organization atlas,cms
<li>computingSite
<li>processingType
<li>workingGroup
<li>prodUserName
<li>specialHandling
</ul>
"""
if int(hours) > 24 * 3:
hours = 24 * 3
title = 'Job summary for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = 'Job summary for the past %s hours' % hours
pars = self.extraValuesFilled()
self.publishTitle(title)
nav = ' <b>type</b>=%s <b>hours</b>=%s' % (jobtype, hours)
if pars != None:
for tp in [
'computingSite', 'processingType', 'workingGroup',
'prodUserName', 'specialHandling'
]:
if pars.get(tp) != None:
nav += ' <b>%s</b>=%s' % (tp, pars[tp])
modes = ('Defined', 'Waiting', 'Active', 'Archived')
rows = []
for mode in modes:
q = pmt.getCloudSummary(
jobtype,
pars.get('VO') if pars != None else None,
hours,
extraConditions=pars,
selectionFields=
'computingSite,processingType,workingGroup, prodUserName,specialHandling,modificationTime',
tablename='ATLAS_PANDA.jobs%s4' % mode)
if len(rows) == 0: header = q['header']
rows += q['rows']
# get getCurrentSiteData
pilotd = pmt.getCurrentSiteData()
### Build the jobsumd dictionary
jobsumd = {}
proctyped = {}
workgroupd = {}
userd = {}
spechandled = {}
item = self.clouds
if region != None:
item = [self._ALL] + self._sites4region.keys()
for c in item:
jobsumd[c] = {}
jobsumd[c][self._ALL] = [0] * self.lstates
if c != self._ALL:
jobsumd[c]['status'] = self.cloudList.get(
c, {'status': 'unknown'})['status']
iCloud = utils.name2Index(header, 'cloud')
iComputingSite = utils.name2Index(header, 'computingSite')
iStatus = utils.name2Index(header, 'jobStatus')
iProctype = utils.name2Index(header, 'processingType')
iSpechandle = utils.name2Index(header, 'specialHandling')
iWorkgroup = utils.name2Index(header, 'workingGroup')
iUser = utils.name2Index(header, 'prodUserName')
iCount = utils.name2Index(header, 'COUNT')
iTime = utils.name2Index(header, 'modificationTime')
iLatest = self._iLatest
def updateTime(oldt, newt):
ol = oldt[iLatest]
if (ol != 0 and ol < newt) or ol == 0: oldt[iLatest] = newt
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
regn = "unassigned"
site = "unassigned"
if region != None: continue
# continue
else:
regn = self._region4sites.get(site, 'unknown')
if region != None and regn == 'unknown': continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
if status == 'cancelled' and site == 'Unknown':
status = 'unassigned'
istatus = utils.name2Index(self.jobstates, status)
proctype = r[iProctype]
spechandle = r[iSpechandle]
workgroup = r[iWorkgroup]
user = r[iUser]
count = r[iCount]
time = r[iTime]
if region != None: cloud = regn
if not cloud in jobsumd:
print 'Unknown cloud: %s for site %s ' % (cloud, site)
continue
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0] * self.lstates
jobsumd[cloud][site][istatus] += count
jobsumd[cloud][self._ALL][istatus] += count
jobsumd[self._ALL][self._ALL][istatus] += count
updateTime(jobsumd[cloud][site], time)
updateTime(jobsumd[cloud][self._ALL], time)
updateTime(jobsumd[self._ALL][self._ALL], time)
proctyped[proctype] = proctyped.get(proctype, 0) + count
workgroupd[workgroup] = workgroupd.get(workgroup, 0) + count
userd[user] = userd.get(user, 0) + count
if spechandle != None:
spechandled[spechandle] = spechandled.get(spechandle,
0) + count
iFailed = self._iFailed
iFinished = self._iFinished
iRation = self._iRation
iPilots = self._iPilots
for cloud in jobsumd:
for site in jobsumd[cloud]:
if site == 'status': continue
allcloud = jobsumd[cloud][site]
ntot = allcloud[iFailed] + allcloud[iFinished]
psite = pilotd.get(site)
if psite:
nplt = pilotd[site]['updateJob'] + pilotd[site]['getJob']
jobsumd[cloud][site][iPilots] = nplt
jobsumd[cloud][self._ALL][iPilots] += nplt
jobsumd[self._ALL][self._ALL][iPilots] += nplt
if ntot > 0:
jobsumd[cloud][site][
iRation] = 100.0 * allcloud[iFailed] / ntot
## Get cloud/site status
main = {}
main['states'] = self.jobstates
main['jobsumd'] = jobsumd
if pars == None: pars = {}
if (hours != None):
pars['hours'] = hours
if (jobtype != None):
pars['jobtype'] = jobtype
if (region != None):
pars['region'] = region
nav += ' <b>region view</a>'
main['params'] = pars
self.publish(main)
self.publishNav(nav)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "monitor/%s.js" % "cloudsummary"),
role=pmRoles.script())
def doQuery(self,jobtype='production',hours=12,region=None):
"""
<ul>
<li>jobtype - production,analysis, test
<li>hours - production,analysis, test
<li>VO - virtual organization atlas,cms
<li>computingSite
<li>processingType
<li>workingGroup
<li>prodUserName
<li>specialHandling
</ul>
"""
if int(hours) > 24 * 3:
hours = 24*3
title = 'Job summary for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = 'Job summary for the past %s hours' % hours
pars = self.extraValuesFilled()
self.publishTitle(title)
nav = ' <b>type</b>=%s <b>hours</b>=%s' % (jobtype, hours )
if pars != None:
for tp in ['computingSite','processingType','workingGroup', 'prodUserName', 'specialHandling']:
if pars.get(tp) !=None:
nav += ' <b>%s</b>=%s' % (tp,pars[tp])
modes = ( 'Defined', 'Waiting', 'Active', 'Archived' )
rows = []
for mode in modes:
q = pmt.getCloudSummary(jobtype,pars.get('VO') if pars!=None else None,hours,extraConditions=pars,selectionFields='computingSite,processingType,workingGroup, prodUserName,specialHandling,modificationTime', tablename='ATLAS_PANDA.jobs%s4' % mode)
if len(rows) ==0: header = q['header']
rows += q['rows']
# get getCurrentSiteData
pilotd = pmt.getCurrentSiteData()
### Build the jobsumd dictionary
jobsumd = {}
proctyped = {}
workgroupd = {}
userd = {}
spechandled = {}
item = self.clouds
if region != None:
item = [self._ALL] + self._sites4region.keys()
for c in item:
jobsumd[c] = {}
jobsumd[c][self._ALL] = [0]*self.lstates
if c != self._ALL :
jobsumd[c]['status'] = self.cloudList.get(c,{'status':'unknown'})['status']
iCloud = utils.name2Index(header,'cloud')
iComputingSite = utils.name2Index(header,'computingSite')
iStatus = utils.name2Index(header,'jobStatus')
iProctype = utils.name2Index(header,'processingType')
iSpechandle = utils.name2Index(header,'specialHandling')
iWorkgroup = utils.name2Index(header,'workingGroup')
iUser = utils.name2Index(header,'prodUserName')
iCount = utils.name2Index(header,'COUNT')
iTime = utils.name2Index(header,'modificationTime')
iLatest = self._iLatest
def updateTime(oldt,newt):
ol = oldt[iLatest]
if (ol != 0 and ol<newt) or ol==0: oldt[iLatest] = newt
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
regn = "unassigned"
site="unassigned"
if region!= None: continue
# continue
else:
regn = self._region4sites.get(site,'unknown')
if region!= None and regn == 'unknown': continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
if status == 'cancelled' and site == 'Unknown': status = 'unassigned'
istatus = utils.name2Index(self.jobstates,status)
proctype = r[iProctype]
spechandle = r[iSpechandle]
workgroup = r[iWorkgroup]
user = r[iUser]
count = r[iCount]
time = r[iTime]
if region!=None: cloud = regn
if not cloud in jobsumd:
print 'Unknown cloud: %s for site %s ' % ( cloud, site)
continue
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0]*self.lstates
jobsumd[cloud][site][istatus] += count
jobsumd[cloud][self._ALL][istatus] += count
jobsumd[self._ALL][self._ALL][istatus] += count
updateTime(jobsumd[cloud][site], time)
updateTime(jobsumd[cloud][self._ALL], time)
updateTime(jobsumd[self._ALL][self._ALL], time)
proctyped[proctype] = proctyped.get(proctype,0) + count
workgroupd[workgroup] = workgroupd.get(workgroup,0) + count
userd[user] = userd.get(user,0) + count
if spechandle != None:
spechandled[spechandle] = spechandled.get(spechandle,0) + count
iFailed = self._iFailed
iFinished = self._iFinished
iRation = self._iRation
iPilots = self._iPilots
for cloud in jobsumd:
for site in jobsumd[cloud]:
if site == 'status': continue
allcloud = jobsumd[cloud][site]
ntot = allcloud[iFailed] + allcloud[iFinished]
psite = pilotd.get(site)
if psite:
nplt = pilotd[site]['updateJob']+ pilotd[site]['getJob']
jobsumd[cloud][site][iPilots] = nplt
jobsumd[cloud][self._ALL][iPilots] += nplt
jobsumd[self._ALL][self._ALL][iPilots] += nplt
if ntot >0:
jobsumd[cloud][site][iRation] = 100.0*allcloud[iFailed]/ntot
## Get cloud/site status
main = {}
main['states'] = self.jobstates
main['jobsumd'] = jobsumd
if pars == None: pars = {}
if (hours!=None):
pars['hours'] = hours
if (jobtype!=None):
pars['jobtype'] = jobtype
if (region!=None):
pars['region'] = region
nav += ' <b>region view</a>'
main['params'] = pars
self.publish(main)
self.publishNav(nav)
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "cloudsummary" ),role=pmRoles.script() )
def doQuery(self,jobtype='production,test',hours=12,cores=2,vo=None,computingSite=None,processingType=None,workingGroup=None,prodUserName=None,specialHandling=None,region=False):
if int(hours) > 24 * 3:
hours = 24 * 3
title = 'Job summary from the MultiCore sites for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = 'Job summary from the MultiCore sites for the last %s hours' % hours
self.publishTitle(title)
modes = ( 'Defined', 'Waiting', 'Active', 'Archived' )
rows = []
cores = cores if cores != None else 0
for mode in modes:
q = pmt.getCloudSummary(jobtype,vo,hours,selectionFields='computingSite,processingType,workingGroup, prodUserName,specialHandling,coreCount', tablename='ATLAS_PANDA.jobs%s4' % mode)
if len(rows) == 0: header = q['header']
rows += q['rows']
### Build the jobsumd dictionary
jobsumd = {}
proctyped = {}
workgroupd = {}
userd = {}
spechandled = {}
for c in self.clouds:
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0]*(self.lstates+1)
if c != 'ALL':
jobsumd[c]['status'] = self.cloudList[c]['status']
iCloud = utils.name2Index(header,'cloud')
print utils.lineInfo(), header
iComputingSite = utils.name2Index(header,'computingSite')
iStatus = utils.name2Index(header,'jobStatus')
iProctype = utils.name2Index(header,'processingType')
iSpechandle = utils.name2Index(header,'specialHandling')
iWorkgroup = utils.name2Index(header,'workingGroup')
iUser = utils.name2Index(header,'prodUserName')
iCore = utils.name2Index(header,'coreCount')
iCount = utils.name2Index(header,'COUNT')
for r in rows:
ncores = r[iCore] if r[iCore] != None else 0
if cores != 0 and ncores < cores: continue
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
if region: continue
site="Unknown"
# continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
istatus = utils.name2Index(self.jobstates,status)
proctype = r[iProctype]
spechandle = r[iSpechandle]
workgroup = r[iWorkgroup]
user = r[iUser]
count = r[iCount]
if not cloud in jobsumd:
print 'Unknown cloud: %s for site %s ' % ( cloud, site)
continue
cloud = 'ALL'
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0]*(self.lstates+1)
jobsumd[cloud][site][self.lstates] = ncores if ncores != None else 0
jobsumd[cloud][site][istatus] += count
if cloud != 'ALL': jobsumd[cloud]['ALL'][istatus] += count
jobsumd['ALL']['ALL'][istatus] += count
proctyped[proctype] = proctyped.get(proctype,0) + count
workgroupd[workgroup] = workgroupd.get(workgroup,0) + count
userd[user] = userd.get(user,0) + count
if spechandle != None:
spechandled[spechandle] = spechandled.get(spechandle,0) + count
## Get cloud/site status
main = {}
main['states'] = self.jobstates
main['jobsumd'] = jobsumd
main['params'] = { 'jobtype': jobtype,'hours': hours,'vo':vo
,'computingSite': computingSite
,'processingType': processingType
,'workingGroup':workingGroup
,'prodUserName':prodUserName
,'specialHandling':specialHandling
,'cores' : cores
,'region' : region}
self.publish(main)
self.publishNav(' type=%s hours=%s' % (jobtype, hours ) )
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "mcore" ),role="script")
def doQuery(self,
jobtype='production,test',
hours=12,
cores=2,
vo=None,
computingSite=None,
processingType=None,
workingGroup=None,
prodUserName=None,
specialHandling=None,
region=False):
if int(hours) > 24 * 3:
hours = 24 * 3
title = 'Job summary from the MultiCore sites for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = 'Job summary from the MultiCore sites for the last %s hours' % hours
self.publishTitle(title)
modes = ('Defined', 'Waiting', 'Active', 'Archived')
rows = []
cores = cores if cores != None else 0
for mode in modes:
q = pmt.getCloudSummary(
jobtype,
vo,
hours,
selectionFields=
'computingSite,processingType,workingGroup, prodUserName,specialHandling,coreCount',
tablename='ATLAS_PANDA.jobs%s4' % mode)
if len(rows) == 0: header = q['header']
rows += q['rows']
### Build the jobsumd dictionary
jobsumd = {}
proctyped = {}
workgroupd = {}
userd = {}
spechandled = {}
for c in self.clouds:
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0] * (self.lstates + 1)
if c != 'ALL':
jobsumd[c]['status'] = self.cloudList[c]['status']
iCloud = utils.name2Index(header, 'cloud')
print utils.lineInfo(), header
iComputingSite = utils.name2Index(header, 'computingSite')
iStatus = utils.name2Index(header, 'jobStatus')
iProctype = utils.name2Index(header, 'processingType')
iSpechandle = utils.name2Index(header, 'specialHandling')
iWorkgroup = utils.name2Index(header, 'workingGroup')
iUser = utils.name2Index(header, 'prodUserName')
iCore = utils.name2Index(header, 'coreCount')
iCount = utils.name2Index(header, 'COUNT')
for r in rows:
ncores = r[iCore] if r[iCore] != None else 0
if cores != 0 and ncores < cores: continue
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
if region: continue
site = "Unknown"
# continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
istatus = utils.name2Index(self.jobstates, status)
proctype = r[iProctype]
spechandle = r[iSpechandle]
workgroup = r[iWorkgroup]
user = r[iUser]
count = r[iCount]
if not cloud in jobsumd:
print 'Unknown cloud: %s for site %s ' % (cloud, site)
continue
cloud = 'ALL'
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0] * (self.lstates + 1)
jobsumd[cloud][site][
self.lstates] = ncores if ncores != None else 0
jobsumd[cloud][site][istatus] += count
if cloud != 'ALL': jobsumd[cloud]['ALL'][istatus] += count
jobsumd['ALL']['ALL'][istatus] += count
proctyped[proctype] = proctyped.get(proctype, 0) + count
workgroupd[workgroup] = workgroupd.get(workgroup, 0) + count
userd[user] = userd.get(user, 0) + count
if spechandle != None:
spechandled[spechandle] = spechandled.get(spechandle,
0) + count
## Get cloud/site status
main = {}
main['states'] = self.jobstates
main['jobsumd'] = jobsumd
main['params'] = {
'jobtype': jobtype,
'hours': hours,
'vo': vo,
'computingSite': computingSite,
'processingType': processingType,
'workingGroup': workingGroup,
'prodUserName': prodUserName,
'specialHandling': specialHandling,
'cores': cores,
'region': region
}
self.publish(main)
self.publishNav(' type=%s hours=%s' % (jobtype, hours))
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "monitor/%s.js" % "mcore"),
role="script")
