def doQuery(self, jobtype='production', hours=12, region=None): """ <ul> <li>jobtype - production,analysis, test <li>hours - production,analysis, test <li>VO - virtual organization atlas,cms <li>computingSite <li>processingType <li>workingGroup <li>prodUserName <li>specialHandling </ul> """ if int(hours) > 24 * 3: hours = 24 * 3 title = 'Job summary for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours ) else: title = 'Job summary for the past %s hours' % hours pars = self.extraValuesFilled() self.publishTitle(title) nav = ' <b>type</b>=%s <b>hours</b>=%s' % (jobtype, hours) if pars != None: for tp in [ 'computingSite', 'processingType', 'workingGroup', 'prodUserName', 'specialHandling' ]: if pars.get(tp) != None: nav += ' <b>%s</b>=%s' % (tp, pars[tp]) modes = ('Defined', 'Waiting', 'Active', 'Archived') rows = [] for mode in modes: q = pmt.getCloudSummary( jobtype, pars.get('VO') if pars != None else None, hours, extraConditions=pars, selectionFields= 'computingSite,processingType,workingGroup, prodUserName,specialHandling,modificationTime', tablename='ATLAS_PANDA.jobs%s4' % mode) if len(rows) == 0: header = q['header'] rows += q['rows'] # get getCurrentSiteData pilotd = pmt.getCurrentSiteData() ### Build the jobsumd dictionary jobsumd = {} proctyped = {} workgroupd = {} userd = {} spechandled = {} item = self.clouds if region != None: item = [self._ALL] + self._sites4region.keys() for c in item: jobsumd[c] = {} jobsumd[c][self._ALL] = [0] * self.lstates if c != self._ALL: jobsumd[c]['status'] = self.cloudList.get( c, {'status': 'unknown'})['status'] iCloud = utils.name2Index(header, 'cloud') iComputingSite = utils.name2Index(header, 'computingSite') iStatus = utils.name2Index(header, 'jobStatus') iProctype = utils.name2Index(header, 'processingType') iSpechandle = utils.name2Index(header, 'specialHandling') iWorkgroup = utils.name2Index(header, 'workingGroup') iUser = utils.name2Index(header, 'prodUserName') iCount = utils.name2Index(header, 'COUNT') iTime = utils.name2Index(header, 'modificationTime') iLatest = self._iLatest def updateTime(oldt, newt): ol = oldt[iLatest] if (ol != 0 and ol < newt) or ol == 0: oldt[iLatest] = newt for r in rows: site = r[iComputingSite] if site == None: # print "Db record doesn't define any site: %s " % r regn = "unassigned" site = "unassigned" if region != None: continue # continue else: regn = self._region4sites.get(site, 'unknown') if region != None and regn == 'unknown': continue cloud = r[iCloud] # utils.getCloud4Site(r) status = r[iStatus] if status == 'cancelled' and site == 'Unknown': status = 'unassigned' istatus = utils.name2Index(self.jobstates, status) proctype = r[iProctype] spechandle = r[iSpechandle] workgroup = r[iWorkgroup] user = r[iUser] count = r[iCount] time = r[iTime] if region != None: cloud = regn if not cloud in jobsumd: print 'Unknown cloud: %s for site %s ' % (cloud, site) continue if not status in self.jobstates: print 'Unknown jobstate:', status continue if not site in jobsumd[cloud]: jobsumd[cloud][site] = {} jobsumd[cloud][site] = [0] * self.lstates jobsumd[cloud][site][istatus] += count jobsumd[cloud][self._ALL][istatus] += count jobsumd[self._ALL][self._ALL][istatus] += count updateTime(jobsumd[cloud][site], time) updateTime(jobsumd[cloud][self._ALL], time) updateTime(jobsumd[self._ALL][self._ALL], time) proctyped[proctype] = proctyped.get(proctype, 0) + count workgroupd[workgroup] = workgroupd.get(workgroup, 0) + count userd[user] = userd.get(user, 0) + count if spechandle != None: spechandled[spechandle] = spechandled.get(spechandle, 0) + count iFailed = self._iFailed iFinished = self._iFinished iRation = self._iRation iPilots = self._iPilots for cloud in jobsumd: for site in jobsumd[cloud]: if site == 'status': continue allcloud = jobsumd[cloud][site] ntot = allcloud[iFailed] + allcloud[iFinished] psite = pilotd.get(site) if psite: nplt = pilotd[site]['updateJob'] + pilotd[site]['getJob'] jobsumd[cloud][site][iPilots] = nplt jobsumd[cloud][self._ALL][iPilots] += nplt jobsumd[self._ALL][self._ALL][iPilots] += nplt if ntot > 0: jobsumd[cloud][site][ iRation] = 100.0 * allcloud[iFailed] / ntot ## Get cloud/site status main = {} main['states'] = self.jobstates main['jobsumd'] = jobsumd if pars == None: pars = {} if (hours != None): pars['hours'] = hours if (jobtype != None): pars['jobtype'] = jobtype if (region != None): pars['region'] = region nav += ' <b>region view</a>' main['params'] = pars self.publish(main) self.publishNav(nav) self.publish( "%s/%s" % (self.server().fileScriptURL(), "monitor/%s.js" % "cloudsummary"), role=pmRoles.script())
def doQuery(self,jobtype='production',hours=12,region=None): """ <ul> <li>jobtype - production,analysis, test <li>hours - production,analysis, test <li>VO - virtual organization atlas,cms <li>computingSite <li>processingType <li>workingGroup <li>prodUserName <li>specialHandling </ul> """ if int(hours) > 24 * 3: hours = 24*3 title = 'Job summary for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours ) else: title = 'Job summary for the past %s hours' % hours pars = self.extraValuesFilled() self.publishTitle(title) nav = ' <b>type</b>=%s <b>hours</b>=%s' % (jobtype, hours ) if pars != None: for tp in ['computingSite','processingType','workingGroup', 'prodUserName', 'specialHandling']: if pars.get(tp) !=None: nav += ' <b>%s</b>=%s' % (tp,pars[tp]) modes = ( 'Defined', 'Waiting', 'Active', 'Archived' ) rows = [] for mode in modes: q = pmt.getCloudSummary(jobtype,pars.get('VO') if pars!=None else None,hours,extraConditions=pars,selectionFields='computingSite,processingType,workingGroup, prodUserName,specialHandling,modificationTime', tablename='ATLAS_PANDA.jobs%s4' % mode) if len(rows) ==0: header = q['header'] rows += q['rows'] # get getCurrentSiteData pilotd = pmt.getCurrentSiteData() ### Build the jobsumd dictionary jobsumd = {} proctyped = {} workgroupd = {} userd = {} spechandled = {} item = self.clouds if region != None: item = [self._ALL] + self._sites4region.keys() for c in item: jobsumd[c] = {} jobsumd[c][self._ALL] = [0]*self.lstates if c != self._ALL : jobsumd[c]['status'] = self.cloudList.get(c,{'status':'unknown'})['status'] iCloud = utils.name2Index(header,'cloud') iComputingSite = utils.name2Index(header,'computingSite') iStatus = utils.name2Index(header,'jobStatus') iProctype = utils.name2Index(header,'processingType') iSpechandle = utils.name2Index(header,'specialHandling') iWorkgroup = utils.name2Index(header,'workingGroup') iUser = utils.name2Index(header,'prodUserName') iCount = utils.name2Index(header,'COUNT') iTime = utils.name2Index(header,'modificationTime') iLatest = self._iLatest def updateTime(oldt,newt): ol = oldt[iLatest] if (ol != 0 and ol<newt) or ol==0: oldt[iLatest] = newt for r in rows: site = r[iComputingSite] if site == None: # print "Db record doesn't define any site: %s " % r regn = "unassigned" site="unassigned" if region!= None: continue # continue else: regn = self._region4sites.get(site,'unknown') if region!= None and regn == 'unknown': continue cloud = r[iCloud] # utils.getCloud4Site(r) status = r[iStatus] if status == 'cancelled' and site == 'Unknown': status = 'unassigned' istatus = utils.name2Index(self.jobstates,status) proctype = r[iProctype] spechandle = r[iSpechandle] workgroup = r[iWorkgroup] user = r[iUser] count = r[iCount] time = r[iTime] if region!=None: cloud = regn if not cloud in jobsumd: print 'Unknown cloud: %s for site %s ' % ( cloud, site) continue if not status in self.jobstates: print 'Unknown jobstate:', status continue if not site in jobsumd[cloud]: jobsumd[cloud][site] = {} jobsumd[cloud][site] = [0]*self.lstates jobsumd[cloud][site][istatus] += count jobsumd[cloud][self._ALL][istatus] += count jobsumd[self._ALL][self._ALL][istatus] += count updateTime(jobsumd[cloud][site], time) updateTime(jobsumd[cloud][self._ALL], time) updateTime(jobsumd[self._ALL][self._ALL], time) proctyped[proctype] = proctyped.get(proctype,0) + count workgroupd[workgroup] = workgroupd.get(workgroup,0) + count userd[user] = userd.get(user,0) + count if spechandle != None: spechandled[spechandle] = spechandled.get(spechandle,0) + count iFailed = self._iFailed iFinished = self._iFinished iRation = self._iRation iPilots = self._iPilots for cloud in jobsumd: for site in jobsumd[cloud]: if site == 'status': continue allcloud = jobsumd[cloud][site] ntot = allcloud[iFailed] + allcloud[iFinished] psite = pilotd.get(site) if psite: nplt = pilotd[site]['updateJob']+ pilotd[site]['getJob'] jobsumd[cloud][site][iPilots] = nplt jobsumd[cloud][self._ALL][iPilots] += nplt jobsumd[self._ALL][self._ALL][iPilots] += nplt if ntot >0: jobsumd[cloud][site][iRation] = 100.0*allcloud[iFailed]/ntot ## Get cloud/site status main = {} main['states'] = self.jobstates main['jobsumd'] = jobsumd if pars == None: pars = {} if (hours!=None): pars['hours'] = hours if (jobtype!=None): pars['jobtype'] = jobtype if (region!=None): pars['region'] = region nav += ' <b>region view</a>' main['params'] = pars self.publish(main) self.publishNav(nav) self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "cloudsummary" ),role=pmRoles.script() )
def doQuery(self,jobtype='production,test',hours=12,cores=2,vo=None,computingSite=None,processingType=None,workingGroup=None,prodUserName=None,specialHandling=None,region=False): if int(hours) > 24 * 3: hours = 24 * 3 title = 'Job summary from the MultiCore sites for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours ) else: title = 'Job summary from the MultiCore sites for the last %s hours' % hours self.publishTitle(title) modes = ( 'Defined', 'Waiting', 'Active', 'Archived' ) rows = [] cores = cores if cores != None else 0 for mode in modes: q = pmt.getCloudSummary(jobtype,vo,hours,selectionFields='computingSite,processingType,workingGroup, prodUserName,specialHandling,coreCount', tablename='ATLAS_PANDA.jobs%s4' % mode) if len(rows) == 0: header = q['header'] rows += q['rows'] ### Build the jobsumd dictionary jobsumd = {} proctyped = {} workgroupd = {} userd = {} spechandled = {} for c in self.clouds: jobsumd[c] = {} jobsumd[c]['ALL'] = [0]*(self.lstates+1) if c != 'ALL': jobsumd[c]['status'] = self.cloudList[c]['status'] iCloud = utils.name2Index(header,'cloud') print utils.lineInfo(), header iComputingSite = utils.name2Index(header,'computingSite') iStatus = utils.name2Index(header,'jobStatus') iProctype = utils.name2Index(header,'processingType') iSpechandle = utils.name2Index(header,'specialHandling') iWorkgroup = utils.name2Index(header,'workingGroup') iUser = utils.name2Index(header,'prodUserName') iCore = utils.name2Index(header,'coreCount') iCount = utils.name2Index(header,'COUNT') for r in rows: ncores = r[iCore] if r[iCore] != None else 0 if cores != 0 and ncores < cores: continue site = r[iComputingSite] if site == None: # print "Db record doesn't define any site: %s " % r if region: continue site="Unknown" # continue cloud = r[iCloud] # utils.getCloud4Site(r) status = r[iStatus] istatus = utils.name2Index(self.jobstates,status) proctype = r[iProctype] spechandle = r[iSpechandle] workgroup = r[iWorkgroup] user = r[iUser] count = r[iCount] if not cloud in jobsumd: print 'Unknown cloud: %s for site %s ' % ( cloud, site) continue cloud = 'ALL' if not status in self.jobstates: print 'Unknown jobstate:', status continue if not site in jobsumd[cloud]: jobsumd[cloud][site] = {} jobsumd[cloud][site] = [0]*(self.lstates+1) jobsumd[cloud][site][self.lstates] = ncores if ncores != None else 0 jobsumd[cloud][site][istatus] += count if cloud != 'ALL': jobsumd[cloud]['ALL'][istatus] += count jobsumd['ALL']['ALL'][istatus] += count proctyped[proctype] = proctyped.get(proctype,0) + count workgroupd[workgroup] = workgroupd.get(workgroup,0) + count userd[user] = userd.get(user,0) + count if spechandle != None: spechandled[spechandle] = spechandled.get(spechandle,0) + count ## Get cloud/site status main = {} main['states'] = self.jobstates main['jobsumd'] = jobsumd main['params'] = { 'jobtype': jobtype,'hours': hours,'vo':vo ,'computingSite': computingSite ,'processingType': processingType ,'workingGroup':workingGroup ,'prodUserName':prodUserName ,'specialHandling':specialHandling ,'cores' : cores ,'region' : region} self.publish(main) self.publishNav(' type=%s hours=%s' % (jobtype, hours ) ) self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "mcore" ),role="script")
def doQuery(self, jobtype='production,test', hours=12, cores=2, vo=None, computingSite=None, processingType=None, workingGroup=None, prodUserName=None, specialHandling=None, region=False): if int(hours) > 24 * 3: hours = 24 * 3 title = 'Job summary from the MultiCore sites for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours ) else: title = 'Job summary from the MultiCore sites for the last %s hours' % hours self.publishTitle(title) modes = ('Defined', 'Waiting', 'Active', 'Archived') rows = [] cores = cores if cores != None else 0 for mode in modes: q = pmt.getCloudSummary( jobtype, vo, hours, selectionFields= 'computingSite,processingType,workingGroup, prodUserName,specialHandling,coreCount', tablename='ATLAS_PANDA.jobs%s4' % mode) if len(rows) == 0: header = q['header'] rows += q['rows'] ### Build the jobsumd dictionary jobsumd = {} proctyped = {} workgroupd = {} userd = {} spechandled = {} for c in self.clouds: jobsumd[c] = {} jobsumd[c]['ALL'] = [0] * (self.lstates + 1) if c != 'ALL': jobsumd[c]['status'] = self.cloudList[c]['status'] iCloud = utils.name2Index(header, 'cloud') print utils.lineInfo(), header iComputingSite = utils.name2Index(header, 'computingSite') iStatus = utils.name2Index(header, 'jobStatus') iProctype = utils.name2Index(header, 'processingType') iSpechandle = utils.name2Index(header, 'specialHandling') iWorkgroup = utils.name2Index(header, 'workingGroup') iUser = utils.name2Index(header, 'prodUserName') iCore = utils.name2Index(header, 'coreCount') iCount = utils.name2Index(header, 'COUNT') for r in rows: ncores = r[iCore] if r[iCore] != None else 0 if cores != 0 and ncores < cores: continue site = r[iComputingSite] if site == None: # print "Db record doesn't define any site: %s " % r if region: continue site = "Unknown" # continue cloud = r[iCloud] # utils.getCloud4Site(r) status = r[iStatus] istatus = utils.name2Index(self.jobstates, status) proctype = r[iProctype] spechandle = r[iSpechandle] workgroup = r[iWorkgroup] user = r[iUser] count = r[iCount] if not cloud in jobsumd: print 'Unknown cloud: %s for site %s ' % (cloud, site) continue cloud = 'ALL' if not status in self.jobstates: print 'Unknown jobstate:', status continue if not site in jobsumd[cloud]: jobsumd[cloud][site] = {} jobsumd[cloud][site] = [0] * (self.lstates + 1) jobsumd[cloud][site][ self.lstates] = ncores if ncores != None else 0 jobsumd[cloud][site][istatus] += count if cloud != 'ALL': jobsumd[cloud]['ALL'][istatus] += count jobsumd['ALL']['ALL'][istatus] += count proctyped[proctype] = proctyped.get(proctype, 0) + count workgroupd[workgroup] = workgroupd.get(workgroup, 0) + count userd[user] = userd.get(user, 0) + count if spechandle != None: spechandled[spechandle] = spechandled.get(spechandle, 0) + count ## Get cloud/site status main = {} main['states'] = self.jobstates main['jobsumd'] = jobsumd main['params'] = { 'jobtype': jobtype, 'hours': hours, 'vo': vo, 'computingSite': computingSite, 'processingType': processingType, 'workingGroup': workingGroup, 'prodUserName': prodUserName, 'specialHandling': specialHandling, 'cores': cores, 'region': region } self.publish(main) self.publishNav(' type=%s hours=%s' % (jobtype, hours)) self.publish( "%s/%s" % (self.server().fileScriptURL(), "monitor/%s.js" % "mcore"), role="script")