def doJson(self,lfn=None, jobs=None,type='input',ds=None,table='new',limit=1000,jobstatus=None,site=None,jobtype='production',days=1,user=None,select=None):
"""
Show the list of Panda id with LFN or versa verse LFNs by Panda IDs
<ul>
<li><code>lfn</code> - the list of the comma separated files
<li><code>ds</code> - the list of the comma separated datasets
<li><code>jobs</code> - the list of the comma separated Panda's job IDs
<li><code>table</code> = "new" (default) look up the records for last 3 days
<br> ="old" - look up the records those more than 3 days old (slow)
<br> ="deep" - look up the "old" and "new" tables (slow)
<li><code>type</code> - the type selector. <br>
= 'input - the default value<br>
= '*' | 'all' - list all types available.
<li><code>jobstatus</code> = the comma separated list of the job status to filter
<br>For example: 'defined, waiting,assigned,activated,sent,starting,running'
<li><code>site</code> = the comma separated list of the sizte to list the jobs from
<br> For example 'UTA_SWT2'
<li><code>jobtype</code> = the comma separated list of the job type to filter
<br> For example, "analysis, production"
<li><code>days</code> = the number of the days to look up the list of the jobs if either 'jobstatus' or 'site' parameter is defined
<li><code>user</code> = the comma separated list of the usernames .
<br>NB. The names may contain the the wildcard symbol '*'. Be aware the wildcard slows the search down
</ul>
"""
title = 'The list of files for the '
if jobstatus and jobstatus.strip() =='': jobstatus = None
if site and site.strip() =='': site = None
if lfn and lfn.strip() =='': lfn = None
if jobs and isinstance(jobs,str) and jobs.strip() =='': jobs = None
if ds and ds.strip() =='': ds=None
if type and type.strip() =='': type='all'
if lfn==None and jobs==None and ds==None and jobstatus==None and site==None:
self.publishTitle("Ambigios query: lfn=%(lfn)s; pandaid=%(pandaid)s either lfn or padaid can be defined. One can not define lfn and pandaid at once" % { 'lfn': lfn, 'pandaid' : jobs} )
self.publish('<h2>Check the input parameters. Click the "?" to see the API documentaion<h2>', role=pmRoles.html())
else:
nav = ''
if limit:
nav += "Limit %s rows." % limit
if type=='*' or type==None: type = 'all'
if lfn != None:
self.publishTitle("The list of the PANDA jobs with the LFN of the '%s' type provided" % type)
if not '*' in lfn: # disregard the jobtype parameter
if utils.isFilled(jobtype):
nav += " Disregarding the jobtype='%s' default parameter" % jobtype
jobtype = None
if ds != None:
self.publishTitle("The list of the PANDA jobs with the DATASETs of the '%s' type provided" % type)
if jobs!=None:
self.publishTitle("The list of the '%s' LFN with the PANDA Job IDs provided" % type)
if utils.isFilled(nav):
self.publishNav(nav)
main = {}
main["buffer"] = {}
main["buffer"]["method"] = "joblfn"
main["buffer"]["params"] = (lfn if lfn!=None else '',jobs if jobs!= None else '' )
if jobs != None: main["buffer"]["jobs"] = jobs
main["buffer"]["type"] = False
if (utils.isFilled(jobstatus) or utils.isFilled(site) or utils.isFilled(user)) and not utils.isFilled(jobs):
tables = ['atlas_panda.jobsArchived4','atlas_panda.jobsActive4','atlas_panda.jobsWaiting4','atlas_panda.jobsDefined4']
r = pmt.getJobIds(site, jobtype,jobstatus,table=tables,days=days,username=user)
jobs = [i[0] for i in r['rows']]
if not utils.isFilled(select):
select = []
if jobs == None or ( not isinstance(jobs,int) and len(jobs) > 1): select.append('pandaid');
select += ['type', 'lfn', 'fsize', 'dataset', 'guid', 'scope', 'destinationse']
else:
select = utils.parseArray(select);
main["buffer"]["data"] = pmt.getJobLFN(select=','.join(select),table=table,lfn=lfn,pandaid=jobs,type=type,ds=ds,limit=limit)
self.publish(main)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskBuffer/%s.js" % "getJobLFN"),role=pmRoles.script())
def doJson(self,table=None,column=None,db='pmt',doc=None):
"""
Describe the selected Panda table / column <br>
<code>table = the Db table name regexp comma-separated patterns to check</code><br>
<code>column = the Db table column name regexp comma-separated patterns to find</code><br>
<code>doc = '*|all' - document all columns selected if available<br>
= 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided
</code><br>
"""
title = 'Describe the Panda Db '
inp = False
if table != None and table != 'None':
title += "'%s*'" % table
inp = True
title +=" table"
else:
title +=" tables"
if column != None and column !='None':
title += " with '%s*' column " % column
inp = True
main = {}
if inp:
self.publishTitle(title)
timer = Stopwatch.Stopwatch()
main["buffer"] = {}
main["buffer"]["db"] = db
dbAccess = self._db.get(db)
if dbAccess != None:
main["buffer"]["method"] = "describe"
main["buffer"]["params"] = (table,column,doc)
main["buffer"]["type"] = False
if not utils.isFilled(doc):
tdes = dbAccess.describe(table,column)
iclmn = utils.name2Index(tdes['header'],'COLUMN_NAME')
rows = tdes['rows']
for h in rows:
h[iclmn] = nmap.get(h[iclmn])
main["buffer"]["data"] = tdes
else:
docrows = []
if utils.isFilled(table):
tdoc = dbAccess.pandatabs(table,comments=doc,select="table_name,'Table Description' as COLUMN_NAME,comments")
iclmn = utils.name2Index(tdoc['header'],'COLUMN_NAME')
docrows = tdoc['rows']
for h in docrows:
h[iclmn] = nmap.get(h[iclmn])
tcol = dbAccess.pandacols(column,table,select='TABLE_NAME,COLUMN_NAME,COMMENTS', comments=doc)
iclmn = utils.name2Index(tcol['header'],'COLUMN_NAME')
for h in tcol['rows']:
h[iclmn] = nmap.get(h[iclmn])
tcol['rows'] += docrows
main["buffer"]["data"] = tcol
self.publish( {'s-maxage':60000,'max-age': 6000}, role=pmRoles.cache())
else:
self.publishTitle("Error: 'describe' - unknown Db name '%s' has been provided" % db)
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
else:
self.publishTitle("Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'")
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
self.publish(main)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskBuffer/%s.js" % "describe"),role=pmRoles.script() )
def doJobs(
self,
pandaid=None,
params=None,
jobparam=None,
table=None,
hours=None,
tstart=None,
tend=None,
field="modificationTime",
days=None,
format=None,
jobtype=None,
testsite=None,
limit=1500,
summary=None,
minjob=1,
dump=None,
region=None,
lfn=None,
showcpu=None,
):
## Perform the query
main = {}
errorcolcheck = False
selection = "*"
vals = self.extraValuesFilled(self._alias)
title = "PanDA Job(s)"
if utils.isFilled(pandaid):
hours = None
days = None
title += " %s " % pandaid
if days != None:
title += " for %.2f Day" % float(days)
if days > 1:
title += "s"
if hours != None:
if days == None:
title += " for "
title += " %d Hour" % int(hours)
if hours > 1:
title += "s"
if vals != None:
for k in vals.keys():
if vals[k] == "undefined":
vals[k] = None
elif (
vals[k] != None
and k.lower() == "jobstatus"
and (
vals[k].lower().find("prerun") >= 0
or vals[k].lower().find("finishing") >= 0
or vals[k].lower().find("unassigned") >= 0
or vals[k].lower().find("retried") >= 0
)
):
stvals = utils.parseArray(vals[k])
statvals = []
for v in stvals:
if v.lower() == "prerun":
statvals += ["defined", "assigned", "waiting", "activated", "pending", "sent", "starting"]
elif v.lower() == "finishing":
statvals += ["holding", "transferring"]
elif v.lower() == "unassigned":
vals["computingSite"] = "NULL"
statvals += ["cancelled"]
elif v.lower() == "retried":
# See the slide 3 from Tadashi: https://indico.cern.ch/getFile.py/access?contribId=13&sessionId=11&resId=0&materialId=slides&confId=119171
vals["taskBufferErrorCode"] = [106, 107]
statvals += ["failed"]
else:
statvals += [v]
vals[k] = ",".join(statvals)
cleanVal = dict((k, v) for k, v in vals.iteritems() if v is not None)
vals = cleanVal
if vals == None:
vals = {}
nav = ""
certificate = None
certusr = None
try:
cert = self.server().certificate()
if cert != None:
certusr = pmt.cleanUserID(cert)
main["ceruser"] = certusr
except:
pass
if vals.get("prodUserName") == "auto" and certusr != None:
if cert != None:
vals["prodUserName"] = certusr
for v in vals:
nav += "<b>%s</b>=%s " % (v, vals[v])
# nav += "<span id=linkjobid stule='display:inline'></span>"
# nav +="<script>$(document).ready(function(){$('#linkjobid').html(utils().linkJobs(s.cloud,jobtype"
# nav += jobtype if jobtype!=None else "undefined"
# nav += ", s.jobStatus, "
# nav += "%d" % hours if hours!=None else "undefined"
# nav += ", s.computingSite,'Classic Page' ))});</script>";
if not utils.isFilled(summary):
nav += "<b>%s</b>=%s " % ("limit", limit)
self.publishNav(nav)
if len(vals) == 0:
vals = None
if vals == None and jobparam == None:
main["Description"] = self.description()
else:
if lfn != None and lfn != "undefined":
lfnselect = "pandaid,lfn"
lfnjobs = pmt.getJobLFN(select=lfnselect, table="new", lfn=lfn, pandaid=pandaid, type="*", limit=limit)
if lfnjobs["rows"] > 0:
pandaid = [ljobs[0] for ljobs in lfnjobs["rows"]]
selection = {}
if vals != None:
selection.update(vals)
if pandaid != None:
selection.update({nmap.get("pandaid"): pandaid})
if region != None:
selection.update({"region": region})
main["Selection"] = selection
conditions = vals
if conditions == None:
conditions = {}
cs = conditions.get("computingSite")
if cs == None:
site4region = self.getSite4Region(region)
if site4region != None and len(site4region) > 0:
conditions["computingSite"] = site4region
else:
pass # ignore the region parameter
if utils.isValid(summary):
jobtype = (
"production"
if jobtype == None
and jobparam.lower().find("taskid") >= 0
and jobparam.lower().find("jeditaskid") < 0
else jobtype
)
jobs = pmt.getJobsAtt(
pandaid,
jobparam,
conditions,
None,
hours,
tstart,
tend,
field,
days,
format,
jobtype,
testsite,
None,
True,
)
rows = jobs["rows"]
indx = []
if len(rows) > 30:
for i, r in enumerate(rows):
try:
if r[1] > minjob:
indx.append(i)
continue
except:
pass
cleanrows = [rows[i] for i in indx]
jobs["rows"] = cleanrows
istatus = utils.name2Index(jobs["header"], "jobstatus")
if istatus != None:
statOrder = utils.name2Index(utils.jobStates)
def byStatus(a, b):
return statOrder[a[istatus]] - statOrder[b[istatus]]
jobs["rows"] = sorted(jobs["rows"], cmp=byStatus)
else:
if jobparam == self._allparams and conditions.get(self._jobParametersName) == None:
conditions[self._jobParametersName] = "any"
jobs = pmt.getJobsAtt(
pandaid,
"PandaID," + jobparam,
conditions,
None,
hours,
tstart,
tend,
field,
days,
format,
jobtype,
testsite,
limit,
)
errorcolcheck = True
if len(jobs) > 0:
header = [nmap.get(h) for h in jobs["header"]]
info = jobs["rows"]
main["header"] = header # + self._extraFields
timecols = []
errdict = None
errorcol = not errorcolcheck
for i, h in enumerate(header):
if "time" in h.lower() and "cpuconsumptiontime" != h.lower():
timecols.append(i)
if not errorcol and "error" in h.lower():
errorcol = True
## # add an artificial retryID
## header += ['retryID']
headindx = utils.name2Index(header)
creationtDict = {}
histograms = {}
prodSourceLabel = headindx.get("prodSourceLabel")
jobStatus = headindx.get("jobStatus")
status4title = conditions.get("jobStatus")
if not utils.isFilled(status4title):
if len(info) == 1 and jobStatus != None:
status4title = info[0][jobStatus]
if utils.isFilled(status4title):
title = status4title[0].upper() + status4title[1:] + " " + title
prodtype = "production" in jobtype.lower() if jobtype != None else True
taskidsum = prodtype
if not utils.isValid(summary) and dump != "yes":
jsets = setutils.factory(header, info, self._errorCodes, self._siteId, self._siteNicks)
main["jobset"] = jsets.jobsets()
for r in info:
if not taskidsum and ((prodSourceLabel and r[prodSourceLabel] == "managed") and prodtype):
taskidsum = True
if errorcol and errorcolcheck:
errdict = self.getErrors(r, headindx, errdict)
for t in timecols:
r[t] = utils.epochTime(r[t])
## r += [None] # retryID
# row = self.jobsetTime(utils.zip(header(info[r])),creationtDict,histograms)
# info[r] = self.addFields(utils.zip(header(info[r])),creationtDict,histograms)
# Action cleanUserID SSL_CLIENT_S_DN
main["info"] = info
if errorcolcheck:
main["errorCodes"] = errdict
main["colNames"] = sorted(header)
main["jobsummary"] = copy.copy(self._jobSum)
if taskidsum:
main["jobsummary"] += [["Task ID", "taskID"]]
# print utils.lineInfo(), "===============", self._description.get("JEDITASKID"), self._description.get("JediTaskID")
# if self._description.get("JEDITASKID") or self._description.get("JediTaskID"):
# main['jobsummary'] += [['Jedi ID','JediTaskID']]
elif jobtype != "jedi":
main["jobsummary"] += [["Jobsets", "jobsetID"]] # need constrain
elif jobtype == "analysis":
main["jobsummary"] += [["Analysis Tasks", "destinationDBlock"]]
if showcpu == "yes":
main["jobsummary"] += [["CPU Type", "cpuConsumptionUnit"]]
self.publishTitle(title)
return main
def doJson(
self,
job=None,
taskid=None,
tstart=None,
tend=None,
hours=71,
days=None,
jobparam=None,
limit=1500,
summary=None,
minjob=1,
jobtype=None,
timeField="modificationTime",
plot="no",
dump=None,
tasktype=None,
status=None,
region=None,
lfn=None,
showcpu=None,
):
"""Display the job information:
<p>Sample queries:
<ul>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours='>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01</a>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours=&processingType=simul'>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01&[email protected]</a>
<li><a href='http://pandamon.cern.ch/jobinfo?job=1246240909'>http://pandamon.cern.ch/jobinfo?job=1246240909</a>
</ul>
<hr>
Supported query parameters are as follows:
<table>
<tr><td>taskid</td><td> - jobset / task id</td></tr>
<tr><td>job</td><td> - comma separated list of jobs' PandaID</td></tr>
<tr><td>jobtype</td><td> - select the 'jobtype' jobs: analysys, production, install, test </td></tr>
<tr><td>tstart=None</td><td> - start time </td></tr>
<tr><td>tend=None</td><td> - end time</td></tr>
<tr><td>hours=N </td><td> - last (N hours + N days) or N hours after 'tstart' or N hours before 'tend' </td></tr>
<tr><td>days=N</td><td> - last (N hours + N days) (see the hours parameter documentation</td></tr>
<tr><td>lfn=</td><td> - Select the PanDA jobs using the files from the comma-separated lfn list </td></tr>
<tr><td>jobparam=</td><td>The comma separated list of the <a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>job parameters</a>.<br> The default should satisfy the interactive users.<br>
One needs to customize jobparam to build his/her <a href='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#Custom_Monitoring_Client_Code'>own client</a></td></tr>
<tr><td>limit=</d><td> - max number of the Db records to fetch from each job Db table. Since there are 4 tables the real number of the rows can be 4 times as many as that limit</td></tr>
<tr><td><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</td><td>- comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.<br>
For example: <ul><li><a href='http://pandamon.cern.ch/jobinfo?processingType=simul'>http://pandamon.cern.ch/jobinfo?processingType=simul</a>
<li> <a href='http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test'>http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test</a></ul>
<tr><td>summary</td><td>Create the summary for the first member of the jobparam list<br>
For example: <a href='http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes'>http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes</a>
</td></tr>
<tr><td>dump</td><td> - 'yes' - produce the flat list of job info records suitable for CLI applications</td></tr>
<tr><td>region</td><td> - the comma separated list of the regions </td></tr>
<tr><td>showcpu</td><td> - 'yes' to show the CPU type used </td></tr>
</table>
"""
# % { 'url' : self.server().branchUrl() }
if job == "undefined":
job = None
if limit == "undefined":
limit = None
if summary == "undefined":
summary = None
if jobparam == "undefined":
jobparam = None
if tstart == "undefined":
tstart = None
if tend == "undefined":
tend = None
if tasktype == "prod":
jobtype = "production"
if jobtype == "null":
jobtype = None
if jobparam == None:
if job != None:
jobparam = self._allparams
else:
jobparam = ",".join(self._colNames)
if timeField == None or timeField == "undefined" or timeField == "":
timeField = "modificationTime"
if self.isValid(jobparam):
self.publish(
self.doJobs(
job,
tstart=tstart,
tend=tend,
hours=hours,
days=days,
jobparam=jobparam,
limit=limit,
field=timeField,
jobtype=jobtype,
summary=summary,
minjob=minjob,
dump=dump,
region=region,
lfn=lfn,
showcpu=showcpu,
)
)
self.publish("%s/%s" % (self.server().fileScriptURL(), "jobInfoRender.js"), role=pmRoles.script())
self.publish({"s-maxage": 600, "max-age": 600}, role=pmRoles.cache())
else:
title = "Job data "
if utils.isFilled(job):
hours = None
days = None
title += " %s " % job
if days != None:
title += " for %.2f day" % float(days)
if days > 1:
title += "s"
if hours != None:
if days == None:
title += " for "
title += " %d hour" % int(hours)
if hours > 1:
title += "s"
self.publishTitle(title)
def doJson(self, table=None, column=None, db='pmt', doc=None):
"""
Describe the selected Panda table / column <br>
<code>table = the Db table name regexp comma-separated patterns to check</code><br>
<code>column = the Db table column name regexp comma-separated patterns to find</code><br>
<code>doc = '*|all' - document all columns selected if available<br>
= 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided
</code><br>
"""
title = 'Describe the Panda Db '
inp = False
if table != None and table != 'None':
title += "'%s*'" % table
inp = True
title += " table"
else:
title += " tables"
if column != None and column != 'None':
title += " with '%s*' column " % column
inp = True
main = {}
if inp:
self.publishTitle(title)
timer = Stopwatch.Stopwatch()
main["buffer"] = {}
main["buffer"]["db"] = db
dbAccess = self._db.get(db)
if dbAccess != None:
main["buffer"]["method"] = "describe"
main["buffer"]["params"] = (table, column, doc)
main["buffer"]["type"] = False
if not utils.isFilled(doc):
tdes = dbAccess.describe(table, column)
iclmn = utils.name2Index(tdes['header'], 'COLUMN_NAME')
rows = tdes['rows']
for h in rows:
h[iclmn] = nmap.get(h[iclmn])
main["buffer"]["data"] = tdes
else:
docrows = []
if utils.isFilled(table):
tdoc = dbAccess.pandatabs(
table,
comments=doc,
select=
"table_name,'Table Description' as COLUMN_NAME,comments"
)
iclmn = utils.name2Index(tdoc['header'], 'COLUMN_NAME')
docrows = tdoc['rows']
for h in docrows:
h[iclmn] = nmap.get(h[iclmn])
tcol = dbAccess.pandacols(
column,
table,
select='TABLE_NAME,COLUMN_NAME,COMMENTS',
comments=doc)
iclmn = utils.name2Index(tcol['header'], 'COLUMN_NAME')
for h in tcol['rows']:
h[iclmn] = nmap.get(h[iclmn])
tcol['rows'] += docrows
main["buffer"]["data"] = tcol
self.publish({
's-maxage': 60000,
'max-age': 6000
},
role=pmRoles.cache())
else:
self.publishTitle(
"Error: 'describe' - unknown Db name '%s' has been provided"
% db)
self.publish({
's-maxage': 0,
'max-age': 0
},
role=pmRoles.cache())
else:
self.publishTitle(
"Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'"
)
self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache())
self.publish(main)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "taskBuffer/%s.js" % "describe"),
role=pmRoles.script())
def doJobs(self,
pandaid=None,
params=None,
jobparam=None,
table=None,
hours=None,
tstart=None,
tend=None,
field='modificationTime',
days=None,
format=None,
jobtype=None,
testsite=None,
limit=1500,
summary=None,
minjob=1,
dump=None,
region=None,
lfn=None,
showcpu=None):
## Perform the query
main = {}
errorcolcheck = False
selection = '*'
vals = self.extraValuesFilled(self._alias)
title = 'PanDA Job(s)'
if utils.isFilled(pandaid):
hours = None
days = None
title += " %s " % pandaid
if days != None:
title += ' for %.2f Day' % float(days)
if days > 1: title += "s"
if hours != None:
if days == None: title += " for "
title += ' %d Hour' % int(hours)
if hours > 1: title += "s"
if vals != None:
for k in vals.keys():
if vals[k] == 'undefined': vals[k] = None
elif vals[k] != None and k.lower() == 'jobstatus' and (
vals[k].lower().find('prerun') >= 0
or vals[k].lower().find('finishing') >= 0
or vals[k].lower().find('unassigned') >= 0
or vals[k].lower().find('retried') >= 0):
stvals = utils.parseArray(vals[k])
statvals = []
for v in stvals:
if v.lower() == 'prerun':
statvals += [
'defined', 'assigned', 'waiting', 'activated',
'pending', 'sent', 'starting'
]
elif v.lower() == 'finishing':
statvals += ['holding', 'transferring']
elif v.lower() == 'unassigned':
vals['computingSite'] = 'NULL'
statvals += ['cancelled']
elif v.lower() == 'retried':
# See the slide 3 from Tadashi: https://indico.cern.ch/getFile.py/access?contribId=13&sessionId=11&resId=0&materialId=slides&confId=119171
vals['taskBufferErrorCode'] = [106, 107]
statvals += ['failed']
else:
statvals += [v]
vals[k] = ','.join(statvals)
cleanVal = dict(
(k, v) for k, v in vals.iteritems() if v is not None)
vals = cleanVal
if vals == None: vals = {}
nav = ''
certificate = None
certusr = None
try:
cert = self.server().certificate()
if cert != None:
certusr = pmt.cleanUserID(cert)
main['ceruser'] = certusr
except:
pass
if vals.get('prodUserName') == 'auto' and certusr != None:
if cert != None: vals['prodUserName'] = certusr
for v in vals:
nav += "<b>%s</b>=%s " % (v, vals[v])
# nav += "<span id=linkjobid stule='display:inline'></span>"
# nav +="<script>$(document).ready(function(){$('#linkjobid').html(utils().linkJobs(s.cloud,jobtype"
# nav += jobtype if jobtype!=None else "undefined"
# nav += ", s.jobStatus, "
# nav += "%d" % hours if hours!=None else "undefined"
# nav += ", s.computingSite,'Classic Page' ))});</script>";
if not utils.isFilled(summary):
nav += "<b>%s</b>=%s " % ('limit', limit)
self.publishNav(nav)
if len(vals) == 0: vals = None
if vals == None and jobparam == None:
main['Description'] = self.description()
else:
if lfn != None and lfn != "undefined":
lfnselect = 'pandaid,lfn'
lfnjobs = pmt.getJobLFN(select=lfnselect,
table='new',
lfn=lfn,
pandaid=pandaid,
type='*',
limit=limit)
if lfnjobs['rows'] > 0:
pandaid = [ljobs[0] for ljobs in lfnjobs['rows']]
selection = {}
if vals != None: selection.update(vals)
if pandaid != None:
selection.update({nmap.get('pandaid'): pandaid})
if region != None: selection.update({'region': region})
main['Selection'] = selection
conditions = vals
if conditions == None: conditions = {}
cs = conditions.get('computingSite')
if cs == None:
site4region = self.getSite4Region(region)
if site4region != None and len(site4region) > 0:
conditions['computingSite'] = site4region
else:
pass # ignore the region parameter
if utils.isValid(summary):
jobtype = 'production' if jobtype == None and jobparam.lower(
).find('taskid') >= 0 and jobparam.lower().find(
'jeditaskid') < 0 else jobtype
jobs = pmt.getJobsAtt(pandaid, jobparam, conditions, None,
hours, tstart, tend, field, days, format,
jobtype, testsite, None, True)
rows = jobs['rows']
indx = []
if len(rows) > 30:
for i, r in enumerate(rows):
try:
if r[1] > minjob:
indx.append(i)
continue
except:
pass
cleanrows = [rows[i] for i in indx]
jobs['rows'] = cleanrows
istatus = utils.name2Index(jobs['header'], 'jobstatus')
if istatus != None:
statOrder = utils.name2Index(utils.jobStates)
def byStatus(a, b):
return statOrder[a[istatus]] - statOrder[b[istatus]]
jobs['rows'] = sorted(jobs['rows'], cmp=byStatus)
else:
if jobparam == self._allparams and conditions.get(
self._jobParametersName) == None:
conditions[self._jobParametersName] = 'any'
jobs = pmt.getJobsAtt(pandaid, 'PandaID,' + jobparam,
conditions, None, hours, tstart, tend,
field, days, format, jobtype, testsite,
limit)
errorcolcheck = True
if len(jobs) > 0:
header = [nmap.get(h) for h in jobs['header']]
info = jobs['rows']
main['header'] = header # + self._extraFields
timecols = []
errdict = None
errorcol = not errorcolcheck
for i, h in enumerate(header):
if 'time' in h.lower(
) and 'cpuconsumptiontime' != h.lower():
timecols.append(i)
if not errorcol and 'error' in h.lower():
errorcol = True
## # add an artificial retryID
## header += ['retryID']
headindx = utils.name2Index(header)
creationtDict = {}
histograms = {}
prodSourceLabel = headindx.get('prodSourceLabel')
jobStatus = headindx.get('jobStatus')
status4title = conditions.get('jobStatus')
if not utils.isFilled(status4title):
if len(info) == 1 and jobStatus != None:
status4title = info[0][jobStatus]
if utils.isFilled(status4title):
title = status4title[0].upper(
) + status4title[1:] + " " + title
prodtype = 'production' in jobtype.lower(
) if jobtype != None else True
taskidsum = prodtype
if not utils.isValid(summary) and dump != 'yes':
jsets = setutils.factory(header, info, self._errorCodes,
self._siteId, self._siteNicks)
main['jobset'] = jsets.jobsets()
for r in info:
if not taskidsum and (
(prodSourceLabel and r[prodSourceLabel] == 'managed')
and prodtype):
taskidsum = True
if errorcol and errorcolcheck:
errdict = self.getErrors(r, headindx, errdict)
for t in timecols:
r[t] = utils.epochTime(r[t])
## r += [None] # retryID
# row = self.jobsetTime(utils.zip(header(info[r])),creationtDict,histograms)
# info[r] = self.addFields(utils.zip(header(info[r])),creationtDict,histograms)
# Action cleanUserID SSL_CLIENT_S_DN
main['info'] = info
if errorcolcheck:
main['errorCodes'] = errdict
main['colNames'] = sorted(header)
main['jobsummary'] = copy.copy(self._jobSum)
if taskidsum:
main['jobsummary'] += [['Task ID', 'taskID']]
# print utils.lineInfo(), "===============", self._description.get("JEDITASKID"), self._description.get("JediTaskID")
# if self._description.get("JEDITASKID") or self._description.get("JediTaskID"):
# main['jobsummary'] += [['Jedi ID','JediTaskID']]
elif jobtype != 'jedi':
main['jobsummary'] += [['Jobsets',
'jobsetID']] # need constrain
elif jobtype == 'analysis':
main['jobsummary'] += [[
'Analysis Tasks', 'destinationDBlock'
]]
if showcpu == 'yes':
main['jobsummary'] += [['CPU Type', 'cpuConsumptionUnit']]
self.publishTitle(title)
return main
def doJson(self,
job=None,
taskid=None,
tstart=None,
tend=None,
hours=71,
days=None,
jobparam=None,
limit=1500,
summary=None,
minjob=1,
jobtype=None,
timeField='modificationTime',
plot='no',
dump=None,
tasktype=None,
status=None,
region=None,
lfn=None,
showcpu=None):
"""Display the job information:
<p>Sample queries:
<ul>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours='>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01</a>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours=&processingType=simul'>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01&[email protected]</a>
<li><a href='http://pandamon.cern.ch/jobinfo?job=1246240909'>http://pandamon.cern.ch/jobinfo?job=1246240909</a>
</ul>
<hr>
Supported query parameters are as follows:
<table>
<tr><td>taskid</td><td> - jobset / task id</td></tr>
<tr><td>job</td><td> - comma separated list of jobs' PandaID</td></tr>
<tr><td>jobtype</td><td> - select the 'jobtype' jobs: analysys, production, install, test </td></tr>
<tr><td>tstart=None</td><td> - start time </td></tr>
<tr><td>tend=None</td><td> - end time</td></tr>
<tr><td>hours=N </td><td> - last (N hours + N days) or N hours after 'tstart' or N hours before 'tend' </td></tr>
<tr><td>days=N</td><td> - last (N hours + N days) (see the hours parameter documentation</td></tr>
<tr><td>lfn=</td><td> - Select the PanDA jobs using the files from the comma-separated lfn list </td></tr>
<tr><td>jobparam=</td><td>The comma separated list of the <a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>job parameters</a>.<br> The default should satisfy the interactive users.<br>
One needs to customize jobparam to build his/her <a href='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#Custom_Monitoring_Client_Code'>own client</a></td></tr>
<tr><td>limit=</d><td> - max number of the Db records to fetch from each job Db table. Since there are 4 tables the real number of the rows can be 4 times as many as that limit</td></tr>
<tr><td><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</td><td>- comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.<br>
For example: <ul><li><a href='http://pandamon.cern.ch/jobinfo?processingType=simul'>http://pandamon.cern.ch/jobinfo?processingType=simul</a>
<li> <a href='http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test'>http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test</a></ul>
<tr><td>summary</td><td>Create the summary for the first member of the jobparam list<br>
For example: <a href='http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes'>http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes</a>
</td></tr>
<tr><td>dump</td><td> - 'yes' - produce the flat list of job info records suitable for CLI applications</td></tr>
<tr><td>region</td><td> - the comma separated list of the regions </td></tr>
<tr><td>showcpu</td><td> - 'yes' to show the CPU type used </td></tr>
</table>
"""
# % { 'url' : self.server().branchUrl() }
if job == 'undefined': job = None
if limit == 'undefined': limit = None
if summary == 'undefined': summary = None
if jobparam == 'undefined': jobparam = None
if tstart == 'undefined': tstart = None
if tend == 'undefined': tend = None
if tasktype == 'prod': jobtype = 'production'
if jobtype == 'null': jobtype = None
if jobparam == None:
if job != None:
jobparam = self._allparams
else:
jobparam = ','.join(self._colNames)
if timeField == None or timeField == 'undefined' or timeField == '':
timeField = 'modificationTime'
if self.isValid(jobparam):
self.publish(
self.doJobs(job,
tstart=tstart,
tend=tend,
hours=hours,
days=days,
jobparam=jobparam,
limit=limit,
field=timeField,
jobtype=jobtype,
summary=summary,
minjob=minjob,
dump=dump,
region=region,
lfn=lfn,
showcpu=showcpu))
self.publish("%s/%s" %
(self.server().fileScriptURL(), "jobInfoRender.js"),
role=pmRoles.script())
self.publish({
's-maxage': 600,
'max-age': 600
},
role=pmRoles.cache())
else:
title = 'Job data '
if utils.isFilled(job):
hours = None
days = None
title += " %s " % job
if days != None:
title += ' for %.2f day' % float(days)
if days > 1: title += "s"
if hours != None:
if days == None: title += " for "
title += ' %d hour' % int(hours)
if hours > 1: title += "s"
self.publishTitle(title)