def doQuery(self,
me='World',
title='PanDA Hello World Example',
cache=600):
""" The simplest Panda Web Modulde to provide the UI as follows:<br>
the URL: 'http:<hostname>/hello' should return the page reading <b>"Hello World !!!"</b> <br>
the URL: 'http:<hostname>/hello?me=All should return the page reading <b>"Hello All !!!"</b><br>
the URL: 'http:<hostname>/hello?me=All&title='Ciao Panda Example' should return the page reading <b>"Hello All !!!</b>"<br>
<p> Arguments:
<ul>
<li>me -- the name of the person to greet, otherwise 'World'
<li>title -- the page title, otherwise 'PanDA Hello World Example'
</ul>
"""
user = self.server().user()
if user:
if me == user:
me = "<h2>Hello %s%s !!! </h2>" % (me[0].upper(), me[1:])
title = "<pre> %s \n</pre><br>Thank you for using the secure communication" % title
else:
me = " <b>Sorry, %s</b>. You entered the wrong name. Please, Try again" % user
title = "Wrong name was used"
else:
me = "Hello unknown guy . Please use the secure connection to communicate me"
title = 'PanDA Hello World Example'
self.publishPage(main="%s" % me, title=title)
self.publish(timedelta(seconds=348), role=pmRoles.cache())
self.publish({
's-maxage': cache,
'max-age': cache
},
role=pmRoles.cache())
def doJson(self, method=None, undoc=False):
"""
List of All <a href="https://svnweb.cern.ch/trac/panda/browser/panda-server/current/pandaserver/taskbuffer/TaskBuffer.py"> TaskBuffer Class </a>Methods
"""
self._doc = None
usr = self.server().user()
if usr == None:
self.publishTitle(
'Only the known user can list All TaskBuffer methods. Please, use https protocol'
)
self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache())
self.publish({})
else:
if not "fine" in usr.lower() and not "wenaus" in usr.lower(
) and undoc:
self.publishTitle(
'List of All TaskBuffer Methods for the User <%s> You can not see the undocumented methods though'
% usr)
undoc = False
else:
self.publishTitle(
'List of All TaskBuffer Methods for the User <%s> '
% usr)
self.doMain(undoc, method)
self.publish({
's-maxage': 90000,
'max-age': 90000
},
role=pmRoles.cache())
def doJson(self, script=None):
"""
doJson(self,script=None) defines the Web API list of parametes and its default values
<p>This Python DocString is to automatically published by Web Server the Web API Help widget
The Web Browser user will be able to access it by clicking the "?" button onto the page menu bar.
The string may contain the regular HTML markups.
script - the file name to download and execute . <br>
It should contain the Javascript function `
<pre>
function _fun_(tag,content) { . . . . }
</pre>
"""
self.publishTitle('Hello JavaScript') # short cut for
self.publishNav("Load and Execute the '%s' Javascript " %
(script if script != None else 'Embedded'))
if script != None:
""" publish the reference to the external JavaScript if any """
self.publish("%s/%s" %
(self.server().fileScriptURL(), "%s.js" % script),
role=pmRoles.script())
""" Create the custom content to be published """
content = {
"Hello":
"This is my own content to be published from %s " %
utils.lineInfo()
}
self.publish(content)
self.publish({'s-maxage': 340, 'max-age': 200}, role=pmRoles.cache())
def doJson(self,hours=6):
"""
Select Parameters For Task Request
"""
self.publishTitle(' Select Parameters For Task Request')
self.doMain(hours)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskdef/%s.js" % 'reqtask1' ),role="script")
self.publish( {'s-maxage':600,'max-age': 600}, role=pmRoles.cache())
def doJson(self,method=None,undoc=False):
"""
List of All <a href="https://svnweb.cern.ch/trac/panda/browser/panda-server/current/pandaserver/taskbuffer/TaskBuffer.py"> TaskBuffer Class </a>Methods
"""
self._doc = None
usr = self.server().user()
if usr == None:
self.publishTitle('Only the known user can list All TaskBuffer methods. Please, use https protocol')
self.publish({'s-maxage':0,'max-age': 0 }, role=pmRoles.cache())
self.publish( {} )
else:
if not "fine" in usr.lower() and not "wenaus" in usr.lower() and undoc:
self.publishTitle('List of All TaskBuffer Methods for the User <%s> You can not see the undocumented methods though' % usr)
undoc=False
else:
self.publishTitle('List of All TaskBuffer Methods for the User <%s> ' % usr)
self.doMain(undoc,method)
self.publish({'s-maxage':90000,'max-age': 90000 }, role=pmRoles.cache())
def doJson(self,
pandaid,
message=None,
host=None,
timestamp=None,
errorcode=None,
test=None):
""" Contact Us Panda Monitor Modules """
usr = self.server().user()
pandaid = utils.parseArray(pandaid)
plid = len(pandaid)
main = {}
if plid >= 1: main['pandaid'] = pandaid[0]
if plid >= 2:
main['taskid'] = pandaid[1]
if plid == 2 and usr != None:
pandaid.append(usr)
plid = len(pandaid)
if plid >= 3:
pandaid[2] = pmt.cleanUserID(pandaid[2]).replace(' ', '+')
main['user'] = pandaid[2]
if usr != None:
mail = self.server().email()
if mail == None and test != None:
mail = '*****@*****.**'
main['username'] = usr
if mail != None:
main['mail'] = mail
msg = self.message(pandaid, message, host, timestamp,
errorcode)
msg += self.signature()
subject = self.subject(pandaid)
toaddrs = ''
if test == None:
toaddrs = '[email protected] [email protected] '
# toaddrs+='%s %s %s' % ('*****@*****.**', '*****@*****.**', mail)
toaddrs += '%s %s' % ('*****@*****.**', mail)
main['message'] = msg
main[
'subject'] = subject if test == None else 'Test: %s ' % subject
main['toaddrs'] = toaddrs
response = pmt.sendmail(msg, subject, toaddrs, fromaddr=mail)
main['response'] = response
self.publishTitle("User %s sent support mail from %s" %
(usr, mail))
else:
self.publishTitle("User %s wanted to send the support mail " %
usr)
self.publish(main)
else:
self.publishTitle(
"Unknown user, please use the Web https protocol.")
self.publish(main)
self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache())
def doJson(self, hours=6):
"""
Select Parameters For Task Request
"""
self.publishTitle(' Select Parameters For Task Request')
self.doMain(hours)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "taskdef/%s.js" % 'reqtask1'),
role="script")
self.publish({'s-maxage': 600, 'max-age': 600}, role=pmRoles.cache())
def doQuery(self,config=None):
""" Process the query request
<ul>
<li>Click <a href="http://pandamon.xxxx.xx/alist">here to see the <b>"List of all available Monitor Modules"</b></a>
</ul>
"""
title = 'PanDA monitor ( new )'
m = self.frontPageNew() + self.frontPageClassic()
m= self.doTab()
self.publishPage(title=title,main=m )
# self.publish(self.leftMenu(),"menu",role=pmRoles.html())
self.publish({'s-maxage': 999999,'max-age':1800},role=pmRoles.cache())
def doJson(self,
release=None,
cloud=None,
site=None,
cache=None,
cmt=None):
"""
Release information displays <br><ul>
<li><code>release = release comma-separated regex name pattern </code>
<li><code>cloud = cloud abbreviation to narrow the search </code>
<li><code>site = siteid to select the release info for that Panda site</code>
<li><code>cache = cache name </code>
<li><code>cmt = cmtconfig comma-separated regex pattern </code>
</ul>
"""
if release != None and site == None:
title = "Sites holding release %s" % release
else:
title = "Release availability "
if site != None: title += " from site: %s" % site
elif cloud != None:
if release == None: title += " at sites "
title += " from cloud: %s" % cloud
else:
title += " at sites"
if cache != None: title += ", cache %s" % cache
self.publishTitle(title)
q = pmt.getReleases(release, cloud, site, cache, cmt)
header = q['header']
rows = q['rows']
main = {}
main['header'] = header if len(rows) > 0 else []
main['info'] = self.compress(header, rows, cache == "None")
main['time'] = {}
main['params'] = {
'release': release,
'cloud': cloud,
'site': site,
'cache': cache
}
self.publish(main)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "monitor/%s.js" % "releaseinfo"),
role=pmRoles.script())
self.publish({'s-maxage': 3600, 'max-age': 3600}, role=pmRoles.cache())
def doJson(self,src='http://ivukotic.web.cern.ch/ivukotic/FAX',date='2012-10-16',selS='ALL',script='fax/map'):
""" FAX configuration viewer
"""
self.reset()
params = "from=%s" % date
if selS != None: params += '&selS=%s' % selS
self.fetchInfo(src,'getLinks',params)
if len(self._sites) == 0:
self.fetchInfo(src,'getSites')
main = {'sites': self._sites, 'links': self._links,'url': src }
self.publish(main)
self.publishTitle('FAX configuration viewer')
self.publishNav(' %s for "%s" ' % ( script, params ) )
self.publish( "%s/%s" % (self.server().fileScriptURL(),"%s.js" % script ),role=pmRoles.script())
self.publish({'s-maxage':340,'max-age': 200},role=pmRoles.cache())
def doJson(self):
"""
Show the non-zero MAXTIME values from schedconfig for each site
( see <a href='http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all'>AGIS schedconf</a> also )
"""
self.publishTitle(
'The schedconfig MAXTIME <font size=-2> ( see <a href="http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json&preset=schedconf.all">AGIS schedconf</a> also)</font>'
)
timer = Stopwatch.Stopwatch()
self.doMain()
self.publishNav('The maxtime from schedconfig: "%s"' % (timer))
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "taskBuffer/%s.js" % 'tbTable'),
role="script")
self.publish({'s-maxage': 600, 'max-age': 600}, role=pmRoles.cache())
def doJson(self,
jobtype='analysis',
timefield='MODIFICATIONTIME',
tstart=None,
tend=None,
days=None,
width=600,
height=400,
options='',
plot=False,
log=False,
file='useract365daysTotal'):
"""
User activity plots <ul>
<li>jobtype - name of the ROOT histogram
<li>file - name the file to get the data from rather from Db
<li>width - the width of the plots in px
<li>height - the height of the plots in px
<li>plot - = 'True' to create the plot; 'False' to use the table view render the data
<li>log - use logarithmic scale for Y-axis
<li>options - some ROOT <a href='http://root.cern.ch/root/html534/THistPainter.html#HP01b'>Paint options</a> or <br>
'M' = to get the "per month" rather "per day" data
</ul>
"""
self.publishTitle('Panda User activities for %(jobtype)s jobs' %
{'jobtype': jobtype})
timer = Stopwatch.Stopwatch()
select = "to_char(MODIFICATIONTIME ,'YYMMDD') as time,PRODUSERID"
if 'M' in options:
select = "to_char(MODIFICATIONTIME ,'YYMM') as time,PRODUSERID"
self.doMain(jobtype, select, timefield, tstart, tend, days, width,
height, options, plot, log, file)
self.publishNav('The User Activities for %s from CERN ( %s)' %
(jobtype, timer))
render = "taskBuffer/%s.js" % (
"tbTable") if not plot else "hello/%s.js" % "pyroot"
self.publish("%s/%s" % (self.server().fileScriptURL(), render),
role=pmRoles.script())
self.publish({
's-maxage': 86400,
'max-age': 86400
},
role=pmRoles.cache())
def doJson(self, pandaid, message=None,host=None,timestamp=None,errorcode=None,test=None):
""" Contact Us Panda Monitor Modules """
usr = self.server().user()
pandaid = utils.parseArray(pandaid)
plid = len(pandaid)
main = {}
if plid >= 1: main['pandaid'] = pandaid[0]
if plid >= 2:
main['taskid'] = pandaid[1]
if plid == 2 and usr != None:
pandaid.append(usr)
plid = len(pandaid)
if plid >= 3:
pandaid[2] = pmt.cleanUserID(pandaid[2]).replace(' ','+')
main['user'] = pandaid[2]
if usr != None:
mail = self.server().email()
if mail == None and test != None:
mail = '*****@*****.**'
main['username']= usr
if mail != None:
main['mail']= mail
msg = self.message(pandaid, message, host,timestamp,errorcode);
msg += self.signature()
subject = self.subject(pandaid)
toaddrs = ''
if test == None: toaddrs = '[email protected] [email protected] '
# toaddrs+='%s %s %s' % ('*****@*****.**', '*****@*****.**', mail)
toaddrs+='%s %s' % ( '*****@*****.**', mail)
main['message'] = msg;
main['subject'] = subject if test==None else 'Test: %s ' % subject
main['toaddrs'] = toaddrs
response = pmt.sendmail(msg,subject,toaddrs,fromaddr=mail)
main['response'] = response
self.publishTitle("User %s sent support mail from %s" % (usr,mail) )
else:
self.publishTitle("User %s wanted to send the support mail " % usr )
self.publish(main)
else:
self.publishTitle("Unknown user, please use the Web https protocol.")
self.publish(main)
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
def doJson(self, hello=None,pattern=None):
"""
List the Panda Monutor Modules:
<ul>
<li><code>hello</code> = "yes" to list the hello appplication
<li><code>pattern</code> = regexp pattern to list the matched modules only
</ul>
"""
modulesDir = os.path.dirname(utils.lineInfo(False,'%(filename)s'))
pat = None
if pattern != None: pat = re.compile(pattern)
filtered = self.listFiles(modulesDir,hello,pat)
# htmlList='<IMG SRC="http://www.mikesfreegifs.com/main4/underconstruction/coolconstr.gif" ALIGN=LEFT BORDER=0 HSPACE=10>'
htmlList = "<table><thead><tr><th>#</th><th>Module Name</th><th>Module Description</th><th>SVN rev</th></tr></thead><tbody>"
usr = self.server().user()
version = self.server().version().get('version')
if version == None: version = ''
else: version = "/~%s" % version
script = version
i = 0
for f in sorted(filtered):
if f == 'taskBufferList' and usr == None:
continue
else:
try:
object = pmModule.factory(f.replace(".py",""),fileonly=True);
doc = object.__doc__
if doc == None: doc = "The module is under development. It has not been documented yet"
i = i+1
htmlList += "<tr><td>%d</td><td>" % i
htmlList +="<a title='Click to start the %(module)s application and see its API doc' href='%(version)s/%(module)s'>%(module)s</a>\n" % {'version':version, 'module':f}
htmlList +="</td><td>%s</td></tr>" % doc
except:
# raise
# htmlList += "<tr><td>-</td><td>"
# htmlList +="<a title='Click to start the %(module)s application and see its API doc' href='%{script}s/%(module)s'>%(module)s</a>\n" % {'script':script, 'module':f}
# htmlList +="</td><td>%s</td></tr>" % f.replace(".py","").replace('/','.')
pass
htmlList += "</tbody></table>"
self.publishTitle("The list of the <a href='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API'>Panda Monitor Modules</a>")
self.publishMain(htmlList)
self.publish( {'s-maxage':9000,'max-age': 9000}, role=pmRoles.cache())
def doJson(self,scope=None,doc=None,code=None):
"""
List the <a href='https://twiki.cern.ch/twiki/bin/viewauth/AtlasComputing/PandaErrorCodes'>Panda error codes</a>
<ui>
<li> scope - scope: 'pilot','ddm' 'transExit','jobDispatcher',list the tasks requested for the last hours
<li> doc - pattern to filter the the code descriptions
<li> code - code
</ul>
"""
if scope == None:
self.publishTitle('%s Codes' % 'PanDA Error')
else:
self.publishTitle('%s Codes' % scope)
if scope == None: self.publish(self._errors)
else:
if not scope in self._errorFields:
raise KeyError(" Unknown scope %s . The list of the valid scopes: %s " %(scope, self._errorFields))
selescope = None
if scope != 'transExit':
searchScope = "%sErrorCode" % scope
else:
searchScope = "%sCode" % scope
for j in self._errors:
if j != searchScope: continue
selescope = {j:self._errors[j]}
break
result = { 'unknown':[ [-1,'unknown','unknown'] ]}
if selescope != None:
if code == None:
result = selescope
else:
for cd in selescope:
codelines = selescope[cd]
for cordsc in codelines:
if code == cordsc[0]:
result = { cd : [cordsc] }
break
self.publish(result)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"monitor/%s.js" % 'errorcodes' ),role=pmRoles.script())
self.publish( {'s-maxage':6000,'max-age': 6000}, role=pmRoles.cache())
def doJson(self,
src='http://ivukotic.web.cern.ch/ivukotic/FAX',
date='2012-10-16',
selS='ALL',
script='fax/map'):
""" FAX configuration viewer
"""
self.reset()
params = "from=%s" % date
if selS != None: params += '&selS=%s' % selS
self.fetchInfo(src, 'getLinks', params)
if len(self._sites) == 0:
self.fetchInfo(src, 'getSites')
main = {'sites': self._sites, 'links': self._links, 'url': src}
self.publish(main)
self.publishTitle('FAX configuration viewer')
self.publishNav(' %s for "%s" ' % (script, params))
self.publish("%s/%s" %
(self.server().fileScriptURL(), "%s.js" % script),
role=pmRoles.script())
self.publish({'s-maxage': 340, 'max-age': 200}, role=pmRoles.cache())
def doJson(self,
pid,
line,
type='filesWithFAX',
name='panda.mon.fax',
levelname='INFO'):
""" Accept the logger information
<p>Supported query parameters are as follows:
<table>
<tr><td><a href='http://pandamon.cern.ch/describe?table=^PANDALOG&doc=*'>PARAM</a>=</td><td>[The value to be recorded]</td></tr>
</table>
"""
user = self.server().user()
#if self.server().user() == None:
# raise ValueError("No user name was present or none secure connection was used" )
if name != 'panda.mon.fax':
raise ValueError("this module is not ready for %s category yet" %
name)
record = self.extraValues()
record['pid'] = pid
record['type'] = type
record['line'] = line
record['name'] = name
record['levelname'] = levelname
# if name == 'panda.mon.fax':
# record['message'] = '%s%s=%s' % (record['message'] ,type,line)
record['LOGUSER'] = user
record['loglevel'] = self._levelmap[levelname]
record['BINTIME'] = datetime.utcnow()
record['TIME'] = record['BINTIME'].strftime('%Y-%m-%d %H:%M:%S')
rupp = {}
for r in record:
rupp[r.upper()] = record[r]
# pmw.addPandaLogRecord(rupp,self._description['rows'])
pmw.addFaxRecord(rupp, self._description['rows'])
self.publishTitle("Panda Logger")
self.publish(record)
self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache())
def doJson(self, jobtype='analysis', timefield='MODIFICATIONTIME', tstart=None, tend=None, days=None,width=600,height=400,options='',plot=False,log=False,file='useract365daysTotal'):
"""
User activity plots <ul>
<li>jobtype - name of the ROOT histogram
<li>file - name the file to get the data from rather from Db
<li>width - the width of the plots in px
<li>height - the height of the plots in px
<li>plot - = 'True' to create the plot; 'False' to use the table view render the data
<li>log - use logarithmic scale for Y-axis
<li>options - some ROOT <a href='http://root.cern.ch/root/html534/THistPainter.html#HP01b'>Paint options</a> or <br>
'M' = to get the "per month" rather "per day" data
</ul>
"""
self.publishTitle('Panda User activities for %(jobtype)s jobs' % {'jobtype': jobtype} )
timer = Stopwatch.Stopwatch()
select = "to_char(MODIFICATIONTIME ,'YYMMDD') as time,PRODUSERID"
if 'M' in options:
select = "to_char(MODIFICATIONTIME ,'YYMM') as time,PRODUSERID"
self.doMain(jobtype, select, timefield, tstart, tend, days,width,height,options,plot,log,file)
self.publishNav('The User Activities for %s from CERN ( %s)' % ( jobtype, timer ) )
render = "taskBuffer/%s.js" % ("tbTable") if not plot else "hello/%s.js" % "pyroot"
self.publish( "%s/%s" % (self.server().fileScriptURL(),render),role=pmRoles.script())
self.publish( {'s-maxage':86400,'max-age': 86400}, role=pmRoles.cache())
def doJson(self,script=None):
"""
doJson(self,script=None) defines the Web API list of parametes and its default values
<p>This Python DocString is to automatically published by Web Server the Web API Help widget
The Web Browser user will be able to access it by clicking the "?" button onto the page menu bar.
The string may contain the regular HTML markups.
script - the file name to download and execute . <br>
It should contain the Javascript function `
<pre>
function _fun_(tag,content) { . . . . }
</pre>
"""
self.publishTitle('Hello JavaScript') # short cut for
self.publishNav("Load and Execute the '%s' Javascript " % ( script if script!= None else 'Embedded' ) )
if script != None:
""" publish the reference to the external JavaScript if any """
self.publish( "%s/%s" % (self.server().fileScriptURL(),"%s.js" % script ),role=pmRoles.script())
""" Create the custom content to be published """
content = {"Hello" : "This is my own content to be published from %s " % utils.lineInfo() }
self.publish(content)
self.publish({'s-maxage':340,'max-age': 200},role=pmRoles.cache())
def doJson(self,method="getAssigningTask",params='',db='pmt'):
"""
Invoke the arbitrary method of <a href="https://svnweb.cern.ch/trac/panda/browser/panda-server/current/pandaserver/taskbuffer/TaskBuffer.py">TaskBuffer</a> class <br>
<code>method = "getAssigningTask"</code><br>
<code>method = "getNumUserSubscriptions"</code><br>
<code>method = "getSiteList"</code> ... etc <br>
<code>params = the comma separated list of the parameters to be passed to the method defined by 'method' parameter</code>
"""
self._doc = None
self._db = db
self.publishTitle('Panda TaskBuffer')
timer = Stopwatch.Stopwatch()
self.doMain(method,params,db)
self.publishNav('The TaskBuffer from CERN: "%s:%s". "%s"' % ( method , params, timer ) ) # I know the cyber security will be mad ;-) VF.
sc = {"getAssigningTask" : "getAssigningTask"
,"getPledgeResourceRatio": "getPledgeResourceRatio"
,"getScriptOfflineRunning": "getScriptOfflineRunning"
,"getFullJobStatus":"getFullJobStatus"
,"fullJobStatus" :"getFullJobStatus"
,"getUsers" : "getUsers"
,'getMembers' : "tbTable"
,'getSiteInfo' : "getSiteInfo"
,'getSnapshot' : "tbTable"
,'getCloudSummary': "tbTable"
,'getSiteSummary' : "tbTable"
,'getCloudList' : "getCloudList"
,"getMCShares" : "getMCShares"
,'getCloudConfig' : "tbTable"
,'getCloudSites' : "tbTable"
,'getjobparam' : "getjobparam"
,'getUserActivity': "tbTable"
,'wnList' : "tbTable"
,'diskUsage' : "tbTable"
,'debugInfo' : "tbTable"
,'corruptFiles' : "tbTable"
,'getVCloudSummary': "tbTable"
,'jobParams' : "tbTable"
,'schedCfg' : "tbTable"
,'getLastDefinedDataset': "tbTable"
,'getTaskMaxLength': "tbTable"
,'listTaskRequests': "tbTable"
,'getUserSubs' : "tbTable"
,'pandatabs' : "tbTable"
,'pandacols' : "tbTable"
,'countReleases': "tbTable"
,'getLogInfo' : "tbTable"
,'getJobsAtt' : "tbTable"
,'getErrorCount': "tbTable"
,'getJediTaskAtt': "tbTable"
,'getJediDSAtt' : "tbTable"
,'describe' : "describe"
}
f = 'default'
for m in sc:
if m in method:
f = sc[m]
break
self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskBuffer/%s.js" % f ),role="script")
cache = {
"getMCShares": {'s-maxage':90000,'max-age': 90000 }
,"getJediTaskAtt": {'s-maxage':900,'max-age': 900 }
}
c = cache.get( method );
if c: self.publish(c, role=pmRoles.cache())
def doQuery(self,
jobtype='production',
hours=24,
vcloud='IAAS',
testJobs=None):
if int(hours) > 24 * 3:
hours = 24 * 3
title = "ATLAS '%s' Virtualized Sites for the Last %s Hours Only." % (
vcloud, hours
) # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = "ATLAS '%s' Virtualized Sites for the last %s hours" % (
vcloud, hours)
self.publishTitle(title)
modes = ('Defined', 'Waiting', 'Active', 'Archived')
rows = []
t = []
for mode in modes:
t += ['ATLAS_PANDA.jobs%s4' % mode]
q = pmt.getVCloudSummary(jobtype,
hours,
table=','.join(t),
vcloud=vcloud,
testsite=(testJobs != None))
header = q['header']
rows = q['rows']
### Build the jobsumd dictionary
jobsumd = {}
for c in [
'ALL',
]:
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0] * (self.lstates + 2)
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0] * (self.lstates + 2)
iCloud = utils.name2Index(header, 'cloud')
iComputingSite = utils.name2Index(header, 'computingSite')
iStatus = utils.name2Index(header, 'jobStatus')
iCount = utils.name2Index(header, 'COUNT')
iCpu = utils.name2Index(header, 'cpuConsumptionTime')
icpu = self.lstates
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
if region: continue
site = "Unknown"
# continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
istatus = utils.name2Index(self.jobstates, status)
count = r[iCount]
cpu = r[iCpu]
if not jobsumd.has_key(cloud):
jobsumd[cloud] = {}
jobsumd[cloud]['ALL'] = [0] * (self.lstates + 1)
if cloud != 'ALL':
jobsumd[cloud]['status'] = self.cloudList[cloud]['status']
if cloud == 'CERN' and vcloud == 'IAAS': # fake site for the time being
jobsumd[cloud]['HELIX_ATOS'] = {}
jobsumd[cloud]['HELIX_ATOS'] = [0] * (self.lstates + 1)
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0] * (self.lstates + 1)
jobsumd[cloud][site][istatus] += count
jobsumd[cloud]['ALL'][istatus] += count
jobsumd['ALL']['ALL'][istatus] += count
jobsumd[cloud][site][icpu] += cpu
jobsumd[cloud]['ALL'][icpu] += cpu
jobsumd['ALL']['ALL'][icpu] += cpu
## Get cloud/site status
main = {}
main['states'] = list(self.jobstates) + ["CPU (hours)"]
main['jobsumd'] = jobsumd
main['params'] = {
'jobtype': jobtype,
'hours': hours,
'computingSite': vcloud
}
if testJobs == None:
main['params']['srcLabel'] = 'managed'
self.publish(main)
self.publishNav(
' <b>type</b>=%s <b>hours</b>=%s <b>date:</b> %s' %
(jobtype, hours, datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')))
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "monitor/%s.js" % "vcloud"),
role=pmRoles.script())
self.publish({'s-maxage': 340, 'max-age': 200}, role=pmRoles.cache())
def doJson(self):
""" Is Alive """
main = {'isalive' : 'yes' }
self.publish(main)
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
def doJson(self, table=None, column=None, db='pmt', doc=None):
"""
Describe the selected Panda table / column <br>
<code>table = the Db table name regexp comma-separated patterns to check</code><br>
<code>column = the Db table column name regexp comma-separated patterns to find</code><br>
<code>doc = '*|all' - document all columns selected if available<br>
= 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided
</code><br>
"""
title = 'Describe the Panda Db '
inp = False
if table != None and table != 'None':
title += "'%s*'" % table
inp = True
title += " table"
else:
title += " tables"
if column != None and column != 'None':
title += " with '%s*' column " % column
inp = True
main = {}
if inp:
self.publishTitle(title)
timer = Stopwatch.Stopwatch()
main["buffer"] = {}
main["buffer"]["db"] = db
dbAccess = self._db.get(db)
if dbAccess != None:
main["buffer"]["method"] = "describe"
main["buffer"]["params"] = (table, column, doc)
main["buffer"]["type"] = False
if not utils.isFilled(doc):
tdes = dbAccess.describe(table, column)
iclmn = utils.name2Index(tdes['header'], 'COLUMN_NAME')
rows = tdes['rows']
for h in rows:
h[iclmn] = nmap.get(h[iclmn])
main["buffer"]["data"] = tdes
else:
docrows = []
if utils.isFilled(table):
tdoc = dbAccess.pandatabs(
table,
comments=doc,
select=
"table_name,'Table Description' as COLUMN_NAME,comments"
)
iclmn = utils.name2Index(tdoc['header'], 'COLUMN_NAME')
docrows = tdoc['rows']
for h in docrows:
h[iclmn] = nmap.get(h[iclmn])
tcol = dbAccess.pandacols(
column,
table,
select='TABLE_NAME,COLUMN_NAME,COMMENTS',
comments=doc)
iclmn = utils.name2Index(tcol['header'], 'COLUMN_NAME')
for h in tcol['rows']:
h[iclmn] = nmap.get(h[iclmn])
tcol['rows'] += docrows
main["buffer"]["data"] = tcol
self.publish({
's-maxage': 60000,
'max-age': 6000
},
role=pmRoles.cache())
else:
self.publishTitle(
"Error: 'describe' - unknown Db name '%s' has been provided"
% db)
self.publish({
's-maxage': 0,
'max-age': 0
},
role=pmRoles.cache())
else:
self.publishTitle(
"Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'"
)
self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache())
self.publish(main)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "taskBuffer/%s.js" % "describe"),
role=pmRoles.script())
def doJson(self):
""" Is Alive """
main = {'isalive': 'yes'}
self.publish(main)
self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache())
def doJson(self,table=None,column=None,db='pmt',doc=None):
"""
Describe the selected Panda table / column <br>
<code>table = the Db table name regexp comma-separated patterns to check</code><br>
<code>column = the Db table column name regexp comma-separated patterns to find</code><br>
<code>doc = '*|all' - document all columns selected if available<br>
= 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided
</code><br>
"""
title = 'Describe the Panda Db '
inp = False
if table != None and table != 'None':
title += "'%s*'" % table
inp = True
title +=" table"
else:
title +=" tables"
if column != None and column !='None':
title += " with '%s*' column " % column
inp = True
main = {}
if inp:
self.publishTitle(title)
timer = Stopwatch.Stopwatch()
main["buffer"] = {}
main["buffer"]["db"] = db
dbAccess = self._db.get(db)
if dbAccess != None:
main["buffer"]["method"] = "describe"
main["buffer"]["params"] = (table,column,doc)
main["buffer"]["type"] = False
if not utils.isFilled(doc):
tdes = dbAccess.describe(table,column)
iclmn = utils.name2Index(tdes['header'],'COLUMN_NAME')
rows = tdes['rows']
for h in rows:
h[iclmn] = nmap.get(h[iclmn])
main["buffer"]["data"] = tdes
else:
docrows = []
if utils.isFilled(table):
tdoc = dbAccess.pandatabs(table,comments=doc,select="table_name,'Table Description' as COLUMN_NAME,comments")
iclmn = utils.name2Index(tdoc['header'],'COLUMN_NAME')
docrows = tdoc['rows']
for h in docrows:
h[iclmn] = nmap.get(h[iclmn])
tcol = dbAccess.pandacols(column,table,select='TABLE_NAME,COLUMN_NAME,COMMENTS', comments=doc)
iclmn = utils.name2Index(tcol['header'],'COLUMN_NAME')
for h in tcol['rows']:
h[iclmn] = nmap.get(h[iclmn])
tcol['rows'] += docrows
main["buffer"]["data"] = tcol
self.publish( {'s-maxage':60000,'max-age': 6000}, role=pmRoles.cache())
else:
self.publishTitle("Error: 'describe' - unknown Db name '%s' has been provided" % db)
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
else:
self.publishTitle("Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'")
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
self.publish(main)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskBuffer/%s.js" % "describe"),role=pmRoles.script() )
def doJson(
self,
job=None,
taskid=None,
tstart=None,
tend=None,
hours=71,
days=None,
jobparam=None,
limit=1500,
summary=None,
minjob=1,
jobtype=None,
timeField="modificationTime",
plot="no",
dump=None,
tasktype=None,
status=None,
region=None,
lfn=None,
showcpu=None,
):
"""Display the job information:
<p>Sample queries:
<ul>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours='>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01</a>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours=&processingType=simul'>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01&[email protected]</a>
<li><a href='http://pandamon.cern.ch/jobinfo?job=1246240909'>http://pandamon.cern.ch/jobinfo?job=1246240909</a>
</ul>
<hr>
Supported query parameters are as follows:
<table>
<tr><td>taskid</td><td> - jobset / task id</td></tr>
<tr><td>job</td><td> - comma separated list of jobs' PandaID</td></tr>
<tr><td>jobtype</td><td> - select the 'jobtype' jobs: analysys, production, install, test </td></tr>
<tr><td>tstart=None</td><td> - start time </td></tr>
<tr><td>tend=None</td><td> - end time</td></tr>
<tr><td>hours=N </td><td> - last (N hours + N days) or N hours after 'tstart' or N hours before 'tend' </td></tr>
<tr><td>days=N</td><td> - last (N hours + N days) (see the hours parameter documentation</td></tr>
<tr><td>lfn=</td><td> - Select the PanDA jobs using the files from the comma-separated lfn list </td></tr>
<tr><td>jobparam=</td><td>The comma separated list of the <a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>job parameters</a>.<br> The default should satisfy the interactive users.<br>
One needs to customize jobparam to build his/her <a href='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#Custom_Monitoring_Client_Code'>own client</a></td></tr>
<tr><td>limit=</d><td> - max number of the Db records to fetch from each job Db table. Since there are 4 tables the real number of the rows can be 4 times as many as that limit</td></tr>
<tr><td><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</td><td>- comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.<br>
For example: <ul><li><a href='http://pandamon.cern.ch/jobinfo?processingType=simul'>http://pandamon.cern.ch/jobinfo?processingType=simul</a>
<li> <a href='http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test'>http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test</a></ul>
<tr><td>summary</td><td>Create the summary for the first member of the jobparam list<br>
For example: <a href='http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes'>http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes</a>
</td></tr>
<tr><td>dump</td><td> - 'yes' - produce the flat list of job info records suitable for CLI applications</td></tr>
<tr><td>region</td><td> - the comma separated list of the regions </td></tr>
<tr><td>showcpu</td><td> - 'yes' to show the CPU type used </td></tr>
</table>
"""
# % { 'url' : self.server().branchUrl() }
if job == "undefined":
job = None
if limit == "undefined":
limit = None
if summary == "undefined":
summary = None
if jobparam == "undefined":
jobparam = None
if tstart == "undefined":
tstart = None
if tend == "undefined":
tend = None
if tasktype == "prod":
jobtype = "production"
if jobtype == "null":
jobtype = None
if jobparam == None:
if job != None:
jobparam = self._allparams
else:
jobparam = ",".join(self._colNames)
if timeField == None or timeField == "undefined" or timeField == "":
timeField = "modificationTime"
if self.isValid(jobparam):
self.publish(
self.doJobs(
job,
tstart=tstart,
tend=tend,
hours=hours,
days=days,
jobparam=jobparam,
limit=limit,
field=timeField,
jobtype=jobtype,
summary=summary,
minjob=minjob,
dump=dump,
region=region,
lfn=lfn,
showcpu=showcpu,
)
)
self.publish("%s/%s" % (self.server().fileScriptURL(), "jobInfoRender.js"), role=pmRoles.script())
self.publish({"s-maxage": 600, "max-age": 600}, role=pmRoles.cache())
else:
title = "Job data "
if utils.isFilled(job):
hours = None
days = None
title += " %s " % job
if days != None:
title += " for %.2f day" % float(days)
if days > 1:
title += "s"
if hours != None:
if days == None:
title += " for "
title += " %d hour" % int(hours)
if hours > 1:
title += "s"
self.publishTitle(title)
def doQuery(self,jobtype='production',hours=24,vcloud='IAAS',testJobs=None):
if int(hours) > 24 * 3:
hours = 24 * 3
title = "ATLAS '%s' Virtualized Sites for the Last %s Hours Only." % (vcloud,hours) # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = "ATLAS '%s' Virtualized Sites for the last %s hours" % (vcloud,hours)
self.publishTitle(title)
modes = ( 'Defined', 'Waiting', 'Active', 'Archived' )
rows = []
t = []
for mode in modes: t += ['ATLAS_PANDA.jobs%s4' % mode]
q = pmt.getVCloudSummary(jobtype,hours,table=','.join(t),vcloud=vcloud,testsite=(testJobs!=None))
header = q['header']
rows = q['rows']
### Build the jobsumd dictionary
jobsumd = {}
for c in ['ALL',]:
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0]*(self.lstates+2)
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0]*(self.lstates+2)
iCloud = utils.name2Index(header,'cloud')
iComputingSite = utils.name2Index(header,'computingSite')
iStatus = utils.name2Index(header,'jobStatus')
iCount = utils.name2Index(header,'COUNT')
iCpu = utils.name2Index(header,'cpuConsumptionTime')
icpu = self.lstates
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
if region: continue
site="Unknown"
# continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
istatus = utils.name2Index(self.jobstates,status)
count = r[iCount]
cpu = r[iCpu]
if not jobsumd.has_key(cloud):
jobsumd[cloud] = {}
jobsumd[cloud]['ALL'] = [0]*(self.lstates+1)
if cloud != 'ALL':
jobsumd[cloud]['status'] = self.cloudList[cloud]['status']
if cloud == 'CERN' and vcloud == 'IAAS': # fake site for the time being
jobsumd[cloud]['HELIX_ATOS'] = {}
jobsumd[cloud]['HELIX_ATOS'] = [0]*(self.lstates+1)
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0]*(self.lstates+1)
jobsumd[cloud][site][istatus] += count
jobsumd[cloud]['ALL'][istatus] += count
jobsumd['ALL']['ALL'][istatus] += count
jobsumd[cloud][site][icpu] += cpu
jobsumd[cloud]['ALL'][icpu] += cpu
jobsumd['ALL']['ALL'][icpu] += cpu
## Get cloud/site status
main = {}
main['states'] = list(self.jobstates) + ["CPU (hours)"]
main['jobsumd'] = jobsumd
main['params'] = { 'jobtype': jobtype,'hours': hours
,'computingSite': vcloud
}
if testJobs == None:
main['params']['srcLabel'] = 'managed'
self.publish(main)
self.publishNav(' <b>type</b>=%s <b>hours</b>=%s <b>date:</b> %s' % (jobtype, hours, datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') ) )
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "vcloud" ),role=pmRoles.script() )
self.publish( {'s-maxage':340,'max-age': 200}, role=pmRoles.cache())
def doQuery(self,site='all',jobtype='all',days=1,hostonly='yes',plot='finished,failed',style='SB', details=None,deep='no'):
""" Get the list of the jobs' status per worker node for selected site for upto 3 last days
<ol>
<li><code>site</code> - site to get the information about or "all"
<li><code>jobtype</code> = 'production" or " analysis" or "all"
<li><code>days</code> - number of days we want to fetch the information for. This parameter must be less or equal 3.
<li><code>hostonly</code> - "yes" . "no" - means provide the information 'per CPU' rather 'per host'
<li><code>plot</code> - the comma separated list of the <a href='https://twiki.cern.ch/twiki/bin/viewauth/Atlas/PandaShiftGuide#Job_state_definitions_in_Panda'>job status names</a> to generate the plot for
<li><code>style</code> the combination of the symbols: 'H' - histogram, 'B' - bar , 'L' - line, 'P' - points, 'S' - stacked<br>
= 'BS' - 'by default'
<li><code>details</code> - 'yes' - provide the status per host<br>
<br> = 'summary' - provide the summary only <br>
default = 'yes' if not site == 'all' else 'no'
<li><code>deep</code> - do not look up the "long job table" . It can be very slow.
default = 'no'
<li> <b><a title='Click to see the full list of the parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</b> - comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.
<li><code>jobStatus</code> - it can be any comma-separated list of the Panda Jobs states patterns or / and <br>
two 'meta' states: "factory" to list the "factory" nodes and "wn" to list the the Worker Nodes
</ol>
"""
vals = self.extraValuesFilled()
if site == None: site='all'
factory = None
wn = None
titletype = "PanDA"
jobStatus = None
if vals != None:
jstat = vals.get('jobStatus')
jobStatus = jstat
if jstat != None:
jstat = [x.lower() for x in utils.parseArray(jstat)]
print utils.lineInfo(), jstat, 'factory' in jstat
if 'factory' in jstat:
factory = self.factory()
jstat += factory
titletype = "Factory"
if 'wn' in jstat:
wn = self.wn()
jstat += wn
titletype = "Worker"
jstat = list(set(jstat))
vals['jobStatus'] = ','.join(jstat)
if deep == None or deep == False: deep = 'no'
deep = not deep.lower() == 'no'
title = ''
hours = (float(days))*24
Deep = 'Deep' if deep else ''
if float(days) > 3 and False and not deep:
days = 3.0
hours = 24*days
title = 'The List of the %s Nodes Activity for the Last %s Hours Only.' % (titletype, int(hours))
else:
title = 'The %s List of the %s Nodes Activity for the %s Hours' % (Deep, titletype,int(hours) )
days = float(days)
try:
if jobtype.lower() == 'all': jobtype=None
except:
pass
self.publishTitle(title)
modes = ( 'Defined', 'Waiting', 'Active', 'Archived' )
tables = []
for mode in modes: tables += ['ATLAS_PANDA.jobs%s4' % mode]
if days>3 and deep: tables += ['ATLAS_PANDAARCH.JOBSARCHIVED']
q = pmt.wnList(site, jobtype, table=tables, days=days,conditions = vals)
try:
if details == None:
details = not (site == None or site == 'None' or site.lower() == 'all' or site == '*' )
# else:
# details = isinstance(details,str) and details.lower() == 'yes'
except:
details = False
## Publish the result
main = {}
main['data'] = self.do(q['header'],q['rows'],site,hostonly,details,factory,wn)
main['params'] = { 'jobtype': jobtype,'days': days,'site': site,'hostonly':hostonly,'jobStatus':jobStatus}
plotpars = []
if plot:
for p in plot.split(","):
if p.lower() == 'prerun' or factory != None:
if wn == None: plotpars += self.factory()
if factory == None:
plotpars += ['pending','sent','starting']
elif p.lower() == 'finishing' and factory == None:
plotpars += ['holding','transferring']
else:
plotpars += [p]
main['params']['plot'] = list(set(plotpars)) # remove duplicates
if style: main['params']['style'] = style
self.publish(main)
self.publishNav(' type=%s days= last %s ' % (jobtype, days ) )
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "wnlist" ),role=pmRoles.script())
self.publish( {'s-maxage':600,'max-age': 600}, role=pmRoles.cache())
def doJson(self, method="getAssigningTask", params='', db='pmt'):
"""
Invoke the arbitrary method of <a href="https://svnweb.cern.ch/trac/panda/browser/panda-server/current/pandaserver/taskbuffer/TaskBuffer.py">TaskBuffer</a> class <br>
<code>method = "getAssigningTask"</code><br>
<code>method = "getNumUserSubscriptions"</code><br>
<code>method = "getSiteList"</code> ... etc <br>
<code>params = the comma separated list of the parameters to be passed to the method defined by 'method' parameter</code>
"""
self._doc = None
self._db = db
self.publishTitle('Panda TaskBuffer')
timer = Stopwatch.Stopwatch()
self.doMain(method, params, db)
self.publishNav(
'The TaskBuffer from CERN: "%s:%s". "%s"' %
(method, params,
timer)) # I know the cyber security will be mad ;-) VF.
sc = {
"getAssigningTask": "getAssigningTask",
"getPledgeResourceRatio": "getPledgeResourceRatio",
"getScriptOfflineRunning": "getScriptOfflineRunning",
"getFullJobStatus": "getFullJobStatus",
"fullJobStatus": "getFullJobStatus",
"getUsers": "getUsers",
'getMembers': "tbTable",
'getSiteInfo': "getSiteInfo",
'getSnapshot': "tbTable",
'getCloudSummary': "tbTable",
'getSiteSummary': "tbTable",
'getCloudList': "getCloudList",
"getMCShares": "getMCShares",
'getCloudConfig': "tbTable",
'getCloudSites': "tbTable",
'getjobparam': "getjobparam",
'getUserActivity': "tbTable",
'wnList': "tbTable",
'diskUsage': "tbTable",
'debugInfo': "tbTable",
'corruptFiles': "tbTable",
'getVCloudSummary': "tbTable",
'jobParams': "tbTable",
'schedCfg': "tbTable",
'getLastDefinedDataset': "tbTable",
'getTaskMaxLength': "tbTable",
'listTaskRequests': "tbTable",
'getUserSubs': "tbTable",
'pandatabs': "tbTable",
'pandacols': "tbTable",
'countReleases': "tbTable",
'getLogInfo': "tbTable",
'getJobsAtt': "tbTable",
'getErrorCount': "tbTable",
'getJediTaskAtt': "tbTable",
'getJediDSAtt': "tbTable",
'describe': "describe"
}
f = 'default'
for m in sc:
if m in method:
f = sc[m]
break
self.publish("%s/%s" %
(self.server().fileScriptURL(), "taskBuffer/%s.js" % f),
role="script")
cache = {
"getMCShares": {
's-maxage': 90000,
'max-age': 90000
},
"getJediTaskAtt": {
's-maxage': 900,
'max-age': 900
}
}
c = cache.get(method)
if c: self.publish(c, role=pmRoles.cache())
def doJson(self,
job=None,
taskid=None,
tstart=None,
tend=None,
hours=71,
days=None,
jobparam=None,
limit=1500,
summary=None,
minjob=1,
jobtype=None,
timeField='modificationTime',
plot='no',
dump=None,
tasktype=None,
status=None,
region=None,
lfn=None,
showcpu=None):
"""Display the job information:
<p>Sample queries:
<ul>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours='>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01</a>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours=&processingType=simul'>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01&[email protected]</a>
<li><a href='http://pandamon.cern.ch/jobinfo?job=1246240909'>http://pandamon.cern.ch/jobinfo?job=1246240909</a>
</ul>
<hr>
Supported query parameters are as follows:
<table>
<tr><td>taskid</td><td> - jobset / task id</td></tr>
<tr><td>job</td><td> - comma separated list of jobs' PandaID</td></tr>
<tr><td>jobtype</td><td> - select the 'jobtype' jobs: analysys, production, install, test </td></tr>
<tr><td>tstart=None</td><td> - start time </td></tr>
<tr><td>tend=None</td><td> - end time</td></tr>
<tr><td>hours=N </td><td> - last (N hours + N days) or N hours after 'tstart' or N hours before 'tend' </td></tr>
<tr><td>days=N</td><td> - last (N hours + N days) (see the hours parameter documentation</td></tr>
<tr><td>lfn=</td><td> - Select the PanDA jobs using the files from the comma-separated lfn list </td></tr>
<tr><td>jobparam=</td><td>The comma separated list of the <a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>job parameters</a>.<br> The default should satisfy the interactive users.<br>
One needs to customize jobparam to build his/her <a href='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#Custom_Monitoring_Client_Code'>own client</a></td></tr>
<tr><td>limit=</d><td> - max number of the Db records to fetch from each job Db table. Since there are 4 tables the real number of the rows can be 4 times as many as that limit</td></tr>
<tr><td><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</td><td>- comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.<br>
For example: <ul><li><a href='http://pandamon.cern.ch/jobinfo?processingType=simul'>http://pandamon.cern.ch/jobinfo?processingType=simul</a>
<li> <a href='http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test'>http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test</a></ul>
<tr><td>summary</td><td>Create the summary for the first member of the jobparam list<br>
For example: <a href='http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes'>http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes</a>
</td></tr>
<tr><td>dump</td><td> - 'yes' - produce the flat list of job info records suitable for CLI applications</td></tr>
<tr><td>region</td><td> - the comma separated list of the regions </td></tr>
<tr><td>showcpu</td><td> - 'yes' to show the CPU type used </td></tr>
</table>
"""
# % { 'url' : self.server().branchUrl() }
if job == 'undefined': job = None
if limit == 'undefined': limit = None
if summary == 'undefined': summary = None
if jobparam == 'undefined': jobparam = None
if tstart == 'undefined': tstart = None
if tend == 'undefined': tend = None
if tasktype == 'prod': jobtype = 'production'
if jobtype == 'null': jobtype = None
if jobparam == None:
if job != None:
jobparam = self._allparams
else:
jobparam = ','.join(self._colNames)
if timeField == None or timeField == 'undefined' or timeField == '':
timeField = 'modificationTime'
if self.isValid(jobparam):
self.publish(
self.doJobs(job,
tstart=tstart,
tend=tend,
hours=hours,
days=days,
jobparam=jobparam,
limit=limit,
field=timeField,
jobtype=jobtype,
summary=summary,
minjob=minjob,
dump=dump,
region=region,
lfn=lfn,
showcpu=showcpu))
self.publish("%s/%s" %
(self.server().fileScriptURL(), "jobInfoRender.js"),
role=pmRoles.script())
self.publish({
's-maxage': 600,
'max-age': 600
},
role=pmRoles.cache())
else:
title = 'Job data '
if utils.isFilled(job):
hours = None
days = None
title += " %s " % job
if days != None:
title += ' for %.2f day' % float(days)
if days > 1: title += "s"
if hours != None:
if days == None: title += " for "
title += ' %d hour' % int(hours)
if hours > 1: title += "s"
self.publishTitle(title)
