def __init__(self,name=None,parent=None,obj=None):
pmModule.__init__(self,name,parent,obj)
self.publishUI(self.doQuery,role=pmRoles.json())
self.publishUI(self.doScript,role=pmRoles.script())
self.publishUI(self.doQuery2,role=pmRoles.json(),alias="site")
self.publishUI(self.doScript2,role=pmRoles.script(),alias="site")
def __init__(self,name=None,parent=None,obj=None):
""" Construct the custom web 'hellojs' application """
pmModule.__init__(self,name,parent,obj) # Initialize the base class
self.publishUI(self.doJson) # Publish the method 'doJson' to delivery
# the Web application content
self.publishUI(self.doScript,role=pmRoles.script()) # Publish the method 'hellojs.doScript'
def doScript(self, undoc=False, method=None):
version = self.server().version().get('version')
if version == None: version = ''
else: version = "/~%s" % version
func = """
function(tag,data) {
$(tag).empty();
var taskBuffer = data.taskBuffer;
var undoc = data.undoc;
var html = "<ol>";
var nUndocumented = 0;
for (var method in taskBuffer ) {
var mn = taskBuffer[method];
var mname = mn[0];
var pars = mn[1];
var doc = mn[2];
if ( mname.indexOf('getProxyKey') < 0 ) {
if (doc == undefined || doc == null) { nUndocumented++; if (! undoc) continue; doc = 'Under construction. To be documented yet!';}
else {doc = "<b>" +doc+ "</b>"; }
html += "<li><a href='%(version)s/taskBuffer?method=" + mname+ "'>" + mname+"(" + pars + "): </a><br> " + doc;
}
}
html += "</ol>";
if (nUndocumented >0) { html += "<hr>" + nUndocumented +" undocumented methods were found but not listed" ; }
$(tag).html(html);
}
""" % {
'version': version
}
self.publish(func, role=pmRoles.script())
def doJson(
self,
hosts='atlascloud.org,pandamon-eu.atlascloud.org,atlas-nightlies-browser.cern.ch,pandamon.cern.ch,atlascloud.org,pandamon-eu.atlascloud.org,pandamon.cern.ch,atlas-nightlies-browser.cern.ch',
query='helloora',
params=None):
"""
Demonstrate the concurrent cross domain communication via <a href='http://en.wikipedia.org/wiki/JSONP'>JSONP</a> format
<ul>
<li><code>hosts</code> = the list of hosts to communicate
<li><code>query</code> = the query to be sent to all hosts
<li><code>params</code>= to be attached to the URL query
</ul>
"""
self.publishTitle('Hello JSONP cross domain communication.')
hosts = "%s,%s" % (hosts, hosts)
h = "%s" % hosts
vr = h.split(",")
ph = ", ".join(vr)
self.publishNav('The JSONP communication with <b>%s</b>.' % ph)
self.publish("%s/%s" %
(self.server().fileScriptURL(), "hello/hellojsonp.js"),
role=pmRoles.script())
self.publish({
'hosts': hosts.split(","),
'query': query,
'params': params
})
def doScript(self,config=None):
script = """
function db(tag,maindata) {
var th = views().jqTag(tag);
var dashboard = $("<div>hello gues</div>");
tag.append(dashboard);
function appendJobSummary(tag,main) {
var _jobsummary = main.jobsummary;
var dhead = main.header;
if (main.info.length > 0 && dhead != undefined ) {
var dinfo = {
"data" : main.info
,"rows" : main.info
,"header" : dhead
,"jobsummary" : _jobsummary
};
var tg = dhead[0] + "_"+tag;
var nxdiv = views().renderJobSummary(dashboard,dinfo);
nxdiv.show();
}
}
function fetchJobSummary() {
var lpars = {};
lpars['summary']=true;
aj = new AjaxRender();
lpars['jobparam'] = 'jobStatus';
aj.download('jobsummaryid',appendJobSummary, "jobinfo",lpars);
}
/* fetchJobSummary(); */
}
"""
print utils.linInfo(), script
self.publishNav(script,pmRoles.script())
def doScript(self,undoc=False,method=None):
version = self.server().version().get('version')
if version == None: version = ''
else: version = "/~%s" % version
func = """
function(tag,data) {
$(tag).empty();
var taskBuffer = data.taskBuffer;
var undoc = data.undoc;
var html = "<ol>";
var nUndocumented = 0;
for (var method in taskBuffer ) {
var mn = taskBuffer[method];
var mname = mn[0];
var pars = mn[1];
var doc = mn[2];
if ( mname.indexOf('getProxyKey') < 0 ) {
if (doc == undefined || doc == null) { nUndocumented++; if (! undoc) continue; doc = 'Under construction. To be documented yet!';}
else {doc = "<b>" +doc+ "</b>"; }
html += "<li><a href='%(version)s/taskBuffer?method=" + mname+ "'>" + mname+"(" + pars + "): </a><br> " + doc;
}
}
html += "</ol>";
if (nUndocumented >0) { html += "<hr>" + nUndocumented +" undocumented methods were found but not listed" ; }
$(tag).html(html);
}
""" % { 'version' : version }
self.publish(func,role=pmRoles.script())
def doJson(self, script=None):
"""
doJson(self,script=None) defines the Web API list of parametes and its default values
<p>This Python DocString is to automatically published by Web Server the Web API Help widget
The Web Browser user will be able to access it by clicking the "?" button onto the page menu bar.
The string may contain the regular HTML markups.
script - the file name to download and execute . <br>
It should contain the Javascript function `
<pre>
function _fun_(tag,content) { . . . . }
</pre>
"""
self.publishTitle('Hello JavaScript') # short cut for
self.publishNav("Load and Execute the '%s' Javascript " %
(script if script != None else 'Embedded'))
if script != None:
""" publish the reference to the external JavaScript if any """
self.publish("%s/%s" %
(self.server().fileScriptURL(), "%s.js" % script),
role=pmRoles.script())
""" Create the custom content to be published """
content = {
"Hello":
"This is my own content to be published from %s " %
utils.lineInfo()
}
self.publish(content)
self.publish({'s-maxage': 340, 'max-age': 200}, role=pmRoles.cache())
def doScript(self, pandaid, message=None,host=None,timestamp=None,errorcode=None,test=None):
""" Contact Us Panda Monitor Modules Render"""
javascript = """
function _anyname_(tag,content) {
$(tag).empty();
$(tag).html('<pre>\n'+JSON.stringify(content,undefined,2)+'\n</pre>');
}
"""
self.publish(javascript,role=pmRoles.script())
def __init__(self, name=None, parent=None, obj=None):
""" Construct the custom web 'hellojs' application """
pmModule.__init__(self, name, parent, obj) # Initialize the base class
self.publishUI(self.doJson) # Publish the method 'doJson' to delivery
# the Web application content
self.publishUI(
self.doScript,
role=pmRoles.script()) # Publish the method 'hellojs.doScript'
def doScript(self,
nJobs=10,
siteName='ANALY_VICTORIA-WG1',
prodSourceLabel='user',
cpu=0,
mem=0,
diskSpace=0,
modificationHost='',
timeout=0,
computingElement=None,
atlasRelease='no use',
prodUserID=None,
getProxyKey=None,
countryGroup='',
workingGroup='',
allowOtherCountry=True):
func = """
function(tag,main) {
$(tag).empty();
var dt = false;
var time = main.time;
var method = main.buffer.method;
var params = main.buffer.params;
var data = main.buffer.data;
var d = $('<div></div>');
var query = "<P>The server spent <b>" + time.query + "</b> to access the CERN Oracle server";
var fetch = " and <p><b>" + time.fetch + "</b> to execute the TaskBuffer.%s:%s query";
d.html(query + fetch);
var tdiv = $("<div></div>");
var tline = '<p><b>' + method + '(' +JSON.stringify(params)+ '): </b><br>';
dt = true;
tline += '<hr>';
tline += '<table id="TaskBuferTableId" class="display"><thead><tr><th>Parameter</th><th>Value</th></thead><tbody>';
for (var i in data) {
var num = data[i];
tline += "<tr><td>" + i + "</td><td>" + JSON.stringify(num) + "</td></tr>";
}
tline += "<tbody></table>";
tdiv.html(tline);
d.append(tdiv);
$(tag).append(d);
if (dt) {
$(document).ready(function(){
$('#TaskBuferTableId').dataTable( { "bJQueryUI": true });
} );
}
}
""" % ('getJobs', (nJobs, siteName, prodSourceLabel, cpu, mem, diskSpace,
modificationHost, timeout, computingElement,
atlasRelease, prodUserID, getProxyKey, countryGroup,
workingGroup, allowOtherCountry))
self.publish(func, role=pmRoles.script())
def doScript(self):
# """ provides the javascript function to client-site data rendering
# function(tag, data) { . . . }
# where tag - is the HTML tag (string in css notation) the data is to be rendered into
# data - the data structure , the fucttion should render
# The default version assumes the data is the "HTML" string to fill the innerHTML
# of the tag provided
# """
deffunc = """
function (tag,data) {
$(tag).empty();
var thisTag = $("<div></div>");
thisTag.append("<p>Total size: " + data.size + " MB");
thisTag.append("<p>Total items: " + (data.items/(1024*1024)).toFixed() + " MB");
thisTag.append("<p>Average item size: " + data.itemsize + " bytes");
thisTag.append("<p>Number of domains: " + data.ndomains);
thisTag.append("<p>Domain info:");
var table = $('<table cellpadding="2" cellspacing="2" border="1" class="display" id="dbstat_table"></table>');
thisTag.append(table);
var h = $("<thead></thead>");
table.append(h);
var hr = $("<tr></tr>");
hr.css( {"text-decoration":"underline", "color":"blue", "cursor":"pointer"});
h.append(hr);
$.each(data.header, function(i,header) { hr.append($("<th>" +header + "</th>")); } );
h = $("<tbody></tbody>");
table.append(h);
$.each(data.info, function(i,r) {
var tr = $('<tr></tr>');
tr.css("font-size","0.6em");
h.append(tr);
$.each(r, function(d,v) {
if (d == 1) v = (v/100000).toFixed(2);
else if (d == 5) v = (v/100000).toFixed(1);
var td = $("<td>" + v + "</td>");
tr.append(td);
if ( d!=0) { td.css("text-align","right") ;}
} );
} );
var h = $("<tfoot></tfoot>");
table.append(h);
var hr = $("<tr></tr>");
h.append(hr);
$.each( data.header , function(i,h) { hr.append($("<th>" + h + "</th>")); } );
$(tag).append(thisTag);
$('#dbstat_table').dataTable({"bProcessing": true,"bJQueryUI": true});
}
"""
self.publish(deffunc, role=pmRoles.script())
def doJson(self,
release=None,
cloud=None,
site=None,
cache=None,
cmt=None):
"""
Release information displays <br><ul>
<li><code>release = release comma-separated regex name pattern </code>
<li><code>cloud = cloud abbreviation to narrow the search </code>
<li><code>site = siteid to select the release info for that Panda site</code>
<li><code>cache = cache name </code>
<li><code>cmt = cmtconfig comma-separated regex pattern </code>
</ul>
"""
if release != None and site == None:
title = "Sites holding release %s" % release
else:
title = "Release availability "
if site != None: title += " from site: %s" % site
elif cloud != None:
if release == None: title += " at sites "
title += " from cloud: %s" % cloud
else:
title += " at sites"
if cache != None: title += ", cache %s" % cache
self.publishTitle(title)
q = pmt.getReleases(release, cloud, site, cache, cmt)
header = q['header']
rows = q['rows']
main = {}
main['header'] = header if len(rows) > 0 else []
main['info'] = self.compress(header, rows, cache == "None")
main['time'] = {}
main['params'] = {
'release': release,
'cloud': cloud,
'site': site,
'cache': cache
}
self.publish(main)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "monitor/%s.js" % "releaseinfo"),
role=pmRoles.script())
self.publish({'s-maxage': 3600, 'max-age': 3600}, role=pmRoles.cache())
def doScript(self):
# """ provides the javascript function to client-site data rendering
# function(tag, data) { . . . }
# where tag - is the HTML tag (string in css notation) the data is to be rendered into
# data - the data structure , the fucttion should render
# The default version assumes the data is the "HTML" string to fill the innerHTML
# of the tag provided
# """
deffunc = """
function (tag,data) {
$(tag).empty();
var thisTag = $("<div></div>");
thisTag.append("<p>Total size: " + data.size + " MB");
thisTag.append("<p>Total items: " + (data.items/(1024*1024)).toFixed() + " MB");
thisTag.append("<p>Average item size: " + data.itemsize + " bytes");
thisTag.append("<p>Number of domains: " + data.ndomains);
thisTag.append("<p>Domain info:");
var table = $('<table cellpadding="2" cellspacing="2" border="1" class="display" id="dbstat_table"></table>');
thisTag.append(table);
var h = $("<thead></thead>");
table.append(h);
var hr = $("<tr></tr>");
hr.css( {"text-decoration":"underline", "color":"blue", "cursor":"pointer"});
h.append(hr);
$.each(data.header, function(i,header) { hr.append($("<th>" +header + "</th>")); } );
h = $("<tbody></tbody>");
table.append(h);
$.each(data.info, function(i,r) {
var tr = $('<tr></tr>');
tr.css("font-size","0.6em");
h.append(tr);
$.each(r, function(d,v) {
if (d == 1) v = (v/100000).toFixed(2);
else if (d == 5) v = (v/100000).toFixed(1);
var td = $("<td>" + v + "</td>");
tr.append(td);
if ( d!=0) { td.css("text-align","right") ;}
} );
} );
var h = $("<tfoot></tfoot>");
table.append(h);
var hr = $("<tr></tr>");
h.append(hr);
$.each( data.header , function(i,h) { hr.append($("<th>" + h + "</th>")); } );
$(tag).append(thisTag);
$('#dbstat_table').dataTable({"bProcessing": true,"bJQueryUI": true});
}
"""
self.publish(deffunc,role=pmRoles.script() )
def doJson(self,src='http://ivukotic.web.cern.ch/ivukotic/FAX',date='2012-10-16',selS='ALL',script='fax/map'):
""" FAX configuration viewer
"""
self.reset()
params = "from=%s" % date
if selS != None: params += '&selS=%s' % selS
self.fetchInfo(src,'getLinks',params)
if len(self._sites) == 0:
self.fetchInfo(src,'getSites')
main = {'sites': self._sites, 'links': self._links,'url': src }
self.publish(main)
self.publishTitle('FAX configuration viewer')
self.publishNav(' %s for "%s" ' % ( script, params ) )
self.publish( "%s/%s" % (self.server().fileScriptURL(),"%s.js" % script ),role=pmRoles.script())
self.publish({'s-maxage':340,'max-age': 200},role=pmRoles.cache())
def doScript(self,
pandaid,
message=None,
host=None,
timestamp=None,
errorcode=None,
test=None):
""" Contact Us Panda Monitor Modules Render"""
javascript = """
function _anyname_(tag,content) {
$(tag).empty();
$(tag).html('<pre>\n'+JSON.stringify(content,undefined,2)+'\n</pre>');
}
"""
self.publish(javascript, role=pmRoles.script())
def doScript(self, site='ALL', From='Jun_2011', To='May_2012', type='7'):
func = """
function(tag,data) {
var files = data.filenames;
$(tag).empty();
var table = "<table>";
table += "<tbody>";
for (var f in files) {
if (!f%2 || true ) { table += "<tr>"; }
table += "<th><img src='http://atlascloud.org/static/images/analytics/analysis_" +data['type'] +"/" +files[f]+ ".png' /></th>";
if (f%2 || true ) { table += "</tr>"; }
}
table += "</tbody><table>";
$(tag).html(table);
}
"""
self.publish(func, role=pmRoles.script())
def doScript(self,
host='World',
title='PanDA Hello Json Word Example',
guest='Your Panda',
hello="Good Morning"):
func = """
function(tag,data) {
$(tag).html('<table cellpadding="0" cellspacing="0" border="0" class="display" id="example"></table>' );
var hdr = [];
for (var h in data['headers'] ) {
hdr.push( {"sTitle":h } );
};
rows = data['rows'];
$('#example').dataTable( { "aaData" : rows, "aoColumns":hdr,"bJQueryUI": true });
}
"""
self.publish(func, role=pmRoles.script())
def doScript(self,site='ALL',From='Jun_2011',To='May_2012',type='7'):
func = """
function(tag,data) {
var files = data.filenames;
$(tag).empty();
var table = "<table>";
table += "<tbody>";
for (var f in files) {
if (!f%2 || true ) { table += "<tr>"; }
table += "<th><img src='http://atlascloud.org/static/images/analytics/analysis_" +data['type'] +"/" +files[f]+ ".png' /></th>";
if (f%2 || true ) { table += "</tr>"; }
}
table += "</tbody><table>";
$(tag).html(table);
}
"""
self.publish(func,role=pmRoles.script())
def doScript(self,
pid,
line,
type='filesWithFAX',
name='panda.mon.fax',
levelname='INFO'):
"""
doScript(self,script=None) publishes the Javascript function to render the content onto the client Web browser
The content is to be published by doJson
NB.The signatures (list of parameters) of the doJson and doScript should be the same
"""
javascript = """
function _anyname_(tag,content) {
/* Render the "content" */
$(tag).empty();
$(tag).html("<pre>"+ JSON.stringify(content,undefined,2)+"\n</pre>");
}
"""
self.publish(javascript, role=pmRoles.script())
def doScript(self,cloud='ALL',style='table'):
func = """
function(tag,data) {
$(tag).empty();
var points = data.points;
if ( data.style == 'table') {
$(tag).html('<table cellpadding="0" cellspacing="0" border="0" id="example"></table>' );
var h = ["data points"];
var hdr = [];
hdr.push( {"sTitle":h } );
hdr.push( {"sTitle":h } );
rows = data['points'];
$('#example').dataTable( { "aaData" : rows, "aoColumns":hdr ,"bJQueryUI": true });
} else {
$(tag).html(" <b> Unknown presentation style:" + data.style + "</b>" );
}
}
"""
self.publish(func,role=pmRoles.script())
def doJson(self,
jobtype='analysis',
timefield='MODIFICATIONTIME',
tstart=None,
tend=None,
days=None,
width=600,
height=400,
options='',
plot=False,
log=False,
file='useract365daysTotal'):
"""
User activity plots <ul>
<li>jobtype - name of the ROOT histogram
<li>file - name the file to get the data from rather from Db
<li>width - the width of the plots in px
<li>height - the height of the plots in px
<li>plot - = 'True' to create the plot; 'False' to use the table view render the data
<li>log - use logarithmic scale for Y-axis
<li>options - some ROOT <a href='http://root.cern.ch/root/html534/THistPainter.html#HP01b'>Paint options</a> or <br>
'M' = to get the "per month" rather "per day" data
</ul>
"""
self.publishTitle('Panda User activities for %(jobtype)s jobs' %
{'jobtype': jobtype})
timer = Stopwatch.Stopwatch()
select = "to_char(MODIFICATIONTIME ,'YYMMDD') as time,PRODUSERID"
if 'M' in options:
select = "to_char(MODIFICATIONTIME ,'YYMM') as time,PRODUSERID"
self.doMain(jobtype, select, timefield, tstart, tend, days, width,
height, options, plot, log, file)
self.publishNav('The User Activities for %s from CERN ( %s)' %
(jobtype, timer))
render = "taskBuffer/%s.js" % (
"tbTable") if not plot else "hello/%s.js" % "pyroot"
self.publish("%s/%s" % (self.server().fileScriptURL(), render),
role=pmRoles.script())
self.publish({
's-maxage': 86400,
'max-age': 86400
},
role=pmRoles.cache())
def doScript(self,script='hellojs'):
"""
doScript(self,script=None) publishes the Javascript function to render the content onto the client Web browser
The content is to be published by doJson
NB.The signatures (list of parameters) of the doJson and doScript should be the same
"""
javascript = """
function _anyname_(tag,content) {
/* Render the "content" */
var thisTag = $("<div style='padding:4px' class='ui-widget ui-widget-content ui-corner-all'></div>");
thisTag.append("<ul>");
for (var i in content) { thisTag.append("<li>" + i + ": " + JSON.stringify(content[i])); }
thisTag.append('</ul>');
$(tag).empty(); /* Clear the HTML tag from the previouse content */
$(tag).append(thisTag);
}
"""
self.publish(javascript,role=pmRoles.script())
def doScript(self, script='hellojs'):
"""
doScript(self,script=None) publishes the Javascript function to render the content onto the client Web browser
The content is to be published by doJson
NB.The signatures (list of parameters) of the doJson and doScript should be the same
"""
javascript = """
function _anyname_(tag,content) {
/* Render the "content" */
var thisTag = $("<div style='padding:4px' class='ui-widget ui-widget-content ui-corner-all'></div>");
thisTag.append("<ul>");
for (var i in content) { thisTag.append("<li>" + i + ": " + JSON.stringify(content[i])); }
thisTag.append('</ul>');
$(tag).empty(); /* Clear the HTML tag from the previouse content */
$(tag).append(thisTag);
}
"""
self.publish(javascript, role=pmRoles.script())
def doJson(self,
src='http://ivukotic.web.cern.ch/ivukotic/FAX',
date='2012-10-16',
selS='ALL',
script='fax/map'):
""" FAX configuration viewer
"""
self.reset()
params = "from=%s" % date
if selS != None: params += '&selS=%s' % selS
self.fetchInfo(src, 'getLinks', params)
if len(self._sites) == 0:
self.fetchInfo(src, 'getSites')
main = {'sites': self._sites, 'links': self._links, 'url': src}
self.publish(main)
self.publishTitle('FAX configuration viewer')
self.publishNav(' %s for "%s" ' % (script, params))
self.publish("%s/%s" %
(self.server().fileScriptURL(), "%s.js" % script),
role=pmRoles.script())
self.publish({'s-maxage': 340, 'max-age': 200}, role=pmRoles.cache())
def __init__(self, name=None, parent=None, obj=None):
pmModule.__init__(self, name, parent, obj)
self._description = pmt.describe('^PANDALOG$')
# add the optional parameters:
params = {}
ci = utils.name2Index(self._description['header'], 'column_name')
collist = 'name message asctime Type PandaID loglevel levelname filename lineno module'.split(
)
exclude = "pid line type name levelname BINTIME LOGUSER".upper().split(
)
nmap = {}
for c in collist:
nmap[c.upper()] = c
for c in self._description['rows']:
p = not c[ci] in exclude
nm = nmap.get(c[ci], c[ci])
c[ci] = nm
if p: params[nm] = ' '
self._allparams = ','.join(params.keys())
self._levelmap = {'INFO': 20, 'WARNING': 30, 'DEBUG': 10, 'ERROR': 40}
## 'publishUI' must be the last init statement !
self.publishUI(self.doJson, params=params)
self.publishUI(self.doScript, role=pmRoles.script())
def doJson(self, jobtype='analysis', timefield='MODIFICATIONTIME', tstart=None, tend=None, days=None,width=600,height=400,options='',plot=False,log=False,file='useract365daysTotal'):
"""
User activity plots <ul>
<li>jobtype - name of the ROOT histogram
<li>file - name the file to get the data from rather from Db
<li>width - the width of the plots in px
<li>height - the height of the plots in px
<li>plot - = 'True' to create the plot; 'False' to use the table view render the data
<li>log - use logarithmic scale for Y-axis
<li>options - some ROOT <a href='http://root.cern.ch/root/html534/THistPainter.html#HP01b'>Paint options</a> or <br>
'M' = to get the "per month" rather "per day" data
</ul>
"""
self.publishTitle('Panda User activities for %(jobtype)s jobs' % {'jobtype': jobtype} )
timer = Stopwatch.Stopwatch()
select = "to_char(MODIFICATIONTIME ,'YYMMDD') as time,PRODUSERID"
if 'M' in options:
select = "to_char(MODIFICATIONTIME ,'YYMM') as time,PRODUSERID"
self.doMain(jobtype, select, timefield, tstart, tend, days,width,height,options,plot,log,file)
self.publishNav('The User Activities for %s from CERN ( %s)' % ( jobtype, timer ) )
render = "taskBuffer/%s.js" % ("tbTable") if not plot else "hello/%s.js" % "pyroot"
self.publish( "%s/%s" % (self.server().fileScriptURL(),render),role=pmRoles.script())
self.publish( {'s-maxage':86400,'max-age': 86400}, role=pmRoles.cache())
def doScript(self,nJobs=10,siteName='ANALY_VICTORIA-WG1',prodSourceLabel='user',cpu=0,mem=0,diskSpace=0,modificationHost='',timeout=0,computingElement=None,
atlasRelease='no use',prodUserID=None,getProxyKey=None,countryGroup='',workingGroup='',allowOtherCountry=True):
func = """
function(tag,main) {
$(tag).empty();
var dt = false;
var time = main.time;
var method = main.buffer.method;
var params = main.buffer.params;
var data = main.buffer.data;
var d = $('<div></div>');
var query = "<P>The server spent <b>" + time.query + "</b> to access the CERN Oracle server";
var fetch = " and <p><b>" + time.fetch + "</b> to execute the TaskBuffer.%s:%s query";
d.html(query + fetch);
var tdiv = $("<div></div>");
var tline = '<p><b>' + method + '(' +JSON.stringify(params)+ '): </b><br>';
dt = true;
tline += '<hr>';
tline += '<table id="TaskBuferTableId" class="display"><thead><tr><th>Parameter</th><th>Value</th></thead><tbody>';
for (var i in data) {
var num = data[i];
tline += "<tr><td>" + i + "</td><td>" + JSON.stringify(num) + "</td></tr>";
}
tline += "<tbody></table>";
tdiv.html(tline);
d.append(tdiv);
$(tag).append(d);
if (dt) {
$(document).ready(function(){
$('#TaskBuferTableId').dataTable( { "bJQueryUI": true });
} );
}
}
""" % ('getJobs', ( nJobs,siteName,prodSourceLabel,cpu,mem,diskSpace,modificationHost,timeout,computingElement,
atlasRelease,prodUserID,getProxyKey,countryGroup,workingGroup,allowOtherCountry))
self.publish(func,role=pmRoles.script())
def doJson(self,script=None):
"""
doJson(self,script=None) defines the Web API list of parametes and its default values
<p>This Python DocString is to automatically published by Web Server the Web API Help widget
The Web Browser user will be able to access it by clicking the "?" button onto the page menu bar.
The string may contain the regular HTML markups.
script - the file name to download and execute . <br>
It should contain the Javascript function `
<pre>
function _fun_(tag,content) { . . . . }
</pre>
"""
self.publishTitle('Hello JavaScript') # short cut for
self.publishNav("Load and Execute the '%s' Javascript " % ( script if script!= None else 'Embedded' ) )
if script != None:
""" publish the reference to the external JavaScript if any """
self.publish( "%s/%s" % (self.server().fileScriptURL(),"%s.js" % script ),role=pmRoles.script())
""" Create the custom content to be published """
content = {"Hello" : "This is my own content to be published from %s " % utils.lineInfo() }
self.publish(content)
self.publish({'s-maxage':340,'max-age': 200},role=pmRoles.cache())
def __init__(self,name=None,parent=None,obj=None):
pmModule.__init__(self,name,parent,obj)
self.publishUI(self.doJson,role=pmRoles.object())
self.publishUI(self.doScript,role=pmRoles.script() )
self._doc = None
def doQuery(self,
jobtype='production',
hours=24,
vcloud='IAAS',
testJobs=None):
if int(hours) > 24 * 3:
hours = 24 * 3
title = "ATLAS '%s' Virtualized Sites for the Last %s Hours Only." % (
vcloud, hours
) # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = "ATLAS '%s' Virtualized Sites for the last %s hours" % (
vcloud, hours)
self.publishTitle(title)
modes = ('Defined', 'Waiting', 'Active', 'Archived')
rows = []
t = []
for mode in modes:
t += ['ATLAS_PANDA.jobs%s4' % mode]
q = pmt.getVCloudSummary(jobtype,
hours,
table=','.join(t),
vcloud=vcloud,
testsite=(testJobs != None))
header = q['header']
rows = q['rows']
### Build the jobsumd dictionary
jobsumd = {}
for c in [
'ALL',
]:
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0] * (self.lstates + 2)
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0] * (self.lstates + 2)
iCloud = utils.name2Index(header, 'cloud')
iComputingSite = utils.name2Index(header, 'computingSite')
iStatus = utils.name2Index(header, 'jobStatus')
iCount = utils.name2Index(header, 'COUNT')
iCpu = utils.name2Index(header, 'cpuConsumptionTime')
icpu = self.lstates
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
if region: continue
site = "Unknown"
# continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
istatus = utils.name2Index(self.jobstates, status)
count = r[iCount]
cpu = r[iCpu]
if not jobsumd.has_key(cloud):
jobsumd[cloud] = {}
jobsumd[cloud]['ALL'] = [0] * (self.lstates + 1)
if cloud != 'ALL':
jobsumd[cloud]['status'] = self.cloudList[cloud]['status']
if cloud == 'CERN' and vcloud == 'IAAS': # fake site for the time being
jobsumd[cloud]['HELIX_ATOS'] = {}
jobsumd[cloud]['HELIX_ATOS'] = [0] * (self.lstates + 1)
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0] * (self.lstates + 1)
jobsumd[cloud][site][istatus] += count
jobsumd[cloud]['ALL'][istatus] += count
jobsumd['ALL']['ALL'][istatus] += count
jobsumd[cloud][site][icpu] += cpu
jobsumd[cloud]['ALL'][icpu] += cpu
jobsumd['ALL']['ALL'][icpu] += cpu
## Get cloud/site status
main = {}
main['states'] = list(self.jobstates) + ["CPU (hours)"]
main['jobsumd'] = jobsumd
main['params'] = {
'jobtype': jobtype,
'hours': hours,
'computingSite': vcloud
}
if testJobs == None:
main['params']['srcLabel'] = 'managed'
self.publish(main)
self.publishNav(
' <b>type</b>=%s <b>hours</b>=%s <b>date:</b> %s' %
(jobtype, hours, datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')))
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "monitor/%s.js" % "vcloud"),
role=pmRoles.script())
self.publish({'s-maxage': 340, 'max-age': 200}, role=pmRoles.cache())
def doJson(self,hosts='atlascloud.org,pandamon-eu.atlascloud.org,atlas-nightlies-browser.cern.ch,pandamon.cern.ch,atlascloud.org,pandamon-eu.atlascloud.org,pandamon.cern.ch,atlas-nightlies-browser.cern.ch',query='helloora',params=None):
"""
Demonstrate the concurrent cross domain communication via <a href='http://en.wikipedia.org/wiki/JSONP'>JSONP</a> format
<ul>
<li><code>hosts</code> = the list of hosts to communicate
<li><code>query</code> = the query to be sent to all hosts
<li><code>params</code>= to be attached to the URL query
</ul>
"""
self.publishTitle('Hello JSONP cross domain communication.')
hosts = "%s,%s" % (hosts,hosts)
h = "%s" % hosts
vr = h.split(",")
ph = ", ".join(vr)
self.publishNav('The JSONP communication with <b>%s</b>.' % ph )
self.publish( "%s/%s" % (self.server().fileScriptURL(),"hello/hellojsonp.js"),role=pmRoles.script())
self.publish({'hosts' : hosts.split(","),'query': query,'params': params })
def __init__(self, name=None, parent=None, obj=None):
pmModule.__init__(self, name, parent, obj)
self.publishUI(self.doJson)
self.publishUI(self.doScript, role=pmRoles.script())
def doQuery(self,jobtype='production',hours=12,region=None):
"""
<ul>
<li>jobtype - production,analysis, test
<li>hours - production,analysis, test
<li>VO - virtual organization atlas,cms
<li>computingSite
<li>processingType
<li>workingGroup
<li>prodUserName
<li>specialHandling
</ul>
"""
if int(hours) > 24 * 3:
hours = 24*3
title = 'Job summary for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = 'Job summary for the past %s hours' % hours
pars = self.extraValuesFilled()
self.publishTitle(title)
nav = ' <b>type</b>=%s <b>hours</b>=%s' % (jobtype, hours )
if pars != None:
for tp in ['computingSite','processingType','workingGroup', 'prodUserName', 'specialHandling']:
if pars.get(tp) !=None:
nav += ' <b>%s</b>=%s' % (tp,pars[tp])
modes = ( 'Defined', 'Waiting', 'Active', 'Archived' )
rows = []
for mode in modes:
q = pmt.getCloudSummary(jobtype,pars.get('VO') if pars!=None else None,hours,extraConditions=pars,selectionFields='computingSite,processingType,workingGroup, prodUserName,specialHandling,modificationTime', tablename='ATLAS_PANDA.jobs%s4' % mode)
if len(rows) ==0: header = q['header']
rows += q['rows']
# get getCurrentSiteData
pilotd = pmt.getCurrentSiteData()
### Build the jobsumd dictionary
jobsumd = {}
proctyped = {}
workgroupd = {}
userd = {}
spechandled = {}
item = self.clouds
if region != None:
item = [self._ALL] + self._sites4region.keys()
for c in item:
jobsumd[c] = {}
jobsumd[c][self._ALL] = [0]*self.lstates
if c != self._ALL :
jobsumd[c]['status'] = self.cloudList.get(c,{'status':'unknown'})['status']
iCloud = utils.name2Index(header,'cloud')
iComputingSite = utils.name2Index(header,'computingSite')
iStatus = utils.name2Index(header,'jobStatus')
iProctype = utils.name2Index(header,'processingType')
iSpechandle = utils.name2Index(header,'specialHandling')
iWorkgroup = utils.name2Index(header,'workingGroup')
iUser = utils.name2Index(header,'prodUserName')
iCount = utils.name2Index(header,'COUNT')
iTime = utils.name2Index(header,'modificationTime')
iLatest = self._iLatest
def updateTime(oldt,newt):
ol = oldt[iLatest]
if (ol != 0 and ol<newt) or ol==0: oldt[iLatest] = newt
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
regn = "unassigned"
site="unassigned"
if region!= None: continue
# continue
else:
regn = self._region4sites.get(site,'unknown')
if region!= None and regn == 'unknown': continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
if status == 'cancelled' and site == 'Unknown': status = 'unassigned'
istatus = utils.name2Index(self.jobstates,status)
proctype = r[iProctype]
spechandle = r[iSpechandle]
workgroup = r[iWorkgroup]
user = r[iUser]
count = r[iCount]
time = r[iTime]
if region!=None: cloud = regn
if not cloud in jobsumd:
print 'Unknown cloud: %s for site %s ' % ( cloud, site)
continue
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0]*self.lstates
jobsumd[cloud][site][istatus] += count
jobsumd[cloud][self._ALL][istatus] += count
jobsumd[self._ALL][self._ALL][istatus] += count
updateTime(jobsumd[cloud][site], time)
updateTime(jobsumd[cloud][self._ALL], time)
updateTime(jobsumd[self._ALL][self._ALL], time)
proctyped[proctype] = proctyped.get(proctype,0) + count
workgroupd[workgroup] = workgroupd.get(workgroup,0) + count
userd[user] = userd.get(user,0) + count
if spechandle != None:
spechandled[spechandle] = spechandled.get(spechandle,0) + count
iFailed = self._iFailed
iFinished = self._iFinished
iRation = self._iRation
iPilots = self._iPilots
for cloud in jobsumd:
for site in jobsumd[cloud]:
if site == 'status': continue
allcloud = jobsumd[cloud][site]
ntot = allcloud[iFailed] + allcloud[iFinished]
psite = pilotd.get(site)
if psite:
nplt = pilotd[site]['updateJob']+ pilotd[site]['getJob']
jobsumd[cloud][site][iPilots] = nplt
jobsumd[cloud][self._ALL][iPilots] += nplt
jobsumd[self._ALL][self._ALL][iPilots] += nplt
if ntot >0:
jobsumd[cloud][site][iRation] = 100.0*allcloud[iFailed]/ntot
## Get cloud/site status
main = {}
main['states'] = self.jobstates
main['jobsumd'] = jobsumd
if pars == None: pars = {}
if (hours!=None):
pars['hours'] = hours
if (jobtype!=None):
pars['jobtype'] = jobtype
if (region!=None):
pars['region'] = region
nav += ' <b>region view</a>'
main['params'] = pars
self.publish(main)
self.publishNav(nav)
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "cloudsummary" ),role=pmRoles.script() )
def doJson(self, table=None, column=None, db='pmt', doc=None):
"""
Describe the selected Panda table / column <br>
<code>table = the Db table name regexp comma-separated patterns to check</code><br>
<code>column = the Db table column name regexp comma-separated patterns to find</code><br>
<code>doc = '*|all' - document all columns selected if available<br>
= 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided
</code><br>
"""
title = 'Describe the Panda Db '
inp = False
if table != None and table != 'None':
title += "'%s*'" % table
inp = True
title += " table"
else:
title += " tables"
if column != None and column != 'None':
title += " with '%s*' column " % column
inp = True
main = {}
if inp:
self.publishTitle(title)
timer = Stopwatch.Stopwatch()
main["buffer"] = {}
main["buffer"]["db"] = db
dbAccess = self._db.get(db)
if dbAccess != None:
main["buffer"]["method"] = "describe"
main["buffer"]["params"] = (table, column, doc)
main["buffer"]["type"] = False
if not utils.isFilled(doc):
tdes = dbAccess.describe(table, column)
iclmn = utils.name2Index(tdes['header'], 'COLUMN_NAME')
rows = tdes['rows']
for h in rows:
h[iclmn] = nmap.get(h[iclmn])
main["buffer"]["data"] = tdes
else:
docrows = []
if utils.isFilled(table):
tdoc = dbAccess.pandatabs(
table,
comments=doc,
select=
"table_name,'Table Description' as COLUMN_NAME,comments"
)
iclmn = utils.name2Index(tdoc['header'], 'COLUMN_NAME')
docrows = tdoc['rows']
for h in docrows:
h[iclmn] = nmap.get(h[iclmn])
tcol = dbAccess.pandacols(
column,
table,
select='TABLE_NAME,COLUMN_NAME,COMMENTS',
comments=doc)
iclmn = utils.name2Index(tcol['header'], 'COLUMN_NAME')
for h in tcol['rows']:
h[iclmn] = nmap.get(h[iclmn])
tcol['rows'] += docrows
main["buffer"]["data"] = tcol
self.publish({
's-maxage': 60000,
'max-age': 6000
},
role=pmRoles.cache())
else:
self.publishTitle(
"Error: 'describe' - unknown Db name '%s' has been provided"
% db)
self.publish({
's-maxage': 0,
'max-age': 0
},
role=pmRoles.cache())
else:
self.publishTitle(
"Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'"
)
self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache())
self.publish(main)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "taskBuffer/%s.js" % "describe"),
role=pmRoles.script())
def doJson(self,table=None,column=None,db='pmt',doc=None):
"""
Describe the selected Panda table / column <br>
<code>table = the Db table name regexp comma-separated patterns to check</code><br>
<code>column = the Db table column name regexp comma-separated patterns to find</code><br>
<code>doc = '*|all' - document all columns selected if available<br>
= 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided
</code><br>
"""
title = 'Describe the Panda Db '
inp = False
if table != None and table != 'None':
title += "'%s*'" % table
inp = True
title +=" table"
else:
title +=" tables"
if column != None and column !='None':
title += " with '%s*' column " % column
inp = True
main = {}
if inp:
self.publishTitle(title)
timer = Stopwatch.Stopwatch()
main["buffer"] = {}
main["buffer"]["db"] = db
dbAccess = self._db.get(db)
if dbAccess != None:
main["buffer"]["method"] = "describe"
main["buffer"]["params"] = (table,column,doc)
main["buffer"]["type"] = False
if not utils.isFilled(doc):
tdes = dbAccess.describe(table,column)
iclmn = utils.name2Index(tdes['header'],'COLUMN_NAME')
rows = tdes['rows']
for h in rows:
h[iclmn] = nmap.get(h[iclmn])
main["buffer"]["data"] = tdes
else:
docrows = []
if utils.isFilled(table):
tdoc = dbAccess.pandatabs(table,comments=doc,select="table_name,'Table Description' as COLUMN_NAME,comments")
iclmn = utils.name2Index(tdoc['header'],'COLUMN_NAME')
docrows = tdoc['rows']
for h in docrows:
h[iclmn] = nmap.get(h[iclmn])
tcol = dbAccess.pandacols(column,table,select='TABLE_NAME,COLUMN_NAME,COMMENTS', comments=doc)
iclmn = utils.name2Index(tcol['header'],'COLUMN_NAME')
for h in tcol['rows']:
h[iclmn] = nmap.get(h[iclmn])
tcol['rows'] += docrows
main["buffer"]["data"] = tcol
self.publish( {'s-maxage':60000,'max-age': 6000}, role=pmRoles.cache())
else:
self.publishTitle("Error: 'describe' - unknown Db name '%s' has been provided" % db)
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
else:
self.publishTitle("Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'")
self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache())
self.publish(main)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskBuffer/%s.js" % "describe"),role=pmRoles.script() )
def doJson(self,
job=None,
taskid=None,
tstart=None,
tend=None,
hours=71,
days=None,
jobparam=None,
limit=1500,
summary=None,
minjob=1,
jobtype=None,
timeField='modificationTime',
plot='no',
dump=None,
tasktype=None,
status=None,
region=None,
lfn=None,
showcpu=None):
"""Display the job information:
<p>Sample queries:
<ul>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours='>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01</a>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours=&processingType=simul'>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01&[email protected]</a>
<li><a href='http://pandamon.cern.ch/jobinfo?job=1246240909'>http://pandamon.cern.ch/jobinfo?job=1246240909</a>
</ul>
<hr>
Supported query parameters are as follows:
<table>
<tr><td>taskid</td><td> - jobset / task id</td></tr>
<tr><td>job</td><td> - comma separated list of jobs' PandaID</td></tr>
<tr><td>jobtype</td><td> - select the 'jobtype' jobs: analysys, production, install, test </td></tr>
<tr><td>tstart=None</td><td> - start time </td></tr>
<tr><td>tend=None</td><td> - end time</td></tr>
<tr><td>hours=N </td><td> - last (N hours + N days) or N hours after 'tstart' or N hours before 'tend' </td></tr>
<tr><td>days=N</td><td> - last (N hours + N days) (see the hours parameter documentation</td></tr>
<tr><td>lfn=</td><td> - Select the PanDA jobs using the files from the comma-separated lfn list </td></tr>
<tr><td>jobparam=</td><td>The comma separated list of the <a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>job parameters</a>.<br> The default should satisfy the interactive users.<br>
One needs to customize jobparam to build his/her <a href='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#Custom_Monitoring_Client_Code'>own client</a></td></tr>
<tr><td>limit=</d><td> - max number of the Db records to fetch from each job Db table. Since there are 4 tables the real number of the rows can be 4 times as many as that limit</td></tr>
<tr><td><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</td><td>- comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.<br>
For example: <ul><li><a href='http://pandamon.cern.ch/jobinfo?processingType=simul'>http://pandamon.cern.ch/jobinfo?processingType=simul</a>
<li> <a href='http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test'>http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test</a></ul>
<tr><td>summary</td><td>Create the summary for the first member of the jobparam list<br>
For example: <a href='http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes'>http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes</a>
</td></tr>
<tr><td>dump</td><td> - 'yes' - produce the flat list of job info records suitable for CLI applications</td></tr>
<tr><td>region</td><td> - the comma separated list of the regions </td></tr>
<tr><td>showcpu</td><td> - 'yes' to show the CPU type used </td></tr>
</table>
"""
# % { 'url' : self.server().branchUrl() }
if job == 'undefined': job = None
if limit == 'undefined': limit = None
if summary == 'undefined': summary = None
if jobparam == 'undefined': jobparam = None
if tstart == 'undefined': tstart = None
if tend == 'undefined': tend = None
if tasktype == 'prod': jobtype = 'production'
if jobtype == 'null': jobtype = None
if jobparam == None:
if job != None:
jobparam = self._allparams
else:
jobparam = ','.join(self._colNames)
if timeField == None or timeField == 'undefined' or timeField == '':
timeField = 'modificationTime'
if self.isValid(jobparam):
self.publish(
self.doJobs(job,
tstart=tstart,
tend=tend,
hours=hours,
days=days,
jobparam=jobparam,
limit=limit,
field=timeField,
jobtype=jobtype,
summary=summary,
minjob=minjob,
dump=dump,
region=region,
lfn=lfn,
showcpu=showcpu))
self.publish("%s/%s" %
(self.server().fileScriptURL(), "jobInfoRender.js"),
role=pmRoles.script())
self.publish({
's-maxage': 600,
'max-age': 600
},
role=pmRoles.cache())
else:
title = 'Job data '
if utils.isFilled(job):
hours = None
days = None
title += " %s " % job
if days != None:
title += ' for %.2f day' % float(days)
if days > 1: title += "s"
if hours != None:
if days == None: title += " for "
title += ' %d hour' % int(hours)
if hours > 1: title += "s"
self.publishTitle(title)
def doQuery(self, jobtype='production', hours=12, region=None):
"""
<ul>
<li>jobtype - production,analysis, test
<li>hours - production,analysis, test
<li>VO - virtual organization atlas,cms
<li>computingSite
<li>processingType
<li>workingGroup
<li>prodUserName
<li>specialHandling
</ul>
"""
if int(hours) > 24 * 3:
hours = 24 * 3
title = 'Job summary for the last %s hours only.' % hours # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = 'Job summary for the past %s hours' % hours
pars = self.extraValuesFilled()
self.publishTitle(title)
nav = ' <b>type</b>=%s <b>hours</b>=%s' % (jobtype, hours)
if pars != None:
for tp in [
'computingSite', 'processingType', 'workingGroup',
'prodUserName', 'specialHandling'
]:
if pars.get(tp) != None:
nav += ' <b>%s</b>=%s' % (tp, pars[tp])
modes = ('Defined', 'Waiting', 'Active', 'Archived')
rows = []
for mode in modes:
q = pmt.getCloudSummary(
jobtype,
pars.get('VO') if pars != None else None,
hours,
extraConditions=pars,
selectionFields=
'computingSite,processingType,workingGroup, prodUserName,specialHandling,modificationTime',
tablename='ATLAS_PANDA.jobs%s4' % mode)
if len(rows) == 0: header = q['header']
rows += q['rows']
# get getCurrentSiteData
pilotd = pmt.getCurrentSiteData()
### Build the jobsumd dictionary
jobsumd = {}
proctyped = {}
workgroupd = {}
userd = {}
spechandled = {}
item = self.clouds
if region != None:
item = [self._ALL] + self._sites4region.keys()
for c in item:
jobsumd[c] = {}
jobsumd[c][self._ALL] = [0] * self.lstates
if c != self._ALL:
jobsumd[c]['status'] = self.cloudList.get(
c, {'status': 'unknown'})['status']
iCloud = utils.name2Index(header, 'cloud')
iComputingSite = utils.name2Index(header, 'computingSite')
iStatus = utils.name2Index(header, 'jobStatus')
iProctype = utils.name2Index(header, 'processingType')
iSpechandle = utils.name2Index(header, 'specialHandling')
iWorkgroup = utils.name2Index(header, 'workingGroup')
iUser = utils.name2Index(header, 'prodUserName')
iCount = utils.name2Index(header, 'COUNT')
iTime = utils.name2Index(header, 'modificationTime')
iLatest = self._iLatest
def updateTime(oldt, newt):
ol = oldt[iLatest]
if (ol != 0 and ol < newt) or ol == 0: oldt[iLatest] = newt
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
regn = "unassigned"
site = "unassigned"
if region != None: continue
# continue
else:
regn = self._region4sites.get(site, 'unknown')
if region != None and regn == 'unknown': continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
if status == 'cancelled' and site == 'Unknown':
status = 'unassigned'
istatus = utils.name2Index(self.jobstates, status)
proctype = r[iProctype]
spechandle = r[iSpechandle]
workgroup = r[iWorkgroup]
user = r[iUser]
count = r[iCount]
time = r[iTime]
if region != None: cloud = regn
if not cloud in jobsumd:
print 'Unknown cloud: %s for site %s ' % (cloud, site)
continue
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0] * self.lstates
jobsumd[cloud][site][istatus] += count
jobsumd[cloud][self._ALL][istatus] += count
jobsumd[self._ALL][self._ALL][istatus] += count
updateTime(jobsumd[cloud][site], time)
updateTime(jobsumd[cloud][self._ALL], time)
updateTime(jobsumd[self._ALL][self._ALL], time)
proctyped[proctype] = proctyped.get(proctype, 0) + count
workgroupd[workgroup] = workgroupd.get(workgroup, 0) + count
userd[user] = userd.get(user, 0) + count
if spechandle != None:
spechandled[spechandle] = spechandled.get(spechandle,
0) + count
iFailed = self._iFailed
iFinished = self._iFinished
iRation = self._iRation
iPilots = self._iPilots
for cloud in jobsumd:
for site in jobsumd[cloud]:
if site == 'status': continue
allcloud = jobsumd[cloud][site]
ntot = allcloud[iFailed] + allcloud[iFinished]
psite = pilotd.get(site)
if psite:
nplt = pilotd[site]['updateJob'] + pilotd[site]['getJob']
jobsumd[cloud][site][iPilots] = nplt
jobsumd[cloud][self._ALL][iPilots] += nplt
jobsumd[self._ALL][self._ALL][iPilots] += nplt
if ntot > 0:
jobsumd[cloud][site][
iRation] = 100.0 * allcloud[iFailed] / ntot
## Get cloud/site status
main = {}
main['states'] = self.jobstates
main['jobsumd'] = jobsumd
if pars == None: pars = {}
if (hours != None):
pars['hours'] = hours
if (jobtype != None):
pars['jobtype'] = jobtype
if (region != None):
pars['region'] = region
nav += ' <b>region view</a>'
main['params'] = pars
self.publish(main)
self.publishNav(nav)
self.publish(
"%s/%s" %
(self.server().fileScriptURL(), "monitor/%s.js" % "cloudsummary"),
role=pmRoles.script())
def doQuery(self,site='all',jobtype='all',days=1,hostonly='yes',plot='finished,failed',style='SB', details=None,deep='no'):
""" Get the list of the jobs' status per worker node for selected site for upto 3 last days
<ol>
<li><code>site</code> - site to get the information about or "all"
<li><code>jobtype</code> = 'production" or " analysis" or "all"
<li><code>days</code> - number of days we want to fetch the information for. This parameter must be less or equal 3.
<li><code>hostonly</code> - "yes" . "no" - means provide the information 'per CPU' rather 'per host'
<li><code>plot</code> - the comma separated list of the <a href='https://twiki.cern.ch/twiki/bin/viewauth/Atlas/PandaShiftGuide#Job_state_definitions_in_Panda'>job status names</a> to generate the plot for
<li><code>style</code> the combination of the symbols: 'H' - histogram, 'B' - bar , 'L' - line, 'P' - points, 'S' - stacked<br>
= 'BS' - 'by default'
<li><code>details</code> - 'yes' - provide the status per host<br>
<br> = 'summary' - provide the summary only <br>
default = 'yes' if not site == 'all' else 'no'
<li><code>deep</code> - do not look up the "long job table" . It can be very slow.
default = 'no'
<li> <b><a title='Click to see the full list of the parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</b> - comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.
<li><code>jobStatus</code> - it can be any comma-separated list of the Panda Jobs states patterns or / and <br>
two 'meta' states: "factory" to list the "factory" nodes and "wn" to list the the Worker Nodes
</ol>
"""
vals = self.extraValuesFilled()
if site == None: site='all'
factory = None
wn = None
titletype = "PanDA"
jobStatus = None
if vals != None:
jstat = vals.get('jobStatus')
jobStatus = jstat
if jstat != None:
jstat = [x.lower() for x in utils.parseArray(jstat)]
print utils.lineInfo(), jstat, 'factory' in jstat
if 'factory' in jstat:
factory = self.factory()
jstat += factory
titletype = "Factory"
if 'wn' in jstat:
wn = self.wn()
jstat += wn
titletype = "Worker"
jstat = list(set(jstat))
vals['jobStatus'] = ','.join(jstat)
if deep == None or deep == False: deep = 'no'
deep = not deep.lower() == 'no'
title = ''
hours = (float(days))*24
Deep = 'Deep' if deep else ''
if float(days) > 3 and False and not deep:
days = 3.0
hours = 24*days
title = 'The List of the %s Nodes Activity for the Last %s Hours Only.' % (titletype, int(hours))
else:
title = 'The %s List of the %s Nodes Activity for the %s Hours' % (Deep, titletype,int(hours) )
days = float(days)
try:
if jobtype.lower() == 'all': jobtype=None
except:
pass
self.publishTitle(title)
modes = ( 'Defined', 'Waiting', 'Active', 'Archived' )
tables = []
for mode in modes: tables += ['ATLAS_PANDA.jobs%s4' % mode]
if days>3 and deep: tables += ['ATLAS_PANDAARCH.JOBSARCHIVED']
q = pmt.wnList(site, jobtype, table=tables, days=days,conditions = vals)
try:
if details == None:
details = not (site == None or site == 'None' or site.lower() == 'all' or site == '*' )
# else:
# details = isinstance(details,str) and details.lower() == 'yes'
except:
details = False
## Publish the result
main = {}
main['data'] = self.do(q['header'],q['rows'],site,hostonly,details,factory,wn)
main['params'] = { 'jobtype': jobtype,'days': days,'site': site,'hostonly':hostonly,'jobStatus':jobStatus}
plotpars = []
if plot:
for p in plot.split(","):
if p.lower() == 'prerun' or factory != None:
if wn == None: plotpars += self.factory()
if factory == None:
plotpars += ['pending','sent','starting']
elif p.lower() == 'finishing' and factory == None:
plotpars += ['holding','transferring']
else:
plotpars += [p]
main['params']['plot'] = list(set(plotpars)) # remove duplicates
if style: main['params']['style'] = style
self.publish(main)
self.publishNav(' type=%s days= last %s ' % (jobtype, days ) )
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "wnlist" ),role=pmRoles.script())
self.publish( {'s-maxage':600,'max-age': 600}, role=pmRoles.cache())
def doJson(
self,
job=None,
taskid=None,
tstart=None,
tend=None,
hours=71,
days=None,
jobparam=None,
limit=1500,
summary=None,
minjob=1,
jobtype=None,
timeField="modificationTime",
plot="no",
dump=None,
tasktype=None,
status=None,
region=None,
lfn=None,
showcpu=None,
):
"""Display the job information:
<p>Sample queries:
<ul>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours='>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01</a>
<li><a href='http://pandamon.cern.ch/jobinfo?tstart=2013-04-07 00:00&tend=2013-04-07 00:03&hours=&processingType=simul'>http://pandamon.cern.ch/jobinfo/?tstart=2011-06-07 00:00&tend=2011-06-07 00:01&[email protected]</a>
<li><a href='http://pandamon.cern.ch/jobinfo?job=1246240909'>http://pandamon.cern.ch/jobinfo?job=1246240909</a>
</ul>
<hr>
Supported query parameters are as follows:
<table>
<tr><td>taskid</td><td> - jobset / task id</td></tr>
<tr><td>job</td><td> - comma separated list of jobs' PandaID</td></tr>
<tr><td>jobtype</td><td> - select the 'jobtype' jobs: analysys, production, install, test </td></tr>
<tr><td>tstart=None</td><td> - start time </td></tr>
<tr><td>tend=None</td><td> - end time</td></tr>
<tr><td>hours=N </td><td> - last (N hours + N days) or N hours after 'tstart' or N hours before 'tend' </td></tr>
<tr><td>days=N</td><td> - last (N hours + N days) (see the hours parameter documentation</td></tr>
<tr><td>lfn=</td><td> - Select the PanDA jobs using the files from the comma-separated lfn list </td></tr>
<tr><td>jobparam=</td><td>The comma separated list of the <a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>job parameters</a>.<br> The default should satisfy the interactive users.<br>
One needs to customize jobparam to build his/her <a href='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#Custom_Monitoring_Client_Code'>own client</a></td></tr>
<tr><td>limit=</d><td> - max number of the Db records to fetch from each job Db table. Since there are 4 tables the real number of the rows can be 4 times as many as that limit</td></tr>
<tr><td><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</td><td>- comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.<br>
For example: <ul><li><a href='http://pandamon.cern.ch/jobinfo?processingType=simul'>http://pandamon.cern.ch/jobinfo?processingType=simul</a>
<li> <a href='http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test'>http://pandamon.cern.ch/jobinfo?limit=2&hours=4&processingType=ganga*,test</a></ul>
<tr><td>summary</td><td>Create the summary for the first member of the jobparam list<br>
For example: <a href='http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes'>http://pandamon.cern.ch/jobinfo?processingType=simul&jobparam=jobStatus&summary=yes</a>
</td></tr>
<tr><td>dump</td><td> - 'yes' - produce the flat list of job info records suitable for CLI applications</td></tr>
<tr><td>region</td><td> - the comma separated list of the regions </td></tr>
<tr><td>showcpu</td><td> - 'yes' to show the CPU type used </td></tr>
</table>
"""
# % { 'url' : self.server().branchUrl() }
if job == "undefined":
job = None
if limit == "undefined":
limit = None
if summary == "undefined":
summary = None
if jobparam == "undefined":
jobparam = None
if tstart == "undefined":
tstart = None
if tend == "undefined":
tend = None
if tasktype == "prod":
jobtype = "production"
if jobtype == "null":
jobtype = None
if jobparam == None:
if job != None:
jobparam = self._allparams
else:
jobparam = ",".join(self._colNames)
if timeField == None or timeField == "undefined" or timeField == "":
timeField = "modificationTime"
if self.isValid(jobparam):
self.publish(
self.doJobs(
job,
tstart=tstart,
tend=tend,
hours=hours,
days=days,
jobparam=jobparam,
limit=limit,
field=timeField,
jobtype=jobtype,
summary=summary,
minjob=minjob,
dump=dump,
region=region,
lfn=lfn,
showcpu=showcpu,
)
)
self.publish("%s/%s" % (self.server().fileScriptURL(), "jobInfoRender.js"), role=pmRoles.script())
self.publish({"s-maxage": 600, "max-age": 600}, role=pmRoles.cache())
else:
title = "Job data "
if utils.isFilled(job):
hours = None
days = None
title += " %s " % job
if days != None:
title += " for %.2f day" % float(days)
if days > 1:
title += "s"
if hours != None:
if days == None:
title += " for "
title += " %d hour" % int(hours)
if hours > 1:
title += "s"
self.publishTitle(title)
def __init__(self,name=None,parent=None,obj=None):
pmModule.__init__(self,name,parent,obj)
self.publishUI(self.doQuery,role='html')
self.publishUI(self.doJson)
self.publishUI(self.doScript,role=pmRoles.script())
def doJson(self,lfn=None, jobs=None,type='input',ds=None,table='new',limit=1000,jobstatus=None,site=None,jobtype='production',days=1,user=None,select=None):
"""
Show the list of Panda id with LFN or versa verse LFNs by Panda IDs
<ul>
<li><code>lfn</code> - the list of the comma separated files
<li><code>ds</code> - the list of the comma separated datasets
<li><code>jobs</code> - the list of the comma separated Panda's job IDs
<li><code>table</code> = "new" (default) look up the records for last 3 days
<br> ="old" - look up the records those more than 3 days old (slow)
<br> ="deep" - look up the "old" and "new" tables (slow)
<li><code>type</code> - the type selector. <br>
= 'input - the default value<br>
= '*' | 'all' - list all types available.
<li><code>jobstatus</code> = the comma separated list of the job status to filter
<br>For example: 'defined, waiting,assigned,activated,sent,starting,running'
<li><code>site</code> = the comma separated list of the sizte to list the jobs from
<br> For example 'UTA_SWT2'
<li><code>jobtype</code> = the comma separated list of the job type to filter
<br> For example, "analysis, production"
<li><code>days</code> = the number of the days to look up the list of the jobs if either 'jobstatus' or 'site' parameter is defined
<li><code>user</code> = the comma separated list of the usernames .
<br>NB. The names may contain the the wildcard symbol '*'. Be aware the wildcard slows the search down
</ul>
"""
title = 'The list of files for the '
if jobstatus and jobstatus.strip() =='': jobstatus = None
if site and site.strip() =='': site = None
if lfn and lfn.strip() =='': lfn = None
if jobs and isinstance(jobs,str) and jobs.strip() =='': jobs = None
if ds and ds.strip() =='': ds=None
if type and type.strip() =='': type='all'
if lfn==None and jobs==None and ds==None and jobstatus==None and site==None:
self.publishTitle("Ambigios query: lfn=%(lfn)s; pandaid=%(pandaid)s either lfn or padaid can be defined. One can not define lfn and pandaid at once" % { 'lfn': lfn, 'pandaid' : jobs} )
self.publish('<h2>Check the input parameters. Click the "?" to see the API documentaion<h2>', role=pmRoles.html())
else:
nav = ''
if limit:
nav += "Limit %s rows." % limit
if type=='*' or type==None: type = 'all'
if lfn != None:
self.publishTitle("The list of the PANDA jobs with the LFN of the '%s' type provided" % type)
if not '*' in lfn: # disregard the jobtype parameter
if utils.isFilled(jobtype):
nav += " Disregarding the jobtype='%s' default parameter" % jobtype
jobtype = None
if ds != None:
self.publishTitle("The list of the PANDA jobs with the DATASETs of the '%s' type provided" % type)
if jobs!=None:
self.publishTitle("The list of the '%s' LFN with the PANDA Job IDs provided" % type)
if utils.isFilled(nav):
self.publishNav(nav)
main = {}
main["buffer"] = {}
main["buffer"]["method"] = "joblfn"
main["buffer"]["params"] = (lfn if lfn!=None else '',jobs if jobs!= None else '' )
if jobs != None: main["buffer"]["jobs"] = jobs
main["buffer"]["type"] = False
if (utils.isFilled(jobstatus) or utils.isFilled(site) or utils.isFilled(user)) and not utils.isFilled(jobs):
tables = ['atlas_panda.jobsArchived4','atlas_panda.jobsActive4','atlas_panda.jobsWaiting4','atlas_panda.jobsDefined4']
r = pmt.getJobIds(site, jobtype,jobstatus,table=tables,days=days,username=user)
jobs = [i[0] for i in r['rows']]
if not utils.isFilled(select):
select = []
if jobs == None or ( not isinstance(jobs,int) and len(jobs) > 1): select.append('pandaid');
select += ['type', 'lfn', 'fsize', 'dataset', 'guid', 'scope', 'destinationse']
else:
select = utils.parseArray(select);
main["buffer"]["data"] = pmt.getJobLFN(select=','.join(select),table=table,lfn=lfn,pandaid=jobs,type=type,ds=ds,limit=limit)
self.publish(main)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskBuffer/%s.js" % "getJobLFN"),role=pmRoles.script())
def doQuery(self,jobtype='production',hours=24,vcloud='IAAS',testJobs=None):
if int(hours) > 24 * 3:
hours = 24 * 3
title = "ATLAS '%s' Virtualized Sites for the Last %s Hours Only." % (vcloud,hours) # Use <a href="%s/?%s">"stats"</a> page to see all %s hours' % ( 24 * 3, utils.monURL, 'mode=sitestats', hours )
else:
title = "ATLAS '%s' Virtualized Sites for the last %s hours" % (vcloud,hours)
self.publishTitle(title)
modes = ( 'Defined', 'Waiting', 'Active', 'Archived' )
rows = []
t = []
for mode in modes: t += ['ATLAS_PANDA.jobs%s4' % mode]
q = pmt.getVCloudSummary(jobtype,hours,table=','.join(t),vcloud=vcloud,testsite=(testJobs!=None))
header = q['header']
rows = q['rows']
### Build the jobsumd dictionary
jobsumd = {}
for c in ['ALL',]:
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0]*(self.lstates+2)
jobsumd[c] = {}
jobsumd[c]['ALL'] = [0]*(self.lstates+2)
iCloud = utils.name2Index(header,'cloud')
iComputingSite = utils.name2Index(header,'computingSite')
iStatus = utils.name2Index(header,'jobStatus')
iCount = utils.name2Index(header,'COUNT')
iCpu = utils.name2Index(header,'cpuConsumptionTime')
icpu = self.lstates
for r in rows:
site = r[iComputingSite]
if site == None:
# print "Db record doesn't define any site: %s " % r
if region: continue
site="Unknown"
# continue
cloud = r[iCloud] # utils.getCloud4Site(r)
status = r[iStatus]
istatus = utils.name2Index(self.jobstates,status)
count = r[iCount]
cpu = r[iCpu]
if not jobsumd.has_key(cloud):
jobsumd[cloud] = {}
jobsumd[cloud]['ALL'] = [0]*(self.lstates+1)
if cloud != 'ALL':
jobsumd[cloud]['status'] = self.cloudList[cloud]['status']
if cloud == 'CERN' and vcloud == 'IAAS': # fake site for the time being
jobsumd[cloud]['HELIX_ATOS'] = {}
jobsumd[cloud]['HELIX_ATOS'] = [0]*(self.lstates+1)
if not status in self.jobstates:
print 'Unknown jobstate:', status
continue
if not site in jobsumd[cloud]:
jobsumd[cloud][site] = {}
jobsumd[cloud][site] = [0]*(self.lstates+1)
jobsumd[cloud][site][istatus] += count
jobsumd[cloud]['ALL'][istatus] += count
jobsumd['ALL']['ALL'][istatus] += count
jobsumd[cloud][site][icpu] += cpu
jobsumd[cloud]['ALL'][icpu] += cpu
jobsumd['ALL']['ALL'][icpu] += cpu
## Get cloud/site status
main = {}
main['states'] = list(self.jobstates) + ["CPU (hours)"]
main['jobsumd'] = jobsumd
main['params'] = { 'jobtype': jobtype,'hours': hours
,'computingSite': vcloud
}
if testJobs == None:
main['params']['srcLabel'] = 'managed'
self.publish(main)
self.publishNav(' <b>type</b>=%s <b>hours</b>=%s <b>date:</b> %s' % (jobtype, hours, datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') ) )
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"monitor/%s.js" % "vcloud" ),role=pmRoles.script() )
self.publish( {'s-maxage':340,'max-age': 200}, role=pmRoles.cache())
def doQuery(self,tstart=None,tend=None,hours=6,days=None, type='FAXrecovery',limit=None,count=None,message=None):
""" Shows Panda logger messages coming from FAX failovers. Messages are summarized per site.
<ul>
<li> tstart - the start time of the period the messages belongs to
<li> tend - the start time of the period the messages belongs to
<li> hours - select the messages for the last '24*days+hours' hours
<li> days - select the messages for the last 'days+hours/24' days
<li> limit - the max number of records to be fetched from the Db table
<li> <b><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</b> - comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.
</ul>
"""
site="panda.mon.fax"
vals = self.extraValuesFilled()
if vals == None: vals = {}
if not vals.has_key("jobStatus"):
vals["jobStatus"] = 'finished,cancelled,failed'
levelcode= ''
hrs = 0
try:
hrs = int(hours)
except:
pass
try:
hrs += int(days)*24
days = None
except:
pass
logwatch = Stopwatch()
jobparam="computingSite,jobStatus,prodUserName"
conditions=None
q = pmt.getFaxInfo(site,type,levelcode, message=message, limit=limit, hours=hrs,select=["PID as PandaID ",'MESSAGE','Time'],selectjobs=jobparam,conditions=conditions,days=days,tstart=tstart,tend=tend,table='ATLAS_PANDA.PANDALOG_FAX tab')
logwatch.stop()
header = utils.normalizeSchema(q['header'])
rows = q['rows']
main = {}
main['header'] = []
main['info'] = rows
try:
count = int(count)
main['count'] = count
except:
pass
jobwatch = None
if len(rows) > 0:
jobwatch = Stopwatch()
main['header'] = header
# extract the "pandaid"
iPandaId = utils.name2Index(header,'pandaid')
if iPandaId ==None: iPandaId = utils.name2Index(header,'pid')
iProdUserName = utils.name2Index(header,'prodUserName')
iComputingSite = utils.name2Index(header,'computingSite')
rows = sorted(rows, key=lambda fax: fax[iPandaId])
selRows = []
for r in rows:
if r[iProdUserName] == 'gangarbt': continue
r[iComputingSite] = self.cloud(r[iComputingSite])
selRows += [r]
rows = selRows
main['info'] = rows
# split the message by variables
header += self._faxVariables
iMessage = utils.name2Index(header,'message')
vIndx = utils.name2Index(header)
for r in rows:
r += [0]*len(self._faxVariables)
try:
faxcolumns = self.toObj(r[iMessage])
except:
print " FAX message {%s} from the job record [%s] is corrupted" % (r[iMessage], r)
continue
for f in faxcolumns:
try:
r[vIndx[f]] = faxcolumns[f]
except:
raise ValueError(" len=%s r=%s f=%s vIndx=%s %s" % (len(r),r,f,vIndx,faxcolumns))
r.pop(iMessage) # removing message column
header.pop(iMessage) # removing message from header
jobwatch.stop()
main['time'] = {'logquery': '%s' % logwatch,'jobquery': '%s' % jobwatch}
main['params'] = {'hours':hrs}
self.publish(main)
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"fax/%s.js" % "failoverfast" ),role=pmRoles.script() )
self.publishTitle("Panda report on jobs failovers to FAX over last %s hours " % hrs )
def doQuery(self,tstart=None,tend=None,hours=6,days=None, type='FAXrecovery',limit=None,count=None,message=None):
""" Shows Panda logger messages coming from FAX failovers. Messages are summarized per site.
<ul>
<li> tstart - the start time of the period the messages belongs to
<li> tend - the start time of the period the messages belongs to
<li> hours - select the messages for the last '24*days+hours' hours
<li> days - select the messages for the last 'days+hours/24' days
<li> limit - the max number of records to be fetched from the Db table
<li> <b><a title='Click to see the full list of the jobinfo parameters available' href='http://pandamon.cern.ch/describe?table=^JOBSARCHIVED4&doc=*'>PARAM</a>=value</b> - comma-separated list of the value <a href ='https://twiki.cern.ch/twiki/bin/view/PanDA/PandaPlatform#API_parameter_value_format'> patterns</a> to filter the job records.
</ul>
"""
site="panda.mon.fax"
vals = self.extraValuesFilled()
if vals == None: vals = {}
if not vals.has_key("jobStatus"):
vals["jobStatus"] = 'finished,cancelled,failed'
levelcode= ''
hrs = 0
try:
hrs = int(hours)
except:
pass
try:
hrs += int(days)*24
days = None
except:
pass
logwatch = Stopwatch()
jobparam="computingSite,jobStatus,prodUserName"
conditions=None
q = pmt.getFaxInfo(site,type,levelcode, message=message, limit=limit, hours=hrs,select=["PID as PandaID ",'MESSAGE','Time'],selectjobs=jobparam,conditions=conditions,days=days,tstart=tstart,tend=tend,table='ATLAS_PANDA.PANDALOG_FAX tab')
logwatch.stop()
header = utils.normalizeSchema(q['header'])
rows = q['rows']
main = {}
main['header'] = []
main['info'] = rows
try:
count = int(count)
main['count'] = count
except:
pass
jobwatch = None
if len(rows) > 0:
jobwatch = Stopwatch()
main['header'] = header
# extract the "pandaid"
iPandaId = utils.name2Index(header,'pandaid')
if iPandaId ==None: iPandaId = utils.name2Index(header,'pid')
iProdUserName = utils.name2Index(header,'prodUserName')
iComputingSite = utils.name2Index(header,'computingSite')
rows = sorted(rows, key=lambda fax: fax[iPandaId])
selRows = []
for r in rows:
if r[iProdUserName] == 'gangarbt': continue
r[iComputingSite] = self.cloud(r[iComputingSite])
selRows += [r]
rows = selRows
main['info'] = rows
# split the message by variables
header += self._faxVariables
iMessage = utils.name2Index(header,'message')
vIndx = utils.name2Index(header)
for r in rows:
r += [0]*len(self._faxVariables)
try:
faxcolumns = self.toObj(r[iMessage])
except:
print " FAX message {%s} from the job record [%s] is corrupted" % (r[iMessage], r)
continue
for f in faxcolumns:
try:
r[vIndx[f]] = faxcolumns[f]
except:
raise ValueError(" len=%s r=%s f=%s vIndx=%s %s" % (len(r),r,f,vIndx,faxcolumns))
r.pop(iMessage) # removing message column
header.pop(iMessage) # removing message from header
jobwatch.stop()
main['time'] = {'logquery': '%s' % logwatch,'jobquery': '%s' % jobwatch}
main['params'] = {'hours':hrs}
self.publish(main)
self.publish( "%s/%s" % ( self.server().fileScriptURL(),"fax/%s.js" % "failoverfast" ),role=pmRoles.script() )
self.publishTitle("Panda report on jobs failovers to FAX over last %s hours " % hrs )
def doJson(self,scope=None,doc=None,code=None):
"""
List the <a href='https://twiki.cern.ch/twiki/bin/viewauth/AtlasComputing/PandaErrorCodes'>Panda error codes</a>
<ui>
<li> scope - scope: 'pilot','ddm' 'transExit','jobDispatcher',list the tasks requested for the last hours
<li> doc - pattern to filter the the code descriptions
<li> code - code
</ul>
"""
if scope == None:
self.publishTitle('%s Codes' % 'PanDA Error')
else:
self.publishTitle('%s Codes' % scope)
if scope == None: self.publish(self._errors)
else:
if not scope in self._errorFields:
raise KeyError(" Unknown scope %s . The list of the valid scopes: %s " %(scope, self._errorFields))
selescope = None
if scope != 'transExit':
searchScope = "%sErrorCode" % scope
else:
searchScope = "%sCode" % scope
for j in self._errors:
if j != searchScope: continue
selescope = {j:self._errors[j]}
break
result = { 'unknown':[ [-1,'unknown','unknown'] ]}
if selescope != None:
if code == None:
result = selescope
else:
for cd in selescope:
codelines = selescope[cd]
for cordsc in codelines:
if code == cordsc[0]:
result = { cd : [cordsc] }
break
self.publish(result)
self.publish( "%s/%s" % (self.server().fileScriptURL(),"monitor/%s.js" % 'errorcodes' ),role=pmRoles.script())
self.publish( {'s-maxage':6000,'max-age': 6000}, role=pmRoles.cache())
