if options.report:
if "RELMON_SA" in os.environ:
from directories2html import directory2html
from dirstructure import Directory
else:
from Utilities.RelMon.directories2html import directory2html
from Utilities.RelMon.dirstructure import Directory
pickle_name=options.pklfile
if len(options.pklfile)==0:
pickle_name=original_pickle_name
print("Reading directory from %s" %(pickle_name))
ifile=open(pickle_name,"rb")
directory=cPickle.load(ifile)
ifile.close()
if os.path.exists(options.outdir_name) and len(directory.name)==0:
os.chdir(options.outdir_name)
# Calculate the results of the tests for each directory
print("Calculating stats for the directory...")
directory.calcStats()
print("Producing html...")
directory2html(directory)
if not (options.report or options.compare):
print("Neither comparison nor report to be executed. A typo?")
def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
"""Create a table, with as rows the directories and as columns the samples.
Each box in the table will contain a pie chart linking to the directory.
"""
#aggregation_rules={}
#aggregation_rules_twiki={}
chdir(indir)
if os.path.isabs(indir):
title = basename(indir)
else:
title=indir
title=title.strip(".")
title=title.strip("/")
# Get the list of pickles
sample_pkls=filter(lambda name: name.endswith(".pkl"),listdir("./"))
# Load directories, build a list of all first level subdirs
dir_unpicklers=[]
n_unpicklers=0
for sample_pkl in sample_pkls:
dir_unpickler=unpickler(sample_pkl)
dir_unpickler.start()
n_unpicklers+=1
dir_unpicklers.append(dir_unpickler)
if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
n_unpicklers=0
for dir_unpickler in dir_unpicklers:
dir_unpickler.join()
dir_dict={}
# create a fake global directory
global_dir=Directory("global","")
for dir_unpickler in dir_unpicklers:
dir_unpickler.join()
directory=dir_unpickler.directory
#directory.prune("Run summary")
#directory.calcStats()
global_dir.meta=directory.meta
dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
global_dir.subdirs.append(directory)
global_dir.calcStats()
directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
rel1=""
rel2=""
try:
rel1,rel2=title.split("VS")
except:
rel1=global_dir.meta.release1.split("-")[0]
rel2=global_dir.meta.release2.split("-")[0]
global_dir.meta.release1=rel1
global_dir.meta.release2=rel2
# union of all subdirs names
all_subdirs=[]
for directory in dir_dict.values():
for subdir_name in directory.get_subdirs_names():
all_subdirs.append(subdir_name)
all_subdirs=sorted(list(set(all_subdirs)))
#print " $$ all_subdirs: %s" %(all_subdirs)
# Get The title
page_html+= '<div class="span-20">'+\
'<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
'</div>'+\
'<div class="span-1">'+\
'<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
'</div>'+\
'<hr>'
page_html+='<div class="span-24"><p></p></div>\n'*3
# Get The summary section
page_html+= get_summary_section(global_dir,False)
# Make the anchor sections
page_html+= '<div class="span-24">'
page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
'<ul>'+\
'<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
'<li><a href="#categories">Categories</a></li>'+\
'<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
'<li><a href="#summary_table">Summary Table</a></li>'+\
'<li><a href="#rank_summary">Ranks Summary</a></li>'+\
'<li><a href="#twiki_table">Twiki Table</a></li>'+\
'</ul>'+\
'</div><hr>'
# Make the CategoriesBar chart
page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
# Make the gauges per categories
page_html+=make_categories_summary(dir_dict,aggregation_rules)
# Make the Directories chart
page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
# Barbarian vertical space. Suggestions are welcome
for i in xrange(2):
page_html+='<div class="span-24"><p></p></div>\n'
# Prepare the table
page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
for i in xrange(5):
page_html+='<div class="span-24"><p></p></div>\n'
page_html+="""
<table border="1" >
<tr>
<td> </td>
"""
# First row with samples
page_html+="""
<td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
sorted_samples=sorted(dir_dict.keys())
for sample in sorted_samples:
sample_nick=sample
## For runs: put only the number after the _
#if "_" in sample:
#run_number=sample.split("_")[-1]
#if (not run_number.isalpha()) and len(run_number)>=6:
#sample_nick=run_number
page_html+="""
<td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
page_html+=" </tr>\n"
# FIRST ROW
# Now the summaries at the beginning of the table
page_html+="<tr>"
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<b>Summary</b></div></td>'
page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
for sample in sorted_samples:
col=dir_dict[sample]
# check if the directory was a top one or not
summary_page_name="RelMonSummary.html"
if col.name!="":
summary_page_name=hash_name(col.name, hashing_flag)+".html"
img_link=col.get_summary_chart_ajax(55,55)
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
page_html+="</tr>"
# Now the content
for subdir_name in all_subdirs:
page_html+=' <tr>\n'
page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
row_summary=Directory("row_summary","")
sample_counter=0
n_samples=len(sorted_samples)
for sample in sorted_samples:
subdirs_dict=directory.get_subdirs_dict()
directory=dir_dict[sample]
dir_is_there=subdirs_dict.has_key(subdir_name)
if dir_is_there:
row_summary.subdirs.append(subdirs_dict[subdir_name])
# one first row for the summary!
row_summary.calcStats()
img_link=row_summary.get_summary_chart_ajax(55,55)
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
for sample in sorted_samples:
sample_counter+=1
directory=dir_dict[sample]
subdirs_dict=directory.get_subdirs_dict()
# Check if the directory is the top one
if directory.name!="":
# We did not run on the topdir
#print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
#print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
if hashing_flag:
summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
else:
summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
else:
#print " ## summary_page: %s"%(directory.name+subdir_name)
#print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
dir_is_there=subdirs_dict.has_key(subdir_name)
img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
img_tooltip="N/A"
if dir_is_there:
#row_summary.subdirs.append(subdirs_dict[subdir_name])
img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
page_html+='<td style="background-color:white;"><div class="span-1">'
if dir_is_there:
page_html+='<a href="%s">'%(summary_page)
page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
if dir_is_there:
page_html+='</a>'
page_html+='</div></td>'
page_html+=" </tr>\n"
page_html+='</table> <a href="#top">Top...</a><hr>'
page_html+=get_rank_section(global_dir)
page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
page_html+=get_page_footer()
return page_html
from os.path import exists
from os import chdir, mkdir
import os
import cPickle
pickle_name = options.pklfile
if len(options.pklfile) == 0:
pickle_name = original_pickle_name
print "Reading directory from %s" % (pickle_name)
ifile = open(pickle_name, "rb")
directory = cPickle.load(ifile)
ifile.close()
if not options.compare:
if not os.path.exists(options.outdir_name):
mkdir(options.outdir_name)
if exists(options.outdir_name) and len(directory.name) == 0:
chdir(options.outdir_name)
# Calculate the results of the tests for each directory
print "Calculating stats for the directory..."
directory.calcStats()
print "Producing html..."
directory2html(directory)
if not (options.report or options.compare):
print "Neither comparison nor report to be executed. A typo?"
def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
"""Create a table, with as rows the directories and as columns the samples.
Each box in the table will contain a pie chart linking to the directory.
"""
#aggregation_rules={}
#aggregation_rules_twiki={}
chdir(indir)
if os.path.isabs(indir):
title = basename(indir)
else:
title=indir
title=title.strip(".")
title=title.strip("/")
# Get the list of pickles
sample_pkls=filter(lambda name: name.endswith(".pkl"),listdir("./"))
# Load directories, build a list of all first level subdirs
dir_unpicklers=[]
n_unpicklers=0
for sample_pkl in sample_pkls:
dir_unpickler=unpickler(sample_pkl)
dir_unpickler.start()
n_unpicklers+=1
dir_unpicklers.append(dir_unpickler)
if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
n_unpicklers=0
for dir_unpickler in dir_unpicklers:
dir_unpickler.join()
dir_dict={}
# create a fake global directory
global_dir=Directory("global","")
for dir_unpickler in dir_unpicklers:
dir_unpickler.join()
directory=dir_unpickler.directory
#directory.prune("Run summary")
#directory.calcStats()
global_dir.meta=directory.meta
dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
global_dir.subdirs.append(directory)
global_dir.calcStats()
directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
rel1=""
rel2=""
try:
rel1,rel2=title.split("VS")
except:
rel1=global_dir.meta.release1.split("-")[0]
rel2=global_dir.meta.release2.split("-")[0]
global_dir.meta.release1=rel1
global_dir.meta.release2=rel2
# union of all subdirs names
all_subdirs=[]
for directory in dir_dict.values():
for subdir_name in directory.get_subdirs_names():
all_subdirs.append(subdir_name)
all_subdirs=sorted(list(set(all_subdirs)))
#print " $$ all_subdirs: %s" %(all_subdirs)
# Get The title
page_html+= '<div class="span-20">'+\
'<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
'</div>'+\
'<div class="span-1">'+\
'<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
'</div>'+\
'<hr>'
page_html+='<div class="span-24"><p></p></div>\n'*3
# Get The summary section
page_html+= get_summary_section(global_dir,False)
# Make the anchor sections
page_html+= '<div class="span-24">'
page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
'<ul>'+\
'<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
'<li><a href="#categories">Categories</a></li>'+\
'<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
'<li><a href="#summary_table">Summary Table</a></li>'+\
'<li><a href="#rank_summary">Ranks Summary</a></li>'+\
'<li><a href="#twiki_table">Twiki Table</a></li>'+\
'</ul>'+\
'</div><hr>'
# Make the CategoriesBar chart
page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
# Make the gauges per categories
page_html+=make_categories_summary(dir_dict,aggregation_rules)
# Make the Directories chart
page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
# Barbarian vertical space. Suggestions are welcome
for i in xrange(2):
page_html+='<div class="span-24"><p></p></div>\n'
# Prepare the table
page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
for i in xrange(5):
page_html+='<div class="span-24"><p></p></div>\n'
page_html+="""
<table border="1" >
<tr>
<td> </td>
"""
# First row with samples
page_html+="""
<td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
sorted_samples=sorted(dir_dict.keys())
for sample in sorted_samples:
sample_nick=sample
## For runs: put only the number after the _
#if "_" in sample:
#run_number=sample.split("_")[-1]
#if (not run_number.isalpha()) and len(run_number)>=6:
#sample_nick=run_number
page_html+="""
<td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
page_html+=" </tr>\n"
# FIRST ROW
# Now the summaries at the beginning of the table
page_html+="<tr>"
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<b>Summary</b></div></td>'
page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
for sample in sorted_samples:
col=dir_dict[sample]
# check if the directory was a top one or not
summary_page_name="RelMonSummary.html"
if col.name!="":
summary_page_name=hash_name(col.name, hashing_flag)+".html"
img_link=col.get_summary_chart_ajax(55,55)
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
page_html+="</tr>"
# Now the content
for subdir_name in all_subdirs:
page_html+=' <tr>\n'
page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
row_summary=Directory("row_summary","")
sample_counter=0
n_samples=len(sorted_samples)
for sample in sorted_samples:
subdirs_dict=directory.get_subdirs_dict()
directory=dir_dict[sample]
dir_is_there=subdirs_dict.has_key(subdir_name)
if dir_is_there:
row_summary.subdirs.append(subdirs_dict[subdir_name])
# one first row for the summary!
row_summary.calcStats()
img_link=row_summary.get_summary_chart_ajax(55,55)
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
for sample in sorted_samples:
sample_counter+=1
directory=dir_dict[sample]
subdirs_dict=directory.get_subdirs_dict()
# Check if the directory is the top one
if directory.name!="":
# We did not run on the topdir
#print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
#print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
if hashing_flag:
summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
else:
summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
else:
#print " ## summary_page: %s"%(directory.name+subdir_name)
#print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
dir_is_there=subdirs_dict.has_key(subdir_name)
img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
img_tooltip="N/A"
if dir_is_there:
#row_summary.subdirs.append(subdirs_dict[subdir_name])
img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
page_html+='<td style="background-color:white;"><div class="span-1">'
if dir_is_there:
page_html+='<a href="%s">'%(summary_page)
page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
if dir_is_there:
page_html+='</a>'
page_html+='</div></td>'
page_html+=" </tr>\n"
page_html+='</table> <a href="#top">Top...</a><hr>'
page_html+=get_rank_section(global_dir)
page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
page_html+=get_page_footer()
return page_html