def run(self): this_dir=DirID(self.directory.name,self.depth) if this_dir in self.black_list: print "Skipping %s since blacklisted!" %this_dir return 0 self.depth+=1 the_test=Statistical_Tests[self.stat_test](self.test_threshold) #print "Test %s with threshold %s" %(self.stat_test,self.test_threshold) directory1=self.base1+"/"+self.directory.mother_dir+"/"+self.directory.name directory2=self.base2+"/"+self.directory.mother_dir+"/"+self.directory.name fetchers =(DirFetcher(self.comm1,directory1),DirFetcher(self.comm2,directory2)) for fetcher in fetchers: fetcher.start() for fetcher in fetchers: fetcher.join() contents1 = fetchers[0].contents contents2 = fetchers[1].contents set1= set(contents1.keys()) set2= set(contents2.keys()) walkers=[] self_directory_directories=self.directory.subdirs self_directory_comparisons=self.directory.comparisons contents_names=list(set1.intersection(set2)) for name in contents_names: content = contents1[name] if "dir" in content["type"]: #if this_dir not in DirWalker.white_list:continue subdir=Directory(name,join(self.directory.mother_dir,self.directory.name)) dirwalker=DirWalkerDB(self.comm1,self.comm2,self.base1,self.base2,subdir,self.depth, self.do_pngs,self.stat_test,self.test_threshold,self.black_list) dirwalker.start() walkers.append(dirwalker) n_threads=activeCount() if n_threads>5: #print >> stderr, "Threads that are running: %s. Joining them." %(n_threads) dirwalker.join() elif content["kind"]=="ROOT": # print directory1,name comparison=Comparison(name, join(self.directory.mother_dir,self.directory.name), literal2root(content["obj_as_string"],content["type"]), literal2root(contents2[name]["obj_as_string"],content["type"]), deepcopy(the_test), do_pngs=self.do_pngs) self_directory_comparisons.append(comparison) for walker in walkers: walker.join() walker_directory=walker.directory if not walker_directory.is_empty(): self_directory_directories.append(walker_directory)
def __init__(self, name, topdirname, rootfilename1, rootfilename2, run=-1, black_list=[], stat_test="KS", test_threshold=.5, draw_success=True, do_pngs=False, black_list_histos=[]): self.name = name self.dqmrootfile1 = DQMRootFile(abspath(rootfilename1)) self.dqmrootfile2 = DQMRootFile(abspath(rootfilename2)) self.run = run self.stat_test = Statistical_Tests[stat_test](test_threshold) self.workdir = getcwd() self.black_list = black_list self.directory = Directory(topdirname) #print "DIRWALKERFILE %s %s" %(draw_success,do_pngs) self.directory.draw_success = draw_success self.directory.do_pngs = do_pngs self.black_list_histos = black_list_histos self.different_histograms = {} self.filename1 = basename(rootfilename2) self.filename2 = basename(rootfilename1)
def __fill_single_dir(self,dir_name,directory,mother_name="",depth=0): #print "MOTHER NAME = +%s+" %mother_name #print "About to study %s (in dir %s)" %(dir_name,getcwd()) # see if in black_list this_dir=DirID(dir_name,depth) #print this_dir if this_dir in self.black_list: #print "Directory %s skipped because black-listed" %dir_name return 0 depth+=1 self.cd(dir_name) #print self.ls() #print "Test %s with thre %s" %(self.stat_test.name, self.stat_test.threshold) contents=self.ls() if depth==1: n_top_contents=len(contents) #print contents cont_counter=1 comparisons=[] for name,obj_type in contents.items(): if obj_type=="TDirectoryFile": #We have a dir, launch recursion! #Some feedback on the progress if depth==1: print "Studying directory %s, %s/%s" %(name,cont_counter,n_top_contents) cont_counter+=1 #print "Studying directory",name # ok recursion on! subdir=Directory(name) subdir.draw_success=directory.draw_success subdir.do_pngs=directory.do_pngs self.__fill_single_dir(name,subdir,join(mother_name,dir_name),depth) if not subdir.is_empty(): if depth==1: print " ->Appending %s..." %name, directory.subdirs.append(subdir) if depth==1: print "Appended." else: # We have probably an histo. Let's make the plot and the png. if obj_type[:2]!="TH" and obj_type[:3]!="TPr" : continue h1,h2=self.getObjs(name) #print "COMPARISON : +%s+%s+" %(mother_name,dir_name) path = join(mother_name,dir_name,name) if path in self.black_list_histos: print " Skipping %s" %(path) directory.comparisons.append(Comparison(name, join(mother_name,dir_name), h1,h2, deepcopy(self.stat_test), draw_success=directory.draw_success, do_pngs=directory.do_pngs, skip=True)) else: directory.comparisons.append(Comparison(name, join(mother_name,dir_name), h1,h2, deepcopy(self.stat_test), draw_success=directory.draw_success, do_pngs=directory.do_pngs, skip=False)) directory.filename1 = self.filename1 directory.filename2 = self.filename2 directory.different_histograms['file1'] = self.different_histograms['file1'] directory.different_histograms['file2'] = self.different_histograms['file2'] self.cd("..")
db_base_url="/data/json/archive/" base1="%s/%s/%s/%s/DQM/" %(db_base_url,options.run1,options.sample,options.cmssw_release1) base2="%s/%s/%s/%s/DQM/" %(db_base_url,options.run2,options.sample,options.cmssw_release2) print("Analysing Histograms located in directory %s at: " %options.dir_name) for base in base1,base2: print(" o %s (server= %s)" %(base,options.dqm_server)) # Set up the communicators comm1 = DQMcommunicator(server=options.dqm_server) comm2 = DQMcommunicator(server=options.dqm_server) # Set up the fake directory structure directory=Directory(options.dir_name) dirwalker=DirWalkerDB(comm1,comm2,base1,base2,directory) # Set the production of pngs on and off dirwalker.do_pngs=options.do_pngs # set the stat test dirwalker.stat_test=options.stat_test dirwalker.test_threshold=options.test_threshold # Set the blacklist, if needed if len(black_list)>0: print("We have a Blacklist:") for dirid in black_list: print(" o %s" %dirid) dirwalker.black_list=black_list
black_list = string2blacklist(options.black_list) #------------------------------------------------------------------------------- print "Analysing Histograms located in directory %s at: " % options.dir_name for filename in rootfilename1, rootfilename2: print " o %s" % filename if len(black_list) > 0: print "We have a Blacklist:" for dirid in black_list: print " o %s" % dirid # Set up the fake directory structure directory = Directory(options.dir_name) dirwalker = DirWalkerFile(fulldirname, options.dir_name, rootfilename1, rootfilename2, run, black_list, options.stat_test, options.test_threshold, not options.no_successes, options.do_pngs) # Start the walker outdir_name = options.outdir_name if run > 1 and options.specify_run: outdir_name += "_%s" % run fulldirname += "_%s" % run print "+" * 30 print "Output Directory will be ", outdir_name options.outdir_name = outdir_name if not exists(outdir_name) and len(outdir_name) > 0: mkdir(outdir_name)
def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag): """Create a table, with as rows the directories and as columns the samples. Each box in the table will contain a pie chart linking to the directory. """ #aggregation_rules={} #aggregation_rules_twiki={} chdir(indir) if os.path.isabs(indir): title = basename(indir) else: title=indir title=title.strip(".") title=title.strip("/") # Get the list of pickles sample_pkls=filter(lambda name: name.endswith(".pkl"),listdir("./")) # Load directories, build a list of all first level subdirs dir_unpicklers=[] n_unpicklers=0 for sample_pkl in sample_pkls: dir_unpickler=unpickler(sample_pkl) dir_unpickler.start() n_unpicklers+=1 dir_unpicklers.append(dir_unpickler) if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu n_unpicklers=0 for dir_unpickler in dir_unpicklers: dir_unpickler.join() dir_dict={} # create a fake global directory global_dir=Directory("global","") for dir_unpickler in dir_unpicklers: dir_unpickler.join() directory=dir_unpickler.directory #directory.prune("Run summary") #directory.calcStats() global_dir.meta=directory.meta dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory global_dir.subdirs.append(directory) global_dir.calcStats() directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory") categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules) page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart) rel1="" rel2="" try: rel1,rel2=title.split("VS") except: rel1=global_dir.meta.release1.split("-")[0] rel2=global_dir.meta.release2.split("-")[0] global_dir.meta.release1=rel1 global_dir.meta.release2=rel2 # union of all subdirs names all_subdirs=[] for directory in dir_dict.values(): for subdir_name in directory.get_subdirs_names(): all_subdirs.append(subdir_name) all_subdirs=sorted(list(set(all_subdirs))) #print " $$ all_subdirs: %s" %(all_subdirs) # Get The title page_html+= '<div class="span-20">'+\ '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\ '</div>'+\ '<div class="span-1">'+\ '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\ '</div>'+\ '<hr>' page_html+='<div class="span-24"><p></p></div>\n'*3 # Get The summary section page_html+= get_summary_section(global_dir,False) # Make the anchor sections page_html+= '<div class="span-24">' page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\ '<ul>'+\ '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\ '<li><a href="#categories">Categories</a></li>'+\ '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\ '<li><a href="#summary_table">Summary Table</a></li>'+\ '<li><a href="#rank_summary">Ranks Summary</a></li>'+\ '<li><a href="#twiki_table">Twiki Table</a></li>'+\ '</ul>'+\ '</div><hr>' # Make the CategoriesBar chart page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>' page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>' # Make the gauges per categories page_html+=make_categories_summary(dir_dict,aggregation_rules) # Make the Directories chart page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>' page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>' # Barbarian vertical space. Suggestions are welcome for i in xrange(2): page_html+='<div class="span-24"><p></p></div>\n' # Prepare the table page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>' for i in xrange(5): page_html+='<div class="span-24"><p></p></div>\n' page_html+=""" <table border="1" > <tr> <td> </td> """ # First row with samples page_html+=""" <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>""" sorted_samples=sorted(dir_dict.keys()) for sample in sorted_samples: sample_nick=sample ## For runs: put only the number after the _ #if "_" in sample: #run_number=sample.split("_")[-1] #if (not run_number.isalpha()) and len(run_number)>=6: #sample_nick=run_number page_html+=""" <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick page_html+=" </tr>\n" # FIRST ROW # Now the summaries at the beginning of the table page_html+="<tr>" page_html+='<td style="background-color:white;"><div class="span-1">' page_html+='<b>Summary</b></div></td>' page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir)) for sample in sorted_samples: col=dir_dict[sample] # check if the directory was a top one or not summary_page_name="RelMonSummary.html" if col.name!="": summary_page_name=hash_name(col.name, hashing_flag)+".html" img_link=col.get_summary_chart_ajax(55,55) page_html+='<td style="background-color:white;"><div class="span-1">' page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col)) page_html+="</tr>" # Now the content for subdir_name in all_subdirs: page_html+=' <tr>\n' page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name row_summary=Directory("row_summary","") sample_counter=0 n_samples=len(sorted_samples) for sample in sorted_samples: subdirs_dict=directory.get_subdirs_dict() directory=dir_dict[sample] dir_is_there=subdirs_dict.has_key(subdir_name) if dir_is_there: row_summary.subdirs.append(subdirs_dict[subdir_name]) # one first row for the summary! row_summary.calcStats() img_link=row_summary.get_summary_chart_ajax(55,55) page_html+='<td style="background-color:white;"><div class="span-1">' page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary)) for sample in sorted_samples: sample_counter+=1 directory=dir_dict[sample] subdirs_dict=directory.get_subdirs_dict() # Check if the directory is the top one if directory.name!="": # We did not run on the topdir #print " ## summary_page: %s"%(directory.name+"_"+subdir_name) #print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag)) if hashing_flag: summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag))) else: summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag))) else: #print " ## summary_page: %s"%(directory.name+subdir_name) #print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag)) summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag))) dir_is_there=subdirs_dict.has_key(subdir_name) img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1" img_tooltip="N/A" if dir_is_there: #row_summary.subdirs.append(subdirs_dict[subdir_name]) img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50) img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name]) page_html+='<td style="background-color:white;"><div class="span-1">' if dir_is_there: page_html+='<a href="%s">'%(summary_page) page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip) if dir_is_there: page_html+='</a>' page_html+='</div></td>' page_html+=" </tr>\n" page_html+='</table> <a href="#top">Top...</a><hr>' page_html+=get_rank_section(global_dir) page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki) page_html+=get_page_footer() return page_html
def __fill_single_dir(self, dir_name, directory, mother_name="", depth=0): #print "MOTHER NAME = +%s+" %mother_name #print "About to study %s (in dir %s)" %(dir_name,getcwd()) # see if in black_list this_dir = DirID(dir_name, depth) #print this_dir if this_dir in self.black_list: #print "Directory %s skipped because black-listed" %dir_name return 0 depth += 1 self.cd(dir_name) #print self.ls() #print "Test %s with thre %s" %(self.stat_test.name, self.stat_test.threshold) contents = self.ls() if depth == 1: n_top_contents = len(contents) #print contents cont_counter = 1 comparisons = [] for name, obj_type in contents.items(): if obj_type == "TDirectoryFile": #We have a dir, launch recursion! #Some feedback on the progress if depth == 1: print "Studying directory %s, %s/%s" % (name, cont_counter, n_top_contents) cont_counter += 1 #print "Studying directory",name # ok recursion on! subdir = Directory(name) subdir.draw_success = directory.draw_success subdir.do_pngs = directory.do_pngs self.__fill_single_dir(name, subdir, join(mother_name, dir_name), depth) if not subdir.is_empty(): if depth == 1: print " ->Appending %s..." % name, directory.subdirs.append(subdir) if depth == 1: print "Appended." else: # We have probably an histo. Let's make the plot and the png. if obj_type[:2] != "TH" and obj_type[:3] != "TPr": continue h1, h2 = self.getObjs(name) #print "COMPARISON : +%s+%s+" %(mother_name,dir_name) path = join(mother_name, dir_name, name) if path in self.black_list_histos: print " Skipping %s" % (path) directory.comparisons.append( Comparison(name, join(mother_name, dir_name), h1, h2, deepcopy(self.stat_test), draw_success=directory.draw_success, do_pngs=directory.do_pngs, skip=True)) else: directory.comparisons.append( Comparison(name, join(mother_name, dir_name), h1, h2, deepcopy(self.stat_test), draw_success=directory.draw_success, do_pngs=directory.do_pngs, skip=False)) self.cd("..")
black_listed = [] #------------------------------------------------------------------------------- print "Analysing Histograms located in directory %s at: " %options.dir_name for filename in rootfilename1,rootfilename2: print " o %s" %filename if len(black_list)>0: print "We have a Blacklist:" for dirid in black_list: print " o %s" %dirid # Set up the fake directory structure directory=Directory(options.dir_name) dirwalker=DirWalkerFile(fulldirname, options.dir_name, rootfilename1,rootfilename2, run, black_list, options.stat_test, options.test_threshold, not options.no_successes, options.do_pngs, set(black_listed) ) # Start the walker outdir_name=options.outdir_name if run>1 and options.specify_run:
db_base_url = "/data/json/archive/" base1 = "%s/%s/%s/%s/DQM/" % (db_base_url, options.run1, options.sample, options.cmssw_release1) base2 = "%s/%s/%s/%s/DQM/" % (db_base_url, options.run2, options.sample, options.cmssw_release2) print "Analysing Histograms located in directory %s at: " % options.dir_name for base in base1, base2: print " o %s (server= %s)" % (base, options.dqm_server) # Set up the communicators comm1 = DQMcommunicator(server=options.dqm_server) comm2 = DQMcommunicator(server=options.dqm_server) # Set up the fake directory structure directory = Directory(options.dir_name) dirwalker = DirWalkerDB(comm1, comm2, base1, base2, directory) # Set the production of pngs on and off dirwalker.do_pngs = options.do_pngs # set the stat test dirwalker.stat_test = options.stat_test dirwalker.test_threshold = options.test_threshold # Set the blacklist, if needed if len(black_list) > 0: print "We have a Blacklist:" for dirid in black_list: print " o %s" % dirid dirwalker.black_list = black_list