def cuttree( oldname, newname, oldtree, branch, check ) : oldfile = r.TFile( oldname ) oldtree = oldfile.Get( oldtree ) nentries = oldtree.GetEntries() nTotVars = oldtree.GetLeaf( branch ).GetLen() chi2data = array('d',[0]*nTotVars) oldtree.SetBranchAddress( branch, chi2data ) newfile = r.TFile( newname, "recreate" ) newfile.cd() newtree = oldtree.CloneTree(0) prog = ProgressBar(0, nentries, 77, mode='fixed') oldprog = str(prog) for i in range(0,nentries) : prog.increment_amount() if oldprog != str(prog): print prog, "\r", stdout.flush() oldprog=str(prog) oldtree.GetEntry(i) if check( chi2data[0] ) : newtree.Fill() newtree.AutoSave() oldfile.Close() newfile.Close()
def calculate_entry_histograms( plots, chain ) : ##assert canvas is not None, "Canvas must be specified in calculate_histograms" # setup our 2d histos histos = [] chi2histos = [] for p in plots : entryhisto, chi2histo = initialize_histo( p ) histos.append(entryhisto) chi2histos.append(chi2histo) nentries = chain.GetEntries() prog = ProgressBar(0, nentries+1, 77, mode='fixed', char='#') for entry in range(0,nentries+1) : prog.increment_amount() print prog,'\r', stdout.flush() chain.GetEntry(entry) for h, c, plot in zip( histos, chi2histos, plots ) : indices = plot.get_indices() vals = [ chain.treeVars["predictions"][ index ] for index in indices ] nbins = plot.bins ibin = h.FindBin(*vals) max_bin = h.FindBin(*plot.max_vals) if ibin != 0 and ibin < max_bin : chi2 = chain.treeVars["predictions"][0] if chi2 < c.GetBinContent(ibin) : c.SetBinContent(ibin, chi2) h.SetBinContent(ibin, entry) print return histos
def solve_5(self, period, N): result = np.identity(N, dtype=np.complex) I = np.identity(N, dtype=np.complex) He = self.matrix_electric Hs = self.matrix_static prog = ProgressBar(0, self.fine_step / 6, 50, mode='fixed', char='#') #python version for i in xrange(0, self.fine_step, 6): prog.increment_amount() print prog, '\r', sys.stdout.flush() k1 = (He * self.E_arr[period][i] + Hs) tmp = He * self.E_arr[period][i + 1] + Hs k2 = np.dot(tmp, I + k1 * 0.25 * self.dt) tmp = He * self.E_arr[period][i + 2] + Hs k3 = np.dot(tmp, I + (k1 * 3.0 / 32.0 + k2 * 9.0 / 32.0) * self.dt) tmp = He * self.E_arr[period][i + 3] + Hs k4 = np.dot( tmp, I + (k1 * 1932.0 - k2 * 7200.0 + k3 * 7296.0) * self.dt / 2197.0) tmp = He * self.E_arr[period][i + 4] + Hs k5 = np.dot( tmp, I + (k1 * 439.0 / 216.0 - k2 * 8.0 + k3 * 3680.0 / 513.0 - k4 * 845.0 / 4104.0) * self.dt) tmp = He * self.E_arr[period][i + 5] + Hs k6 = np.dot( tmp, I + (-1.0 * k1 * 8.0 / 27.0 - k2 * 2.0 - k3 * 3544.0 / 2565.0 + k4 * 1859.0 / 4104.0 - k5 * 11.0 / 40.0) * self.dt) result = np.dot( I + (k1 * 16.0 / 135.0 + k3 * 6656.0 / 12825.0 + k4 * 28561.0 / 56430.0 - k5 * 9.0 / 50.0 + k6 * 2.0 / 55.0) * self.dt, result) return result
def delete_files(self, file_list ): '''create files. The filenames are supplied as a list''' count = 0 total = len(file_list) if self.verbose: print "\nTrying to delete " + str(total) + " files" prog = ProgressBar(count, total, 77, mode='fixed', char='#') before = time.time() for file_name in file_list: ret = self.delete_file( file_name ) if self.verbose: count += 1 prog.increment_amount() print prog, '\r', sys.stdout.flush() if ret == False: print "Failed to delete file " + str( file_name ) return False print after = time.time() self.profiling_printer("Deleting " + str(len(file_list)) + " files", after - before) return True
def compute_initial_allocation(self,disp_progress = True): self.to_assign = np.random.permutation(range(self.N)).tolist() if disp_progress: prog = ProgressBar(0, len(self.to_assign), 77, mode='fixed') oldprog = str(prog) while True: if disp_progress: #<--display progress prog.increment_amount() if oldprog != str(prog): print prog, "\r", sys.stdout.flush() oldprog=str(prog) #--> i = self.to_assign.pop() dists = sdis.cdist(np.atleast_2d(self.X[i]),self.centroids,self.metric).ravel() sorted_centroids = np.argsort(dists) for c in sorted_centroids: if self.weight_per_cluster[c]+self.weights[i]<weight_limit: self.clusters[c].append(i) self.weight_per_cluster[c]+=self.weights[i] break if not(self.to_assign): break
def do_preprocess_simulation(heap, pre_event_list, event_list): prog = ProgressBar(0, len(pre_event_list) + len(event_list), 77, mode='fixed', char='#', autoprint=True) frees_and_reallocs = {} preprocessed_memory = 0 for event in event_list: if event.event_type == Event.eRealloc: frees_and_reallocs[event.pDataPrev] = 0 elif event.event_type == Event.eFree: frees_and_reallocs[event.pData] = 0 prog.increment_amount() for event in pre_event_list: if event.event_type == event.eAlloc: if frees_and_reallocs.has_key(event.pData): heap.track(event) preprocessed_memory += event.allocSize elif event.event_type == Event.eRealloc: if frees_and_reallocs.has_key(event.pData): heap.track(event) preprocessed_memory += event.allocSize prog.increment_amount() print "\n\npreprocessed_memory = %s" % locale.format( '%d', preprocessed_memory, True)
def fill_and_save_data_hists( mcf, plots,entry_hists, modes, contribs,predicts ) : axes = [ "X", "Y", "Z" ] chain = MCAnalysisChain( mcf ) nentries = chain.GetEntries() KOhack=KOhack_class(mcf) for p , h in zip(plots,entry_hists) : ############################################# if check_entry_KO_hack(p,KOhack): KOhack.init_hack(p) ############################################# histo_cont = {} contrib_cont = {} predict_cont = {} print p.short_names firstbin, lastbin = get_histogram_bin_range(h,space=p) for mode in modes : # here need to add in check on contrib and make one for each contribution hname = histo_name( p.short_names, entry_histo_prefix )+ "_" + mode histo_cont[mode] =initialize_histo( p,hname,data =True ) base_val = 1e9 if mode == "pval" : base_val = 0.0 for bin in range( firstbin, lastbin + 1 ) : histo_cont[mode].SetBinContent( bin, base_val ) for c in contribs : # contribs is a list of Contribution objects hname = histo_name( p.short_names, entry_histo_prefix )+ "_dX_" + c.short_name contrib_cont[c.short_name] = initialize_histo( p,hname,data =True ) for bin in range( firstbin, lastbin + 1 ) : contrib_cont[c.short_name].SetBinContent( bin, 0.0 ) for pred in predicts : # predicts is a list of Contribution objects hname = histo_name( p.short_names, entry_histo_prefix )+ "_pred_" + pred.short_name predict_cont[pred.short_name] = initialize_histo( p,hname,data =True ) for bin in range( firstbin, lastbin + 1 ) : predict_cont[pred.short_name].SetBinContent( bin, 0.0 ) prog = ProgressBar(0, (lastbin-firstbin)+1, 77, mode='fixed', char='#') for i in range( firstbin, lastbin+1 ) : prog.increment_amount() print prog,'\r', stdout.flush() entry = int( h.GetBinContent(i) ) if entry > 0 : chain.GetEntry(entry) ############################################# if check_entry_KO_hack(p,KOhack): KOhack.set_ssi_bin_centre(h,i) ############################################# fill_bins( histo_cont, contrib_cont,predict_cont, contribs,predicts , i, chain, mcf, KOhack ) perform_zero_offset( histo_cont["dchi"],space=p ) print save_hdict_to_root_file( histo_cont, mcf.FileName, mcf.DataDirectory) save_hdict_to_root_file( contrib_cont, mcf.FileName, mcf.DataDirectory) save_hdict_to_root_file( predict_cont, mcf.FileName, mcf.DataDirectory)
def sweep_multiprocessing(self,sweep_n,start,end,points,filename='./test.txt'): """ nu[sweep_n] is sweeped. Sweep the frequency and output the result to filename. """ ############################### ##multiprocessing preparation ############################## core = 10 points = points//core*core # points per thread self.result = [[0.0 for i in range(self.n+1)]for j in range(points)]#this is the matrix which store the result, it will be saved to file later. job = self.allocate_job(start,end,points,core) ################################ ##This are codes for progress bar ############################### prog = ProgressBar(0, points, 50, mode='fixed', char='#') ##the linear algebra start here a = np.zeros(self.N) a[self.N-1] = 1 #1 because rho_11+rho_22 ... =1 a = np.matrix(a) a = a.T done_queue = multiprocessing.Queue() process_list = [] for x in range(core): process_list.append(multiprocessing.Process(target = sweep_mp,args = (job[x],self.system,self.nu2,a,self.add_freq,self.index,sweep_n,self.n,done_queue))) tStart = time.time() print 'start' for p in process_list: p.start() stop_num = 0 while stop_num != core: a = done_queue.get() if a == 'STOP': stop_num += 1 else: self.result[a[0]] = a[1] prog.increment_amount() print prog, '\r', sys.stdout.flush() print '\n' for p in process_list: p.join() print "%s.exitcode = %s" %(p.name, p.exitcode) tStop = time.time() print"spend",(tStop - tStart),"second" self.sweep_save_file(filename,points)
def generate_hierarchy(event_list, stack_dict): prog = ProgressBar(0, len(event_list), 77, mode='fixed', char='#', autoprint=True) #create a hierarchal track of the allocation event root = CallStatistics(0) #create a heap to track live allocated blocks and current memory status overall heap = Heap() for event in event_list: # print event.text() # if the event is a free then we need to find out the size of the block we are freeing # @todo this could be part of the event stream when using debug CRT but probably not for release builds? if event.event_type == Event.eFree: size_free = heap.block_size(event.pData) elif event.event_type == Event.eRealloc: size_free = heap.block_size(event.pDataPrev) else: size_free = 0 if event.event_type == Event.eAlloc or event.event_type == Event.eRealloc: size_alloc = event.allocSize else: size_alloc = 0 # only now that we have our size information on potential free blocks do we # let heap track event (since the block in question might get freed) heap.track(event) # start at the tree root and walk each stack call in this event and tally the event with this call and it's callee's stat_obj = root # start at the first call and walk down until we hit the desired leaf stack = stack_dict[event.stack_id] for index in range(len(stack.calls) - 1, stack.leaf_index - 2, -1): call = stack.calls[index] child_obj = stat_obj.add_child_call(call) # add stats for the child to the parent stat_obj.sample_child_event(size_alloc, size_free) if index == stack.leaf_index - 1: # add leaf stats on the child leaf node child_obj.sample_self_event(size_alloc, size_free) stat_obj = child_obj # step down the tree for next call prog.increment_amount() print "\n\n" return root
def build_callstack_detail_html(stack_dict): html = "" prog = ProgressBar(0, len(stack_dict), 77, mode='fixed', char='#', autoprint=True) stacks_sorted = sorted(stack_dict.values(), key=operator.attrgetter('display_id')) for i, stack in enumerate(stacks_sorted): if MemTrack.call_stack_write_limit > 0 and i > MemTrack.call_stack_write_limit: break html += stack.html_table() html += "\n\n" prog.increment_amount() return html
def summarize(cmds, key, type, stack_dict): #create a summary list based off of accumulating stats based on the given key prog = ProgressBar(0, len(cmds), 77, mode='fixed', char='#', autoprint=True) all_total = 0 summary_dict = {} null_summary_key = CallStatistics(0) for cmd in cmds: if type == Event.eAny or cmd.event_type == type: size = cmd.allocSize all_total += size stack = stack_dict.get( cmd.stack_id) # lookup the stack for this event # check stack for key first if stack.__dict__.has_key(key): summary_key = stack.__dict__[key] elif SymbolDB.symbol_cache.has_key(stack.leaf()): addr_info = SymbolDB.symbol_cache[stack.leaf()] summary_key = addr_info.__dict__[key] else: summary_key = null_summary_key summary = summary_dict.setdefault(summary_key, Summary()) summary.sample(cmd, stack) summary.key = summary_key prog.increment_amount() print "\n\n" summaries_sorted = sorted(summary_dict.items(), key=lambda (k, v): v.accumulator.value, reverse=True) # after all the summaries are collected go ahead and generate some of the # final stats in each summary summaries = [summary[1] for summary in summaries_sorted] for summary in summaries: summary.prepare(all_total) return summaries
def calculate_entry_histograms( plots, chain ) : ##assert canvas is not None, "Canvas must be specified in calculate_histograms" # setup our 2d histos vars = v.mc_variables() # KOhack class gets initiated, because it has to be checked "if KOhack is applied" KOhack=KOhack_class(plots[0].mcf) histos = [] chi2histos = [] for p in plots : hname = histo_name( p.short_names, entry_histo_prefix ) cname = histo_name( p.short_names, chi2_histo_prefix ) entryhisto = initialize_histo( p,hname,entry=True ) chi2histo = initialize_histo( p,cname,chi2 =True ) histos.append(entryhisto) chi2histos.append(chi2histo) if check_entry_KO_hack(p,KOhack): KOhack.init_hack(p) nentries = chain.GetEntries() prog = ProgressBar(0, nentries+1, 77, mode='fixed', char='#') for entry in range(0,nentries+1) : prog.increment_amount() print prog,'\r', stdout.flush() chain.GetEntry(entry) for h, c, plot in zip( histos, chi2histos, plots ) : vals_list = get_values_list_from_chain_and_histo(chain,plot,vars,s,KOhack,h) for vals in vals_list: nbins = plot.bins ibin = h.FindBin(*vals) max_bin = h.FindBin(*plot.max_vals) if ibin != 0 and ibin < max_bin : chi2 = get_modified_entry_chi2(vals,chain,KOhack) if chi2 < c.GetBinContent(ibin) : c.SetBinContent(ibin, chi2) h.SetBinContent(ibin, entry) print return histos
def create_files(self, file_list ): '''create files. The filenames are supplied as a list''' count = 0 total = len(file_list) if self.verbose: print "\nTrying to create " + str(total) + " files" prog = ProgressBar(count, total, 77, mode='fixed', char='#') counter = 0 last_time =time.time() before = time.time() for file_name in file_list: ret = self.create_empty_file( file_name ) if self.verbose: count += 1 prog.increment_amount() print prog, '\r', sys.stdout.flush() if ret == False: print "Failed to create file " + str( file_name ) return False else: counter+=1 if counter % 1000 == 0 and self.profiling: diff = time.time() - last_time last_time = time.time() f = open("/tmp/basicStat.csv","a") f.write(str(counter) + "," + str(diff) + "\n") f.flush() f.close() print after = time.time() files_per_second = str( len(file_list) / (after-before) ) self.profiling_printer("Creating " + str(len(file_list)) + " files (" + files_per_second + " per second)", after - before) return True
def solve_5(self, period,N): result = np.identity(N,dtype = np.complex) I = np.identity(N,dtype = np.complex) He = self.matrix_electric Hs = self.matrix_static prog = ProgressBar(0, self.fine_step/6, 50, mode='fixed', char='#') #python version for i in xrange(0,self.fine_step,6): prog.increment_amount() print prog, '\r', sys.stdout.flush() k1 = (He*self.E_arr[period][i] + Hs) tmp = He*self.E_arr[period][i+1]+Hs k2 = np.dot(tmp,I+k1*0.25*self.dt) tmp = He*self.E_arr[period][i+2]+Hs k3 = np.dot(tmp,I+(k1*3.0/32.0+k2*9.0/32.0)*self.dt) tmp = He*self.E_arr[period][i+3]+Hs k4 = np.dot(tmp,I+(k1*1932.0-k2*7200.0+k3*7296.0)*self.dt/2197.0) tmp = He*self.E_arr[period][i+4]+Hs k5 = np.dot(tmp,I+(k1*439.0/216.0-k2*8.0+k3*3680.0/513.0-k4*845.0/4104.0)*self.dt) tmp = He*self.E_arr[period][i+5]+Hs k6 = np.dot(tmp,I+(-1.0*k1*8.0/27.0-k2*2.0-k3*3544.0/2565.0+k4*1859.0/4104.0-k5*11.0/40.0)*self.dt) result = np.dot(I+(k1*16.0/135.0+k3*6656.0/12825.0+k4*28561.0/56430.0-k5*9.0/50.0+k6*2.0/55.0)*self.dt,result) return result
def greedy_for_bound(self,best_in_next = 100, direction = 'west', disp_progress=True,width=40,wgpenalty=0,start=-180,init=True): if init: print 'initialization...' self.to_assign = np.random.permutation(range(self.N)).tolist() ''' if not(np.all([self.centroids[i][1]<=self.centroids[i+1][1] for i in range(len(self.centroids)-1)])): print 'warning: centroids were not sorted by longitude. I reorder them' self.centroids = np.array(sorted(self.centroids, key=lambda x: x[1])) self.init_clusters_with_centroids() ''' self.centroids = np.zeros((0,2)) self.K = 0 shiftedlong = np.where(self.X[self.to_assign][:,1]<start,self.X[self.to_assign][:,1]+360,self.X[self.to_assign][:,1]) if direction=='west': print 'sorting gifts per longitude...' self.to_assign = np.array(self.to_assign)[np.argsort(shiftedlong)] elif direction=='east': print 'sorting gifts per longitude...' self.to_assign = np.array(self.to_assign)[np.argsort(shiftedlong)[::-1]] elif direction=='south': print 'sorting gifts per latitude...' self.to_assign = np.array(self.to_assign)[np.argsort(self.X[self.to_assign][:,0])[::-1]] elif direction=='north': print 'sorting gifts per latitude...' self.to_assign = np.array(self.to_assign)[np.argsort(self.X[self.to_assign][:,0])] else: raise ValueError('direction not implemented') self.to_assign = self.to_assign.tolist() print 'done.' if disp_progress: prog = ProgressBar(0, len(self.to_assign), 77, mode='fixed') oldprog = str(prog) while True: if disp_progress: #<--display progress prog.increment_amount() if oldprog != str(prog): print prog, "\r", sys.stdout.flush() oldprog=str(prog) #--> if self.K == 0: self.create_new_cluster(self.to_assign[0]) del self.to_assign[0] continue candidates = self.to_assign[:best_in_next] bounds_inc = [] for i in candidates: km = np.searchsorted(self.centroids[:,1], self.X[i][1]-width) kp = np.searchsorted(self.centroids[:,1], self.X[i][1]+width) bounds_inc.extend([(self.bound_increase_for_adding_gift_in_cluster(i,k),i,k) for k in range(km,kp) if self.weight_per_cluster[k]+self.weights[i]<weight_limit-wgpenalty]) if not bounds_inc: self.create_new_cluster(self.to_assign[0]) del self.to_assign[0] continue sorted_bounds_inc = sorted(bounds_inc) assigned = False for inc,i,c in sorted_bounds_inc: if inc> 2*self.distances_to_pole[i]*sleigh_weight: #import pdb;pdb.set_trace() self.create_new_cluster(self.to_assign[0]) assigned = True del self.to_assign[0] #print 'one more clust '+str(self.K) #if self.K>1500: # import pdb;pdb.set_trace()#TMP break if self.weight_per_cluster[c]+self.weights[i]<weight_limit-wgpenalty: self.add_in_tour(i,c) assigned = True self.to_assign.remove(i) break if not assigned: raise Exception('not able to assign a trip in this window of longitudes.') if not(self.to_assign): break
linecount = 0 revblock = "" isrevisionblock = False ispageblock = False isknownline = False pagetitle = "" pageid = "" pagecount = 0 pageRedirect = False prog = ProgressBar(linecount, numlinesinfile, 100, mode='fixed', char='#') for txtline in open(xmlfile): try: isknownline = False prog.increment_amount() linecount += 1 if isrevisionblock: revblock = revblock + txtline #start: this is what we do when we've finished a revision if txtline.find("</revision>") > 0: isrevisionblock = False isknownline = True try: processRevision(cleanString(revblock)) except: exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() log.write("Error Processing Revision with stacktrace: " + str(traceback.print_tb(exceptionTraceback))) log.write("\n161 Unexpected error:" + str(sys.exc_info()[0]) + "Line " + str(linecount) + ": Error Processing Revision:\n" + revblock) errorxmlfile.write(revblock) #print sys.exc_info()
def main(): # define command line arguments/parser parser = optparse.OptionParser('%%prog %s' % __version__) parser.add_option("-i", "--input", help="Input log file path", metavar="FILE") parser.add_option("-d", "--dict", help="Input dictionary file path", metavar="FILE") parser.add_option("-a", "--address_db", help="Input address symbols sql db file path", metavar="FILE") parser.add_option("-x", "--histogram", help="Report histogram", metavar="ENABLE", action="store_true", default=True) parser.add_option("-X", "--nohistogram", help="DO NOT report histogram", metavar="ENABLE", action="store_false", dest="histogram") parser.add_option("-c", "--callhier", help="Report call hierarchy", metavar="ENABLE", action="store_true", default=True) parser.add_option("-C", "--nocallhier", help="Do NOT Report call hierarchy", metavar="ENABLE", action="store_false", dest="callhier") parser.add_option("-e", "--errors", help="Report errors", metavar="ENABLE", action="store_true", default=True) parser.add_option("-E", "--noerrors", help="Do NOT Report errors", metavar="ENABLE", action="store_false", dest="errors") parser.add_option("-y", "--callstack_detail", help="Write callstack detail to report", metavar="ENABLE", action="store_true", default=True) parser.add_option("-Y", "--no_callstack_detail", help="Do NOT Report callstack detail", metavar="ENABLE", action="store_false", dest="callstack_detail") parser.add_option("-p", "--prune_callstacks", help="Prune callstacks above memory manager functions", metavar="ENABLE", action="store_true", default=True) parser.add_option("-P", "--no_prune_callstacks", help="Do NOT prune callstacks", metavar="ENABLE", action="store_false", dest="prune_callstacks") parser.add_option("-l", "--limit", help="size limit for reporting information", type="int", default=0x10000) parser.add_option("", "--limit_event_reads", help="allocation limit for reporting branch", type="int", default=0x7FFFFFF) parser.add_option( "", "--call_stack_write_limit", help= "limit number of callstacks written to report (development feature)", type="int", default=-1) parser.add_option("", "--begin_marker", help="Marker ID to start simulation", type="int", default=-1) parser.add_option("", "--end_marker", help="Marker ID to end simulation", type="int", default=-1) parser.add_option("-S", "--simple_processing", help="Do simple total allocated/freed analysis", metavar="ENABLE", action="store_true", default=False) parser.add_option("-v", "--verbose", help="Do verbose logging", metavar="ENABLE", action="store_true") opts, args = parser.parse_args() in_name = 'memtrack_log.dat' in_dict = 'memtrack_dict.dat' in_address = 'address.db' ########################## # apply command line arbuments # pprint.pprint( opts ) verbose = opts.verbose if opts.input: in_name = opts.input if opts.dict: in_dict = opts.dict if opts.address_db: in_address = opts.address_db if not os.path.exists(in_name): print >> sys.stderr, 'Could not find input file: %s' % in_name sys.exit(128) if not os.path.exists(in_dict): print >> sys.stderr, 'Could not find dictionary: %s' % in_dict sys.exit(128) if not os.path.exists(in_address): print >> sys.stderr, 'Could not find symbol db: %s' % in_address sys.exit(128) MemTrack.event_log_read_limit = opts.limit_event_reads MemTrack.reporting_size_limit = opts.limit MemTrack.prune_callstacks = opts.prune_callstacks MemTrack.call_stack_write_limit = opts.call_stack_write_limit ########################## print "Loading...\n\n" ### cache the address symbol db in a dict for ease of use print "Caching symbols...\n" SymbolDB.symbol_cache = build_address_symbol_cache(in_address) print "\n\n" ### Read in the call stack dictionary print "Reading callstack dictionary...\n" stack_dict = read_callstack_dictionary(in_dict) print "\n\n" ### Read the event log print "Reading event log..." event_list = read_event_log(in_name) print "\n\n" ########################## pre_event_list = [] if opts.begin_marker >= 0 or opts.end_marker >= 0: print "Pruning list...\n\n" begin_index = 0 if opts.begin_marker >= 0: begin_index = find_marker(event_list, opts.begin_marker) if begin_index == -1: begin_index = 0 else: print " Found begin marker: %d at index %d" % ( opts.begin_marker, begin_index) end_index = len(event_list) if opts.end_marker >= 0: end_index = find_marker(event_list, opts.end_marker) if end_index == -1: end_index = len(event_list) else: print " Found end marker : %d at index %d" % ( opts.end_marker, end_index) if begin_index > 0: pre_event_list = event_list[:begin_index] event_list = event_list[begin_index:end_index] print "\n Pruned events that will be tracked: %d\n" % len(event_list) gc.collect() if len(event_list) == 0: return if opts.simple_processing: do_simple_processing(pre_event_list, event_list) return ########################## print "Processing...\n\n" lpath, ltail = os.path.split(in_name) lname, lext = os.path.splitext(ltail) outName = lname.split(' ')[0] outName = outName.split('_')[0] ### histogram if opts.histogram: print "<histogram feature not available w/o html and graph generation.>\n\n" print "\n\n" ### heap simulation #create a heap to track live allocated blocks and current memory status overall heap = Heap() if len(pre_event_list) > 0: print "Preprocessing events...\n\n" do_preprocess_simulation(heap, pre_event_list, event_list) print "\n\n" print "Simulating events...\n\n" prog = ProgressBar(0, len(event_list), 77, mode='fixed', char='#', autoprint=True) for event in event_list: heap.track(event) prog.increment_amount() print "\n" if len(heap.errors()): print "simulation errors: %d of %d" % (len( heap.errors()), len(event_list)) else: print "simulation errors: NONE" print "\n\n" ### call hierarchy if opts.callhier: print "Generating call hierarchy allocation tree...\n\n" hierarchy_root = generate_hierarchy(event_list, stack_dict) print "entry points: %d" % len(hierarchy_root.children) ########################## print "complete."
def do_simple_processing(pre_event_list, event_list): allocated = 0 reallocated = 0 reallocated_freed = 0 freed = 0 num_events = 0 blocks = {} print "Preprocessing (looking for frees and reallocs)..." prog = ProgressBar(0, len(event_list), 77, mode='fixed', char='#', autoprint=True) # find all free's and realloc's. # the logic is that there will be a lot less free's and realloc's than alloc's # and so we only track the addresses which will be free'd or realloc'd instead # of all of the addresses which are alloc'd # NOTE: we only care about free's and realloc's within our begin/end markers for event in event_list: if event.event_type == Event.eRealloc: blocks[event.pDataPrev] = 0 elif event.event_type == Event.eFree: blocks[event.pData] = 0 prog.increment_amount() print "\n\nSimulating events..." prog = ProgressBar(0, len(pre_event_list) + len(event_list), 77, mode='fixed', char='#', autoprint=True) for event in pre_event_list: if event.event_type == event.eAlloc: if blocks.has_key(event.pData): blocks[event.pData] = event.allocSize elif event.event_type == Event.eRealloc: if blocks.has_key(event.pData): blocks[event.pData] = event.allocSize prog.increment_amount() for event in event_list: num_events += 1 if event.event_type == event.eAlloc: allocated += event.allocSize if blocks.has_key(event.pData): blocks[event.pData] = event.allocSize elif event.event_type == Event.eRealloc: reallocated += event.allocSize reallocated_freed += blocks[event.pDataPrev] blocks[event.pData] = 0 if blocks.has_key(event.pData): blocks[event.pData] = event.allocSize elif event.event_type == Event.eFree: freed += blocks[event.pData] blocks[event.pData] = 0 prog.increment_amount() total = allocated + reallocated - reallocated_freed - freed print "\n\n" print "allocated = %15s" % locale.format('%d', allocated, True) print "reallocated = %15s" % locale.format('%d', reallocated, True) print "reallocated_freed = %15s" % locale.format('%d', reallocated_freed, True) print "freed = %15s" % locale.format('%d', freed, True) print "-----------------------------------" print "total = %15s" % locale.format('%d', total, True) print "\n" print "total events simulated = %s" % locale.format( '%d', num_events, True)
def fill_and_save_data_hists( mcf, modes, hlist, contribs,predicts ) : axes = [ "X", "Y", "Z" ] chain = MCAnalysisChain( mcf ) nentries = chain.GetEntries() for h in hlist : histo_cont = {} contrib_cont = {} predict_cont = {} h_dim = get_histogram_dimension(h) dim_range = range(h_dim) axis_nbins = [] axis_mins = [] axis_maxs = [] axis_bins = [] axis_titles = [] th_arg_list = [] user_notify_format = "" user_notify = [] title_format = "%s" title_items = [ h.GetTitle() ] for axis in dim_range : axis_nbins.append( eval( "h.GetNbins%s()" % axes[axis] ) ) axis_mins.append( eval( "h.Get%saxis().GetXmin()" % axes[axis] ) ) axis_maxs.append( eval( "h.Get%saxis().GetXmax()" % axes[axis] ) ) axis_bins.append( eval( "h.Get%saxis().GetXbins().GetArray()" % axes[axis] ) ) axis_titles.append( eval( "h.Get%saxis().GetTitle()" % axes[axis] ) ) th_arg_list.append( axis_nbins[-1] ) th_arg_list.append( axis_mins[-1] ) th_arg_list.append( axis_maxs[-1] ) user_notify_format += ": [ %.2e, %.2e ] :" user_notify.append( axis_mins[-1] ) user_notify.append( axis_maxs[-1] ) title_format += ";%s" title_items.append( axis_titles[-1] ) print user_notify_format % tuple(user_notify) title = title_format % tuple(title_items) firstbin, lastbin = get_histogram_bin_range(h) for mode in modes : # here need to add in check on contrib and make one for each contribution histo_cont[mode] = eval( 'r.TH%dD( h.GetName() + "_" + mode, title, *th_arg_list )' % h_dim ) base_val = 1e9 if mode == "pval" : base_val = 0.0 for bin in range( firstbin, lastbin + 1 ) : histo_cont[mode].SetBinContent( bin, base_val ) for c in contribs : # contribs is a list of Contribution objects contrib_cont[c.short_name] = eval( 'r.TH%dD( h.GetName() + "_dX_" + c.short_name, title, *th_arg_list )' % h_dim ) for bin in range( firstbin, lastbin + 1 ) : contrib_cont[c.short_name].SetBinContent( bin, 0.0 ) for p in predicts : # predicts is a list of Contribution objects predict_cont[p.short_name] = eval( 'r.TH%dD( h.GetName() + "_pred_" + p.short_name, title, *th_arg_list )' % h_dim ) for bin in range( firstbin, lastbin + 1 ) : predict_cont[p.short_name].SetBinContent( bin, 0.0 ) print "yes", p prog = ProgressBar(0, (lastbin-firstbin)+1, 77, mode='fixed', char='#') for i in range( firstbin, lastbin+1 ) : prog.increment_amount() print prog,'\r', stdout.flush() entry = int( h.GetBinContent(i) ) if entry > 0 : chain.GetEntry(entry) fill_bins( histo_cont, contrib_cont,predict_cont, contribs,predicts , i, chain, mcf ) perform_zero_offset( histo_cont["dchi"] ) print save_hdict_to_root_file( histo_cont, mcf.FileName, mcf.DataDirectory) save_hdict_to_root_file( contrib_cont, mcf.FileName, mcf.DataDirectory) save_hdict_to_root_file( predict_cont, mcf.FileName, mcf.DataDirectory)
def recalc_to_file( collection, output_file = "" ) : model = models.get_model_from_file(collection) lhoods = models.get_lhood_from_file(collection) outfile = collection.FileName if output_file == "" else output_file print "Output file is %s" % outfile # initialise the MC-variables MCVdict=v.mc_variables() chain = MCRecalcChain( collection ) nentries = chain.GetEntries() begin = getattr( collection, "StartEntry", 0) end = getattr( collection, "EndEntry", nentries+1) total_delta = 0 # create trees in scope of outfile out = r.TFile(outfile,"recreate") chi2tree = chain.chains["predictions"].CloneTree(0) # might need to do address of on contirbvars nTotVars = chain.nTotVars["predictions"] contribvars = array('d',[0.0]*nTotVars) contribtree = r.TTree( 'contribtree', 'chi2 contributions') varsOutName = "vars[%d]/D" % ( nTotVars ) contribtree.SetMaxTreeSize(10*chi2tree.GetMaxTreeSize()) contribtree.Branch("vars",contribvars,varsOutName) # same with lhood nLHoods = len(lhoods.keys()) lhoodvars = array('d',[0.0]*nLHoods) lhoodtree = r.TTree( 'lhoodtree', 'lhood contributions') varsOutName = "vars[%d]/D" % ( nLHoods ) lhoodtree.SetMaxTreeSize(10*chi2tree.GetMaxTreeSize()) lhoodtree.Branch("vars",lhoodvars,varsOutName) # want to save best fit point entry number: create new tree and branch bfname = getattr( collection, "BestFitEntryName", "BestFitEntry" ) bft=r.TTree(bfname, "Entry") bfn=array('i',[0]) bft.Branch('EntryNo',bfn,'EntryNo/I') # and the minChi minEntry minChi=1e9 minEntry=-1 count=-1 # becuase the first entry has number 0 prog = ProgressBar(begin, end, 77, mode='fixed', char='#') for entry in range(begin,end) : prog.increment_amount() print prog,'\r', stdout.flush() chain.GetEntry(entry) if good_point( chain.treeVars["predictions"], collection ) : delta = 0. chi2 = 0 for constraint in model : MCV=MCVdict[constraint.short_name] v_index = MCV.get_index(collection) chi2_t = constraint.get_chi2( chain.treeVars["predictions"][v_index] ) contribvars[v_index] = chi2_t chi2 += chi2_t for i,lh in enumerate(lhoods.values()) : chi2_t = lh.get_chi2( chain.treeVars["predictions"] ) lhoodvars[i] = chi2_t chi2 += chi2_t chi2 += spectrum_constraints( chain.treeVars["predictions"], collection ) if chi2 > getattr(collection, "MinChi2", 0 ) and \ chi2 < getattr(collection, "MaxChi2", 1e9 ) : # This was inserted to check on if there was a significant # calculation error ( average deltachi2 per entry: 1e-15 ) if __DEBUG : delta_chi2_val = chi2_t - chain.treeVars["contributions"][key] delta = delta + delta_chi2_val total_delta = total_delta + abs(delta) chain.treeVars["predictions"][0] = chi2 contribvars[0] = chi2 chi2tree.Fill() contribtree.Fill() lhoodtree.Fill() count+=1 #dealing with minChi if chi2 < minChi: minChi=chi2 minEntry=count #Saving best fit Entry number bft.GetEntry(0) bfn[0]=minEntry bft.Fill() bft.AutoSave() chi2tree.AutoSave() contribtree.AutoSave() lhoodtree.AutoSave() out.Close() if __DEBUG : print "\n--------------------------\n" print " TOTAL ( MEAN )" print "%10e(%10e)" % ( total_delta, (total_delta/(end-begin)) ) print "\n--------------------------\n"