def FitTheHistograms(): ## ============================== ## fit the histograms ## ============================== hmap = HistoMap() hmap.read(input_file) print "FITTING THE HISTOGRAMS" bad, low, nfit = analyse( ## recieve a dictionary of the bad, not fit histograms and the ones with low statistics hmap, ## map of the filled histograms lmap, ## map of the calibration coefficients fitted=False, ## not fitted yet manager=None, ## parallel processing manager #nHistos = 60 ## number of histograms per core/machine ) ## Print statistics print 'BAD-CELLS : #', len(bad) print 'LOW-CELLS : #', len(low) print 'NFIT-CELLS: #', len(nfit) print "Fitting completed" LambdasPath = pt.lambdas_location_an( ) ## path to save the calibration coefficients print "Saving lambdas to ", LambdasPath lmap.save(LambdasPath) os.remove(input_file)
def process(self, chunk): from KaliCalo.MassDistribution import HistoMap histos = HistoMap() self.output = histos for entry in chunk: efile = entry[0] enum = entry[1] histos.read(efile) print 'Processed %s \t#%d/%d\tEntries:%d\tHistos:%d' % ( efile, enum, self.total, histos.entries(), len(histos))
def GetOutput(): ## ## Create the resulting histos ## from KaliCalo.FitUtils import fitPi0, getPi0Params from KaliCalo.Cells import CellID import ROOT nit = PassIt / 2 Conv = {} Sigm = {} histofile = ROOT.TFile( os.path.join(pt.store_location(), "OutputHistograms.root"), 'recreate') for j in range(1, 1 + MaxIt / PassIt): MassInner = [] MassMiddle = [] MassOuter = [] Mass = [] Histos = [] #if j == 2: nit = nit-1 Conv['Pass%i' % j] = ROOT.TH1F('Conv%i' % j, 'Convergency', nit, 0, nit) Sigm['Pass%i' % j] = ROOT.TH1F('Sigm%i' % j, 'Resolution', nit, 0, nit) for i in range(1, nit + 1): #for i in range(1,nit+1): Histos.append(HistoMap()) ni = i - 1 Histos[ni].read(StoredHistoPath % (j, i)) MassInner.append( ROOT.TH1F('MassInnerPass%iIt%i' % (j, i), 'Inner mass at Pass%i Iteration N%i' % (j, i), 250, 0, 250)) MassMiddle.append( ROOT.TH1F('MassMiddlePass%iIt%i' % (j, i), 'Middle mass at Pass%i Iteration N%i' % (j, i), 250, 0, 250)) MassOuter.append( ROOT.TH1F('MassOuterPass%iIt%i' % (j, i), 'Outer mass at Pass%i Iteration N%i' % (j, i), 250, 0, 250)) Mass.append( ROOT.TH1F('MassPass%iIt%i' % (j, i), 'Mass at Pass%i Iteration N%i' % (j, i), 250, 0, 250)) for ck in range(0, 3): MassInner[ni].Add(Histos[ni][CellID('Ecal', 'Inner', 31, 31)][ck]) MassMiddle[ni].Add(Histos[ni][CellID('Ecal', 'Middle', 31, 31)][ck]) MassOuter[ni].Add(Histos[ni][CellID('Ecal', 'Outer', 31, 31)][ck]) Mass[ni].Add(MassInner[ni]) Mass[ni].Add(MassMiddle[ni]) Mass[ni].Add(MassOuter[ni]) fitPi0(Mass[ni]) mp = getPi0Params(Mass[ni]) print "All:", mp Conv['Pass%i' % j].SetBinContent(i, mp[1].value()) Conv['Pass%i' % j].SetBinError(i, mp[1].error()) Sigm['Pass%i' % j].SetBinContent(i, mp[2].value()) Sigm['Pass%i' % j].SetBinError(i, mp[2].error()) fitPi0(MassInner[ni]) mpi = getPi0Params(MassInner[ni]) print "Inner:", mpi fitPi0(MassMiddle[ni]) print "Middle:", getPi0Params(MassMiddle[ni]) fitPi0(MassOuter[ni]) print "Outer:", getPi0Params(MassOuter[ni]) MassInner[ni].Write() MassMiddle[ni].Write() MassOuter[ni].Write() Mass[ni].Write() Conv['Pass%i' % j].Write() Sigm['Pass%i' % j].Write() histofile.Close() lmap = LambdaMap.LambdaMap() lmap.read(pt.lambdas_location_it() % (1, 1)) lmap2 = LambdaMap.LambdaMap() lmap2.read(pt.lambdas_location_it() % (MaxIt / PassIt, nit)) #lmap2.read(pt.lambdas_location_it()%(MaxIt/PassIt,3)) lams = lmap.lambdas() lams2 = lmap2.lambdas() ROOT.gStyle.SetPalette(1) It12distr = ROOT.TH1F('It12distr', 'Calibration constants: All ', 100, 0.5, 1.5) It12distrInner = ROOT.TH1F('It12distrInner', 'Calibration constants: Inner', 100, 0.5, 1.5) It12distrMiddle = ROOT.TH1F('It12distrMiddle', 'Calibration constants: Middle', 100, 0.5, 1.5) It12distrOuter = ROOT.TH1F('It12distrOuter', 'Calibration constants: Outer', 100, 0.5, 1.5) It1distr = ROOT.TH1F('It1distr', 'Calibration constants: All', 100, 0.5, 1.5) It1distrInner = ROOT.TH1F('It1distrInner', 'Calibration constants: Inner', 100, 0.5, 1.5) It1distrMiddle = ROOT.TH1F('It1distrMiddle', 'Calibration constants: Middle', 100, 0.5, 1.5) It1distrOuter = ROOT.TH1F('It1distrOuter', 'Calibration constants: Outer', 100, 0.5, 1.5) It2distr = ROOT.TH1F('It2distr', 'Calibration constants: All', 100, 0.5, 1.5) It2distrInner = ROOT.TH1F('It2distrInner', 'Calibration constants: Inner', 100, 0.5, 1.5) It2distrMiddle = ROOT.TH1F('It2distrMiddle', 'Calibration constants: Middle', 100, 0.5, 1.5) It2distrOuter = ROOT.TH1F('It2distrOuter', 'Calibration constants: Outer', 100, 0.5, 1.5) comp2DFaceInner = ROOT.TH2F('comp2DFaceInner', '', 48, -970, 970, 36, -725, 725) comp2DFaceInner.SetMinimum(0.5) comp2DFaceInner.SetMaximum(1.5) comp2DFaceMiddle = ROOT.TH2F('comp2DFaceMiddle', '', 64, -1940, 1940, 40, -1210, 1210) comp2DFaceMiddle.SetMinimum(0.5) comp2DFaceMiddle.SetMaximum(1.5) comp2DFaceOuter = ROOT.TH2F('comp2DFaceOuter', '', 64, -3880, 3880, 52, -3150, 3150) comp2DFaceOuter.SetMinimum(0.5) comp2DFaceOuter.SetMaximum(1.5) lamsa = {} filtxt = open(os.path.join(pt.store_location(), 'CalibCoefficients.txt'), 'w') for ic in lams.keys(): if not lams2.has_key(ic): lams2[ic] = 1. print ic lamsa[ic] = lams[ic] * lams2[ic] print >> filtxt, ic.index(), lamsa[ic] for ic2 in lams2.keys(): if ic2 in lamsa.keys(): continue print ic2 lamsa[ic2] = lams2[ic2] print >> filtxt, ic2.index(), lamsa[ic2] filtxt.close() #cf = FakeCells.RectRings() #for kk in RealPrsCoefs.keys(): # #aa = CellID(kk[0],kk[1],kk[2],kk[3]) # #ic = cf(aa) # if not lamsa.has_key(ic): continue # if ic.area() == 2: # #print ic.col(),ic.row() # comp2DFaceInner.SetBinContent(aa.col()-7,aa.row()-13,lamsa[ic]) # if ic.area() == 1: # #print ic.col(),ic.row() # comp2DFaceMiddle.SetBinContent(aa.col()+1,aa.row()-11,lamsa[ic]) # if ic.area() == 0: # #print ic.col(),ic.row() # comp2DFaceOuter.SetBinContent(aa.col()+1,aa.row()-5,lamsa[ic]) for ic in lamsa.keys(): #lamsa[ic] = lams2[ic]*lams[ic] #try2D.Fill(lams[ic],lams2[ic]) if ic.area() == 2: #print ic.col(),ic.row() comp2DFaceInner.SetBinContent(ic.col() - 7, ic.row() - 13, lamsa[ic]) if ic in lams2.keys(): It2distr.Fill(lams2[ic]) It2distrInner.Fill(lams2[ic]) if ic in lams.keys(): It1distr.Fill(lams[ic]) It1distrInner.Fill(lams[ic]) It12distr.Fill(lamsa[ic]) It12distrInner.Fill(lamsa[ic]) if ic.area() == 1: #print ic.col(),ic.row() comp2DFaceMiddle.SetBinContent(ic.col() + 1, ic.row() - 11, lamsa[ic]) if ic in lams2.keys(): It2distr.Fill(lams2[ic]) It2distrMiddle.Fill(lams2[ic]) if ic in lams.keys(): It1distr.Fill(lams[ic]) It1distrMiddle.Fill(lams[ic]) It12distr.Fill(lamsa[ic]) It12distrMiddle.Fill(lamsa[ic]) if ic.area() == 0: #print ic.col(),ic.row() comp2DFaceOuter.SetBinContent(ic.col() + 1, ic.row() - 5, lamsa[ic]) if ic in lams2.keys(): It2distr.Fill(lams2[ic]) It2distrOuter.Fill(lams2[ic]) if ic in lams.keys(): It1distr.Fill(lams[ic]) It1distrOuter.Fill(lams[ic]) It12distr.Fill(lamsa[ic]) It12distrOuter.Fill(lamsa[ic]) It2distr.Fit("gaus", '', '', 0.5, 1.5) It2distrInner.Fit("gaus", '', '', 0.5, 1.5) It2distrMiddle.Fit("gaus", '', '', 0.5, 1.5) It2distrOuter.Fit("gaus", '', '', 0.5, 1.5) It12distr.Fit("gaus", '', '', 0.5, 1.5) It12distrInner.Fit("gaus", '', '', 0.5, 1.5) It12distrMiddle.Fit("gaus", '', '', 0.5, 1.5) It12distrOuter.Fit("gaus", '', '', 0.5, 1.5) It1distr.Fit("gaus", '', '', 0.5, 1.5) It1distrInner.Fit("gaus", '', '', 0.5, 1.5) It1distrMiddle.Fit("gaus", '', '', 0.5, 1.5) It1distrOuter.Fit("gaus", '', '', 0.5, 1.5) fillam = ROOT.TFile( os.path.join(pt.store_location(), "OutputCoefficients.root"), 'recreate') It2distr.Write() It2distrInner.Write() It2distrMiddle.Write() It2distrOuter.Write() It12distr.Write() It12distrInner.Write() It12distrMiddle.Write() It12distrOuter.Write() It1distr.Write() It1distrInner.Write() It1distrMiddle.Write() It1distrOuter.Write() comp2DFaceInner.Write() comp2DFaceMiddle.Write() comp2DFaceOuter.Write() fillam.Close()
def SplitHistos(): ## first join the filled histograms from all the workers #print "Splitting histograms" #import MergeHistos histos = pt.gethistomaps() histos.sort() #hmap = MergeHistos.mergeDBs ( histos ) hmap = HistoMap() ## add all the others for hm in histos: hmap.updateFromDB(hm) print "Adding histograms", hm, hmap.entries() #hmap.save(pt.histos_location()) ## find the last one saved and ## save current as the next one ih = 1 Pass = 1 while True: if ih > PassIt / 2: Pass += 1 ih = 1 if os.path.exists(StoredHistoPath % (Pass, ih)): ## look through all the existing files print StoredHistoPath % (Pass, ih), 'exists' ih += 1 continue else: ## write to the file next to the latest existing print 'Saving histograms to ', StoredHistoPath % (Pass, ih) hmap.save(StoredHistoPath % (Pass, ih)) os.system('chmod a+rwx ' + StoredHistoPath % (Pass, ih)) break ## calculate a number of histograms per worker nworkers = len(histos) nhistos = len(hmap.keys()) histos_per_worker = nhistos / nworkers + 1 #print nworkers, nhistos, histos_per_worker #print "SPLITTING THE HISTOGRAMS BETWEEN ", nworkers, "WORKERS" #print histos_per_worker, "PER EACH" data = hmap.split(histos_per_worker) ## get 'All-Ecal' histograms hA = hmap[AZ] ## inner area hI = hmap[IZ] ## middle area hM = hmap[MZ] ## outer area hO = hmap[OZ] ## To have the global histograms in each histomap #print len(data) for dt in data: if not hA in dt[0]: dt[0].append(hA) if not hI in dt[0]: dt[0].append(hI) if not hM in dt[0]: dt[0].append(hM) if not hO in dt[0]: dt[0].append(hO) #print len(dt[0]),"HISTOGRAMS" ## rewrite existing databases by the ## parts of the split histo map num = len(data) if num < nworkers: data.append(([], 0)) for ih in range(nworkers): splitmap = HistoMap() hm = histos[ih] for jh in data[ih][0]: splitmap.insert(jh) print "Clearing previous location", hm os.remove(hm) splitmap.save(hm)
def SplitHistos(): ## first join the filled histograms from all the workers import MergeHistos histos = pt.gethistomaps() histos.sort() print '# of histograms = ',len(histos) loggerIt.warning('TRY TO MERGE %d histograms ' % len( histos )) sys.stdout.flush() hmap = MergeHistos.mergeDBs ( histos , 60 ) loggerIt.warning('RESULT Entries:%d Histos:%s' % ( hmap.entries() , len(hmap) )) ## find the last one saved and ## save current as the next one ih = 1 Pass = 1 while True: if ih > PassIt/2: Pass += 1 ih = 1 if os.path.exists(StoredHistoPath%(Pass,ih)): ## look through all the existing files print StoredHistoPath%(Pass,ih), 'exists' ih += 1 continue else: ## write to the file next to the latest existing print 'Saving histograms to ', StoredHistoPath%(Pass,ih) hmap.save(StoredHistoPath%(Pass,ih)) os.system('chmod a+rwx '+StoredHistoPath%(Pass,ih)) break ## calculate a number of histograms per worker nworkers = len(histos) nhistos = len(hmap.keys()) Nmin = nhistos/nworkers Nmax = Nmin+1 rst = (float(nhistos)/float(nworkers)-Nmin)*nworkers rst = int(rst) gh = 0 group_max = HistoMap() group_min = HistoMap() loggerIt.warning("SPLITTING %s HISTOGRAMS BETWEEN %s WORKERS" %(nhistos,nworkers)) loggerIt.warning("%s WORKERS WILL HAVE %s HISTOGRAMS PER EACH"%(rst, Nmax)) loggerIt.warning("%s WORKERS WILL HAVE %s HISTOGRAMS PER EACH"%(nworkers-rst,Nmin)) for hk in hmap.keys(): if gh < (rst*Nmax): group_max.insert(hmap[hk]) else: group_min.insert(hmap[hk]) gh += 1 print len(group_max),"+", len(group_min),"=", len(group_max)+len(group_min), "=", nhistos d1 = group_max.split(Nmax) d2 = group_min.split(Nmin) data = [] data += d1 data += d2 print len(d1),"+",len(d2),"=",len(data),"=",nworkers,"?" ## get 'All-Ecal' histograms hA = hmap [ AZ ] ## inner area hI = hmap [ IZ ] ## middle area hM = hmap [ MZ ] ## outer area hO = hmap [ OZ ] ## To have the global histograms in each histomap #print len(data) for dt in data: if not hA in dt[0]: dt[0].append(hA) if not hI in dt[0]: dt[0].append(hI) if not hM in dt[0]: dt[0].append(hM) if not hO in dt[0]: dt[0].append(hO) #print len(dt[0]),"HISTOGRAMS" ## rewrite existing databases by the ## parts of the split histo map for ih in range(nworkers): splitmap = HistoMap() hm = histos[ih] print ih, data[ih][0] for jh in data[ih][0]: splitmap.insert(jh) #print "Clearing previous location", hm os.remove(hm) splitmap.save(hm)