from ROOT import * from linearADC import * import os from array import array # Fixes seg fault from importing Quiet try: Quiet(TObject()) except NameError: from utils import Quiet bin0startLevel = -0.5 # def CleanGraph(graph, i_range): # vOffset = i_range*64 # points = range(graph.GetN()) # return graph def getPedestals(graphs_shunt, shuntMult_list, histoList, dirName, date, run, verbose=False): pedestalVals = {} if verbose: print "graphs_shunt = ", graphs_shunt if not os.path.exists("%s/PedestalPlots" % dirName):
def doFit_combined(graphList, saveGraph=False, qieNumber=0, qieBarcode="", qieUniqueID="", useCalibrationMode=True, outputDir='', shuntMult=1, pedestalVals={ "low": [0, 0, 0, 0], "high": [0, 0, 0, 0], "shunt": {} }, verbose=False): fitLines = [] slopes = [] offsets = [] maxResiduals = [] maxCharges = [] minCharges = [] # pedestal = [0]*4 linearizedGraphList = [] #print graphList outputTGraphs = TFile( outputDir.replace("outputPlots", "") + "fitResults_%s.root" % qieUniqueID, "update") saveName = None if shuntMult == 1: ranges = range(4) else: ranges = range(2) #change for i_range in ranges: #choose the pedestal to subtract according to the whether it is low current or high current #if i_range == 0 and shuntMult==1: # pedestal = pedestalVals["low"] #elif shuntMult==1: ####################################################### #if shuntMult == 1: # pedestal = pedestalVals["high"] #else: # pedestal = pedestalVals["shunts"][shuntMult] ####################################################### pedestal = pedestalVals["shunts"][shuntMult] # vOffset = i_range*64 graphs = graphList[i_range] if graphs == None: fitLines.append(None) continue else: fitLines.append([]) maxResiduals.append([]) minCharges.append([]) maxCharges.append([]) # if pedestal==None: # pedestal = [] for i_capID in range(4): #print i_graph nominalgraph = graphs[i_capID] #print nominalgraph.GetName() # if shuntMult == 1: # outputTGraphs.cd("adcVsCharge") # else: # outputTGraphs.cd("Shunted_adcVsCharge") # nominalgraph.Write() graph = nominalgraph.Clone("%s_linearized" % nominalgraph.GetName()) graph.SetNameTitle("%s_linearized" % nominalgraph.GetName(), "%s_linearized" % nominalgraph.GetName()) points = range(graph.GetN()) points.reverse() maxCharge = -9e9 minCharge = 9e9 for p in points: # x = graph.GetX()[p]-vOffset # nominalgraph.GetX()[p] -= pedestal[i_capID] graph.GetX()[p] -= pedestal[i_capID] graph.GetXaxis().SetTitle("Charge (fC)") graph.GetYaxis().SetTitle("Linearized ADC") outputTGraphs.cd("LinadcVsCharge") # if shuntMult == 1: # else: # outputTGraphs.cd("Shunted_LinadcVsCharge") if verbose: print "Pedestals Used" print pedestal if graph.GetN() > 1: if verbose: print "TOTAL:", graph.GetN() # f1= TF1("f1","pol1",200,600); # f1.FixParameter(0,-0.5) #if (i_range==0 and shuntMult==1): # graph.Fit("f1","R0") #else: # graph.Fit("pol1","0") if not verbose: graph.Fit("pol1", "0QF") else: graph.Fit("pol1", "0F") linearizedGraphList.append(graph) #if (i_range==0 and shuntMult==1): # fitLine = graph.GetFunction("f1") #else: # fitLine = graph.GetFunction("pol1") fitLine = graph.GetFunction("pol1") fitLine.SetNameTitle("fit_%s" % graph.GetName(), "fit_%s" % graph.GetName()) fitLines[-1].append(fitLine) # if qieNumber==10: # N = graph.GetN() # print qieNumber, i_range, i_capID, graph.GetN(), graph.GetX()[0], graph.GetX()[int(N/2)], graph.GetX()[N-1], pedestal[i_capID] # print "\t",fitLine.GetParameter(0), fitLine.GetParameter(1) else: linearizedGraphList.append(graph) fitLine = TF1("fit_%s" % graph.GetName(), "pol1", -999, 999) fitLine.SetParameter(0, 0) fitLine.SetParameter(1, 0) fitLine.SetParError(0, 999) fitLine.SetParError(1, 999) fitLine.SetNameTitle("fit_%s" % graph.GetName(), "fit_%s" % graph.GetName()) fitLines[-1].append(fitLine) print 'PROBLEM' print graph.GetName() continue graph.Write() if saveGraph or saveResiduals: xVals = graph.GetX() exVals = graph.GetEX() yVals = graph.GetY() eyVals = graph.GetEY() residuals = [] residualErrors = [] #residualsY = [] #residualErrorsY = [] eUp = [] eDown = [] N = graph.GetN() x = [] y = [] absResidual = [] for i in range(N): absResidual.append((yVals[i] - fitLine.Eval(xVals[i]))) residuals.append((yVals[i] - fitLine.Eval(xVals[i])) / max(yVals[i], 0.001)) xLow = (xVals[i] - exVals[i]) xHigh = (xVals[i] + exVals[i]) residualErrors.append((eyVals[i] / max(yVals[i], 0.001))) x.append(xVals[i]) maxResidual = max(absResidual, key=abs) maxResiduals[-1].append(abs(maxResidual)) if N > 2: minCharges[-1].append(xVals[0]) maxCharges[-1].append(xVals[N - 1]) else: minCharges[-1].append(0) maxCharges[-1].append(0) if saveGraph: qieInfo = "" saveName = outputDir if saveName[-1] != '/': saveName += '/' saveName += "plots/" if qieBarcode != "": qieInfo += ", Barcode " + qieBarcode if qieUniqueID != "": qieInfo += " UID " + qieUniqueID else: qieUniqueID = "UnknownID" saveName += qieUniqueID if not os.path.exists(saveName): os.system("mkdir -p %s" % saveName) saveName += "/LinADCvsfC" if qieNumber != 0: qieInfo += ", QIE " + str(qieNumber) saveName += "_qie" + str(qieNumber) qieInfo += ", CapID " + str(i_capID) saveName += "_range" + str(i_range) saveName += "_capID" + str(i_capID) saveName += "_shunt_" + str(shuntMult).replace(".", "_") if not useCalibrationMode: saveName += "_NotCalMode" saveName += ".pdf" graph.SetTitle( "LinADC vs Charge, Range %i Shunt %s%s" % (i_range, str(shuntMult).replace(".", "_"), qieInfo)) graph.GetYaxis().SetTitle("Lin ADC") graph.GetYaxis().SetTitleOffset(1.2) graph.GetXaxis().SetTitle("Charge fC") resArray = array('d', residuals) resErrArray = array('d', residualErrors) resErrUpArray = array('d', eUp) resErrDownArray = array('d', eDown) xArray = array('d', x) xErrorsArray = array('d', [0] * len(x)) if verbose and i_range == 1: print "the length of residuals are:", len(resArray) print "the charge length :", len(x) residualGraphX = TGraphErrors(len(x), xArray, resArray, xErrorsArray, resErrArray) residualGraphX.SetTitle("") c1 = TCanvas() p1 = TPad("", "", 0, 0.2, 0.9, 1) p2 = TPad("", "", 0., 0., 0.9, 0.2) p1.Draw() p2.Draw() p1.SetFillColor(kWhite) p2.SetFillColor(kWhite) p1.cd() p1.SetBottomMargin(0) p1.SetRightMargin(0) # p2.SetTopMargin(0) # p2.SetBottomMargin(0.3) graph.Draw("ap") fitLine.SetLineColor(kRed) fitLine.SetLineWidth(1) fitLine.Draw("same") xmin = graph.GetXaxis().GetXmin() xmax = graph.GetXaxis().GetXmax() ymin = graph.GetYaxis().GetXmin() ymax = graph.GetYaxis().GetXmax() text = TPaveText(xmin + (xmax - xmin) * .2, ymax - (ymax - ymin) * (.3), xmin + (xmax - xmin) * .6, ymax - (ymax - ymin) * .1) text.SetFillColor(kWhite) text.SetTextSize(0.75 * text.GetTextSize()) text.SetFillStyle(8000) text.AddText("Slope = %.4f +- %.4f LinADC/fC" % (fitLine.GetParameter(1), fitLine.GetParError(1))) text.AddText("Offset = %.2f +- %.2f LinADC" % (fitLine.GetParameter(0), fitLine.GetParError(0))) text.AddText("Chisquare = %e " % (fitLine.GetChisquare())) text.Draw("same") p2.cd() p2.SetTopMargin(0) p2.SetRightMargin(0) p2.SetBottomMargin(0.35) residualGraphX.Draw("ap") zeroLine = TF1("zero", "0", -9e9, 9e9) zeroLine.SetLineColor(kBlack) zeroLine.SetLineWidth(1) zeroLine.Draw("same") # xmin = xmin-10 # xmax = xmax+10 if minCharge < 10: minCharge = -10 graph.GetXaxis().SetRangeUser(xmin * 0.9, xmax * 1.1) graph.GetYaxis().SetRangeUser(ymin * .9, ymax * 1.1) residualGraphX.GetXaxis().SetRangeUser(xmin * 0.9, xmax * 1.1) residualGraphX.GetYaxis().SetRangeUser(-0.1, 0.1) residualGraphX.SetMarkerStyle(7) residualGraphX.GetYaxis().SetNdivisions(3, 5, 0) residualGraphX.GetXaxis().SetLabelSize(0.15) residualGraphX.GetYaxis().SetLabelSize(0.15) residualGraphX.GetYaxis().SetTitle("Residuals") residualGraphX.GetXaxis().SetTitle("Charge (fC)") residualGraphX.GetXaxis().SetTitleSize(0.15) residualGraphX.GetYaxis().SetTitleSize(0.15) residualGraphX.GetYaxis().SetTitleOffset(0.33) p1.cd() if not verbose: Quiet(c1.SaveAs)(saveName) else: c1.SaveAs(saveName) #c1.SaveAs(saveName) if shuntMult == 1: ranges = range(4) else: ranges = range(2) #change params = [[], [], [], [], [], []] unshunted_params = [[], [], [], [], [], []] for irange in ranges: if fitLines[irange] == None: for icapID in range(4): params[irange].append([-1, -1]) continue for icapID in range(4): # if shuntMult==1: # high_range[irange].append([fitLines[0][icapID].GetParameter(1),fitLines[0][icapID].GetParameter(0),fitLines[0][icapID].GetParError(1)]) #################################################################### #if irange==0 and shuntMult==1: # offset = -0.5 #else: # offset = fitLines[irange][icapID].GetParameter(0) #################################################################### offset = fitLines[irange][icapID].GetParameter(0) slope = fitLines[irange][icapID].GetParameter(1) uncertainty = fitLines[irange][icapID].GetParError(1) params[irange].append([ slope, offset, uncertainty, maxResiduals[irange][icapID], minCharges[irange][icapID], maxCharges[irange][icapID] ]) if shuntMult == 1: unshunted_params[irange].append( [slope, offset, uncertainty, maxResiduals[irange][icapID]]) # print high_range #sys.exit() # outputTGraphs.cd("LinadcVsCharge") # for graph in linearizedGraphList: # graph.Write() outputTGraphs.cd("fitLines") if shuntMult == 1: ranges = range(4) else: # outputTGraphs.cd("Shunted_fitLines") ranges = range(3) #change for i_range in ranges: if graphList[i_range] == None: continue # print 'Writing' for fitLine in fitLines[i_range]: fitLine.SetNpx(1000) fitLine.Write() if saveGraph: if saveName == None: continue saveName = saveName.replace("_capID" + str(i_capID), "") c1 = TCanvas() slopes = [] offsets = [] for i_capID in range(4): graph = graphList[i_range][i_capID] fitLine = fitLines[i_range][i_capID] # graph.SetMarkerStyle(20+i_capID) fitLine.SetLineColor(lineColors[i_capID]) fitLine.SetLineWidth(2) slopes.append( (fitLine.GetParameter(0), fitLine.GetParError(0))) offsets.append( (fitLine.GetParameter(1), fitLine.GetParError(1))) if i_capID == 0: graph.Draw("ap") if shuntMult == -1: graph.SetTitle( "LinADC vs Charge, Range %i, %s, QIE %.1f" % (i_range, qieUniqueID, qieNumber)) else: graph.SetTitle( "LinADC vs Charge, Range %i, %s, QIE %i,Shunt %.1f" % (i_range, qieUniqueID, qieNumber, shuntMult)) graph.GetYaxis().SetRangeUser( ymin - graphOffset[i_range], graph.GetYaxis().GetXmax() + graphOffset[i_range] * 4) else: N_ = graph.GetN() x_ = graph.GetX() y_ = graph.GetY() for n in range(N_): graph.SetPoint( n, x_[n], y_[n] + (graphOffset[i_range] * i_capID)) fitLine.SetParameter( 1, fitLine.GetParameter(1) + (graphOffset[i_range] * i_capID)) graph.Draw("p, same") fitLine.Draw("same") if not i_range == 3: text = TPaveText( xmin + 5, ymax + 3 * graphOffset[i_range] - (ymax - ymin) * (.7), xmin + 50, ymax + 3.75 * graphOffset[i_range]) else: text = TPaveText( xmin + 25, ymax + 2 * graphOffset[i_range] - (ymax - ymin) * (.7), xmin + 75, ymax + 3.75 * graphOffset[i_range]) text.SetFillColor(kWhite) text.SetFillStyle(4000) text.SetTextAlign(11) text.AddText("CapID 0:") text.AddText(" Slope = %.2f +- %.8f fC/ADC" % slopes[0]) text.AddText(" Offset = %.2f +- %.8f fC" % offsets[0]) text.AddText("CapID 1:") text.AddText(" Slope = %.2f +- %.2f fC/ADC" % slopes[1]) text.AddText(" Offset = %.2f +- %.2f fC" % offsets[1]) text.AddText("CapID 2:") text.AddText(" Slope = %.2f +- %.2f fC/ADC" % slopes[2]) text.AddText(" Offset = %.2f +- %.2f fC" % offsets[2]) text.AddText("CapID 3:") text.AddText(" Slope = %.2f +- %.2f fC/ADC" % slopes[3]) text.AddText(" Offset = %.2f +- %.2f fC" % offsets[3]) text.Draw("same") if not verbose: Quiet(c1.SaveAs)(saveName) else: c1.SaveAs(saveName) #c1.SaveAs(saveName) # directory = saveName.split("Lin")[0] # os.system("gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=%s/Plots_range_%i_shunt_%s.pdf %s/LinADCvsfC_qie*_range%i_capID*_shunt_%s_NotCalMode.pdf"%(directory,i_range, str(shuntMult).replace(".","_"),directory,i_range, str(shuntMult).replace(".","_"))) return params, unshunted_params
def saveTGraph(rootFile, shuntMult, i_range, qieNumber, i_capID, maxResi, isIncreasing, verbose=False): tGraphDir = rootFile.Get("LinadcVsCharge") fitLineDir = rootFile.Get("fitLines") fName = rootFile.GetName() outputDir = os.path.dirname(fName) gName = "LinADCvsfC_%s_range_%s_shunt_%s_capID_%s_linearized" % ( qieNumber, i_range, str("%.1f" % shuntMult).replace(".", "_"), i_capID) #print rootFile #print rootFile.GetName() #print tGraphDir #print gName graph = tGraphDir.Get(gName) #print graph fitLine = fitLineDir.Get("fit_%s" % gName) qieInfo = "" qieBarcode = "" qieUniqueID = "" useCalibrationMode = True saveName = outputDir if saveName[-1] != '/': saveName += '/' saveName += "plots" if qieBarcode != "": qieInfo += ", Barcode " + qieBarcode if qieUniqueID != "": qieInfo += " UID " + qieUniqueID else: qieUniqueID = "UnknownID" #saveName += qieUniqueID if not os.path.exists(saveName): os.system("mkdir -p %s" % saveName) saveName += "/LinADCvsfC" if qieNumber != 0: qieInfo += ", QIE " + str(qieNumber) saveName += "_qie" + str(qieNumber) qieInfo += ", CapID " + str(i_capID) saveName += "_range" + str(i_range) saveName += "_capID" + str(i_capID) saveName += "_shunt_" + str(shuntMult).replace(".", "_") if not useCalibrationMode: saveName += "_NotCalMode" saveName += ".png" graph.SetTitle("LinADC vs Charge, Range %i Shunt %s%s" % (i_range, str(shuntMult).replace(".", "_"), qieInfo)) graph.GetYaxis().SetTitle("Lin ADC") graph.GetYaxis().SetTitleOffset(1.2) graph.GetXaxis().SetTitle("Charge fC") graph.SetMarkerStyle(7) xVals = graph.GetX() exVals = graph.GetEX() yVals = graph.GetY() eyVals = graph.GetEY() residuals = [] residualErrors = [] #residualsY = [] #residualErrorsY = [] eUp = [] eDown = [] N = graph.GetN() x = [] y = [] for i in range(N): residuals.append((yVals[i] - fitLine.Eval(xVals[i]))) xLow = (xVals[i] - exVals[i]) xHigh = (xVals[i] + exVals[i]) residualErrors.append((eyVals[i])) x.append(xVals[i]) resArray = array('d', residuals) resErrArray = array('d', residualErrors) resErrUpArray = array('d', eUp) resErrDownArray = array('d', eDown) xArray = array('d', x) xErrorsArray = array('d', [0] * len(x)) if verbose and i_range == 1: print "the length of residuals are:", len(resArray) print "the charge length :", len(x) residualGraphX = TGraphErrors(len(x), xArray, resArray, xErrorsArray, resErrArray) residualGraphX.SetTitle("") c1 = TCanvas() p1 = TPad("", "", 0, 0.2, 0.9, 1) p2 = TPad("", "", 0., 0., 0.9, 0.2) p1.Draw() p2.Draw() p1.SetFillColor(kWhite) p2.SetFillColor(kWhite) p1.cd() p1.SetBottomMargin(0) p1.SetRightMargin(0) p2.SetTopMargin(0) p2.SetBottomMargin(0.3) minEl = TMath.MinElement(graph.GetN(), graph.GetX()) maxEl = TMath.MaxElement(graph.GetN(), graph.GetX()) graph.GetXaxis().SetRangeUser( min(chargeRange[i_range][shuntMult]['min'], minEl), max(chargeRange[i_range][shuntMult]['max'], maxEl)) graph.Draw("ap") fitLine.SetLineColor(kRed) fitLine.SetLineWidth(1) fitLine.Draw("same") xmin = graph.GetXaxis().GetXmin() xmax = graph.GetXaxis().GetXmax() ymin = graph.GetYaxis().GetXmin() ymax = graph.GetYaxis().GetXmax() text = TPaveText(xmin + (xmax - xmin) * .1, ymax - (ymax - ymin) * (.5), xmin + (xmax - xmin) * .6, ymax - (ymax - ymin) * .1) text.SetFillColor(kWhite) text.SetTextSize(0.75 * text.GetTextSize()) text.SetFillStyle(8000) ######### Add in Cut Values ############# text.AddText("Slope = %.4f +- %.4f LinADC/fC [%.4f,%.4f]" % (fitLine.GetParameter(1), fitLine.GetParError(1), failureconds[shuntMult][0], failureconds[shuntMult][1])) text.AddText("Offset = %.2f +- %.2f LinADC [%.1f,%.1f]" % (fitLine.GetParameter(0), fitLine.GetParError(0), -1 * failcondo[i_range][0], failcondo[i_range][0])) text.AddText("Max Residuals = %f [%f]" % (maxResi, maxResiduals[i_range])) text.AddText("Chisquare/NDF = %.2f " % (fitLine.GetChisquare() / fitLine.GetNDF())) if not isIncreasing: text.AddText("NOT INCREASING") text.Draw("same") p2.cd() p2.SetTopMargin(0) p2.SetRightMargin(0) p2.SetBottomMargin(0.35) residualGraphX.Draw("ap") zeroLine = TF1("zero", "0", -9e9, 9e9) zeroLine.SetLineColor(kBlack) zeroLine.SetLineWidth(1) zeroLine.Draw("same") upResidualLine = TLine(-9e9, maxResiduals[i_range], 9e9, maxResiduals[i_range]) upResidualLine.SetLineColor(kRed) upResidualLine.SetLineWidth(1) upResidualLine.Draw("same") downResidualLine = TLine(-9e9, -1 * maxResiduals[i_range], 9e9, -1 * maxResiduals[i_range]) downResidualLine.SetLineColor(kRed) downResidualLine.SetLineWidth(1) downResidualLine.Draw("same") # xmin = xmin-10 # xmax = xmax+10 #if minCharge < 10: minCharge = -10 resiPlotBoundaries = [ min(-1 * maxResi, -1 * maxResiduals[i_range] - 0.1 * maxResiduals[i_range]), max(maxResi, maxResiduals[i_range] + 0.1 * maxResiduals[i_range]) ] graph.GetXaxis().SetRangeUser(xmin * 0.9, xmax * 1.1) graph.GetYaxis().SetRangeUser(ymin * .9, ymax * 1.1) residualGraphX.GetXaxis().SetRangeUser(xmin * 0.9, xmax * 1.1) residualGraphX.GetYaxis().SetRangeUser(resiPlotBoundaries[0], resiPlotBoundaries[1]) residualGraphX.SetMarkerStyle(7) residualGraphX.GetYaxis().SetNdivisions(3, 5, 0) residualGraphX.GetXaxis().SetLabelSize(0.15) residualGraphX.GetYaxis().SetLabelSize(0.15) residualGraphX.GetYaxis().SetTitle("Residuals") residualGraphX.GetXaxis().SetTitle("Charge (fC)") residualGraphX.GetXaxis().SetTitleSize(0.15) residualGraphX.GetYaxis().SetTitleSize(0.15) residualGraphX.GetYaxis().SetTitleOffset(0.33) p1.cd() if not verbose: Quiet(c1.SaveAs)(saveName) else: c1.SaveAs(saveName)
def SummaryPlot(runAll=False, dbnames=None, uid=None, total=False, idir = None, hist2D=False, shFac=False, adapterTest=False,images=False, verbose=False, slVqie=False, tester1 = "Shelton",logoutput=False): # Get required arguments from options tester = tester1 indir = idir #if idir == str: # indir = idir # else: # indir = idir[0] # if indir[-1] != "/": # indir += "/" # date = indir[5:15] # run = indir[-1] # print indir # if indir == None: # print "invalid indirectory" # exit # if indir[-1] != "/": # indir+="/" # run = indir[20:-2] if not '/' in indir: print "invalid directory structure, date and run number information expected" exit dirInfo = indir.split("/") hasRun = False hasDate = False for value in dirInfo: if '2018' in value: date = value hasDate = True if "Run_" in value: run = int(value[4:]) hasRun = True if not hasRun or not hasDate: print "invalid directory structure, date and run number information expected" exit if type(tester) == type([]): tester = tester[0] if tester in people: tester = people[tester] elif type(tester) == type(""): if tester in people: tester = people[tester] elif tester in people.values(): tester = tester else: print "Tester not in list of testers" else: print "Tester type error" gROOT.SetBatch(True) qieList = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16] #Canvases c = [] c2 = [] #Histogram Lists histoffset = [] histshunt = [] histslopes = [] histSlopeNvSlope1 = [] histShuntFactor = [] histSlvQie = [] #Total Histograms totalhist = [] #Max - min Variables maximum = 0 minimum = 0 #Failure failure = False Result = True FailedCards = [] FailedSlopes =[] FailedOffset = [] FailedRange = [] #Grab File Names if not os.path.exists("%s/SummaryPlots"%(indir)): os.makedirs("%s/SummaryPlots"%(indir)) else: shutil.rmtree("%s/SummaryPlots"%(indir)) os.makedirs("%s/SummaryPlots"%(indir)) if(runAll or total or not uid is None): files = glob.glob("%sqieCalibrationParameters*.db"%(indir)) elif(len(dbnames) != 0): files = [] for f in dbnames: files.append(glob.glob("%s/%s"%(indir,f))[0]) print files MergeDatabases(files, indir,"MergedDatabaseRun%i.db"%run) xyz1234 = sqlite3.connect("%sMergedDatabaseRun%i.db"%(indir,run)) cursor = xyz1234.cursor() TGaxis.SetMaxDigits(3) #files = cursor.excute("Select distinct runDirectory from qieshuntparams").Fetchall() idlist = cursor.execute("Select distinct id from qieshuntparams").fetchall() # Get Ranges bins = cursor.execute("SELECT DISTINCT range FROM qieshuntparams").fetchall() # Get Shunts shunts = cursor.execute("SELECT DISTINCT shunt FROM qieshuntparams").fetchall() #if (runAll): for nameList in idlist: Result = True name = nameList[0] nameid = name.replace("u","") name = nameid.replace("'","") if not os.path.exists("%s/SummaryPlots/%s/ImagesOutput"%(indir,name)): os.makedirs("%s/SummaryPlots/%s/ImagesOutput"%(indir,name)) FailedCards = [] FailedSlope =[] FailedOffset = [] poorfits = [] FailedRange = [] OffsetMean = [] FailedIsMonotonic = [] if not uid is None: if name not in uid: continue FailedCards = [] if logoutput: originalSTDOUT = sys.stdout stdOutDump = open("%s/SummaryPlots/SummaryPlot.stdout"%(indir), 'w+') sys.stdout = stdOutDump #if not os.path.exists("data/%s/Run_%s/SummaryPlots/TotalPlots"%(date, run)): #os.makedirs("data/%s/Run_%s/SummaryPlots/TotalPlots"%(date, run)) # Modify rootout change title of output ROOT file rootout = TFile("%sfitResults_%s.root" %(indir, name), "update") rootout.cd("SummaryPlots") if(verbose): print "Now analyzing card %s" %nameid if(runAll): for ra in bins: r = ra[0] for shu in shunts: sh = shu[0] if (r == 2 or r == 3) and (sh != 1): continue # Fetch the values of slope and offset for the corresponding shunt and range #values = cursor.execute("select slope,offset from qieshuntparams where range=%i and shunt=%.1f and id = '%s';" % (r, sh,name)).fetchall() values = cursor.execute("select slope, range, offset, qie, capid, maxX,minX, id, maxResidual, isMonotonic, (SELECT slope from qieshuntparams where id=p.id and qie=p.qie and capID=p.capID and range=p.range and shunt=1) from qieshuntparams as p where range = %i and shunt = %.1f and id = '%s';"%(r,sh,name)).fetchall() # Fetch Max and minimum values for slope of shunt maxmin = cursor.execute("select max(slope),min(slope) from qieshuntparams where range=%i and shunt = %.1f and id = '%s';" % (r, sh,name)).fetchall() maximum, minimum = maxmin[0] maximums = max(plotBoundaries_slope[1]/sh, maximum+0.01) minimums = min(plotBoundaries_slope[0]/sh, minimum-0.01) if sh == 1: maximum1 = maximums minimum1 = minimums #Creates Canvases for each Shunt and Range(TH1D) c.append(TCanvas("Card_%s_Shunt_%.1f_Range_%i" % (name, sh, r), "histo")) c[-1].Divide(2,1) c[-1].cd(1) #Create Histograms for the shunt slopes histshunt.append(TH1D("SLOPE_Sh_%s_R_%i" %(str(sh).replace(".",""),r),"%s Shunt %.1f - Range %i" % (name, sh, r), 100, minimums, maximums)) histshunt[-1].SetTitle("SLOPE SH: %.1f R: %d"%(sh,r)) histshunt[-1].GetXaxis().SetTitle("Slope") histshunt[-1].GetYaxis().SetTitle("Frequency") gPad.SetLogy(1) #Create 2D histogram of slope of shunt N vs slope of shunt 1 if(hist2D): histSlopeNvSlope1.append(TH2D("Slope_Shunt_%s_vs_Shunt_1_R_%i"%(str(sh).replace(".",""),r),"%s Slope of Shunt %.1f vs Shunt 1 - Range %i"%(name,sh,r),100,minimum1,maximum1,100,minimums,maximums)) histSlopeNvSlope1[-1].GetXaxis().SetTitle("Shunt 1 Slope") histSlopeNvSlope1[-1].GetYaxis().SetTitle("Shunt %.1f Slope"%sh) #Create 2D histogram of slope vs qie if(slVqie): histSlvQie.append(TH2D("SlopeVsQIE_Shunt_%s_Range_%d"%(str(sh).replace(".",""),r),"%s Slope Vs QIE Shunt %.1f Range %d"%(name,sh,r),16,0.5,16.5,40,minimums,maximums)) histSlvQie[-1].GetXaxis().SetTitle("QIE") histSlvQie[-1].GetYaxis().SetTitle("Slope") #Create histogram of shunt factor if(shFac): histShuntFactor.append(TH1D("ShuntFactor_Sh_%s_R_%.i"%(str(sh).replace(".",""),r),"Shunt Factor Shunt %.1f Range %i"%(sh,r),100,sh-1,sh+1)) histShuntFactor[-1].GetXaxis().SetTitle("Shunt Factor") histShuntFactor[-1].GetYaxis().SetTitle("Frequency") #Create Histograms for the Offsets maxmin = cursor.execute("select max(offset),min(offset) from qieshuntparams where range=%i and shunt = %.1f and id = '%s';" % (r, sh,name)).fetchall() maximum,minimum = maxmin[0] maximumo = max(plotBoundaries_offset[r], maximum) minimumo = min(-1*plotBoundaries_offset[r], minimum) test = [] c[-1].cd(2) histoffset.append(TH1D("OFFSET_Sh_%s_R_%i" %(str(sh).replace(".",""),r),"%s Shunt %.1f - Range %d" %(name, sh, r), 41, minimumo, maximumo)) histoffset[-1].SetTitle("OFFSET SH: %.1f R: %d"%(sh,r)) histoffset[-1].GetXaxis().SetTitle("Offset") histoffset[-1].GetYaxis().SetTitle("Frequency") gPad.SetLogy(1) hline = TLine(0,0,0,0) hline.SetLineColor(2) lline = TLine(0,0,0,0) lline.SetLineColor(2) loline = TLine(0,0,0,0) loline.SetLineColor(2) holine = TLine(0,0,0,0) holine.SetLineColor(2) if adapterTest: if sh not in hslopes.keys(): hslopes[sh] = {} if r not in hslopes[sh].keys(): hslopes[sh][r] = {"total":{}, "front":{}, "back":{}} for ty in ['total','front','back']: hslopes[sh][r][ty] = TH1D("Slopes_shunt_%s_range_%d" % (str(sh).replace(".","_"), r), "Slopes Shunt %.1f Range %d" % (sh,r), 100, minimums, maximums) hslopes[sh][r][ty].SetDirectory(0) hslopes[sh][r][ty].GetXaxis().SetTitle("Slope (LinADC / fC)") hslopes[sh][r][ty].GetYaxis().SetTitle("QIE Channels") hslopes[sh][r]['front'].SetTitle("Slopes Front Adapter Shunt %.1f Range %d" % (sh,r)) hslopes[sh][r]['back'].SetTitle("Slopes Back Adapter Shunt %.1f Range %d" % (sh,r)) if sh not in hoffsets.keys(): hoffsets[sh] = {} if r not in hoffsets[sh].keys(): hoffsets[sh][r] = {"total":{}, "front":{}, "back":{}} for ty in ['total','front','back']: hoffsets[sh][r][ty] = TH1D("Offsets_shunt_%s_range_%d" % (str(sh).replace(".","_"), r), "Offsets Shunt %.1f Range %d" % (sh,r), 100, minimumo, maximumo) hoffsets[sh][r][ty].SetDirectory(0) hoffsets[sh][r][ty].GetXaxis().SetTitle("Offset (LinADC)") hoffsets[sh][r][ty].GetYaxis().SetTitle("QIE Channels") hoffsets[sh][r]['front'].SetTitle("Slopes Front Adapter Shunt %.1f Range %d" % (sh,r)) hoffsets[sh][r]['back'].SetTitle("Slopes Back Adapter Shunt %.1f Range %d" % (sh,r)) # Fills the histograms with the values fetched above for val in values: #slope, offset = val slope, rang, offset,qie,capid ,minX,maxX, _id,maxr, slSh1, _isMonotonic = val if XrangeFail(sh,rang,minX,maxX): FailedRange.append((sh,rang,qie,capid)) Result = False if slopeFailH(sh,rang,_id,slope): FailedSlope.append((sh,rang,qie,capid)) Result = False if(verbose): print "Slope in CAPID %i in QIE %i in Shunt %.1f and Range %i"%(capid,qie,sh,r) elif offsetFail(rang,offset,_id): FailedOffset.append((sh,rang,qie,capid)) Result = False if(verbose): print "Offset in CAPID %i in QIE %i in Shunt %.1f and Range %i"%(capid,qie,sh,r) if poorfit(maxr,rang): Result = False poorfits.append((sh,rang,qie,capid)) if (verbose): print "Poor fitting results in CAPID %i in QIE %i in Shunt %.1f and Range %i"%(capid,qie,sh,r) if not _isMonotonic: FailedIsMonotonic.append((sh,rang,qie,capid)) c[-1].cd(1) histshunt[-1].Fill(slope) histshunt[-1].Draw() hline.DrawLine(failureconds[sh][1],0,failureconds[sh][1],histshunt[-1].GetMaximum()+1) hline.Draw("same") lline.DrawLine(failureconds[sh][0],0,failureconds[sh][0],histshunt[-1].GetMaximum()+1) lline.Draw("same") c[-1].cd(2) histoffset[-1].Fill(offset) histoffset[-1].Draw() if adapterTest: hslopes[sh][r]['total'].Fill(slope) hoffsets[sh][r]['total'].Fill(offset) if qie in backAdapter: hslopes[sh][r]['back'].Fill(slope) hoffsets[sh][r]['back'].Fill(offset) else: hslopes[sh][r]['front'].Fill(slope) hoffsets[sh][r]['front'].Fill(offset) #c[-1].cd(3) if(slVqie): histSlvQie[-1].Fill(qie,slope) if(hist2D): histSlopeNvSlope1[-1].Fill(slSh1,slope) if(shFac): try: histShuntFactor[-1].Fill(slSh1/slope) except ZeroDivisionError: print "Divide by Zero Error: %s Shunt %.1f Range %d"%(name,sh,r) if r == 0: holine.DrawLine(-.5,0,-.5,histoffset[-1].GetMaximum()+1) holine.Draw("same") else: holine.DrawLine(failcondo[r][0],0,failcondo[r][0],histoffset[-1].GetMaximum()+1) holine.Draw("same") loline.DrawLine(-failcondo[r][0],0,-failcondo[r][0],histoffset[-1].GetMaximum()+1) loline.Draw("same") histshunt[-1].Write() histoffset[-1].Write() c[-1].Update() if(images): Quiet(c[-1].SaveAs)("%s/SummaryPlots/%s/ImagesOutput/%s_SHUNT_%s_RANGE_%i.png"%(indir, name,name, str(sh).replace(".",""), r)) if(hist2D): histSlopeNvSlope1[-1].Write() if(shFac): histShuntFactor[-1].Write() if(slVqie): histSlvQie[-1].Write() if(verbose): print "Card %s Shunt %.1f Range %d Finished"%(name,sh,r) for ran in bins: for sh in shunts: r =ran[0] s = sh[0] if (r>1) and s!=1: continue offset1 = cursor.execute("Select avg(offset) from qieshuntparams where shunt =%.1f and id ='%s' and range = %d"%(s,name,r)).fetchall() offset = offset1[0] if offset[0] < rangemean[r][0] or offset[0] > rangemean[r][1]: OffsetMean.append((s,r)) Result = False if(verbose): print "qie and capid is indicative of a failure in the mean of the Offset" rootout.Close() FailedCards.append({name:{'Offset':FailedOffset,'Slope':FailedSlope,'poor fit': poorfit,'Bad Mean Offset':OffsetMean,'Range Failures':FailedRange,"FailIncreasing":FailedIsMonotonic}}) cardplaceholder = {'Result':Result,'date':date, 'run':run, 'Tester':tester, 'Comments':{'Offset':FailedOffset,'Slope':FailedSlope, 'Poor fit':poorfits,'Bad Mean Offset':OffsetMean,'Range Failures':FailedRange,'FailIncreasing':FailedIsMonotonic}} file1 = open("%s/SummaryPlots/%s/%s.json"%(indir,name,name),"w+") json.dump(cardplaceholder, file1) if (adapterTest): rundir = "%s/SummaryPlots" %indir outdir = "adapterTests" os.system("mkdir -p %s/%s" % (rundir, outdir)) c.append(TCanvas("c","c",1600,1200)) ranges = xrange(4) gStyle.SetOptStat(0) if adapterTest: for ra in bins: r=ra[0] for shu in shunts: sh = shu[0] if (r == 2 or r == 3) and (sh != 1): continue l = TLegend(0.75, 0.75, 0.9, 0.9) print hslopes print sh, r l.AddEntry(hslopes[sh][r]['front'], "Front adapter") l.AddEntry(hslopes[sh][r]['back'], "Back adapter") hslopes[sh][r]['front'].SetLineColor(2) hslopes[sh][r]['front'].SetLineWidth(2) hslopes[sh][r]['back'].SetLineColor(4) hslopes[sh][r]['back'].SetLineWidth(2) hslopes[sh][r]['back'].SetTitle("Slopes Shunt %.1f Range %d" % (sh,r)) hslopes[sh][r]['back'].Draw("HIST") hslopes[sh][r]['front'].Draw("HIST SAME") l.Draw("SAME") if(images): (c[-1].SaveAs)("%s/%s/slopes_shunt_%s_range_%d.png" % (rundir,outdir,str(sh).replace(".","_"),r)) lo = TLegend(0.75, 0.75, 0.9, 0.9) lo.AddEntry(hslopes[sh][r]['front'], "Front adapter") lo.AddEntry(hslopes[sh][r]['back'], "Back adapter") hoffsets[sh][r]['front'].SetLineColor(2) hoffsets[sh][r]['front'].SetLineWidth(2) hoffsets[sh][r]['back'].SetLineColor(4) hoffsets[sh][r]['back'].SetLineWidth(2) hoffsets[sh][r]['back'].SetTitle("Offsets Shunt %.1f Range %d" %(sh,r)) hoffsets[sh][r]['back'].Draw("HIST") hoffsets[sh][r]['front'].Draw("HIST SAME") lo.Draw("SAME") if(images): Quiet(c[-1].SaveAs)("%s/%s/offsets_shunt_%s_range_%d.png" % (rundir,outdir,str(sh).replace(".","_"),r)) if (total): hline = TLine(0,0,0,0) hline.SetLineColor(2) lline = TLine(0,0,0,0) lline.SetLineColor(2) loline = TLine(0,0,0,0) loline.SetLineColor(2) holine = TLine(0,0,0,0) holine.SetLineColor(2) if not os.path.exists("%s/SummaryPlots"%indir): os.makedirs("%s/SummaryPlots"%indir) if not os.path.exists("%s/SummaryPlots/TotalOutput"%indir): os.makedirs("%s/SummaryPlots/TotalOutput"%indir) # Modify rootout change title of output ROOT file rootout = TFile("%s/SummaryPlots/summary_plot_total.root" %indir, "recreate") for ra in bins: r =ra[0] for shu in shunts: sh = shu[0] if (r == 2 or r == 3) and (sh != 1): continue # Fetch the values of slope and offset for the corresponding shunt and range # values = cursor.execute("select slope,offset from qieshuntparams where range=%i and shunt=%.1f ;" % (r, sh)).fetchall() values = cursor.execute("select slope, offset from qieshuntparams as p where range = %i and shunt = %.1f;"%(r,sh)).fetchall() # Fetch Max and minimum values for slope of shunt maxmin = cursor.execute("select max(slope),min(slope) from qieshuntparams where range=%i and shunt = %.1f;" % (r,sh)).fetchall() maximum, minimum = maxmin[0] maximums = max(plotBoundaries_slope[1]/sh, maximum+0.01) minimums = min(plotBoundaries_slope[0]/sh, minimum-0.01) if sh == 1: maximum1 = maximums minimum1 = minimums #Creates Canvases for each Shunt and Range(TH1D) c.append(TCanvas("Shunt %.1f - Range %i" % (sh, r), "histo")) c[-1].Divide(2,1) c[-1].cd(1) #Create Histograms for the shunt slopes histshunt.append(TH1D("SLOPE_Sh:_%.1f_RANGE_r:_%d" %(sh,r),"SLOPE Sh: %.1f RANGE r: %d" %(sh,r), 100, minimums, maximums)) #histshunt[-1].SetTitle("SLOPE SH: %.1f "%(sh)) histshunt[-1].GetXaxis().SetTitle("Slope") histshunt[-1].GetYaxis().SetTitle("Frequency") gPad.SetLogy(1) #Create 2D histogram of slope of shunt N vs slope of shunt 1 if(hist2D): histSlopeNvSlope1.append(TH2D("Slope_Shunt_%s_vs_Shunt_1_R_%i"%(str(sh).replace(".",""),r),"Slope of Shunt %.1f vs Shunt 1 - Range %i"%(sh,r),100,minimum1,maximum1,100,minimums,maximums)) histSlopeNvSlope1[-1].GetXaxis().SetTitle("Shunt 1 Slope") histSlopeNvSlope1[-1].GetYaxis().SetTitle("Shunt %.1f Slope"%sh) #Create histogram of shunt factor if(shFac): histShuntFactor.append(TH1D("ShuntFactor_Sh_%s_R_%.i"%(str(sh).replace(".",""),r),"Shunt Factor Shunt %.1f Range %i"%(sh,r),100,sh-1,sh+1)) histShuntFactor[-1].GetXaxis().SetTitle("Shunt Factor") histShuntFactor[-1].GetYaxis().SetTitle("Frequency") #Create Histograms for the Offsets maxmin = cursor.execute("select max(offset),min(offset) from qieshuntparams where range=%i and shunt = %.1f;" % (r, sh)).fetchall() maximum, minimum = maxmin[0] maximumo = max(plotBoundaries_offset[r], maximum) minimumo = min(-1*plotBoundaries_offset[r], minimum) c[-1].cd(2) histoffset.append(TH1D("OFFSET Sh: %.1f - R: %i" %(sh, r),"Shunt %.1f - Range %d" %(sh, r), 40, minimumo, maximumo)) histoffset[-1].SetTitle("OFFSET SH: %.1f R: %d"%(sh,r)) histoffset[-1].GetXaxis().SetTitle("Offset") histoffset[-1].GetYaxis().SetTitle("Frequency") gPad.SetLogy(1) # Fills the histograms with the values fetched above for val in values: try: slope, offset = val except: print val c[-1].cd(1) histshunt[-1].Fill(slope) histshunt[-1].Draw() hline.DrawLine(failureconds[sh][1],0,failureconds[sh][1],histshunt[-1].GetMaximum()+1) hline.Draw("same") lline.DrawLine(failureconds[sh][0],0,failureconds[sh][0],histshunt[-1].GetMaximum()+1) lline.Draw("same") c[-1].cd(2) histoffset[-1].Fill(offset) histoffset[-1].Draw() if r == 0: holine.DrawLine(-.49,0,-.49,histoffset[-1].GetMaximum()+1) holine.Draw("same") loline.DrawLine(-.51,0,-.51,histoffset[-1].GetMaximum()+1) loline.Draw("same") else: holine.DrawLine(failcondo[r][0],0,failcondo[r][0],histoffset[-1].GetMaximum()+1) holine.Draw("same") loline.DrawLine(-failcondo[r][0],0,-failcondo[r][0],histoffset[-1].GetMaximum()+1) loline.Draw("same") histshunt[-1].Write() histoffset[-1].Write() if(hist2D): histSlopeNvSlope1[-1].Fill(slSh1,slope) if(shFac): try: histShuntFactor[-1].Fill(slSh1/slope) except ZeroDivisionError: pass # Write the histograms to the file, saving them for later # histshunt[-1].Draw() # histoffset[-1].Draw() # c2[-1].Write() c[-1].Update() #c[-1].SaveAs("data/%s/Run_%s/SummaryPlots/ImagesOutput/CARD_%s_SHUNT_%s_RANGE_%i.png"%(date, run, name, str(sh).replace(".",""), r)) if(images): c[-1].Print("%s/SummaryPlots/TotalOutput/Total_SHUNT_%s_RANGE_%i.png"%(indir, str(sh).replace(".",""), r)) c[-1].Write() if(hist2D): histSlopeNvSlope1[-1].Write() if(shFac): histShuntFactor[-1].Write() if(verbose): print "Total Plots Shunt %.1f Range %d Finished"%(sh,r) if len(FailedCards) >=1: outputText = open("%s/SummaryPlots/Failed_Shunts_and_Ranges.txt"%indir,"w+") pprint.pprint(FailedCards, outputText) outputText.close() rootout.Close() if logoutput: sys.stdout = originalSTDOUT
def saveOnFail(inputDir): fileList = [] for root, dirnames, filenames in os.walk( os.path.join(inputDir, "Submission")): for filename in fnmatch.filter(filenames, '*.json'): fileList.append(os.path.join(root, filename)) for fName in fileList: rootFile = TFile.Open( fName.replace(".json", ".root").replace("70/", "70/fitResults_"), 'read') dbFile = sqlite3.connect( fName.replace(".json", ".db").replace("70/", "70/qieCalibrationParameters_")) cursor = dbFile.cursor() outputDir = os.path.dirname(fName) ################################################### # Values to get from json file ################################################### qieBarcode = "" qieUniqueID = "" qieNumber = 1 i_capID = 0 i_range = 0 shuntMult = 1 useCalibrationMode = True verbose = False ################################################### # Getting Values From json ################################################### inFile = open(fName, "r") jsonFile = json.load(inFile) qieUniqueID = os.path.splitext(os.path.basename(fName))[0] if jsonFile['Result']: continue failModeList = [] for failModeOffset in jsonFile['Comments']['Offset']: if failModeOffset not in failModeList: failModeList.append(failModeOffset) for failModeSlope in jsonFile['Comments']['Slope']: if failModeSlope not in failModeList: failModeList.append(failModeSlope) for failModeFit in jsonFile['Comments']['Poor fit']: if failModeFit not in failModeList: failModeList.append(failModeFit) for failModeRange in jsonFile['Comments']['Range Failures']: if failModeRange not in failModeList: failModeList.append(failModeRange) for failModeMonotonic in jsonFile['Comments']['FailIncreasing']: if failModeMonotonic not in failModeList: failModeList.append(failModeMonotonic) qieList = [x[2] for x in failModeList] qieCounter = Counter(qieList) tooManyErrors = [] for badChip in qieCounter: # print badChip,": ",qieCounter[badChip] if qieCounter[badChip] >= 16: saveName = outputDir if saveName[-1] != '/': saveName += '/' saveName += "plots" saveName += "/999_LinADCvsfC" if qieNumber != 0: saveName += "_qie" + str(badChip) saveName += ".png" canv = TCanvas("c") text = TPaveText(0.05, 0.1, 0.95, 0.8) text.AddText("QIE %s Contains More Than 16 Errors!" % badChip) text.GetListOfLines().Last().SetTextColor(2) if verbose: print "%s QIE %s Contains More That 16 Errors!" % ( qieUniqueID, badChip) text.Draw() canv.Update() if not verbose: Quiet(canv.SaveAs)(saveName) else: canv.SaveAs(saveName) failModeList = [ failMode for failMode in failModeList if failMode[2] != badChip ] for failMode in failModeList: shuntMult, i_range, qieNumber, i_capID = failMode maxResi, isMonotonic = cursor.execute( "select maxResidual,isMonotonic from qieshuntparams where shunt = %f and range = %d and qie =%d and capID =%d" % (shuntMult, i_range, qieNumber, i_capID)).fetchone() saveTGraph(rootFile, shuntMult, i_range, qieNumber, i_capID, maxResi, isMonotonic)