def zfix(s, cut='llh', nbins=100, thetaMax=40., minE=5.0, plot=False, out=False): from icecube.photospline import spglam as glam thetaMax *= degree zbins = np.linspace(1, np.cos(thetaMax), nbins+1)[::-1] ebins = getEbins() t = np.log10(s['MC_energy']) r = np.log10(s['ML_energy']) z = np.pi - s['zenith'] # Calculate cut values c0 = np.logical_not(np.isnan(r)) ecut = r >= minE c0 *= ecut if cut != None: c0 *= s['cuts'][cut] # Store median and standard deviation info x1 = np.cos(z)[c0] if x1.min() > zbins.min(): zbins = zbins[zbins >= x1.min()] y = (r - t)[c0] medians, sigL, sigR, vars = getMedian(x1, y, zbins) w = 1/vars nknots = 30 step_scale = 2/5. step = (zbins.max() - zbins.min()) * step_scale mids = (zbins[1:] + zbins[:-1]) / 2. axes = [mids] knots = [np.linspace(zbins.min()-step, zbins.max()+step, nknots)] tab = glam.fit(medians, w, axes, knots, order=(4), penalties={2:1e-4}) if plot: fig, ax = plt.subplots() #ax.set_title('Energy Resolution vs Reconstructed Zenith', fontsize=18) ax.set_xlabel(r'$\cos(\mathrm{\theta_{reco}})$', fontsize=16) ax.set_ylabel(r'$\log_{10}(E_{\mathrm{LLH}}/E_{\mathrm{true}})$', fontsize=16) lw = 2 ms = 7*lw pltParams = dict(fmt='.', lw=lw, ms=ms) # Energy resolution vs zenith zbins = np.linspace(1, np.cos(thetaMax), 21)[::-1] x = getMids(zbins) medians, sigL, sigR, vars = getMedian(x1, y, zbins) ax.errorbar(x, medians, yerr=(sigL,sigR), **pltParams) # Spline fit fitx = np.linspace(x.min(), x.max(), len(x)*3) fit = glam.grideval(tab, [fitx]) ax.plot(fitx, fit) ax.set_xlim(0.8,1) if out: plt.savefig(out) plt.show() fit = glam.grideval(tab, [np.cos(z)]) return r - fit
def spline_fit(data, bins = None, range = None, weights = None, order = None, filename = 'spline.fits'): if bins is None: bins = data.shape[1]*[10] counts,bin_arrays = np.histogramdd(data,range=range,weights=weights) vars,bin_arrays = np.histogramdd(data,range=range,weights=weights**2) else: counts,bin_arrays = np.histogramdd(data,bins=bins,range=range,weights=weights) vars,bin_arrays = np.histogramdd(data,bins=bins,range=range,weights=weights**2) coords = [(b[1:]+b[:-1])/2. for b in bin_arrays] if order == None: order = list(np.zeros_like(bins)) knots = pad_knots(bin_arrays, order) w = 1./np.sqrt(vars) w[~np.isfinite(w)] = np.nanmin(w) result = glam.fit(counts,w,coords,knots,order,0) if not filename is None: if os.path.exists(filename): os.system('rm '+filename) if filename[-5:]=='.fits': splinefitstable.write(result,filename) else: splinefitstable.write(result,filename+'.fits') return result
def spltest(s, bin=40, nk=2, npoly=3, clean=1): Emids = getEmids() cut = s['cuts']['llh'] p = getProbs(s, cut, clean=clean) p0 = p['Rp|Tf'] st = len(Emids) - len(p0[0]) p1 = p['Rp|Tf'][:,bin] oldsum = p1.sum() axes = Emids knots = getKnots(axes, nk, npoly) # Increment all 0's by a fraction of the minimum value min_value = p1[p1!=0].min() min_value /= 100. min_mat = min_value * (p1==0) p1 += min_mat # Move to log space for smoother fit p1 = np.log10(p1) # Weight bins by probability wts = np.ones(p1.shape) weight = -1./p1 minwt = weight.min() weight[weight==minwt] = 0 # bins with no data get 0 weight # Others weighted by variance #cut = s['cuts']['llh'] #x = np.log10(s['MC_energy'][cut]) #y = np.log10(s['ML_energy'][cut]) #h, xedges, yedges = histogram2d(y, x, bins=(Ebins, Ebins[st:])) #sigma = np.sqrt(np.var(h, axis=0))[bin] #weight[weight!=0] /= sigma wts *= weight pen = 10**(-8) * minwt spl = glam.fit(p1, wts, [axes], [knots], order=[npoly], penalties={2:[pen]}) test = glam.grideval(spl, [axes]) newsum = (10**test).sum() print 'original', oldsum print 'splined', newsum f = plt.figure() ax1 = f.add_subplot(1,1,1) ax1.set_title('') ax1.set_xlabel('reco') ax1.set_ylabel('prob') plt.plot(Emids, test, '.', label='spline') plt.plot(Emids, p1, 'x', label='orig') #plt.legend(loc='upper left') plt.ylim(-7, 0) plt.show()
def makeTable(bintype='logdist', plot=False): # Starting parameters my.setupShowerLLH(verbose=False) s = load_sim(bintype=bintype) outFile = '%s/IT73_sim/Zfix_%s.fits' % (my.llh_data, bintype) nbins = 100 thetaMax = 40. minE = 4.0 thetaMax *= np.pi / 180. zbins = np.linspace(1, np.cos(thetaMax), nbins+1)[::-1] ebins = getEbins() t = np.log10(s['MC_energy']) r = np.log10(s['ML_energy']) z = np.pi - s['zenith'] # Calculate cut values c0 = s['cuts']['llh'] ecut = r >= minE c0 *= ecut # Store median and standard deviation info x1 = np.cos(z)[c0] if x1.min() > zbins.min(): zbins = zbins[zbins >= x1.min()] y = (r - t)[c0] medians, sigL, sigR, vars = getMedian(x1, y, zbins) w = 1/vars nknots = 30 step_scale = 2/5. step = (zbins.max() - zbins.min()) * step_scale mids = (zbins[1:] + zbins[:-1]) / 2. axes = [mids] knots = [np.linspace(zbins.min()-step, zbins.max()+step, nknots)] tab = glam.fit(medians, w, axes, knots, order=(4), penalties={2:1e-4}) if os.path.exists(outFile): os.remove(outFile) splinefitstable.write(tab, outFile) if plot: fig, ax = plt.subplots() ax.set_title('Energy Resolution vs Reconstructed Zenith', fontsize=18) ax.set_xlabel('Cos(zenith)', fontsize=16) ax.set_ylabel('Ereco - Etrue (median)', fontsize=16) lw = 2 ms = 7*lw pltParams = dict(fmt='.', lw=lw, ms=ms) # Energy resolution vs zenith x = getMids(zbins) ax.errorbar(x, medians, yerr=(sigL,sigR), **pltParams) # Spline fit fitx = np.linspace(x.min(), x.max(), len(x)*3) fit = glam.grideval(tab, [fitx]) ax.plot(fitx, fit) plt.show()
def spline(s, p0, nk=2, npoly=3, plot=False): Ebins, Emids = getEbins(), getEmids st = len(Emids) - len(p0[0]) axes = [Emids, Emids[st:]] knots = [getKnots(axes[0], nk, npoly), getKnots(axes[1], nk, npoly)] # Increment all 0's by a fraction of the minimum value min_value = p0[p0!=0].min() min_value /= 100. min_mat = min_value * (p0==0) p0 += min_mat # Move to log space for smoother fit p0 = np.log10(p0) # Weight bins wts = np.ones(p0.shape) x = np.log10(s['MC_energy'][s['cuts']['llh']]) y = np.log10(s['ML_energy'][s['cuts']['llh']]) h, xedges, yedges = histogram2d(y, x, bins=(Ebins, Ebins[st:])) sigma = np.sqrt(np.var(h, axis=0)) ntrue = np.sum(h, axis=0) # Deal with points with no hits weight = -1/p0 minwt = weight.min() # Set the corners #weight[0][len(p0[0])-1] *= 10**(-4) #weight[len(p0)-1][0] *= 10**(-4) #weight[weight==minwt] = 0 #weight[weight==minwt] *= 10**(-3) #weight /= sigma weight *= np.log10(ntrue) wts *= weight pen = 10**(-4) * weight.min() spl = glam.fit(p0, wts, axes, knots, order=(npoly,npoly), periods=(0,0), penalties={2:[pen,pen]}) if plot: f = plt.figure() ax1 = f.add_subplot(1,1,1) ax1.set_title('') ax1.set_xlabel('true') ax1.set_ylabel('reco') x, y = [Emids, Emids[st:]] t0 = glam.grideval(spl, [x, y]) X, Y = np.meshgrid(x, y) im = ax1.imshow(t0, interpolation='bilinear', vmin=-4.3, vmax=0, origin='lower', extent=[Ebins[st], 9.5, Ebins[0], 9.5]) f.colorbar(im) plt.show() test = glam.grideval(spl, axes) return test
# reproducibility considered good numpy.random.seed(42) z, edges = numpy.histogramdd(numpy.random.multivariate_normal([0, 0, 0], numpy.eye(3), size=1000), bins=[numpy.linspace(-3, 3, 101)] * 3) z = z.cumsum(axis=2).astype(float) w = numpy.ones(z.shape) # evaluate CDF at right-hand bin edges centers = [0.5 * (edges[i][:-1] + edges[i][1:]) for i in range(2)] + [edges[-1][1:]] knots = [pad_knots(numpy.linspace(-3, 3, 5))] * 3 order = [2, 2, 3] smooth = 1. spline = glam.fit(z, w, centers, knots, order, smooth, penalties={2: [smooth] * 3}, monodim=2) y = glam.grideval(spline, centers) residual = ((z - y)**2).sum() numpy.testing.assert_almost_equal(residual, 50791.31, 2)
try: input = raw_input except NameError: pass numpts = 500 knots = [numpy.linspace(-8, 35, 30)] order = 2 smooth = 3.14159 #x1 = numpy.sort(numpy.random.normal(15,4,size=numpts)) x1 = numpy.sort(numpy.random.uniform(-4, 25, size=numpts)) z = numpy.random.poisson( numpy.cos(x1) * numpy.cos(x1) + (x1 - 3.0) * (x1 - 3.0) + 10) result = glam.fit(z, 1. + z, [x1], knots, 2, smooth, [0]) gp = Gnuplot.Gnuplot() xfine = numpy.sort(numpy.random.uniform(-5, 35, size=1000)) rawdat = Gnuplot.Data(x1, z, title="Data") fitdat = Gnuplot.Data(x1, glam.grideval(result, [x1]), title="Fit") spline = Gnuplot.Data(xfine, [ sum([ result.coefficients[n] * bspline(knots[0], x, n, order) for n in range(0, len(knots[0]) - 2 - 1) ]) for x in xfine ], with_="lines", title="Spline") gp.plot(rawdat, fitdat, spline)
# XXX HACK: don't believe anything that happens outside the # tracking volume of the table #scalp(table, w, low=opts.ice_bottom, high=opts.ice_top) order, penalties, knots = spline_spec(3) bin_centers = [b.copy() for b in table.bin_centers[:3]] bin_widths = [b.copy() for b in table.bin_widths[:3]] axis_scale = rescale_axes(knots, bin_centers, bin_widths) print('Number of knots used: ', [len(a) for a in knots]) print("Beginning spline fit for abs table...") spline = glam.fit(z, w, bin_centers, knots, order, smooth, penalties=penalties) spline.geometry = Geometry.SPHERICAL spline.extents = extents[:3] spline.ngroup = table.header['n_group'] print("Saving table to %s..." % abs_outputfile) spline.knots = [ spline.knots[i] * axis_scale[i] for i in range(0, len(spline.knots)) ] check_exists(abs_outputfile) splinefitstable.write(spline, abs_outputfile) # clean up
pad_knots(numpy.linspace(0, 1, 11), 2), # cos(theta) pad_knots(numpy.linspace(1, 3, 11), 2), # vertical depth [km] pad_knots(numpy.linspace(0, 20, 21), 2), # log(energy) [log(E/GeV)] ] smooth = weights[ weights > 0].mean() # make the smoothing term proportional to the weights order = [2, 2, 2] penalties = { 2: [smooth / 1e3, smooth / 1e4, smooth / 1e2] } # Penalize curvature if opts.single: spline = glam.fit(hes.bincontent, weights, hes._h_bincenters, knots, order, smooth, penalties=penalties) spline.bias = bias else: # The depth structure in the energy spectrum is mostly a function of # attenuation at the bundle rim, which we now have an extra dimension for. # Reduce the knot density in depth accordingly. knots[1] = pad_knots(numpy.linspace(1, 3, 7), 2) knots = knots[0:2] + [ pad_knots(numpy.linspace(numpy.sqrt(2), numpy.sqrt(100), 5)** 2), # bundle multiplicity pad_knots(numpy.linspace(0, numpy.sqrt(250), 11)** 2), # distance to shower axis ] + [knots[-1]]
def spline(config, inFile, outFile, plot=False): # Load input median file d = np.load(inFile) d = d.item() xbins, ybins = d['xbins'], d['ybins'] energies = d['medians'] vars = d['var'] vars[vars<1e-15] = np.inf # Variances sufficiently close to 0 aren't real w = 1/vars emin, emax = energies[energies!=0].min(), energies.max() ebins = [emin] + getEbins() + [emax] axes, knots = [],[] binList = [xbins, ybins] nknots = 30 step_scale = 2/5. for bins in binList: mids = (bins[1:]+bins[:-1])/2. axes += [mids] step = (bins.max() - bins.min()) * step_scale knots += [np.linspace(bins.min()-step, bins.max()+step, nknots)] tab = glam.fit(energies, w, axes, knots, order=(4), \ penalties={2:1e-3}) if plot: # Look at spline fit with finer binning fitaxes = [[],[]] fitaxes[0] = np.linspace(xbins.min(),xbins.max(),len(xbins)*3) fitaxes[1] = np.linspace(ybins.min(),ybins.max(),len(ybins)*3) fit = glam.grideval(tab, fitaxes) #err_fit = 10**glam.grideval(err_tab,axes) fig = plt.figure(figsize=(17,6)) mpl.rc("font", family="serif") X, Y = np.meshgrid(axes[0], axes[1]) fitX, fitY = np.meshgrid(fitaxes[0], fitaxes[1]) # Setup custom colormap cmap = plt.cm.jet cmap = cmap_discretize(cmap, ebins) cmap.set_under('white') ax = fig.add_subplot(121) p = ax.pcolor(fitX, fitY, fit.T, cmap=cmap, vmin=emin, vmax=emax) cb = fig.colorbar(p, ax=ax) ax.set_title('Median Energy') ax.set_xlabel('cos(zenith)') ax.set_ylabel('log10(Nchannel)') ax.set_xlim(xbins.min(), xbins.max()) ax.set_ylim(ybins.min(), ybins.max()) ax = fig.add_subplot(122) p = ax.pcolor(X, Y, energies.T, cmap=cmap, vmin=emin, vmax=emax) ax.set_title('Median Energy') ax.set_xlabel('cos(zenith)') ax.set_ylabel('log10(Nchannel)') ax.set_xlim(xbins.min(), xbins.max()) ax.set_ylim(ybins.min(), ybins.max()) plt.show() if outFile: if os.path.exists(outFile): os.remove(outFile) splinefitstable.write(tab, outFile)