def zfix(s, cut='llh', nbins=100, thetaMax=40., minE=5.0,
plot=False, out=False):
from icecube.photospline import spglam as glam
thetaMax *= degree
zbins = np.linspace(1, np.cos(thetaMax), nbins+1)[::-1]
ebins = getEbins()
t = np.log10(s['MC_energy'])
r = np.log10(s['ML_energy'])
z = np.pi - s['zenith']
# Calculate cut values
c0 = np.logical_not(np.isnan(r))
ecut = r >= minE
c0 *= ecut
if cut != None:
c0 *= s['cuts'][cut]
# Store median and standard deviation info
x1 = np.cos(z)[c0]
if x1.min() > zbins.min():
zbins = zbins[zbins >= x1.min()]
y = (r - t)[c0]
medians, sigL, sigR, vars = getMedian(x1, y, zbins)
w = 1/vars
nknots = 30
step_scale = 2/5.
step = (zbins.max() - zbins.min()) * step_scale
mids = (zbins[1:] + zbins[:-1]) / 2.
axes = [mids]
knots = [np.linspace(zbins.min()-step, zbins.max()+step, nknots)]
tab = glam.fit(medians, w, axes, knots, order=(4), penalties={2:1e-4})
if plot:
fig, ax = plt.subplots()
#ax.set_title('Energy Resolution vs Reconstructed Zenith', fontsize=18)
ax.set_xlabel(r'$\cos(\mathrm{\theta_{reco}})$', fontsize=16)
ax.set_ylabel(r'$\log_{10}(E_{\mathrm{LLH}}/E_{\mathrm{true}})$', fontsize=16)
lw = 2
ms = 7*lw
pltParams = dict(fmt='.', lw=lw, ms=ms)
# Energy resolution vs zenith
zbins = np.linspace(1, np.cos(thetaMax), 21)[::-1]
x = getMids(zbins)
medians, sigL, sigR, vars = getMedian(x1, y, zbins)
ax.errorbar(x, medians, yerr=(sigL,sigR), **pltParams)
# Spline fit
fitx = np.linspace(x.min(), x.max(), len(x)*3)
fit = glam.grideval(tab, [fitx])
ax.plot(fitx, fit)
ax.set_xlim(0.8,1)
if out:
plt.savefig(out)
plt.show()
fit = glam.grideval(tab, [np.cos(z)])
return r - fit
def spline(s, p0, nk=2, npoly=3, plot=False):
Ebins, Emids = getEbins(), getEmids
st = len(Emids) - len(p0[0])
axes = [Emids, Emids[st:]]
knots = [getKnots(axes[0], nk, npoly), getKnots(axes[1], nk, npoly)]
# Increment all 0's by a fraction of the minimum value
min_value = p0[p0!=0].min()
min_value /= 100.
min_mat = min_value * (p0==0)
p0 += min_mat
# Move to log space for smoother fit
p0 = np.log10(p0)
# Weight bins
wts = np.ones(p0.shape)
x = np.log10(s['MC_energy'][s['cuts']['llh']])
y = np.log10(s['ML_energy'][s['cuts']['llh']])
h, xedges, yedges = histogram2d(y, x, bins=(Ebins, Ebins[st:]))
sigma = np.sqrt(np.var(h, axis=0))
ntrue = np.sum(h, axis=0)
# Deal with points with no hits
weight = -1/p0
minwt = weight.min()
# Set the corners
#weight[0][len(p0[0])-1] *= 10**(-4)
#weight[len(p0)-1][0] *= 10**(-4)
#weight[weight==minwt] = 0
#weight[weight==minwt] *= 10**(-3)
#weight /= sigma
weight *= np.log10(ntrue)
wts *= weight
pen = 10**(-4) * weight.min()
spl = glam.fit(p0, wts, axes, knots, order=(npoly,npoly), periods=(0,0),
penalties={2:[pen,pen]})
if plot:
f = plt.figure()
ax1 = f.add_subplot(1,1,1)
ax1.set_title('')
ax1.set_xlabel('true')
ax1.set_ylabel('reco')
x, y = [Emids, Emids[st:]]
t0 = glam.grideval(spl, [x, y])
X, Y = np.meshgrid(x, y)
im = ax1.imshow(t0, interpolation='bilinear', vmin=-4.3, vmax=0,
origin='lower', extent=[Ebins[st], 9.5, Ebins[0], 9.5])
f.colorbar(im)
plt.show()
test = glam.grideval(spl, axes)
return test
def spltest(s, bin=40, nk=2, npoly=3, clean=1):
Emids = getEmids()
cut = s['cuts']['llh']
p = getProbs(s, cut, clean=clean)
p0 = p['Rp|Tf']
st = len(Emids) - len(p0[0])
p1 = p['Rp|Tf'][:,bin]
oldsum = p1.sum()
axes = Emids
knots = getKnots(axes, nk, npoly)
# Increment all 0's by a fraction of the minimum value
min_value = p1[p1!=0].min()
min_value /= 100.
min_mat = min_value * (p1==0)
p1 += min_mat
# Move to log space for smoother fit
p1 = np.log10(p1)
# Weight bins by probability
wts = np.ones(p1.shape)
weight = -1./p1
minwt = weight.min()
weight[weight==minwt] = 0 # bins with no data get 0 weight
# Others weighted by variance
#cut = s['cuts']['llh']
#x = np.log10(s['MC_energy'][cut])
#y = np.log10(s['ML_energy'][cut])
#h, xedges, yedges = histogram2d(y, x, bins=(Ebins, Ebins[st:]))
#sigma = np.sqrt(np.var(h, axis=0))[bin]
#weight[weight!=0] /= sigma
wts *= weight
pen = 10**(-8) * minwt
spl = glam.fit(p1, wts, [axes], [knots], order=[npoly],
penalties={2:[pen]})
test = glam.grideval(spl, [axes])
newsum = (10**test).sum()
print 'original', oldsum
print 'splined', newsum
f = plt.figure()
ax1 = f.add_subplot(1,1,1)
ax1.set_title('')
ax1.set_xlabel('reco')
ax1.set_ylabel('prob')
plt.plot(Emids, test, '.', label='spline')
plt.plot(Emids, p1, 'x', label='orig')
#plt.legend(loc='upper left')
plt.ylim(-7, 0)
plt.show()
def makeTable(bintype='logdist', plot=False):
# Starting parameters
my.setupShowerLLH(verbose=False)
s = load_sim(bintype=bintype)
outFile = '%s/IT73_sim/Zfix_%s.fits' % (my.llh_data, bintype)
nbins = 100
thetaMax = 40.
minE = 4.0
thetaMax *= np.pi / 180.
zbins = np.linspace(1, np.cos(thetaMax), nbins+1)[::-1]
ebins = getEbins()
t = np.log10(s['MC_energy'])
r = np.log10(s['ML_energy'])
z = np.pi - s['zenith']
# Calculate cut values
c0 = s['cuts']['llh']
ecut = r >= minE
c0 *= ecut
# Store median and standard deviation info
x1 = np.cos(z)[c0]
if x1.min() > zbins.min():
zbins = zbins[zbins >= x1.min()]
y = (r - t)[c0]
medians, sigL, sigR, vars = getMedian(x1, y, zbins)
w = 1/vars
nknots = 30
step_scale = 2/5.
step = (zbins.max() - zbins.min()) * step_scale
mids = (zbins[1:] + zbins[:-1]) / 2.
axes = [mids]
knots = [np.linspace(zbins.min()-step, zbins.max()+step, nknots)]
tab = glam.fit(medians, w, axes, knots, order=(4), penalties={2:1e-4})
if os.path.exists(outFile):
os.remove(outFile)
splinefitstable.write(tab, outFile)
if plot:
fig, ax = plt.subplots()
ax.set_title('Energy Resolution vs Reconstructed Zenith', fontsize=18)
ax.set_xlabel('Cos(zenith)', fontsize=16)
ax.set_ylabel('Ereco - Etrue (median)', fontsize=16)
lw = 2
ms = 7*lw
pltParams = dict(fmt='.', lw=lw, ms=ms)
# Energy resolution vs zenith
x = getMids(zbins)
ax.errorbar(x, medians, yerr=(sigL,sigR), **pltParams)
# Spline fit
fitx = np.linspace(x.min(), x.max(), len(x)*3)
fit = glam.grideval(tab, [fitx])
ax.plot(fitx, fit)
plt.show()
# reproducibility considered good
numpy.random.seed(42)
z, edges = numpy.histogramdd(numpy.random.multivariate_normal([0, 0, 0],
numpy.eye(3),
size=1000),
bins=[numpy.linspace(-3, 3, 101)] * 3)
z = z.cumsum(axis=2).astype(float)
w = numpy.ones(z.shape)
# evaluate CDF at right-hand bin edges
centers = [0.5 * (edges[i][:-1] + edges[i][1:])
for i in range(2)] + [edges[-1][1:]]
knots = [pad_knots(numpy.linspace(-3, 3, 5))] * 3
order = [2, 2, 3]
smooth = 1.
spline = glam.fit(z,
w,
centers,
knots,
order,
smooth,
penalties={2: [smooth] * 3},
monodim=2)
y = glam.grideval(spline, centers)
residual = ((z - y)**2).sum()
numpy.testing.assert_almost_equal(residual, 50791.31, 2)
except NameError:
pass
numpts = 500
knots = [numpy.linspace(-8, 35, 30)]
order = 2
smooth = 3.14159
#x1 = numpy.sort(numpy.random.normal(15,4,size=numpts))
x1 = numpy.sort(numpy.random.uniform(-4, 25, size=numpts))
z = numpy.random.poisson(
numpy.cos(x1) * numpy.cos(x1) + (x1 - 3.0) * (x1 - 3.0) + 10)
result = glam.fit(z, 1. + z, [x1], knots, 2, smooth, [0])
gp = Gnuplot.Gnuplot()
xfine = numpy.sort(numpy.random.uniform(-5, 35, size=1000))
rawdat = Gnuplot.Data(x1, z, title="Data")
fitdat = Gnuplot.Data(x1, glam.grideval(result, [x1]), title="Fit")
spline = Gnuplot.Data(xfine, [
sum([
result.coefficients[n] * bspline(knots[0], x, n, order)
for n in range(0,
len(knots[0]) - 2 - 1)
]) for x in xfine
],
with_="lines",
title="Spline")
gp.plot(rawdat, fitdat, spline)
input("Press ENTER to continue")
def spline(config, inFile, outFile, plot=False):
# Load input median file
d = np.load(inFile)
d = d.item()
xbins, ybins = d['xbins'], d['ybins']
energies = d['medians']
vars = d['var']
vars[vars<1e-15] = np.inf # Variances sufficiently close to 0 aren't real
w = 1/vars
emin, emax = energies[energies!=0].min(), energies.max()
ebins = [emin] + getEbins() + [emax]
axes, knots = [],[]
binList = [xbins, ybins]
nknots = 30
step_scale = 2/5.
for bins in binList:
mids = (bins[1:]+bins[:-1])/2.
axes += [mids]
step = (bins.max() - bins.min()) * step_scale
knots += [np.linspace(bins.min()-step, bins.max()+step, nknots)]
tab = glam.fit(energies, w, axes, knots, order=(4), \
penalties={2:1e-3})
if plot:
# Look at spline fit with finer binning
fitaxes = [[],[]]
fitaxes[0] = np.linspace(xbins.min(),xbins.max(),len(xbins)*3)
fitaxes[1] = np.linspace(ybins.min(),ybins.max(),len(ybins)*3)
fit = glam.grideval(tab, fitaxes)
#err_fit = 10**glam.grideval(err_tab,axes)
fig = plt.figure(figsize=(17,6))
mpl.rc("font", family="serif")
X, Y = np.meshgrid(axes[0], axes[1])
fitX, fitY = np.meshgrid(fitaxes[0], fitaxes[1])
# Setup custom colormap
cmap = plt.cm.jet
cmap = cmap_discretize(cmap, ebins)
cmap.set_under('white')
ax = fig.add_subplot(121)
p = ax.pcolor(fitX, fitY, fit.T, cmap=cmap, vmin=emin, vmax=emax)
cb = fig.colorbar(p, ax=ax)
ax.set_title('Median Energy')
ax.set_xlabel('cos(zenith)')
ax.set_ylabel('log10(Nchannel)')
ax.set_xlim(xbins.min(), xbins.max())
ax.set_ylim(ybins.min(), ybins.max())
ax = fig.add_subplot(122)
p = ax.pcolor(X, Y, energies.T, cmap=cmap, vmin=emin, vmax=emax)
ax.set_title('Median Energy')
ax.set_xlabel('cos(zenith)')
ax.set_ylabel('log10(Nchannel)')
ax.set_xlim(xbins.min(), xbins.max())
ax.set_ylim(ybins.min(), ybins.max())
plt.show()
if outFile:
if os.path.exists(outFile):
os.remove(outFile)
splinefitstable.write(tab, outFile)
