def my_curve_fit():
path = [[5, 117], [12, 110], [12, 105], [13, 104], [15, 102], [20, 97], [22, 95], [22, 73], [24, 71], [29, 66], [30, 66], [31, 65], [32, 64], [37, 59], [41, 55], [41, 54], [42, 53], [44, 51], [44, 35], [45, 34], [50, 29], [57, 29], [58, 28], [63, 23], [70, 16], [84, 16], [85, 15]]
points = np.array(path)
x = points[:,0]
x = [float(a) for a in x]
b = set()
for i in range(len(x)):
if x[i] in b:
x[i] = x[i] + 0.1
else:
b.add(x[i])
y = points[:,1]
plt.plot(x, y, "og")
spl = UnivariateSpline(x, points[:,1], s=0)
kn = spl.get_knots()
lst = []
for i in range(4):
lst.append(1/jiecheng(i))
for i in range(len(kn)-1):
cf = lst * spl.derivatives(kn[i])
print("For {0} <= x <= {1}, p(x) = {5}*(x-{0})^3 + {4}*(x-{0})^2 + {3}*(x-{0}) + {2}".format(kn[i], kn[i+1], *cf))
dx = np.linspace(kn[i], kn[i+1], 10)
dy = []
for item in dx:
dy.append(pow((item-kn[i]),3)*cf[3]+pow((item-kn[i]),2)*cf[2]+(item-kn[i])*cf[1]+cf[0])
plt.plot(dx, dy, "-r")
def to_parametric_spline(spline: UnivariateSpline,
x_start: Optional[float] = None,
x_end: Optional[float] = None) -> Derivable:
"""
Convert a regular 's(x) = y' spline into a 's(t) = (x, y)' parametric spline.
Parameters
----------
spline : UnivariateSpline
Regular spline.
x_start : float, optional
X coordinate of left start point of the spline. Will be accessible by
t = 0.
x_end : float, optional
X coordinate of right end point of the spline. Will be accessible by
t = 1.
Returns
-------
spline : Derivable
Parametric spline.
"""
if x_start is None:
knots = spline.get_knots()
x_start = knots[0]
if x_end is None:
knots = spline.get_knots()
x_end = knots[-1]
interpolator = interp1d((0, 1), (x_start, x_end), fill_value='extrapolate')
def parametric_spline(t, d=0):
x = interpolator(t)
if d > 0:
x_dt = derivative(interpolator, t, d)
else:
x_dt = x
y_dt = spline(x, d)
return np.array([x_dt, y_dt])
return parametric_spline
def testWiki():
yk3 = UnivariateSpline(xx, yy, k=2, s=0)
knots = yk3.get_knots()
coeffs = yk3.get_coeffs()
#ipdb.set_trace()
r1 = yk3(xxx)
r2 = gety(xxx, xx, yy, coeffs)
plt.plot(xx, yy, "k.", markerSize=10)
plt.plot(xxx, r1, "g.-", markerSize=1)
plt.plot(xxx, r2, "r.-", markerSize=1)
plt.show(0)
def demo():
x = np.arange(6)
y = np.array([3, 1, 4, 1, 5, 9])
spl = UnivariateSpline(x, y, s=0)
kn = spl.get_knots()
for i in range(len(kn)-1):
cf = [1, 1, 1/2, 1/6] * spl.derivatives(kn[i])
print("For {0} <= x <= {1}, p(x) = {5}*(x-{0})^3 + {4}*(x-{0})^2 + {3}*(x-{0}) + {2}".format(kn[i], kn[i+1], *cf))
dx = np.linspace(kn[i], kn[i+1], 100)
dy = []
for item in dx:
dy.append(pow((item-kn[i]),3)*cf[3]+pow((item-kn[i]),2)*cf[2]+(item-kn[i])*cf[1]+cf[0])
plt.plot(dx, dy, "-r")
plt.plot(x, y, "og")
plt.plot(x, y, "-b")
def fit_and_plot_mpl_bspline(x,y):
# Fit B-spline with matplotlib and plot it
SPLINE_ORDER = 3
spl = UnivariateSpline(x,y,k=SPLINE_ORDER)
plt.plot(x,y,'r')
plt.plot(x,spl(x),'g')
knots = spl.get_knots()
coeffs = spl.get_coeffs()
yr = plt.ylim()
for knot in knots:
plt.axvline(knot,color='g')
#for coeff in coeffs:
#plt.text(???,(yr[0]+yr[1])/2,"%f" % coeff)
plt.show()
def spline(self, k=3, s=0.5): """ Interpolates uneven observation data into daily data using scipy.interpolate.UnivariateSpline with parameters k and s given""" # requires filtered_data to exist, check by making sure it isn't empty if len(self.filtered_data) == 0: raise ValueError # first we need to redo the dates as days from start for patient in self.filtered_data: self.filtered_data[patient]['days'] = [] for date in self.filtered_data[patient]['dates']: since_beginning = (date - self.filtered_data[patient]['dates'][0]).days self.filtered_data[patient]['days'].append(since_beginning) # now we spline error = 0 splined_data = dict() for patient in self.filtered_data: day_indices = np.array(self.filtered_data[patient]['days']) splined_data[patient] = [] for question in range(self.num_items): item_data = np.array([i[question] for i in self.filtered_data[patient]['data']]) if len(item_data) == 0: print "Uh, no responses for question %d patient %s" % (question+1, patient) raise ValueError # Now we need to take out the data with None values... not certain how to handle it if the patient ends up not having enough data # because of this filtering, but for UPittSSRI data (primary focus as of 7/29/13) we don't need to worry about it good_indices = np.array([ind for ind, resp in enumerate(item_data) if resp != None]) filtered_days = day_indices[good_indices] filtered_data = item_data[good_indices] full_space = np.linspace(0,self.keep_days,num=self.keep_days) spl = UnivariateSpline(filtered_days, filtered_data, k=k, s=s) self.residual = spl.get_residual() self.knots = spl.get_knots() splined = spl(full_space) splined_data[patient].append(splined) # Measure error ms = math.sqrt(sum((splined[filtered_days] - filtered_data)**2)/len(filtered_data)) error += ms splined_data[patient] = np.array(splined_data[patient]).T error /= (self.num_items)*(len(splined_data)) self.spline_err = error self.splined_data = splined_data self.data_splined = True self.data = splined_data return splined_data
def find_spline_with_numberofknots(data_x, data_y, desired_number_of_knots, threshold=0, max_iterations=100):
max_f = 1000
min_f = 0
iteration = 0
closesed = None
slm = UnivariateSpline(data_x, data_y, s=len(data_x) * max_f, k=1)
# determine max
while slm.get_knots().size > desired_number_of_knots:
max_f = max_f * 2
slm = UnivariateSpline(data_x, data_y, s=len(data_x) * max_f, k=1)
factor = max_f
while not (abs(desired_number_of_knots - slm.get_knots().size) <= threshold
and slm.get_knots().size <= desired_number_of_knots):
slm = UnivariateSpline(data_x, data_y, s=len(data_x) * factor, k=1)
if slm.get_knots().size > desired_number_of_knots:
min_f = factor
if max_f - factor < 0.01:
max_f = max_f * 2
factor = factor + (max_f - factor) / 2
elif slm.get_knots().size < desired_number_of_knots:
if closesed is None \
or desired_number_of_knots - closesed[1] > desired_number_of_knots - slm.get_knots().size:
closesed = (slm, slm.get_knots().size)
if factor - min_f < 0.01:
min_f = min_f - 1
max_f = factor
factor = factor - (factor - min_f) / 2
iteration += 1
if iteration > max_iterations:
MetricLogger().warning(
"cant find normal spline current number of knots:{} closeset spline I will use is:{}".format(
slm.get_knots().size,
-1 if closesed is None else
closesed[1]))
if closesed is None or closesed[1] > desired_number_of_knots:
raise RuntimeError("cant find normal spline")
else:
slm = closesed[0]
break
# xHat = np.linspace(min(data_x), max(data_x), num=10000)
# yHat = slm(xHat)
# # plot the results
# plt.figure()
# plt.plot(data_x, data_y, 'o')
# plt.plot(xHat, yHat, '-')
# plt.show()
return slm
def show(self):
"""
Draw the line on to the plot
"""
if self.graph.options.sig_enh:
norm_frames = min(len(self.yvalues),
self.graph.options.norm_frames)
mean = np.mean(self.yvalues[:norm_frames])
plot_values = self.yvalues / mean - 1
else:
plot_values = self.yvalues
if self.graph.options.smooth:
indexes = range(len(plot_values))
# Tolerance does not scale by data value to scale input
spline = UnivariateSpline(indexes,
plot_values / plot_values.max(),
s=0.1,
k=4)
self.debug("Number of knots in B-spline smoothing: %i",
len(spline.get_knots()))
line_values = spline(indexes) * plot_values.max()
else:
line_values = plot_values
self.hide()
# Plot line and points separately as line may be smoothed
line = self.axes.plot(self.xvalues,
line_values,
pen=pg.mkPen(color=self.line_col,
width=self.line_width,
style=self.line_style))
pts = self.axes.plot(self.xvalues,
plot_values,
pen=None,
symbolBrush=self.point_brush,
symbolPen=self.point_col,
symbolSize=self.point_size)
self.legend_item = line
self.graphics_items = (line, pts)
def break_spline(x, y, **kwargs):
'''
Calculate the break in 2 linear trends using a spline.
'''
s = UnivariateSpline(x, y, **kwargs)
knots = s.get_knots()
if len(knots) > 3:
print "Many knots"
knot_expec = -0.8
posn = np.argmin(knots - knot_expec)
return knots[posn] # Return the knot closest to the expected value
elif len(knots) == 2:
print "No knots"
return -0.6 # Set the threshold at the expected
else:
return knots[1]
def dist2spline(data, xmin=None, xmax=None, **kwargs):
"""
Estimate a smooth 1D distribution from data with weights in the interval [xmin, xmax]
and returns the knots and coefficients of a fitting spline.
"""
from pyqt_fit.kde import KDE1D
from scipy.interpolate import UnivariateSpline
if xmin is None:
xmin = np.min(data)
if xmax is None:
xmax = np.max(data)
kde = KDE1D(data, lower=xmin, upper=xmax, **kwargs)
xx = np.linspace(xmin, xmax, 200)
kdexx = kde(xx)
s = UnivariateSpline(xx, kdexx, s=kde.bandwidth)
xs = s.get_knots()
ys = s(xs)
return xs, ys, s
def spline_anomaly(df,
smooth=0.5,
plot=False,
B=False,
turning=False,
filtered=False,
threshold=1):
"""
Accepts:
a Pandas dataframe of shape:
obmt rate w1_rate
1. float float float
or equivalent.
This dataframe should be a hit neighbourhood as generated by
isolate_anomaly(). The function will work on larger datasets but
there is nothing to be gained by splining these - and the time taken
to do so would be significantly larger.
Runs scipy's splining algorithms on the hit neighbourhoods to
generate a spline to fit the data.
Kwargs:
smooth (float, default=0.5):
smoothing factor for the generated splines.
plot (bool, default=False):
if True, generates a plot of the data and the spline.
B (bool, default=False):
if True, generates B splines.
turning (bool, default=False):
if True, plots all the turning points alongside a normal
plot.
filtered (bool, default=False):
if True, plots the filterd turning points alongside a normal
plot.
threshold (float, default=1.0):
the threshold for filtering turning points
Returns:
tuple of the arrays of knots and coefficients (in that order) of
the fitted spline.
"""
# Create spline.
spl = UnivariateSpline(df['obmt'], df['rate'] - df['w1_rate'])
spl.set_smoothing_factor(smooth)
knots, coeffs = (spl.get_knots(), spl.get_coeffs())
xs = np.linspace(df['obmt'].tolist()[0], df['obmt'].tolist()[-1], 10000)
if B:
spl = BSpline(knots, coeffs, 3)
# Plot original data and spline.
if plot or turning or filtered:
plt.scatter(df['obmt'], df['rate'] - df['w1_rate'])
plt.plot(xs, spl(xs))
if filtered or turning:
# Calls get_turning_points() to isolate the turning points.
if turning:
turning_points = get_turning_points(df)
elif filtered:
turning_points = filter_turning_points(get_turning_points(df),
threshold=threshold)
plt.scatter(turning_points['obmt'],
turning_points['rate'] - turning_points['w1_rate'],
color='red')
plt.show()
return (knots, coeffs)
MSEs_train.append(MSE)
MSEs_CV.append(MSE_CV)
poly_params = np.polyfit(X, Y, 5)
plt.plot(X, Y, 'black')
plt.plot(X, np.polyval(poly_params, X), 'r-')
plt.show()
"""
plt.plot(degrees,MSEs_train,'black')#,label="train MSE")
plt.plot(degrees,MSEs_CV,'red')#,label="CV MSE")
plt.show()
"""
import numpy as np
from scipy.interpolate import UnivariateSpline
degree = 3 # In range 0 .. 5
smoothing = 1 # Lower bound for SSE
Xinv = X[::-1]
Yinv = Y[::-1]
s = UnivariateSpline(Xinv, Yinv, k=degree, s=4e9)
#get_coeffs(), get_knots(), get_residual()
plt.plot(Xinv, Yinv, "black")
plt.plot(Xinv, s(Xinv), "blue")
plt.show()
print len(s.get_knots())
print s.get_residual()
class DualSplineSmoother(object):
'''
claselfdocs
'''
def __init__(self, yp, workdir, scale, sm=200):
'''
Constructor
'''
yp = np.array(yp)
self.l = len(yp)/2
self.xPos = (self.l-2)/2 #fPos = (self.l-2)/2 + 2
tnsc = 2/scale
print tnsc
plt.rcParams['font.size'] = 24
plt.rcParams['lines.linewidth'] = 2.4
self.workdir = workdir
avProfilePoints = yp[:self.l]
self.avx = np.append(np.append([0], np.sort(np.tanh(tnsc*avProfilePoints[:self.xPos]))),[1])
self.av = avProfilePoints[self.xPos:]
sigmaProfilePoints = yp[self.l:]
self.sigmax = np.append(np.append([0], np.sort(np.tanh(tnsc*sigmaProfilePoints[:self.xPos]))),[1])
self.sigma = sigmaProfilePoints[self.xPos:]
self.m = UnivariateSpline(self.avx, self.av)
print "Created spline with " + str(len(self.m.get_knots())) + " knots"
self.s = UnivariateSpline(self.sigmax, self.sigma)
print "Created spline with " + str(len(self.s.get_knots())) + " knots"
def saveSpline(self, filename):
tp = np.linspace(0, 1, 1000)
with open(filename ,"w+") as f:
for i in range(0, 1000):
f.write( str(tp[i]) + " , " + str(self.m(tp[i])) )
if i < 999:
f.write("\n")
f.close()
def saveSigmaSpline(self, filename):
tp = np.linspace(0, 1, 1000)
with open(filename ,"w+") as f:
for i in range(0, 1000):
f.write( str(tp[i]) + " , " + str(self.s(tp[i])) )
if i < 999:
f.write("\n")
f.close()
def showSpline(self, order=0):
plt.clf()
print "Spline full information:"
print self.m.get_knots()
print self.m.get_coeffs()
print self.m.get_residual()
tp = np.linspace(0, 1, 1000)
plt.subplot(211)
plt.scatter(self.avx,self.av)
plt.plot(tp,self.m(tp))
plt.subplot(212)
plt.scatter(self.sigmax,self.sigma)
plt.plot(tp,self.s(tp))
plt.savefig(self.workdir+"/splineFit.pdf")
if order > 0:
plt.clf()
plt.subplot(211)
plt.plot(tp,self.m(tp,1))
plt.subplot(212)
plt.plot(tp,self.s(tp,1))
plt.savefig(self.workdir+"/splineDerivative.pdf")
def plotSplineData(self, dataContainer, yscale):
plt.clf()
plt.xlim(0,1)
plt.ylim(0,yscale)
tp = np.linspace(0, 1, 100)
plt.scatter(dataContainer.points[0],dataContainer.points[1]+dataContainer.background, c='b', marker='o', s=5)
plt.plot(tp, self.m(tp)+dataContainer.background,'r', linewidth=2)
plt.plot(tp, self.m(tp)+np.sqrt(self.s(tp))+dataContainer.background,'r--', linewidth=2)
plt.plot(tp, self.m(tp)-np.sqrt(self.s(tp))+dataContainer.background,'r--', linewidth=2)
plt.plot(tp, np.zeros(100) + dataContainer.background, '--', c='#BBBBBB', alpha=0.8)
plt.savefig(self.workdir+"/splineVsData.pdf")
def plotBinnedData(self, dataContainer):
plt.clf()
tp = np.linspace(0, 1, dataContainer.numBins)
tpHD = np.linspace(0, 1, 500)
plt.plot(self.m(tpHD), self.s(tpHD))
plt.plot(dataContainer.avs, np.power(dataContainer.stds,2), 'o')
plt.savefig(self.workdir+"/noiseVsBins.pdf")
plt.clf()
plt.plot(tpHD, self.m(tpHD),'r', linewidth=2)
plt.plot(tp, dataContainer.avs, 'o')
plt.savefig(self.workdir+"/splineVsBins.pdf")
plt.clf()
plt.plot(tpHD, self.s(tpHD),'r', linewidth=2)
plt.plot(tp, np.power(dataContainer.stds,2), 'o')
plt.savefig(self.workdir+"/spatialNoiseVsBins.pdf")
def plotFisherInfo(self, dataContainer, ymax, ymaxsq):
plt.clf()
t = np.linspace(0, 1, 500)
minf = lambda x: -1 * self.m(x)
minx = fminbound(minf, 0, 1)
fval = self.m(minx)
noisemean = UnivariateSpline(self.m(t)/fval, self.s(t)/fval/fval)
self.se = noisemean(0)
fi = lambda a, sa, sp: 2*np.power(sa, 2)/ (np.power(a,2)*2*sa+np.power(sp,2))
fiapp = lambda a, sa, sp: sa / (np.power(a,2))
plt.xlim(0, 1)
plt.ylim(0, ymaxsq)
print 'whop whop'
plt.plot(t, fi(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval))
plt.plot(t, fiapp(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval), 'r')
plt.savefig(self.workdir+"/variance.pdf")
plt.clf()
plt.ylim(0, ymax)
plt.plot(t, np.sqrt(fi(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval)))
plt.plot(t, np.sqrt(fiapp(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval)), 'r')
plt.savefig(self.workdir+"/stddev.pdf")
plt.clf()
plt.plot(t, 1/fi(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval))
plt.plot(t, 1/fiapp(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval), 'r')
plt.savefig(self.workdir+"/fisherInfo.pdf")
1.82380809e-01, 2.66344008e+00, 1.18164677e+01, 3.01811501e+01,
5.78812583e+01, 9.40718450e+01
])
## Now get scipy's spline fit.
k = 3
tck = interpolate.splrep(x_nodes, y_nodes, k=k, s=0)
knots = tck[0]
coeffs = tck[1]
print('knot points=', knots)
print('coefficients=', coeffs)
## Now try scipy's object-oriented version. The result is exactly the same as "tck": the knots are the
## same and the coeffs are the same, they are just queried in a different way.
uspline = UnivariateSpline(x_nodes, y_nodes, s=0)
uspline_knots = uspline.get_knots()
uspline_coeffs = uspline.get_coeffs()
## Here are scipy's native spline evaluation methods. Again, "ytck" and "y_uspline" are exactly equal.
ytck = interpolate.splev(x, tck)
y_uspline = uspline(x)
y_knots = uspline(knots)
## Now let's try our manually-calculated evaluation function.
y_eval = bspleval(x, knots, coeffs, k, debug=False)
plt.plot(x, ytck, label='tck')
plt.plot(x, y_uspline, label='uspline')
plt.plot(x, y_eval, label='manual')
## Next plot the knots and nodes.
def fit_trace(trace, deg=5):
spl = UnivariateSpline(trace.sample_f, trace.trace_f, k=deg)
return [deg, spl.get_coeffs().tolist(), spl.get_knots().tolist()]
# -*- coding: utf-8 -*-
"""
# scipy.interpolate - інтерполяція
Інтерполяція - це спосіб знаходження проміжних значень величини за її відомим дискретним набором значень. Для інтерполяції і апроксимації сплайнами застосовують функції з модуля `scipy.interpolate` (`interp1d`, `UnivariateSpline`, `interp2d`, `SmoothBivariateSpline` та інші). Сплайн - це функція, область визначення якої розбита на частини, на кожній з яких функція є певним поліномом.
"""
import numpy as np
from scipy.interpolate import interp1d, UnivariateSpline, interp2d
f = lambda x: x**x # функція
x = np.arange(4)
y = f(x)
print x, y # дискретний набір значень
y1 = interp1d(x, y, kind='linear') # лінійна інтерполяція
y2 = interp1d(x, y, kind='quadratic') # інтерполяція квадратичним сплайном
y2 = UnivariateSpline(x, y, k=2, s=0) # або (s - коеф. згладжування)
kn = y2.get_knots() # вузли сплайна
print y1(2.5), y2(2.5), f(
2.5) # інтерпольовані і дійсне значення в точці x=2.5
import matplotlib.pyplot as plt
x = np.linspace(0, x.max(), 100)
plt.plot(x, f(x), 'k-', x, y1(x), 'k--', x, y2(x), 'k:') # графіки
plt.scatter(kn, y2(kn)) # вузли кв. сплайна
plt.xlabel('x'), plt.ylabel('y'), plt.grid(), plt.show()
print "Рисунок – Функція (-) та її лінійна (--) і квадратична (..) інтерполяції сплайнами"
f = lambda x, y: x**2 + y**2 # функція двох змінних
x = np.array([0, 1, 2])
y = np.array([0, 1, 2])
xx, yy = np.meshgrid(x, y) # сітка
z = f(xx, yy) # значення функції у вузлах сітки
z1 = interp2d(x, y, z,
def onedim_pixtopix_variations_spline(flat,
knots=9,
guess=13,
savefits=True,
path=None,
debug_level=0):
"""
This routine fits a smoothing spline to an observed flat field in order to determine the pixel-to-pixel sensitivity variations
as well as the fringing pattern in the red orders. This is done in 1D, ie for the already extracted spectrum.
INPUT:
'flat' : dictionary / np.array containing the extracted flux from the flat field (master white) (keys = orders)
'knots' : the desired number of knots for the smoothing spline
'guess' : starting guess for the "smoothing factor" that determines the number of knots: s = 10^guess (see online documentation for scipy.interpolate.UnivariateSpline)
OUTPUT:
'pix_sens' : dictionary / np.array of the pixel-to-pixel sensitivities (keys = orders)
'smoothed_flat' : dictionary / np.array of the smoothed (ie filtered) whites (keys = orders)
MODHIST:
13/12/2019 - CMB create (clone of onedim_pixtopix_variations)
"""
if savefits:
assert path is not None, 'ERROR: path variable is not set!'
# check whether it's a numpy array (eg from FITS file), or a python dictionary
if flat.__class__ == np.ndarray:
# make sure that they are either quick-extracted spectra (order, pixel), or optimal-extracted spectra (order, fibre, pixel)
assert len(flat.shape) in [2,
3], 'ERROR: shape of flat not recognized!!!'
assert flat.shape[
-1] == 4112, 'ERROR: there are NOT exactly 4112 pixels in dispersion direction'
if debug_level >= 1:
print('Fitting smoothing spline with ' + str(knots) +
' knots to...')
xx = np.arange(flat.shape[-1])
pix_sens = np.zeros(flat.shape) - 1.
smoothed_flat = np.zeros(flat.shape) - 1.
# loop over all orders
for o in range(flat.shape[0]):
if debug_level >= 1:
print('Order_' + str(o + 1).zfill(2))
# are they optimal 3a extracted spectra?
if len(flat.shape) == 3:
# loop over all fibres
for fib in range(flat.shape[1]):
if debug_level >= 2:
print('Fibre ' + str(fib + 1).zfill(2))
# determine smoothing parameter to get desired number of knots
spl = UnivariateSpline(xx, flat[o, fib, :], s=10**guess)
nk = len(spl.get_knots())
if nk < knots:
add = 0
while nk < knots:
add -= 0.01
spl = UnivariateSpline(xx,
flat[o, fib, :],
s=10**(guess + add))
nk = len(spl.get_knots())
# print(add, nk)
elif nk > knots:
add = 0
while nk > knots:
add += 0.01
spl = UnivariateSpline(xx,
flat[o, fib, :],
s=10**(guess + add))
nk = len(spl.get_knots())
# print(add, nk)
smoothed_flat[o, fib, :] = spl(xx)
pix_sens[o,
fib, :] = flat[o, fib, :] / smoothed_flat[o,
fib, :]
# or are they just quick-extracted spectra
else:
# determine smoothing parameter to get desired number of knots
spl = UnivariateSpline(xx, flat[o, :], s=10**guess)
nk = len(spl.get_knots())
if nk < knots:
add = 0
while nk < knots:
add -= 0.01
spl = UnivariateSpline(xx,
flat[o, :],
s=10**(guess + add))
nk = len(spl.get_knots())
# print(add, nk)
elif nk > knots:
add = 0
while nk > knots:
add += 0.01
spl = UnivariateSpline(xx,
flat[o, :],
s=10**(guess + add))
nk = len(spl.get_knots())
# print(add, nk)
smoothed_flat[o, :] = spl(xx)
pix_sens[o, :] = flat[o, :] / smoothed_flat[o, :]
if savefits:
date = path.split('/')[-2]
# get header from master white
h = pyfits.getheader(path + date + '_master_white.fits')
# add requested number of knots to header
h['N_KNOTS'] = (knots, 'number of knots used in smoothing spline')
pyfits.writeto(path + date + '_smoothed_flat.fits',
np.float32(smoothed_flat), h)
pyfits.writeto(path + date + '_pixel_sensitivity.fits',
np.float32(pix_sens), h)
elif flat.__class__ == dict:
print('This has not been implemented yet for dictionaries...')
return
# pix_sens = {}
# smoothed_flat = {}
# # loop over all orders
# for ord in sorted(flat.keys()):
else:
print('ERROR: data type / variable class not recognized')
return
if debug_level >= 1:
print('DONE!!!')
return smoothed_flat, pix_sens
def fit_pspec(self, breaks=None, log_break=True, low_cut=None,
high_cut=None, fit_verbose=False, bootstrap=False,
**bootstrap_kwargs):
'''
Fit the 1D Power spectrum using a segmented linear model. Note that
the current implementation allows for only 1 break point in the
model. If the break point is estimated via a spline, the breaks are
tested, starting from the largest, until the model finds a good fit.
Parameters
----------
breaks : float or None, optional
Guesses for the break points. If given as a list, the length of
the list sets the number of break points to be fit. If a choice is
outside of the allowed range from the data, Lm_Seg will raise an
error. If None, a spline is used to estimate the breaks.
log_break : bool, optional
Sets whether the provided break estimates are log-ed values.
low_cut : `~astropy.units.Quantity`, optional
Lowest frequency to consider in the fit.
high_cut : `~astropy.units.Quantity`, optional
Highest frequency to consider in the fit.
fit_verbose : bool, optional
Enables verbose mode in Lm_Seg.
bootstrap : bool, optional
Bootstrap using the model residuals to estimate the standard
errors.
bootstrap_kwargs : dict, optional
Pass keyword arguments to `~turbustat.statistics.fitting_utils.residual_bootstrap`.
'''
# Make the data to fit to
if low_cut is None:
# Default to the largest frequency, since this is just 1 pixel
# But cut out fitting the total power point.
self.low_cut = 0.98 / (float(self.data.shape[0]) * u.pix)
else:
self.low_cut = \
self._spectral_freq_unit_conversion(low_cut, u.pix**-1)
if high_cut is None:
# Set to something larger than
self.high_cut = (self.freqs.max().value * 1.01) / u.pix
else:
self.high_cut = \
self._spectral_freq_unit_conversion(high_cut, u.pix**-1)
# We keep both the real and imag frequencies, but we only need the real
# component when fitting. We'll cut off the total power frequency too
# in case it causes an extra break point (which seems to happen).
shape = self.freqs.size
rfreqs = self.freqs[1:shape // 2].value
ps1D = self.ps1D[1:shape // 2]
y = np.log10(ps1D[clip_func(rfreqs, self.low_cut.value,
self.high_cut.value)])
x = np.log10(rfreqs[clip_func(rfreqs, self.low_cut.value,
self.high_cut.value)])
if breaks is None:
from scipy.interpolate import UnivariateSpline
# Need to order the points
spline = UnivariateSpline(x, y, k=1, s=1)
# The first and last are just the min and max of x
breaks = spline.get_knots()[1:-1]
if fit_verbose:
print("Breaks found from spline are: " + str(breaks))
# Take the number according to max_breaks starting at the
# largest x.
breaks = breaks[::-1]
# Ensure a break doesn't fall at the max or min.
if breaks.size > 0:
if breaks[0] == x.max():
breaks = breaks[1:]
if breaks.size > 0:
if breaks[-1] == x.min():
breaks = breaks[:-1]
if x.size <= 3 or y.size <= 3:
raise Warning("There are no points to fit to. Try lowering "
"'lg_scale_cut'.")
# If no breaks, set to half-way before last point
if breaks.size == 0:
x_copy = np.sort(x.copy())
breaks = np.array([0.5 * (x_copy[-1] + x_copy[-3])])
# Now try these breaks until a good fit including the break is
# found. If none are found, it accepts that there wasn't a good
# break and continues on.
i = 0
while True:
self.fit = \
Lm_Seg(x, y, breaks[i])
self.fit.fit_model(verbose=fit_verbose, cov_type='HC3')
if self.fit.params.size == 5:
# Success!
breaks = breaks[i]
break
i += 1
if i >= breaks.size:
warnings.warn("No good break point found. Returned fit\
does not include a break!")
break
# return self
if not log_break:
breaks = np.log10(breaks)
# Fit the final model with whichever breaks were passed.
self.fit = Lm_Seg(x, y, breaks)
self.fit.fit_model(verbose=fit_verbose, cov_type='HC3')
if bootstrap:
stderrs = residual_bootstrap(self.fit.fit,
**bootstrap_kwargs)
self._slope_errs = stderrs[1:-1]
else:
self._slope_errs = self.fit.slope_errs
self._slope = self.fit.slopes
self._bootstrap_flag = bootstrap
import numpy as np import matplotlib.pyplot as plt from scipy.interpolate import UnivariateSpline nItems = 500 x = np.linspace(-3, 3, nItems) y = np.exp(-x**2) + 0.1 * np.random.randn(nItems) plt.plot(x, y, 'ro', ms=5) smoothFactor = float(nItems) / 50.0 spl = UnivariateSpline(x, y, k = 2, s = smoothFactor) xs = np.linspace(-3, 3, 1000) plt.plot(xs, spl(xs), 'g', lw=3) coeffs = spl.get_coeffs() knots = spl.get_knots() print 'coeffs', coeffs print 'knots', knots knotsY = spl(knots) plt.plot(knots, knotsY, 'bo') #spl.set_smoothing_factor(0.5) #print 'coeffs', spl.get_coeffs() #print 'knots', spl.get_knots() #plt.plot(xs, spl(xs), 'b', lw=3) plt.show()
class SplineSmoother(object):
'''
claselfdocs
'''
def __init__(self, yp, workdir, scale, sm=200):
'''
Constructor
'''
self.workdir = workdir
yp = np.array(yp)
self.l = len(yp)
self.xPos = (self.l-2)/2 #fPos = (self.l-2)/2 + 2
tnsc = 2/scale
print tnsc
plt.rcParams['font.size'] = 24
plt.rcParams['lines.linewidth'] = 2.4
self.workdir = workdir
self.avx = np.append(np.append([0], np.sort(np.tanh(tnsc*yp[:self.xPos]))),[1])
self.av = yp[self.xPos:]
self.m = UnivariateSpline(self.avx, self.av)
plt.rcParams['font.size'] = 24
plt.rcParams['lines.linewidth'] = 2.4
print "Created spline with " + str(len(self.m.get_knots())) + " knots"
def saveSpline(self, filename):
tp = np.linspace(0, 1, 1000)
with open(filename ,"w+") as f:
for i in range(0, 1000):
f.write( str(tp[i]) + " , " + str(self.m(tp[i])) )
if i < 999:
f.write("\n")
f.close()
def showSpline(self, order=0):
plt.clf()
print "Spline full information:"
print self.m.get_knots()
print self.m.get_coeffs()
print self.m.get_residual()
tp = np.linspace(0, 1, 1000)
plt.scatter(self.avx, self.av)
plt.plot(tp,self.m(tp))
plt.savefig(self.workdir+"/splineFit.pdf")
if order > 0:
plt.plot(tp,self.m(tp,1))
if order > 1:
plt.plot(tp,self.m(tp,2))
plt.savefig(self.workdir+"/splineDerivative.pdf")
def plotSplineData(self, dataContainer, s, p, se, yscale):
plt.clf()
plt.xlim(0,1)
plt.ylim(0,yscale)
tp = np.linspace(0, 1, 500)
plt.scatter(dataContainer.points[0],dataContainer.points[1]+dataContainer.background, c='b', marker='o', s=5)
plt.plot(tp, self.m(tp)+dataContainer.background,'r', linewidth=2)
plt.plot(tp, self.m(tp)+np.sqrt(s*(p*self.m(tp)*self.m(tp)+self.m(tp)+se))+dataContainer.background,'r--', linewidth=2)
plt.plot(tp, self.m(tp)-np.sqrt(s*(p*self.m(tp)*self.m(tp)+self.m(tp)+se))+dataContainer.background,'r--', linewidth=2)
plt.plot(tp, np.zeros(500) + dataContainer.background, '--', c='#BBBBBB', alpha=0.8)
plt.savefig(self.workdir+"/splineVsData.pdf")
def plotBinnedData(self, dataContainer, s, p, se, xmin, xmax):
plt.clf()
sigma = lambda x: s * (p*x*x + x + se)
t = np.linspace(xmin, xmax, 500)
plt.xlim(xmin, xmax)
plt.plot(t, sigma(t))
plt.plot(dataContainer.avs, np.power(dataContainer.stds,2), 'o')
plt.savefig(self.workdir+"/noiseVsBins.pdf")
plt.clf()
tp = np.linspace(0, 1, dataContainer.numBins)
plt.plot(tp, self.m(tp),'r', linewidth=2)
plt.plot(tp, dataContainer.avs, 'o')
plt.savefig(self.workdir+"/splineVsBins.pdf")
def plotFisherInfo(self, dataContainer, s, p, se, ymax, ymaxsq):
plt.clf()
t = np.linspace(0, 1, 1000)
minf = lambda x: -1 * self.m(x)
minx = fminbound(minf, 0, 1)
fval = self.m(minx)
s = s/fval
se = se/fval
p = p*fval
print fval, s, p, se
fi = lambda m, mp: 2*np.power(s * (p * np.power(m,2) + m),2)/(np.power(mp,2) *
(2 * s * (p * np.power(m,2) + m) + np.power(s * (2 * p * m + 1), 2)))
fiapp = lambda m, mp: s * (p * np.power(m,2) + m) / np.power(mp,2)
plt.xlim(0, 1)
plt.ylim(0, ymaxsq)
plt.plot(t, fi(self.m(t)/fval, self.m(t, 1)/fval))
plt.plot(t, fiapp(self.m(t)/fval, self.m(t, 1)/fval), 'r')
plt.savefig(self.workdir+"/variance.pdf")
plt.clf()
plt.ylim(0, ymax)
plt.plot(t, np.sqrt(fi(self.m(t)/fval, self.m(t, 1)/fval)))
plt.plot(t, np.sqrt(fiapp(self.m(t)/fval, self.m(t, 1)/fval)), 'r')
plt.savefig(self.workdir+"/stddev.pdf")
plt.clf()
plt.plot(t, 1/fi(self.m(t)/fval, self.m(t, 1)/fval))
plt.plot(t, 1/fiapp(self.m(t)/fval, self.m(t, 1)/fval), 'r')
plt.savefig(self.workdir+"/fisherInfo.pdf")
with open(self.workdir+"/variance.csv" ,"w+") as f:
fisher=np.sqrt(fi(self.m(t)/fval, self.m(t, 1)/fval))
for i in range(0, 1000):
f.write( str(t[i]) + " , " + str(fisher[i]) )
if i < 999:
f.write("\n")
f.close()
'/Volumes/Transcend/pumd_data_files/processed_data_5yrs_bucket_jun20/mtbi_avg_spend_intrvw_2011_to_2015.csv'
)
test_pipe = test_pipe[test_pipe['UCC'] == ucc]
x = test_pipe['AGE_REF']
y = test_pipe['AVG_SPEND']
plt.plot(x, y, 'ro', ms=5)
# c_spline = CubicSpline(x, y)
# xs = np.linspace(20, 80, 1000)
# plt.plot(xs, c_spline(xs), 'b')
# s=33500000
for i in range(0, 99999999, 1000):
u_spline = UnivariateSpline(x, y, s=i)
knot_count = len(u_spline.get_knots())
print(knot_count)
if knot_count <= 8:
break
xs = np.linspace(20, 80, 1000)
plt.plot(xs, u_spline(xs), 'b', lw=3)
# u_spline.set_smoothing_factor(0.5)
# plt.plot(xs, u_spline(xs), 'g')
plt.title("Special lump sum mortgage payment (owned home) [2011-2015]")
plt.xlabel('Age')
plt.ylabel('Average Spend in $')
plt.show()
# x = np.linspace(30, 80, 50)
# y = np.exp(-x**3) + 0.1 * np.random.randn(50)
def update_adapt(self, x, p, x1_base, delta):
# basic sanity check of input estimates
# don't update if base estimate doesn't exist
if ~np.isfinite(x1_base):
return
## DICTIONARY management
# add to dictionary
if self.x1_d is None: # need to create dictionary
self.x1_d = x - x1_base
self.p_d = p
self.delta_d = np.repeat(delta, len(x)) #uncertainty of base
self.time_d = np.repeat(self.i, len(x)) #time of point
self.x1_base = np.repeat(x1_base, len(x)) #base estimate of point
else:
self.x1_d = np.append(self.x1_d, x - x1_base)
self.p_d = np.append(self.p_d, p)
self.delta_d = np.append(self.delta_d, np.repeat(delta, len(x)))
self.time_d = np.append(self.time_d, np.repeat(self.i, len(x)))
self.x1_base = np.append(self.x1_base, np.repeat(x1_base, len(x)))
# if over budget prune
if self.p_d.shape[0] > self.budget:
N_rmv = len(self.x1_d) - self.budget
dff = np.diff(self.x1_d)
mdff = .5 * (dff[1:len(dff)] + dff[0:(len(dff) - 1)])
mdff = np.concatenate([[dff[0]], mdff, [dff[len(dff) - 1]]])
eps = self.delta_d / mdff # accuracy of base estimate / closeness to nearby points
rmv = np.where(
np.logical_or(eps >= -np.sort(-eps)[N_rmv - 1],
~np.isfinite(eps)))[0]
if len(rmv) > N_rmv:
rmv = np.random.choice(a=rmv, size=N_rmv, replace=False)
# remove pts from dictionary and assoc. information
self.x1_d = np.delete(self.x1_d, rmv, axis=0)
self.p_d = np.delete(self.p_d, rmv, axis=0)
self.delta_d = np.delete(self.delta_d, rmv, axis=0)
self.time_d = np.delete(self.time_d, rmv, axis=0)
self.x1_base = np.delete(self.x1_base, rmv, axis=0)
# sort and remove non-unique values (more stable fitting)
# remove non-unique from dict
ux, iux = np.unique(self.x1_d, return_index=True)
self.x1_d = ux
self.p_d = self.p_d[iux]
self.delta_d = self.delta_d[iux]
self.time_d = self.time_d[iux]
self.x1_base = self.x1_base[iux]
# sort dict
ordx = np.argsort(self.x1_d, axis=0).reshape(-1)
self.x1_d = self.x1_d[ordx]
self.p_d = self.p_d[ordx]
self.delta_d = self.delta_d[ordx]
self.time_d = self.time_d[ordx]
self.x1_base = self.x1_base[ordx]
## ADAPTIVE PROFILE fitting
# need more data than knots
if len(self.x1_d) <= self.K:
return
# determine location of knots based on some heuristics
if len(self.x1_d) > 5 and self.n_knots > 0:
l3len = int(np.ceil(len(self.x1_d) // 3))
if self.n_knots is not None:
qs = np.linspace(.25, .75, np.min([l3len, self.n_knots]))
else:
qs = np.linspace(.25, .75, l3len)
kseq = np.quantile(self.x1_d, qs)
self.kseq = kseq[1:(len(kseq) - 1)]
else:
self.kseq = []
self.x_design = self.make_adapt_design(self.x1_d, update=True)
self.p_mean = np.mean(self.p_d)
self.p_std = np.std(self.p_d)
self.p_z = (self.p_d - self.p_mean) / self.p_std
# fit model and its derivative
#just make sure everything is monotonic (for stability)
mono_fit = IsotonicRegression().fit(self.x_design, self.p_z)
mono_p = mono_fit.predict(self.x_design)
if self.n_knots < 0:
def err_mod(mod): #RSS for fit model
return np.sum((mod(self.x1_d) - self.p_z)**2) / len(self.p_z)
smooth_spl = UnivariateSpline(self.x_design,
mono_p,
k=self.K,
ext=0,
s=len(self.p_z) * self.gamma)
# some heuristics on what knots to keep
knts = smooth_spl.get_knots()
keep_knts = ~np.logical_or(knts < np.quantile(self.x1_d, .1),
knts > np.quantile(self.x1_d, .9))
knts = knts[keep_knts]
linear_mod = LSQUnivariateSpline(self.x_design,
mono_p,
k=self.K,
ext=0,
t=[])
self.adapt = LSQUnivariateSpline(self.x_design,
mono_p,
k=self.K,
ext=0,
t=knts)
adapt_err = err_mod(self.adapt)
lin_err = err_mod(linear_mod)
#if its not twice as good as a simple linear model just use the latter
if lin_err > 0 and adapt_err / lin_err > 0.5:
self.adapt = linear_mod
else: #n_knots > 0
self.adapt = LSQUnivariateSpline(self.x_design,
mono_p,
k=self.K,
ext=0,
t=self.kseq)
self.adapt_deriv = self.adapt.derivative(1)
def fit_pspec(self,
breaks=None,
log_break=True,
lg_scale_cut=2,
verbose=False):
'''
Fit the 1D Power spectrum using a segmented linear model. Note that
the current implementation allows for only 1 break point in the
model. If the break point is estimated via a spline, the breaks are
tested, starting from the largest, until the model finds a good fit.
Parameters
----------
breaks : float or None, optional
Guesses for the break points. If given as a list, the length of
the list sets the number of break points to be fit. If a choice is
outside of the allowed range from the data, Lm_Seg will raise an
error. If None, a spline is used to estimate the breaks.
log_break : bool, optional
Sets whether the provided break estimates are log-ed values.
lg_scale_cut : int, optional
Cuts off largest scales, which deviate from the powerlaw.
verbose : bool, optional
Enables verbose mode in Lm_Seg.
'''
if breaks is None:
from scipy.interpolate import UnivariateSpline
# Need to order the points
shape = self.vel_freqs.size
spline_y = np.log10(self.ps1D[1:shape / 2])
spline_x = np.log10(self.vel_freqs[1:shape / 2])
spline = UnivariateSpline(spline_x, spline_y, k=1, s=1)
# The first and last are just the min and max of x
breaks = spline.get_knots()[1:-1]
if verbose:
print "Breaks found from spline are: " + str(breaks)
# Take the number according to max_breaks starting at the
# largest x.
breaks = breaks[::-1]
x = np.log10(self.vel_freqs[lg_scale_cut + 1:-lg_scale_cut])
y = np.log10(self.ps1D[lg_scale_cut + 1:-lg_scale_cut])
# Now try these breaks until a good fit including the break is
# found. If none are found, it accept that there wasn't a good
# break and continues on.
i = 0
while True:
self.fit = \
Lm_Seg(x, y, breaks[i])
self.fit.fit_model(verbose=verbose)
if self.fit.params.size == 5:
break
i += 1
if i >= breaks.shape:
warnings.warn("No good break point found. Returned fit\
does not include a break!")
break
return self
if not log_break:
breaks = np.log10(breaks)
self.fit = \
Lm_Seg(np.log10(self.vel_freqs[lg_scale_cut+1:-lg_scale_cut]),
np.log10(self.ps1D[lg_scale_cut+1:-lg_scale_cut]), breaks)
self.fit.fit_model(verbose=verbose)
def fit_pspec(self, breaks=None, log_break=True, lg_scale_cut=2, verbose=False):
"""
Fit the 1D Power spectrum using a segmented linear model. Note that
the current implementation allows for only 1 break point in the
model. If the break point is estimated via a spline, the breaks are
tested, starting from the largest, until the model finds a good fit.
Parameters
----------
breaks : float or None, optional
Guesses for the break points. If given as a list, the length of
the list sets the number of break points to be fit. If a choice is
outside of the allowed range from the data, Lm_Seg will raise an
error. If None, a spline is used to estimate the breaks.
log_break : bool, optional
Sets whether the provided break estimates are log-ed values.
lg_scale_cut : int, optional
Cuts off largest scales, which deviate from the powerlaw.
verbose : bool, optional
Enables verbose mode in Lm_Seg.
"""
if breaks is None:
from scipy.interpolate import UnivariateSpline
# Need to order the points
shape = self.vel_freqs.size
spline_y = np.log10(self.ps1D[1 : shape / 2])
spline_x = np.log10(self.vel_freqs[1 : shape / 2])
spline = UnivariateSpline(spline_x, spline_y, k=1, s=1)
# The first and last are just the min and max of x
breaks = spline.get_knots()[1:-1]
if verbose:
print "Breaks found from spline are: " + str(breaks)
# Take the number according to max_breaks starting at the
# largest x.
breaks = breaks[::-1]
x = np.log10(self.vel_freqs[lg_scale_cut + 1 : -lg_scale_cut])
y = np.log10(self.ps1D[lg_scale_cut + 1 : -lg_scale_cut])
# Now try these breaks until a good fit including the break is
# found. If none are found, it accept that there wasn't a good
# break and continues on.
i = 0
while True:
self.fit = Lm_Seg(x, y, breaks[i])
self.fit.fit_model(verbose=verbose)
if self.fit.params.size == 5:
break
i += 1
if i >= breaks.shape:
warnings.warn(
"No good break point found. Returned fit\
does not include a break!"
)
break
return self
if not log_break:
breaks = np.log10(breaks)
self.fit = Lm_Seg(
np.log10(self.vel_freqs[lg_scale_cut + 1 : -lg_scale_cut]),
np.log10(self.ps1D[lg_scale_cut + 1 : -lg_scale_cut]),
breaks,
)
self.fit.fit_model(verbose=verbose)
return self
class SplineSmoother(object):
'''
claselfdocs
'''
def __init__(self, yp, workdir, scale, sm=200):
'''
Constructor
'''
self.workdir = workdir
yp = np.array(yp)
self.l = len(yp)
self.xPos = (self.l - 2) / 2 #fPos = (self.l-2)/2 + 2
tnsc = 2 / scale
print tnsc
plt.rcParams['font.size'] = 24
plt.rcParams['lines.linewidth'] = 2.4
self.workdir = workdir
self.avx = np.append(
np.append([0], np.sort(np.tanh(tnsc * yp[:self.xPos]))), [1])
self.av = yp[self.xPos:]
self.m = UnivariateSpline(self.avx, self.av)
plt.rcParams['font.size'] = 24
plt.rcParams['lines.linewidth'] = 2.4
print "Created spline with " + str(len(self.m.get_knots())) + " knots"
def saveSpline(self, filename):
tp = np.linspace(0, 1, 1000)
with open(filename, "w+") as f:
for i in range(0, 1000):
f.write(str(tp[i]) + " , " + str(self.m(tp[i])))
if i < 999:
f.write("\n")
f.close()
def showSpline(self, order=0):
plt.clf()
print "Spline full information:"
print self.m.get_knots()
print self.m.get_coeffs()
print self.m.get_residual()
tp = np.linspace(0, 1, 1000)
plt.scatter(self.avx, self.av)
plt.plot(tp, self.m(tp))
plt.savefig(self.workdir + "/splineFit.pdf")
if order > 0:
plt.plot(tp, self.m(tp, 1))
if order > 1:
plt.plot(tp, self.m(tp, 2))
plt.savefig(self.workdir + "/splineDerivative.pdf")
def plotSplineData(self, dataContainer, s, p, se, yscale):
plt.clf()
plt.xlim(0, 1)
plt.ylim(0, yscale)
tp = np.linspace(0, 1, 500)
plt.scatter(dataContainer.points[0],
dataContainer.points[1] + dataContainer.background,
c='b',
marker='o',
s=5)
plt.plot(tp, self.m(tp) + dataContainer.background, 'r', linewidth=2)
plt.plot(tp,
self.m(tp) +
np.sqrt(s * (p * self.m(tp) * self.m(tp) + self.m(tp) + se)) +
dataContainer.background,
'r--',
linewidth=2)
plt.plot(tp,
self.m(tp) -
np.sqrt(s * (p * self.m(tp) * self.m(tp) + self.m(tp) + se)) +
dataContainer.background,
'r--',
linewidth=2)
plt.plot(tp,
np.zeros(500) + dataContainer.background,
'--',
c='#BBBBBB',
alpha=0.8)
plt.savefig(self.workdir + "/splineVsData.pdf")
def plotBinnedData(self, dataContainer, s, p, se, xmin, xmax):
plt.clf()
sigma = lambda x: s * (p * x * x + x + se)
t = np.linspace(xmin, xmax, 500)
plt.xlim(xmin, xmax)
plt.plot(t, sigma(t))
plt.plot(dataContainer.avs, np.power(dataContainer.stds, 2), 'o')
plt.savefig(self.workdir + "/noiseVsBins.pdf")
plt.clf()
tp = np.linspace(0, 1, dataContainer.numBins)
plt.plot(tp, self.m(tp), 'r', linewidth=2)
plt.plot(tp, dataContainer.avs, 'o')
plt.savefig(self.workdir + "/splineVsBins.pdf")
def plotFisherInfo(self, dataContainer, s, p, se, ymax, ymaxsq):
plt.clf()
t = np.linspace(0, 1, 1000)
minf = lambda x: -1 * self.m(x)
minx = fminbound(minf, 0, 1)
fval = self.m(minx)
s = s / fval
se = se / fval
p = p * fval
print fval, s, p, se
fi = lambda m, mp: 2 * np.power(s * (p * np.power(m, 2) + m), 2) / (
np.power(mp, 2) *
(2 * s *
(p * np.power(m, 2) + m) + np.power(s * (2 * p * m + 1), 2)))
fiapp = lambda m, mp: s * (p * np.power(m, 2) + m) / np.power(mp, 2)
plt.xlim(0, 1)
plt.ylim(0, ymaxsq)
plt.plot(t, fi(self.m(t) / fval, self.m(t, 1) / fval))
plt.plot(t, fiapp(self.m(t) / fval, self.m(t, 1) / fval), 'r')
plt.savefig(self.workdir + "/variance.pdf")
plt.clf()
plt.ylim(0, ymax)
plt.plot(t, np.sqrt(fi(self.m(t) / fval, self.m(t, 1) / fval)))
plt.plot(t, np.sqrt(fiapp(self.m(t) / fval, self.m(t, 1) / fval)), 'r')
plt.savefig(self.workdir + "/stddev.pdf")
plt.clf()
plt.plot(t, 1 / fi(self.m(t) / fval, self.m(t, 1) / fval))
plt.plot(t, 1 / fiapp(self.m(t) / fval, self.m(t, 1) / fval), 'r')
plt.savefig(self.workdir + "/fisherInfo.pdf")
with open(self.workdir + "/variance.csv", "w+") as f:
fisher = np.sqrt(fi(self.m(t) / fval, self.m(t, 1) / fval))
for i in range(0, 1000):
f.write(str(t[i]) + " , " + str(fisher[i]))
if i < 999:
f.write("\n")
f.close()
from scipy.interpolate import LSQUnivariateSpline, UnivariateSpline import matplotlib.pyplot as plt x = np.linspace(-3, 3, 50) y = np.exp(-x**2) + 0.1 * np.random.randn(50) # Fit a smoothing spline with a pre-defined internal knots: t = [-1, 0, 1] spl = LSQUnivariateSpline(x, y, t) xs = np.linspace(-3, 3, 1000) plt.plot(x, y, 'ro', ms=5) plt.plot(xs, spl(xs), 'g-', lw=3) plt.show() # Check the knot vector: spl.get_knots() # array([-3., -1., 0., 1., 3.]) # Constructing lsq spline using the knots from another spline: x = np.arange(10) s = UnivariateSpline(x, x, s=0) s.get_knots() # array([ 0., 2., 3., 4., 5., 6., 7., 9.]) knt = s.get_knots() s1 = LSQUnivariateSpline(x, x, knt[1:-1]) # Chop 1st and last knot s1.get_knots() # array([ 0., 2., 3., 4., 5., 6., 7., 9.])
# Nonlinear regression exponential
b = optimize.curve_fit(exp_model,
t,
v,
p0=[v[0], 1, v.min()],
bounds=[[0, 0, v.min()],
[v.max(), np.inf,
v.max()]])
yhat = exp_model(t, b[0][0], b[0][1], b[0][2])
res_exp.append((v - yhat) / v)
# B-spline
spl = UnivariateSpline(t, v, k=3, s=1e6)
# yhat_spl.append(spl(t))
res_spl.append((v - spl(t)) / v)
nknots.append(len(spl.get_knots()))
# plt.subplot(2,3,counter+1)
# plt.plot(t,spl(t),'g')
# plt.plot(t,v,'b')
except:
print 'Fitting failed for ', c.iloc[0].CellID
#auto_res_exp = np.hstack(res_exp)
auto_res_spl = np.hstack(res_spl)
auto_res_exp = np.hstack(res_exp)
plt.figure(1)
weights = np.ones_like(auto_res_exp) / float(len(auto_res_exp))
bins = np.linspace(-1, 1, 25)
def fit_pspec(self,
breaks=None,
log_break=True,
low_cut=None,
high_cut=None,
fit_verbose=False):
'''
Fit the 1D Power spectrum using a segmented linear model. Note that
the current implementation allows for only 1 break point in the
model. If the break point is estimated via a spline, the breaks are
tested, starting from the largest, until the model finds a good fit.
Parameters
----------
breaks : float or None, optional
Guesses for the break points. If given as a list, the length of
the list sets the number of break points to be fit. If a choice is
outside of the allowed range from the data, Lm_Seg will raise an
error. If None, a spline is used to estimate the breaks.
log_break : bool, optional
Sets whether the provided break estimates are log-ed values.
low_cut : `~astropy.units.Quantity`, optional
Lowest frequency to consider in the fit.
high_cut : `~astropy.units.Quantity`, optional
Highest frequency to consider in the fit.
fit_verbose : bool, optional
Enables verbose mode in Lm_Seg.
'''
# Make the data to fit to
if low_cut is None:
# Default to the largest frequency, since this is just 1 pixel
# But cut out fitting the total power point.
self.low_cut = 0.98 / (float(self.data.shape[0]) * u.pix)
else:
self.low_cut = \
self._spectral_freq_unit_conversion(low_cut, u.pix**-1)
if high_cut is None:
# Set to something larger than
self.high_cut = (self.freqs.max().value * 1.01) / u.pix
else:
self.high_cut = \
self._spectral_freq_unit_conversion(high_cut, u.pix**-1)
# We keep both the real and imag frequencies, but we only need the real
# component when fitting. We'll cut off the total power frequency too
# in case it causes an extra break point (which seems to happen).
shape = self.freqs.size
rfreqs = self.freqs[1:shape // 2].value
ps1D = self.ps1D[1:shape // 2]
y = np.log10(ps1D[clip_func(rfreqs, self.low_cut.value,
self.high_cut.value)])
x = np.log10(rfreqs[clip_func(rfreqs, self.low_cut.value,
self.high_cut.value)])
if breaks is None:
from scipy.interpolate import UnivariateSpline
# Need to order the points
spline = UnivariateSpline(x, y, k=1, s=1)
# The first and last are just the min and max of x
breaks = spline.get_knots()[1:-1]
if fit_verbose:
print("Breaks found from spline are: " + str(breaks))
# Take the number according to max_breaks starting at the
# largest x.
breaks = breaks[::-1]
# Ensure a break doesn't fall at the max or min.
if breaks.size > 0:
if breaks[0] == x.max():
breaks = breaks[1:]
if breaks.size > 0:
if breaks[-1] == x.min():
breaks = breaks[:-1]
if x.size <= 3 or y.size <= 3:
raise Warning("There are no points to fit to. Try lowering "
"'lg_scale_cut'.")
# If no breaks, set to half-way before last point
if breaks.size == 0:
x_copy = np.sort(x.copy())
breaks = np.array([0.5 * (x_copy[-1] + x_copy[-3])])
# Now try these breaks until a good fit including the break is
# found. If none are found, it accepts that there wasn't a good
# break and continues on.
i = 0
while True:
self.fit = \
Lm_Seg(x, y, breaks[i])
self.fit.fit_model(verbose=fit_verbose)
if self.fit.params.size == 5:
# Success!
break
i += 1
if i >= breaks.size:
warnings.warn("No good break point found. Returned fit\
does not include a break!")
break
return self
if not log_break:
breaks = np.log10(breaks)
# Fit the final model with whichever breaks were passed.
self.fit = Lm_Seg(x, y, breaks)
self.fit.fit_model(verbose=fit_verbose)
# get the knots from the most difficult one
spl_0 = UnivariateSpline(hist_per_pe.bin_centers[220:511:1],
hist_per_pe.data[10][220:511:1],
s=0,
k=3)
'''
spl_0 = []
for i,pe in enumerate(pes):
spl_0.append(UnivariateSpline(hist_per_pe.bin_centers[220:511:1], hist_per_pe.data[i][220:511:1], s=0,k=3))
print(pe,len(spl_0[-1].get_knots()))
'''
#plt.figure(0)
#plt.step(hist_per_pe.bin_centers[220:511:1],hist_per_pe.data[-1][220:511:1])
#plt.plot(xs, spl_0(xs), 'k-', lw=1)
knt0 = spl_0.get_knots()
wei0 = spl_0.get_coeffs()
print(knt0)
#knt0 = hist_per_pe.bin_centers[221:510:1]
#print(knt0)
#print(5./0)
list_coef = []
for i, pe in enumerate(pes):
if i > 99: continue
#t = hist_per_pe.bin_centers[236:509:2]
spl.append(
LSQUnivariateSpline(hist_per_pe.bin_centers[220:511:1],
hist_per_pe.data[i][220:511:1], knt0[1:-1]))
list_coef.append(list(spl[-1].get_coeffs()))