def my_curve_fit():
    path = [[5, 117], [12, 110], [12, 105], [13, 104], [15, 102], [20, 97], [22, 95], [22, 73], [24, 71], [29, 66], [30, 66], [31, 65], [32, 64], [37, 59], [41, 55], [41, 54], [42, 53], [44, 51], [44, 35], [45, 34], [50, 29], [57, 29], [58, 28], [63, 23], [70, 16], [84, 16], [85, 15]]

    points = np.array(path)
    x = points[:,0]
    x = [float(a) for a in x]
    b = set()
    for i in range(len(x)):
        if x[i] in b:
            x[i] = x[i] + 0.1
        else:
            b.add(x[i])
    y = points[:,1]
    plt.plot(x, y, "og")
    spl = UnivariateSpline(x, points[:,1], s=0)
    kn = spl.get_knots()
    lst = []
    for i in range(4):
        lst.append(1/jiecheng(i))
    for i in range(len(kn)-1):
        cf = lst * spl.derivatives(kn[i])
        print("For {0} <= x <= {1}, p(x) = {5}*(x-{0})^3 + {4}*(x-{0})^2 + {3}*(x-{0}) + {2}".format(kn[i], kn[i+1], *cf))
        dx = np.linspace(kn[i], kn[i+1], 10)
        dy = []
        for item in dx:
            dy.append(pow((item-kn[i]),3)*cf[3]+pow((item-kn[i]),2)*cf[2]+(item-kn[i])*cf[1]+cf[0])
        plt.plot(dx, dy, "-r")
Exemple #2
0
def to_parametric_spline(spline: UnivariateSpline,
                         x_start: Optional[float] = None,
                         x_end: Optional[float] = None) -> Derivable:
    """
    Convert a regular 's(x) = y' spline into a 's(t) = (x, y)' parametric spline.

    Parameters
    ----------
    spline : UnivariateSpline
        Regular spline.
    x_start : float, optional
        X coordinate of left start point of the spline. Will be accessible by
        t = 0.
    x_end : float, optional
        X coordinate of right end point of the spline. Will be accessible by
        t = 1.

    Returns
    -------
    spline : Derivable
        Parametric spline.
    """
    if x_start is None:
        knots = spline.get_knots()
        x_start = knots[0]

    if x_end is None:
        knots = spline.get_knots()
        x_end = knots[-1]

    interpolator = interp1d((0, 1), (x_start, x_end), fill_value='extrapolate')

    def parametric_spline(t, d=0):
        x = interpolator(t)
        if d > 0:
            x_dt = derivative(interpolator, t, d)
        else:
            x_dt = x
        y_dt = spline(x, d)
        return np.array([x_dt, y_dt])

    return parametric_spline
def testWiki():
    yk3 = UnivariateSpline(xx, yy, k=2, s=0)
    knots = yk3.get_knots()
    coeffs = yk3.get_coeffs()
    #ipdb.set_trace()
    r1 = yk3(xxx)
    r2 = gety(xxx, xx, yy, coeffs)
    plt.plot(xx, yy, "k.", markerSize=10)
    plt.plot(xxx, r1, "g.-", markerSize=1)
    plt.plot(xxx, r2, "r.-", markerSize=1)
    plt.show(0)
def demo():
    x = np.arange(6)
    y = np.array([3, 1, 4, 1, 5, 9])
    spl = UnivariateSpline(x, y, s=0)
    kn = spl.get_knots()
    for i in range(len(kn)-1):
        cf = [1, 1, 1/2, 1/6] * spl.derivatives(kn[i])
        print("For {0} <= x <= {1}, p(x) = {5}*(x-{0})^3 + {4}*(x-{0})^2 + {3}*(x-{0}) + {2}".format(kn[i], kn[i+1], *cf))
        dx = np.linspace(kn[i], kn[i+1], 100)
        dy = []
        for item in dx:
            dy.append(pow((item-kn[i]),3)*cf[3]+pow((item-kn[i]),2)*cf[2]+(item-kn[i])*cf[1]+cf[0])
        plt.plot(dx, dy, "-r")
        plt.plot(x, y, "og")
        plt.plot(x, y, "-b")
def fit_and_plot_mpl_bspline(x,y):
# Fit B-spline with matplotlib and plot it
    SPLINE_ORDER = 3
    spl = UnivariateSpline(x,y,k=SPLINE_ORDER)
    plt.plot(x,y,'r')
    plt.plot(x,spl(x),'g')

    knots  = spl.get_knots()
    coeffs = spl.get_coeffs()
    yr = plt.ylim()

    for knot in knots:
        plt.axvline(knot,color='g')
    #for coeff in coeffs:
        #plt.text(???,(yr[0]+yr[1])/2,"%f" % coeff)
    plt.show()
Exemple #6
0
	def spline(self, k=3, s=0.5):
		""" Interpolates uneven observation data into daily data using scipy.interpolate.UnivariateSpline with parameters k and s given"""
		# requires filtered_data to exist, check by making sure it isn't empty
		if len(self.filtered_data) == 0:
			raise ValueError

		# first we need to redo the dates as days from start
		for patient in self.filtered_data:
			self.filtered_data[patient]['days'] = []
			for date in self.filtered_data[patient]['dates']:
				since_beginning = (date - self.filtered_data[patient]['dates'][0]).days
				self.filtered_data[patient]['days'].append(since_beginning)

		# now we spline
		error = 0
		splined_data = dict()
		for patient in self.filtered_data:
			day_indices = np.array(self.filtered_data[patient]['days'])
			splined_data[patient] = []
			for question in range(self.num_items):
				item_data = np.array([i[question] for i in self.filtered_data[patient]['data']])
				if len(item_data) == 0:
					print "Uh, no responses for question %d patient %s" % (question+1, patient)
					raise ValueError
				# Now we need to take out the data with None values... not certain how to handle it if the patient ends up not having enough data
				# because of this filtering, but for UPittSSRI data (primary focus as of 7/29/13) we don't need to worry about it
				good_indices = np.array([ind for ind, resp in enumerate(item_data) if resp != None])
				filtered_days = day_indices[good_indices]
				filtered_data = item_data[good_indices]
				full_space = np.linspace(0,self.keep_days,num=self.keep_days)
				spl = UnivariateSpline(filtered_days, filtered_data, k=k, s=s)
				self.residual = spl.get_residual()
				self.knots = spl.get_knots()
				splined = spl(full_space)
				splined_data[patient].append(splined)
				# Measure error
				ms = math.sqrt(sum((splined[filtered_days] - filtered_data)**2)/len(filtered_data))
				error += ms
			splined_data[patient] = np.array(splined_data[patient]).T
		error /= (self.num_items)*(len(splined_data))
		self.spline_err = error

		self.splined_data = splined_data
		self.data_splined = True
		self.data = splined_data
		return splined_data
Exemple #7
0
def find_spline_with_numberofknots(data_x, data_y, desired_number_of_knots, threshold=0, max_iterations=100):
    max_f = 1000
    min_f = 0
    iteration = 0
    closesed = None
    slm = UnivariateSpline(data_x, data_y, s=len(data_x) * max_f, k=1)
    # determine max
    while slm.get_knots().size > desired_number_of_knots:
        max_f = max_f * 2
        slm = UnivariateSpline(data_x, data_y, s=len(data_x) * max_f, k=1)
    factor = max_f
    while not (abs(desired_number_of_knots - slm.get_knots().size) <= threshold
               and slm.get_knots().size <= desired_number_of_knots):
        slm = UnivariateSpline(data_x, data_y, s=len(data_x) * factor, k=1)
        if slm.get_knots().size > desired_number_of_knots:
            min_f = factor
            if max_f - factor < 0.01:
                max_f = max_f * 2
            factor = factor + (max_f - factor) / 2
        elif slm.get_knots().size < desired_number_of_knots:
            if closesed is None \
                    or desired_number_of_knots - closesed[1] > desired_number_of_knots - slm.get_knots().size:
                closesed = (slm, slm.get_knots().size)
            if factor - min_f < 0.01:
                min_f = min_f - 1
            max_f = factor
            factor = factor - (factor - min_f) / 2
        iteration += 1
        if iteration > max_iterations:
            MetricLogger().warning(
                "cant find normal spline current number of knots:{} closeset spline I will use is:{}".format(
                    slm.get_knots().size,
                    -1 if closesed is None else
                    closesed[1]))
            if closesed is None or closesed[1] > desired_number_of_knots:
                raise RuntimeError("cant find normal spline")
            else:
                slm = closesed[0]
                break
    # xHat = np.linspace(min(data_x), max(data_x), num=10000)
    # yHat = slm(xHat)
    # # plot the results
    # plt.figure()
    # plt.plot(data_x, data_y, 'o')
    # plt.plot(xHat, yHat, '-')
    # plt.show()
    return slm
Exemple #8
0
    def show(self):
        """
        Draw the line on to the plot
        """
        if self.graph.options.sig_enh:
            norm_frames = min(len(self.yvalues),
                              self.graph.options.norm_frames)
            mean = np.mean(self.yvalues[:norm_frames])
            plot_values = self.yvalues / mean - 1
        else:
            plot_values = self.yvalues

        if self.graph.options.smooth:
            indexes = range(len(plot_values))
            # Tolerance does not scale by data value to scale input
            spline = UnivariateSpline(indexes,
                                      plot_values / plot_values.max(),
                                      s=0.1,
                                      k=4)
            self.debug("Number of knots in B-spline smoothing: %i",
                       len(spline.get_knots()))
            line_values = spline(indexes) * plot_values.max()
        else:
            line_values = plot_values

        self.hide()
        # Plot line and points separately as line may be smoothed
        line = self.axes.plot(self.xvalues,
                              line_values,
                              pen=pg.mkPen(color=self.line_col,
                                           width=self.line_width,
                                           style=self.line_style))

        pts = self.axes.plot(self.xvalues,
                             plot_values,
                             pen=None,
                             symbolBrush=self.point_brush,
                             symbolPen=self.point_col,
                             symbolSize=self.point_size)

        self.legend_item = line
        self.graphics_items = (line, pts)
Exemple #9
0
def break_spline(x, y, **kwargs):
    '''
    Calculate the break in 2 linear trends using a spline.
    '''

    s = UnivariateSpline(x, y, **kwargs)
    knots = s.get_knots()

    if len(knots) > 3:
        print "Many knots"
        knot_expec = -0.8
        posn = np.argmin(knots - knot_expec)
        return knots[posn]  # Return the knot closest to the expected value

    elif len(knots) == 2:
        print "No knots"
        return -0.6  # Set the threshold at the expected

    else:
        return knots[1]
Exemple #10
0
def dist2spline(data, xmin=None, xmax=None, **kwargs):
    """
    Estimate a smooth 1D distribution from data with weights in the interval [xmin, xmax]
    and returns the knots and coefficients of a fitting spline.

    """
    from pyqt_fit.kde import KDE1D
    from scipy.interpolate import UnivariateSpline

    if xmin is None:
        xmin = np.min(data)
    if xmax is None:
        xmax = np.max(data)

    kde = KDE1D(data, lower=xmin, upper=xmax, **kwargs)

    xx = np.linspace(xmin, xmax, 200)
    kdexx = kde(xx)

    s = UnivariateSpline(xx, kdexx, s=kde.bandwidth)

    xs = s.get_knots()
    ys = s(xs)
    return xs, ys, s
Exemple #11
0
def spline_anomaly(df,
                   smooth=0.5,
                   plot=False,
                   B=False,
                   turning=False,
                   filtered=False,
                   threshold=1):
    """
    Accepts:

        a Pandas dataframe of shape:

                obmt    rate    w1_rate
            1.  float   float   float

        or equivalent.

    This dataframe should be a hit neighbourhood as generated by
    isolate_anomaly(). The function will work on larger datasets but
    there is nothing to be gained by splining these - and the time taken
    to do so would be significantly larger.

    Runs scipy's splining algorithms on the hit neighbourhoods to
    generate a spline to fit the data.

    Kwargs:

        smooth (float, default=0.5):
            smoothing factor for the generated splines.

        plot (bool, default=False):
            if True, generates a plot of the data and the spline.

        B (bool, default=False):
            if True, generates B splines.

        turning (bool, default=False):
            if True, plots all the turning points alongside a normal
            plot.

        filtered (bool, default=False):
            if True, plots the filterd turning points alongside a normal
            plot.

        threshold (float, default=1.0):
            the threshold for filtering turning points

    Returns:

        tuple of the arrays of knots and coefficients (in that order) of
        the fitted spline.
    """

    # Create spline.
    spl = UnivariateSpline(df['obmt'], df['rate'] - df['w1_rate'])
    spl.set_smoothing_factor(smooth)
    knots, coeffs = (spl.get_knots(), spl.get_coeffs())

    xs = np.linspace(df['obmt'].tolist()[0], df['obmt'].tolist()[-1], 10000)

    if B:
        spl = BSpline(knots, coeffs, 3)

    # Plot original data and spline.
    if plot or turning or filtered:
        plt.scatter(df['obmt'], df['rate'] - df['w1_rate'])
        plt.plot(xs, spl(xs))

        if filtered or turning:
            # Calls get_turning_points() to isolate the turning points.
            if turning:
                turning_points = get_turning_points(df)

            elif filtered:
                turning_points = filter_turning_points(get_turning_points(df),
                                                       threshold=threshold)

            plt.scatter(turning_points['obmt'],
                        turning_points['rate'] - turning_points['w1_rate'],
                        color='red')

        plt.show()

    return (knots, coeffs)
Exemple #12
0
    MSEs_train.append(MSE)
    MSEs_CV.append(MSE_CV)

poly_params = np.polyfit(X, Y, 5)
plt.plot(X, Y, 'black')
plt.plot(X, np.polyval(poly_params, X), 'r-')
plt.show()
"""
plt.plot(degrees,MSEs_train,'black')#,label="train MSE")
plt.plot(degrees,MSEs_CV,'red')#,label="CV MSE")
plt.show()
"""

import numpy as np
from scipy.interpolate import UnivariateSpline

degree = 3  # In range 0 .. 5
smoothing = 1  # Lower bound for SSE

Xinv = X[::-1]
Yinv = Y[::-1]

s = UnivariateSpline(Xinv, Yinv, k=degree, s=4e9)

#get_coeffs(), get_knots(), get_residual()
plt.plot(Xinv, Yinv, "black")
plt.plot(Xinv, s(Xinv), "blue")
plt.show()

print len(s.get_knots())
print s.get_residual()
Exemple #13
0
class DualSplineSmoother(object):
	'''
	claselfdocs
	'''


	def __init__(self, yp, workdir, scale, sm=200):
		'''
		Constructor
		'''
		yp = np.array(yp)
		self.l = len(yp)/2
		self.xPos = (self.l-2)/2 #fPos = (self.l-2)/2 + 2
		tnsc = 2/scale
		print tnsc
		plt.rcParams['font.size'] = 24
		plt.rcParams['lines.linewidth'] = 2.4
		self.workdir = workdir
		
		avProfilePoints = yp[:self.l]
		self.avx = np.append(np.append([0], np.sort(np.tanh(tnsc*avProfilePoints[:self.xPos]))),[1])
		self.av = avProfilePoints[self.xPos:]
		
		sigmaProfilePoints = yp[self.l:]
		self.sigmax = np.append(np.append([0], np.sort(np.tanh(tnsc*sigmaProfilePoints[:self.xPos]))),[1])
		self.sigma = sigmaProfilePoints[self.xPos:]
		
		self.m = UnivariateSpline(self.avx, self.av)
		print "Created spline with " + str(len(self.m.get_knots())) + " knots"

		self.s = UnivariateSpline(self.sigmax, self.sigma)
		print "Created spline with " + str(len(self.s.get_knots())) + " knots"

	def saveSpline(self, filename):
		tp = np.linspace(0, 1, 1000)
		with open(filename ,"w+") as f:
			for i in range(0, 1000):
				f.write( str(tp[i]) + " , " + str(self.m(tp[i])) )
				if i < 999:
					f.write("\n")
		f.close()
	
	def saveSigmaSpline(self, filename):
		tp = np.linspace(0, 1, 1000)
		with open(filename ,"w+") as f:
			for i in range(0, 1000):
				f.write( str(tp[i]) + " , " + str(self.s(tp[i])) )
				if i < 999:
					f.write("\n")
		f.close()
	
	def showSpline(self, order=0):
		plt.clf()
		print "Spline full information:"
		print self.m.get_knots()
		print self.m.get_coeffs()
		print self.m.get_residual()
		tp = np.linspace(0, 1, 1000)
		plt.subplot(211)
		plt.scatter(self.avx,self.av)
		plt.plot(tp,self.m(tp))
		plt.subplot(212)
		plt.scatter(self.sigmax,self.sigma)
		plt.plot(tp,self.s(tp))
		plt.savefig(self.workdir+"/splineFit.pdf")
		if order > 0:
			plt.clf()
			plt.subplot(211)
			plt.plot(tp,self.m(tp,1))
			plt.subplot(212)
			plt.plot(tp,self.s(tp,1))
			plt.savefig(self.workdir+"/splineDerivative.pdf")
	
	def plotSplineData(self, dataContainer, yscale):
		plt.clf()
		plt.xlim(0,1)
		plt.ylim(0,yscale)
		tp = np.linspace(0, 1, 100)
		plt.scatter(dataContainer.points[0],dataContainer.points[1]+dataContainer.background, c='b', marker='o', s=5)
		plt.plot(tp, self.m(tp)+dataContainer.background,'r', linewidth=2)
		plt.plot(tp, self.m(tp)+np.sqrt(self.s(tp))+dataContainer.background,'r--', linewidth=2)
		plt.plot(tp, self.m(tp)-np.sqrt(self.s(tp))+dataContainer.background,'r--', linewidth=2)
		plt.plot(tp, np.zeros(100) + dataContainer.background, '--', c='#BBBBBB', alpha=0.8)
		plt.savefig(self.workdir+"/splineVsData.pdf")
		
	def plotBinnedData(self, dataContainer):
		plt.clf()
		tp = np.linspace(0, 1, dataContainer.numBins)
		tpHD = np.linspace(0, 1, 500)
		plt.plot(self.m(tpHD), self.s(tpHD))
		plt.plot(dataContainer.avs, np.power(dataContainer.stds,2), 'o')
		plt.savefig(self.workdir+"/noiseVsBins.pdf")
		plt.clf()
		plt.plot(tpHD, self.m(tpHD),'r', linewidth=2)
		plt.plot(tp, dataContainer.avs, 'o')
		plt.savefig(self.workdir+"/splineVsBins.pdf")
		plt.clf()
		plt.plot(tpHD, self.s(tpHD),'r', linewidth=2)
		plt.plot(tp, np.power(dataContainer.stds,2), 'o')
		plt.savefig(self.workdir+"/spatialNoiseVsBins.pdf")
	
	def plotFisherInfo(self, dataContainer, ymax, ymaxsq):
		plt.clf()
		t = np.linspace(0, 1, 500)
		
		minf = lambda x: -1 * self.m(x)
		minx = fminbound(minf, 0, 1)
		fval = self.m(minx)
		
		noisemean = UnivariateSpline(self.m(t)/fval, self.s(t)/fval/fval)
		self.se = noisemean(0)
		
		fi = lambda a, sa, sp: 2*np.power(sa, 2)/ (np.power(a,2)*2*sa+np.power(sp,2))
		fiapp = lambda a, sa, sp: sa / (np.power(a,2))
		plt.xlim(0, 1)
		plt.ylim(0, ymaxsq)
		print 'whop whop'
		plt.plot(t, fi(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval))
		plt.plot(t, fiapp(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval), 'r')
		plt.savefig(self.workdir+"/variance.pdf")
		plt.clf()
		plt.ylim(0, ymax)
		plt.plot(t, np.sqrt(fi(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval)))
		plt.plot(t, np.sqrt(fiapp(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval)), 'r')
		plt.savefig(self.workdir+"/stddev.pdf")
		plt.clf()
		plt.plot(t, 1/fi(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval))
		plt.plot(t, 1/fiapp(self.m(t,1)/fval, self.s(t)/fval/fval-self.se, self.s(t, 1)/fval/fval), 'r')
		plt.savefig(self.workdir+"/fisherInfo.pdf")
Exemple #14
0
    1.82380809e-01, 2.66344008e+00, 1.18164677e+01, 3.01811501e+01,
    5.78812583e+01, 9.40718450e+01
])

## Now get scipy's spline fit.
k = 3
tck = interpolate.splrep(x_nodes, y_nodes, k=k, s=0)
knots = tck[0]
coeffs = tck[1]
print('knot points=', knots)
print('coefficients=', coeffs)

## Now try scipy's object-oriented version. The result is exactly the same as "tck": the knots are the
## same and the coeffs are the same, they are just queried in a different way.
uspline = UnivariateSpline(x_nodes, y_nodes, s=0)
uspline_knots = uspline.get_knots()
uspline_coeffs = uspline.get_coeffs()

## Here are scipy's native spline evaluation methods. Again, "ytck" and "y_uspline" are exactly equal.
ytck = interpolate.splev(x, tck)
y_uspline = uspline(x)
y_knots = uspline(knots)

## Now let's try our manually-calculated evaluation function.
y_eval = bspleval(x, knots, coeffs, k, debug=False)

plt.plot(x, ytck, label='tck')
plt.plot(x, y_uspline, label='uspline')
plt.plot(x, y_eval, label='manual')

## Next plot the knots and nodes.
Exemple #15
0
def fit_trace(trace, deg=5):
    spl = UnivariateSpline(trace.sample_f, trace.trace_f, k=deg)
    return [deg, spl.get_coeffs().tolist(), spl.get_knots().tolist()]
# -*- coding: utf-8 -*-
"""
# scipy.interpolate - інтерполяція
Інтерполяція - це спосіб знаходження проміжних значень величини за її відомим дискретним набором значень. Для інтерполяції і апроксимації сплайнами застосовують функції з модуля `scipy.interpolate` (`interp1d`, `UnivariateSpline`, `interp2d`, `SmoothBivariateSpline` та інші). Сплайн - це функція, область визначення якої розбита на частини, на кожній з яких функція є певним поліномом.
"""
import numpy as np
from scipy.interpolate import interp1d, UnivariateSpline, interp2d
f = lambda x: x**x  # функція
x = np.arange(4)
y = f(x)
print x, y  # дискретний набір значень
y1 = interp1d(x, y, kind='linear')  # лінійна інтерполяція
y2 = interp1d(x, y, kind='quadratic')  # інтерполяція квадратичним сплайном
y2 = UnivariateSpline(x, y, k=2, s=0)  # або (s - коеф. згладжування)
kn = y2.get_knots()  # вузли сплайна
print y1(2.5), y2(2.5), f(
    2.5)  # інтерпольовані і дійсне значення в точці x=2.5

import matplotlib.pyplot as plt
x = np.linspace(0, x.max(), 100)
plt.plot(x, f(x), 'k-', x, y1(x), 'k--', x, y2(x), 'k:')  # графіки
plt.scatter(kn, y2(kn))  # вузли кв. сплайна
plt.xlabel('x'), plt.ylabel('y'), plt.grid(), plt.show()
print "Рисунок – Функція (-) та її лінійна (--) і квадратична (..) інтерполяції сплайнами"

f = lambda x, y: x**2 + y**2  # функція двох змінних
x = np.array([0, 1, 2])
y = np.array([0, 1, 2])
xx, yy = np.meshgrid(x, y)  # сітка
z = f(xx, yy)  # значення функції у вузлах сітки
z1 = interp2d(x, y, z,
def onedim_pixtopix_variations_spline(flat,
                                      knots=9,
                                      guess=13,
                                      savefits=True,
                                      path=None,
                                      debug_level=0):
    """
    This routine fits a smoothing spline to an observed flat field in order to determine the pixel-to-pixel sensitivity variations
    as well as the fringing pattern in the red orders. This is done in 1D, ie for the already extracted spectrum.
    
    INPUT:
    'flat'          : dictionary / np.array containing the extracted flux from the flat field (master white) (keys = orders)
    'knots'         : the desired number of knots for the smoothing spline
    'guess'         : starting guess for the "smoothing factor" that determines the number of knots:  s = 10^guess  (see online documentation for scipy.interpolate.UnivariateSpline)
    
    OUTPUT:
    'pix_sens'      : dictionary / np.array of the pixel-to-pixel sensitivities (keys = orders)
    'smoothed_flat' : dictionary / np.array of the smoothed (ie filtered) whites (keys = orders)
    
    MODHIST:
    13/12/2019 - CMB create (clone of onedim_pixtopix_variations)
    """

    if savefits:
        assert path is not None, 'ERROR: path variable is not set!'

    # check whether it's a numpy array (eg from FITS file), or a python dictionary
    if flat.__class__ == np.ndarray:

        # make sure that they are either quick-extracted spectra (order, pixel), or optimal-extracted spectra (order, fibre, pixel)
        assert len(flat.shape) in [2,
                                   3], 'ERROR: shape of flat not recognized!!!'
        assert flat.shape[
            -1] == 4112, 'ERROR: there are NOT exactly 4112 pixels in dispersion direction'

        if debug_level >= 1:
            print('Fitting smoothing spline with ' + str(knots) +
                  ' knots to...')

        xx = np.arange(flat.shape[-1])
        pix_sens = np.zeros(flat.shape) - 1.
        smoothed_flat = np.zeros(flat.shape) - 1.

        # loop over all orders
        for o in range(flat.shape[0]):
            if debug_level >= 1:
                print('Order_' + str(o + 1).zfill(2))
            # are they optimal 3a extracted spectra?
            if len(flat.shape) == 3:
                # loop over all fibres
                for fib in range(flat.shape[1]):
                    if debug_level >= 2:
                        print('Fibre ' + str(fib + 1).zfill(2))
                    # determine smoothing parameter to get desired number of knots
                    spl = UnivariateSpline(xx, flat[o, fib, :], s=10**guess)
                    nk = len(spl.get_knots())
                    if nk < knots:
                        add = 0
                        while nk < knots:
                            add -= 0.01
                            spl = UnivariateSpline(xx,
                                                   flat[o, fib, :],
                                                   s=10**(guess + add))
                            nk = len(spl.get_knots())
#                             print(add, nk)
                    elif nk > knots:
                        add = 0
                        while nk > knots:
                            add += 0.01
                            spl = UnivariateSpline(xx,
                                                   flat[o, fib, :],
                                                   s=10**(guess + add))
                            nk = len(spl.get_knots())
#                             print(add, nk)
                    smoothed_flat[o, fib, :] = spl(xx)
                    pix_sens[o,
                             fib, :] = flat[o, fib, :] / smoothed_flat[o,
                                                                       fib, :]
            # or are they just quick-extracted spectra
            else:
                # determine smoothing parameter to get desired number of knots
                spl = UnivariateSpline(xx, flat[o, :], s=10**guess)
                nk = len(spl.get_knots())
                if nk < knots:
                    add = 0
                    while nk < knots:
                        add -= 0.01
                        spl = UnivariateSpline(xx,
                                               flat[o, :],
                                               s=10**(guess + add))
                        nk = len(spl.get_knots())
#                         print(add, nk)
                elif nk > knots:
                    add = 0
                    while nk > knots:
                        add += 0.01
                        spl = UnivariateSpline(xx,
                                               flat[o, :],
                                               s=10**(guess + add))
                        nk = len(spl.get_knots())
#                         print(add, nk)
                smoothed_flat[o, :] = spl(xx)
                pix_sens[o, :] = flat[o, :] / smoothed_flat[o, :]

        if savefits:
            date = path.split('/')[-2]
            # get header from master white
            h = pyfits.getheader(path + date + '_master_white.fits')
            # add requested number of knots to header
            h['N_KNOTS'] = (knots, 'number of knots used in smoothing spline')
            pyfits.writeto(path + date + '_smoothed_flat.fits',
                           np.float32(smoothed_flat), h)
            pyfits.writeto(path + date + '_pixel_sensitivity.fits',
                           np.float32(pix_sens), h)

    elif flat.__class__ == dict:
        print('This has not been implemented yet for dictionaries...')
        return


#         pix_sens = {}
#         smoothed_flat = {}
#         # loop over all orders
#         for ord in sorted(flat.keys()):

    else:
        print('ERROR: data type / variable class not recognized')
        return

    if debug_level >= 1:
        print('DONE!!!')

    return smoothed_flat, pix_sens
Exemple #18
0
    def fit_pspec(self, breaks=None, log_break=True, low_cut=None,
                  high_cut=None, fit_verbose=False, bootstrap=False,
                  **bootstrap_kwargs):
        '''
        Fit the 1D Power spectrum using a segmented linear model. Note that
        the current implementation allows for only 1 break point in the
        model. If the break point is estimated via a spline, the breaks are
        tested, starting from the largest, until the model finds a good fit.

        Parameters
        ----------
        breaks : float or None, optional
            Guesses for the break points. If given as a list, the length of
            the list sets the number of break points to be fit. If a choice is
            outside of the allowed range from the data, Lm_Seg will raise an
            error. If None, a spline is used to estimate the breaks.
        log_break : bool, optional
            Sets whether the provided break estimates are log-ed values.
        low_cut : `~astropy.units.Quantity`, optional
            Lowest frequency to consider in the fit.
        high_cut : `~astropy.units.Quantity`, optional
            Highest frequency to consider in the fit.
        fit_verbose : bool, optional
            Enables verbose mode in Lm_Seg.
        bootstrap : bool, optional
            Bootstrap using the model residuals to estimate the standard
            errors.
        bootstrap_kwargs : dict, optional
            Pass keyword arguments to `~turbustat.statistics.fitting_utils.residual_bootstrap`.
        '''

        # Make the data to fit to
        if low_cut is None:
            # Default to the largest frequency, since this is just 1 pixel
            # But cut out fitting the total power point.
            self.low_cut = 0.98 / (float(self.data.shape[0]) * u.pix)
        else:
            self.low_cut = \
                self._spectral_freq_unit_conversion(low_cut, u.pix**-1)

        if high_cut is None:
            # Set to something larger than
            self.high_cut = (self.freqs.max().value * 1.01) / u.pix
        else:
            self.high_cut = \
                self._spectral_freq_unit_conversion(high_cut, u.pix**-1)

        # We keep both the real and imag frequencies, but we only need the real
        # component when fitting. We'll cut off the total power frequency too
        # in case it causes an extra break point (which seems to happen).
        shape = self.freqs.size
        rfreqs = self.freqs[1:shape // 2].value
        ps1D = self.ps1D[1:shape // 2]

        y = np.log10(ps1D[clip_func(rfreqs, self.low_cut.value,
                                    self.high_cut.value)])
        x = np.log10(rfreqs[clip_func(rfreqs, self.low_cut.value,
                                      self.high_cut.value)])

        if breaks is None:
            from scipy.interpolate import UnivariateSpline

            # Need to order the points
            spline = UnivariateSpline(x, y, k=1, s=1)

            # The first and last are just the min and max of x
            breaks = spline.get_knots()[1:-1]

            if fit_verbose:
                print("Breaks found from spline are: " + str(breaks))

            # Take the number according to max_breaks starting at the
            # largest x.
            breaks = breaks[::-1]

            # Ensure a break doesn't fall at the max or min.
            if breaks.size > 0:
                if breaks[0] == x.max():
                    breaks = breaks[1:]
            if breaks.size > 0:
                if breaks[-1] == x.min():
                    breaks = breaks[:-1]

            if x.size <= 3 or y.size <= 3:
                raise Warning("There are no points to fit to. Try lowering "
                              "'lg_scale_cut'.")

            # If no breaks, set to half-way before last point
            if breaks.size == 0:
                x_copy = np.sort(x.copy())
                breaks = np.array([0.5 * (x_copy[-1] + x_copy[-3])])

            # Now try these breaks until a good fit including the break is
            # found. If none are found, it accepts that there wasn't a good
            # break and continues on.
            i = 0
            while True:
                self.fit = \
                    Lm_Seg(x, y, breaks[i])
                self.fit.fit_model(verbose=fit_verbose, cov_type='HC3')

                if self.fit.params.size == 5:
                    # Success!
                    breaks = breaks[i]
                    break
                i += 1
                if i >= breaks.size:
                    warnings.warn("No good break point found. Returned fit\
                                   does not include a break!")
                    break

            # return self

        if not log_break:
            breaks = np.log10(breaks)

        # Fit the final model with whichever breaks were passed.
        self.fit = Lm_Seg(x, y, breaks)
        self.fit.fit_model(verbose=fit_verbose, cov_type='HC3')

        if bootstrap:
            stderrs = residual_bootstrap(self.fit.fit,
                                         **bootstrap_kwargs)

            self._slope_errs = stderrs[1:-1]
        else:
            self._slope_errs = self.fit.slope_errs

        self._slope = self.fit.slopes

        self._bootstrap_flag = bootstrap
Exemple #19
0
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import UnivariateSpline
nItems = 500
x = np.linspace(-3, 3, nItems)
y = np.exp(-x**2) + 0.1 * np.random.randn(nItems)
plt.plot(x, y, 'ro', ms=5)

smoothFactor = float(nItems) / 50.0

spl = UnivariateSpline(x, y, k = 2, s = smoothFactor)
xs = np.linspace(-3, 3, 1000)
plt.plot(xs, spl(xs), 'g', lw=3)

coeffs = spl.get_coeffs()
knots  = spl.get_knots()

print 'coeffs', coeffs
print 'knots', knots

knotsY = spl(knots)
plt.plot(knots, knotsY, 'bo')

#spl.set_smoothing_factor(0.5)
#print 'coeffs', spl.get_coeffs()
#print 'knots', spl.get_knots()
#plt.plot(xs, spl(xs), 'b', lw=3)
plt.show()

class SplineSmoother(object):
	'''
	claselfdocs
	'''


	def __init__(self, yp, workdir, scale, sm=200):
		'''
		Constructor
		'''
		
		self.workdir = workdir
		yp = np.array(yp)
		self.l = len(yp)
		self.xPos = (self.l-2)/2 #fPos = (self.l-2)/2 + 2
		tnsc = 2/scale
		print tnsc
		plt.rcParams['font.size'] = 24
		plt.rcParams['lines.linewidth'] = 2.4
		self.workdir = workdir
		
		self.avx = np.append(np.append([0], np.sort(np.tanh(tnsc*yp[:self.xPos]))),[1])
		self.av = yp[self.xPos:]
		self.m = UnivariateSpline(self.avx, self.av)
		
		plt.rcParams['font.size'] = 24
		plt.rcParams['lines.linewidth'] = 2.4

		print "Created spline with " + str(len(self.m.get_knots())) + " knots"

	def saveSpline(self, filename):
		tp = np.linspace(0, 1, 1000)
		with open(filename ,"w+") as f:
			for i in range(0, 1000):
				f.write( str(tp[i]) + " , " + str(self.m(tp[i])) )
				if i < 999:
					f.write("\n")
		f.close()
	
	def showSpline(self, order=0):
		plt.clf()
		print "Spline full information:"
		print self.m.get_knots()
		print self.m.get_coeffs()
		print self.m.get_residual()
		tp = np.linspace(0, 1, 1000)
		plt.scatter(self.avx, self.av)
		plt.plot(tp,self.m(tp))
		plt.savefig(self.workdir+"/splineFit.pdf")
		if order > 0:
			plt.plot(tp,self.m(tp,1))
			if order > 1:
				plt.plot(tp,self.m(tp,2))
			plt.savefig(self.workdir+"/splineDerivative.pdf")
	
	def plotSplineData(self, dataContainer, s, p, se, yscale):
		plt.clf()
		plt.xlim(0,1)
		plt.ylim(0,yscale)
		tp = np.linspace(0, 1, 500)
		plt.scatter(dataContainer.points[0],dataContainer.points[1]+dataContainer.background, c='b', marker='o', s=5)
		plt.plot(tp, self.m(tp)+dataContainer.background,'r', linewidth=2)
		plt.plot(tp, self.m(tp)+np.sqrt(s*(p*self.m(tp)*self.m(tp)+self.m(tp)+se))+dataContainer.background,'r--', linewidth=2)
		plt.plot(tp, self.m(tp)-np.sqrt(s*(p*self.m(tp)*self.m(tp)+self.m(tp)+se))+dataContainer.background,'r--', linewidth=2)
		plt.plot(tp, np.zeros(500) + dataContainer.background, '--', c='#BBBBBB', alpha=0.8)
		plt.savefig(self.workdir+"/splineVsData.pdf")
		
	def plotBinnedData(self, dataContainer, s, p, se, xmin, xmax):
		plt.clf()
		sigma = lambda x: s * (p*x*x + x + se)
		t = np.linspace(xmin, xmax, 500)
		plt.xlim(xmin, xmax)
		plt.plot(t, sigma(t))
		plt.plot(dataContainer.avs, np.power(dataContainer.stds,2), 'o')
		plt.savefig(self.workdir+"/noiseVsBins.pdf")
		plt.clf()
		tp = np.linspace(0, 1, dataContainer.numBins)
		plt.plot(tp, self.m(tp),'r', linewidth=2)
		plt.plot(tp, dataContainer.avs, 'o')
		plt.savefig(self.workdir+"/splineVsBins.pdf")
	
	def plotFisherInfo(self, dataContainer, s, p, se, ymax, ymaxsq):
		plt.clf()
		t = np.linspace(0, 1, 1000)
		
		minf = lambda x: -1 * self.m(x)
		minx = fminbound(minf, 0, 1)
		fval = self.m(minx)
		s = s/fval
		se = se/fval
		p = p*fval
		print fval, s, p, se
		fi = lambda m, mp: 2*np.power(s * (p * np.power(m,2) + m),2)/(np.power(mp,2) * 
		(2 * s * (p * np.power(m,2) + m) + np.power(s * (2 * p * m + 1), 2)))
		fiapp = lambda m, mp: s * (p * np.power(m,2) + m) / np.power(mp,2)
		plt.xlim(0, 1)
		plt.ylim(0, ymaxsq)
		plt.plot(t, fi(self.m(t)/fval, self.m(t, 1)/fval))
		plt.plot(t, fiapp(self.m(t)/fval, self.m(t, 1)/fval), 'r')
		plt.savefig(self.workdir+"/variance.pdf")
		plt.clf()
		plt.ylim(0, ymax)
		plt.plot(t, np.sqrt(fi(self.m(t)/fval, self.m(t, 1)/fval)))
		plt.plot(t, np.sqrt(fiapp(self.m(t)/fval, self.m(t, 1)/fval)), 'r')
		plt.savefig(self.workdir+"/stddev.pdf")
		plt.clf()
		plt.plot(t, 1/fi(self.m(t)/fval, self.m(t, 1)/fval))
		plt.plot(t, 1/fiapp(self.m(t)/fval, self.m(t, 1)/fval), 'r')
		plt.savefig(self.workdir+"/fisherInfo.pdf")
		with open(self.workdir+"/variance.csv" ,"w+") as f:
			fisher=np.sqrt(fi(self.m(t)/fval, self.m(t, 1)/fval))
			for i in range(0, 1000):
				f.write( str(t[i]) + " , " + str(fisher[i]) )
				if i < 999:
					f.write("\n")
		f.close()
    '/Volumes/Transcend/pumd_data_files/processed_data_5yrs_bucket_jun20/mtbi_avg_spend_intrvw_2011_to_2015.csv'
)
test_pipe = test_pipe[test_pipe['UCC'] == ucc]

x = test_pipe['AGE_REF']
y = test_pipe['AVG_SPEND']
plt.plot(x, y, 'ro', ms=5)

# c_spline = CubicSpline(x, y)
# xs = np.linspace(20, 80, 1000)
# plt.plot(xs, c_spline(xs), 'b')

# s=33500000
for i in range(0, 99999999, 1000):
    u_spline = UnivariateSpline(x, y, s=i)
    knot_count = len(u_spline.get_knots())
    print(knot_count)
    if knot_count <= 8:
        break
xs = np.linspace(20, 80, 1000)
plt.plot(xs, u_spline(xs), 'b', lw=3)

# u_spline.set_smoothing_factor(0.5)
# plt.plot(xs, u_spline(xs), 'g')
plt.title("Special lump sum mortgage payment (owned home) [2011-2015]")
plt.xlabel('Age')
plt.ylabel('Average Spend in $')
plt.show()

# x = np.linspace(30, 80, 50)
# y = np.exp(-x**3) + 0.1 * np.random.randn(50)
Exemple #22
0
    def update_adapt(self, x, p, x1_base, delta):
        # basic sanity check of input estimates
        # don't update if base estimate doesn't exist
        if ~np.isfinite(x1_base):
            return

        ## DICTIONARY management
        # add to dictionary
        if self.x1_d is None:  # need to create dictionary
            self.x1_d = x - x1_base
            self.p_d = p
            self.delta_d = np.repeat(delta, len(x))  #uncertainty of base
            self.time_d = np.repeat(self.i, len(x))  #time of point
            self.x1_base = np.repeat(x1_base, len(x))  #base estimate of point
        else:
            self.x1_d = np.append(self.x1_d, x - x1_base)
            self.p_d = np.append(self.p_d, p)
            self.delta_d = np.append(self.delta_d, np.repeat(delta, len(x)))
            self.time_d = np.append(self.time_d, np.repeat(self.i, len(x)))
            self.x1_base = np.append(self.x1_base, np.repeat(x1_base, len(x)))

        # if over budget prune
        if self.p_d.shape[0] > self.budget:
            N_rmv = len(self.x1_d) - self.budget
            dff = np.diff(self.x1_d)
            mdff = .5 * (dff[1:len(dff)] + dff[0:(len(dff) - 1)])
            mdff = np.concatenate([[dff[0]], mdff, [dff[len(dff) - 1]]])
            eps = self.delta_d / mdff  # accuracy of base estimate / closeness to nearby points
            rmv = np.where(
                np.logical_or(eps >= -np.sort(-eps)[N_rmv - 1],
                              ~np.isfinite(eps)))[0]
            if len(rmv) > N_rmv:
                rmv = np.random.choice(a=rmv, size=N_rmv, replace=False)
            # remove pts from dictionary and assoc. information
            self.x1_d = np.delete(self.x1_d, rmv, axis=0)
            self.p_d = np.delete(self.p_d, rmv, axis=0)
            self.delta_d = np.delete(self.delta_d, rmv, axis=0)
            self.time_d = np.delete(self.time_d, rmv, axis=0)
            self.x1_base = np.delete(self.x1_base, rmv, axis=0)

        # sort and remove non-unique values (more stable fitting)
        # remove non-unique from dict
        ux, iux = np.unique(self.x1_d, return_index=True)
        self.x1_d = ux
        self.p_d = self.p_d[iux]
        self.delta_d = self.delta_d[iux]
        self.time_d = self.time_d[iux]
        self.x1_base = self.x1_base[iux]
        # sort dict
        ordx = np.argsort(self.x1_d, axis=0).reshape(-1)
        self.x1_d = self.x1_d[ordx]
        self.p_d = self.p_d[ordx]
        self.delta_d = self.delta_d[ordx]
        self.time_d = self.time_d[ordx]
        self.x1_base = self.x1_base[ordx]

        ## ADAPTIVE PROFILE fitting
        # need more data than knots
        if len(self.x1_d) <= self.K:
            return
        # determine location of knots based on some heuristics
        if len(self.x1_d) > 5 and self.n_knots > 0:
            l3len = int(np.ceil(len(self.x1_d) // 3))
            if self.n_knots is not None:
                qs = np.linspace(.25, .75, np.min([l3len, self.n_knots]))
            else:
                qs = np.linspace(.25, .75, l3len)
            kseq = np.quantile(self.x1_d, qs)
            self.kseq = kseq[1:(len(kseq) - 1)]
        else:
            self.kseq = []

        self.x_design = self.make_adapt_design(self.x1_d, update=True)

        self.p_mean = np.mean(self.p_d)
        self.p_std = np.std(self.p_d)
        self.p_z = (self.p_d - self.p_mean) / self.p_std

        # fit model and its derivative
        #just make sure everything is monotonic (for stability)
        mono_fit = IsotonicRegression().fit(self.x_design, self.p_z)
        mono_p = mono_fit.predict(self.x_design)
        if self.n_knots < 0:

            def err_mod(mod):  #RSS for fit model
                return np.sum((mod(self.x1_d) - self.p_z)**2) / len(self.p_z)

            smooth_spl = UnivariateSpline(self.x_design,
                                          mono_p,
                                          k=self.K,
                                          ext=0,
                                          s=len(self.p_z) * self.gamma)
            # some heuristics on what knots to keep
            knts = smooth_spl.get_knots()
            keep_knts = ~np.logical_or(knts < np.quantile(self.x1_d, .1),
                                       knts > np.quantile(self.x1_d, .9))
            knts = knts[keep_knts]
            linear_mod = LSQUnivariateSpline(self.x_design,
                                             mono_p,
                                             k=self.K,
                                             ext=0,
                                             t=[])
            self.adapt = LSQUnivariateSpline(self.x_design,
                                             mono_p,
                                             k=self.K,
                                             ext=0,
                                             t=knts)
            adapt_err = err_mod(self.adapt)
            lin_err = err_mod(linear_mod)
            #if its not twice as good as a simple linear model just use the latter
            if lin_err > 0 and adapt_err / lin_err > 0.5:
                self.adapt = linear_mod
        else:  #n_knots > 0
            self.adapt = LSQUnivariateSpline(self.x_design,
                                             mono_p,
                                             k=self.K,
                                             ext=0,
                                             t=self.kseq)
        self.adapt_deriv = self.adapt.derivative(1)
Exemple #23
0
    def fit_pspec(self,
                  breaks=None,
                  log_break=True,
                  lg_scale_cut=2,
                  verbose=False):
        '''
        Fit the 1D Power spectrum using a segmented linear model. Note that
        the current implementation allows for only 1 break point in the
        model. If the break point is estimated via a spline, the breaks are
        tested, starting from the largest, until the model finds a good fit.

        Parameters
        ----------
        breaks : float or None, optional
            Guesses for the break points. If given as a list, the length of
            the list sets the number of break points to be fit. If a choice is
            outside of the allowed range from the data, Lm_Seg will raise an
            error. If None, a spline is used to estimate the breaks.
        log_break : bool, optional
            Sets whether the provided break estimates are log-ed values.
        lg_scale_cut : int, optional
            Cuts off largest scales, which deviate from the powerlaw.
        verbose : bool, optional
            Enables verbose mode in Lm_Seg.
        '''

        if breaks is None:
            from scipy.interpolate import UnivariateSpline

            # Need to order the points
            shape = self.vel_freqs.size
            spline_y = np.log10(self.ps1D[1:shape / 2])
            spline_x = np.log10(self.vel_freqs[1:shape / 2])

            spline = UnivariateSpline(spline_x, spline_y, k=1, s=1)

            # The first and last are just the min and max of x
            breaks = spline.get_knots()[1:-1]

            if verbose:
                print "Breaks found from spline are: " + str(breaks)

            # Take the number according to max_breaks starting at the
            # largest x.
            breaks = breaks[::-1]

            x = np.log10(self.vel_freqs[lg_scale_cut + 1:-lg_scale_cut])
            y = np.log10(self.ps1D[lg_scale_cut + 1:-lg_scale_cut])

            # Now try these breaks until a good fit including the break is
            # found. If none are found, it accept that there wasn't a good
            # break and continues on.
            i = 0
            while True:
                self.fit = \
                    Lm_Seg(x, y, breaks[i])
                self.fit.fit_model(verbose=verbose)

                if self.fit.params.size == 5:
                    break
                i += 1
                if i >= breaks.shape:
                    warnings.warn("No good break point found. Returned fit\
                                   does not include a break!")
                    break

            return self

        if not log_break:
            breaks = np.log10(breaks)

        self.fit = \
            Lm_Seg(np.log10(self.vel_freqs[lg_scale_cut+1:-lg_scale_cut]),
                   np.log10(self.ps1D[lg_scale_cut+1:-lg_scale_cut]), breaks)
        self.fit.fit_model(verbose=verbose)
Exemple #24
0
    def fit_pspec(self, breaks=None, log_break=True, lg_scale_cut=2, verbose=False):
        """
        Fit the 1D Power spectrum using a segmented linear model. Note that
        the current implementation allows for only 1 break point in the
        model. If the break point is estimated via a spline, the breaks are
        tested, starting from the largest, until the model finds a good fit.

        Parameters
        ----------
        breaks : float or None, optional
            Guesses for the break points. If given as a list, the length of
            the list sets the number of break points to be fit. If a choice is
            outside of the allowed range from the data, Lm_Seg will raise an
            error. If None, a spline is used to estimate the breaks.
        log_break : bool, optional
            Sets whether the provided break estimates are log-ed values.
        lg_scale_cut : int, optional
            Cuts off largest scales, which deviate from the powerlaw.
        verbose : bool, optional
            Enables verbose mode in Lm_Seg.
        """

        if breaks is None:
            from scipy.interpolate import UnivariateSpline

            # Need to order the points
            shape = self.vel_freqs.size
            spline_y = np.log10(self.ps1D[1 : shape / 2])
            spline_x = np.log10(self.vel_freqs[1 : shape / 2])

            spline = UnivariateSpline(spline_x, spline_y, k=1, s=1)

            # The first and last are just the min and max of x
            breaks = spline.get_knots()[1:-1]

            if verbose:
                print "Breaks found from spline are: " + str(breaks)

            # Take the number according to max_breaks starting at the
            # largest x.
            breaks = breaks[::-1]

            x = np.log10(self.vel_freqs[lg_scale_cut + 1 : -lg_scale_cut])
            y = np.log10(self.ps1D[lg_scale_cut + 1 : -lg_scale_cut])

            # Now try these breaks until a good fit including the break is
            # found. If none are found, it accept that there wasn't a good
            # break and continues on.
            i = 0
            while True:
                self.fit = Lm_Seg(x, y, breaks[i])
                self.fit.fit_model(verbose=verbose)

                if self.fit.params.size == 5:
                    break
                i += 1
                if i >= breaks.shape:
                    warnings.warn(
                        "No good break point found. Returned fit\
                                   does not include a break!"
                    )
                    break

            return self

        if not log_break:
            breaks = np.log10(breaks)

        self.fit = Lm_Seg(
            np.log10(self.vel_freqs[lg_scale_cut + 1 : -lg_scale_cut]),
            np.log10(self.ps1D[lg_scale_cut + 1 : -lg_scale_cut]),
            breaks,
        )
        self.fit.fit_model(verbose=verbose)

        return self
Exemple #25
0
class SplineSmoother(object):
    '''
	claselfdocs
	'''
    def __init__(self, yp, workdir, scale, sm=200):
        '''
		Constructor
		'''

        self.workdir = workdir
        yp = np.array(yp)
        self.l = len(yp)
        self.xPos = (self.l - 2) / 2  #fPos = (self.l-2)/2 + 2
        tnsc = 2 / scale
        print tnsc
        plt.rcParams['font.size'] = 24
        plt.rcParams['lines.linewidth'] = 2.4
        self.workdir = workdir

        self.avx = np.append(
            np.append([0], np.sort(np.tanh(tnsc * yp[:self.xPos]))), [1])
        self.av = yp[self.xPos:]
        self.m = UnivariateSpline(self.avx, self.av)

        plt.rcParams['font.size'] = 24
        plt.rcParams['lines.linewidth'] = 2.4

        print "Created spline with " + str(len(self.m.get_knots())) + " knots"

    def saveSpline(self, filename):
        tp = np.linspace(0, 1, 1000)
        with open(filename, "w+") as f:
            for i in range(0, 1000):
                f.write(str(tp[i]) + " , " + str(self.m(tp[i])))
                if i < 999:
                    f.write("\n")
        f.close()

    def showSpline(self, order=0):
        plt.clf()
        print "Spline full information:"
        print self.m.get_knots()
        print self.m.get_coeffs()
        print self.m.get_residual()
        tp = np.linspace(0, 1, 1000)
        plt.scatter(self.avx, self.av)
        plt.plot(tp, self.m(tp))
        plt.savefig(self.workdir + "/splineFit.pdf")
        if order > 0:
            plt.plot(tp, self.m(tp, 1))
            if order > 1:
                plt.plot(tp, self.m(tp, 2))
            plt.savefig(self.workdir + "/splineDerivative.pdf")

    def plotSplineData(self, dataContainer, s, p, se, yscale):
        plt.clf()
        plt.xlim(0, 1)
        plt.ylim(0, yscale)
        tp = np.linspace(0, 1, 500)
        plt.scatter(dataContainer.points[0],
                    dataContainer.points[1] + dataContainer.background,
                    c='b',
                    marker='o',
                    s=5)
        plt.plot(tp, self.m(tp) + dataContainer.background, 'r', linewidth=2)
        plt.plot(tp,
                 self.m(tp) +
                 np.sqrt(s * (p * self.m(tp) * self.m(tp) + self.m(tp) + se)) +
                 dataContainer.background,
                 'r--',
                 linewidth=2)
        plt.plot(tp,
                 self.m(tp) -
                 np.sqrt(s * (p * self.m(tp) * self.m(tp) + self.m(tp) + se)) +
                 dataContainer.background,
                 'r--',
                 linewidth=2)
        plt.plot(tp,
                 np.zeros(500) + dataContainer.background,
                 '--',
                 c='#BBBBBB',
                 alpha=0.8)
        plt.savefig(self.workdir + "/splineVsData.pdf")

    def plotBinnedData(self, dataContainer, s, p, se, xmin, xmax):
        plt.clf()
        sigma = lambda x: s * (p * x * x + x + se)
        t = np.linspace(xmin, xmax, 500)
        plt.xlim(xmin, xmax)
        plt.plot(t, sigma(t))
        plt.plot(dataContainer.avs, np.power(dataContainer.stds, 2), 'o')
        plt.savefig(self.workdir + "/noiseVsBins.pdf")
        plt.clf()
        tp = np.linspace(0, 1, dataContainer.numBins)
        plt.plot(tp, self.m(tp), 'r', linewidth=2)
        plt.plot(tp, dataContainer.avs, 'o')
        plt.savefig(self.workdir + "/splineVsBins.pdf")

    def plotFisherInfo(self, dataContainer, s, p, se, ymax, ymaxsq):
        plt.clf()
        t = np.linspace(0, 1, 1000)

        minf = lambda x: -1 * self.m(x)
        minx = fminbound(minf, 0, 1)
        fval = self.m(minx)
        s = s / fval
        se = se / fval
        p = p * fval
        print fval, s, p, se
        fi = lambda m, mp: 2 * np.power(s * (p * np.power(m, 2) + m), 2) / (
            np.power(mp, 2) *
            (2 * s *
             (p * np.power(m, 2) + m) + np.power(s * (2 * p * m + 1), 2)))
        fiapp = lambda m, mp: s * (p * np.power(m, 2) + m) / np.power(mp, 2)
        plt.xlim(0, 1)
        plt.ylim(0, ymaxsq)
        plt.plot(t, fi(self.m(t) / fval, self.m(t, 1) / fval))
        plt.plot(t, fiapp(self.m(t) / fval, self.m(t, 1) / fval), 'r')
        plt.savefig(self.workdir + "/variance.pdf")
        plt.clf()
        plt.ylim(0, ymax)
        plt.plot(t, np.sqrt(fi(self.m(t) / fval, self.m(t, 1) / fval)))
        plt.plot(t, np.sqrt(fiapp(self.m(t) / fval, self.m(t, 1) / fval)), 'r')
        plt.savefig(self.workdir + "/stddev.pdf")
        plt.clf()
        plt.plot(t, 1 / fi(self.m(t) / fval, self.m(t, 1) / fval))
        plt.plot(t, 1 / fiapp(self.m(t) / fval, self.m(t, 1) / fval), 'r')
        plt.savefig(self.workdir + "/fisherInfo.pdf")
        with open(self.workdir + "/variance.csv", "w+") as f:
            fisher = np.sqrt(fi(self.m(t) / fval, self.m(t, 1) / fval))
            for i in range(0, 1000):
                f.write(str(t[i]) + " , " + str(fisher[i]))
                if i < 999:
                    f.write("\n")
        f.close()
Exemple #26
0
from scipy.interpolate import LSQUnivariateSpline, UnivariateSpline
import matplotlib.pyplot as plt
x = np.linspace(-3, 3, 50)
y = np.exp(-x**2) + 0.1 * np.random.randn(50)

# Fit a smoothing spline with a pre-defined internal knots:

t = [-1, 0, 1]
spl = LSQUnivariateSpline(x, y, t)

xs = np.linspace(-3, 3, 1000)
plt.plot(x, y, 'ro', ms=5)
plt.plot(xs, spl(xs), 'g-', lw=3)
plt.show()

# Check the knot vector:

spl.get_knots()
# array([-3., -1., 0., 1., 3.])

# Constructing lsq spline using the knots from another spline:

x = np.arange(10)
s = UnivariateSpline(x, x, s=0)
s.get_knots()
# array([ 0.,  2.,  3.,  4.,  5.,  6.,  7.,  9.])
knt = s.get_knots()
s1 = LSQUnivariateSpline(x, x, knt[1:-1])  # Chop 1st and last knot
s1.get_knots()
# array([ 0.,  2.,  3.,  4.,  5.,  6.,  7.,  9.])
Exemple #27
0
            # Nonlinear regression exponential
            b = optimize.curve_fit(exp_model,
                                   t,
                                   v,
                                   p0=[v[0], 1, v.min()],
                                   bounds=[[0, 0, v.min()],
                                           [v.max(), np.inf,
                                            v.max()]])
            yhat = exp_model(t, b[0][0], b[0][1], b[0][2])
            res_exp.append((v - yhat) / v)

            # B-spline
            spl = UnivariateSpline(t, v, k=3, s=1e6)
            #            yhat_spl.append(spl(t))
            res_spl.append((v - spl(t)) / v)
            nknots.append(len(spl.get_knots()))

#            plt.subplot(2,3,counter+1)
#            plt.plot(t,spl(t),'g')
#            plt.plot(t,v,'b')

        except:
            print 'Fitting failed for ', c.iloc[0].CellID

#auto_res_exp = np.hstack(res_exp)
auto_res_spl = np.hstack(res_spl)
auto_res_exp = np.hstack(res_exp)

plt.figure(1)
weights = np.ones_like(auto_res_exp) / float(len(auto_res_exp))
bins = np.linspace(-1, 1, 25)
Exemple #28
0
    def fit_pspec(self,
                  breaks=None,
                  log_break=True,
                  low_cut=None,
                  high_cut=None,
                  fit_verbose=False):
        '''
        Fit the 1D Power spectrum using a segmented linear model. Note that
        the current implementation allows for only 1 break point in the
        model. If the break point is estimated via a spline, the breaks are
        tested, starting from the largest, until the model finds a good fit.

        Parameters
        ----------
        breaks : float or None, optional
            Guesses for the break points. If given as a list, the length of
            the list sets the number of break points to be fit. If a choice is
            outside of the allowed range from the data, Lm_Seg will raise an
            error. If None, a spline is used to estimate the breaks.
        log_break : bool, optional
            Sets whether the provided break estimates are log-ed values.
        low_cut : `~astropy.units.Quantity`, optional
            Lowest frequency to consider in the fit.
        high_cut : `~astropy.units.Quantity`, optional
            Highest frequency to consider in the fit.
        fit_verbose : bool, optional
            Enables verbose mode in Lm_Seg.
        '''

        # Make the data to fit to
        if low_cut is None:
            # Default to the largest frequency, since this is just 1 pixel
            # But cut out fitting the total power point.
            self.low_cut = 0.98 / (float(self.data.shape[0]) * u.pix)
        else:
            self.low_cut = \
                self._spectral_freq_unit_conversion(low_cut, u.pix**-1)

        if high_cut is None:
            # Set to something larger than
            self.high_cut = (self.freqs.max().value * 1.01) / u.pix
        else:
            self.high_cut = \
                self._spectral_freq_unit_conversion(high_cut, u.pix**-1)

        # We keep both the real and imag frequencies, but we only need the real
        # component when fitting. We'll cut off the total power frequency too
        # in case it causes an extra break point (which seems to happen).
        shape = self.freqs.size
        rfreqs = self.freqs[1:shape // 2].value
        ps1D = self.ps1D[1:shape // 2]

        y = np.log10(ps1D[clip_func(rfreqs, self.low_cut.value,
                                    self.high_cut.value)])
        x = np.log10(rfreqs[clip_func(rfreqs, self.low_cut.value,
                                      self.high_cut.value)])

        if breaks is None:
            from scipy.interpolate import UnivariateSpline

            # Need to order the points
            spline = UnivariateSpline(x, y, k=1, s=1)

            # The first and last are just the min and max of x
            breaks = spline.get_knots()[1:-1]

            if fit_verbose:
                print("Breaks found from spline are: " + str(breaks))

            # Take the number according to max_breaks starting at the
            # largest x.
            breaks = breaks[::-1]

            # Ensure a break doesn't fall at the max or min.
            if breaks.size > 0:
                if breaks[0] == x.max():
                    breaks = breaks[1:]
            if breaks.size > 0:
                if breaks[-1] == x.min():
                    breaks = breaks[:-1]

            if x.size <= 3 or y.size <= 3:
                raise Warning("There are no points to fit to. Try lowering "
                              "'lg_scale_cut'.")

            # If no breaks, set to half-way before last point
            if breaks.size == 0:
                x_copy = np.sort(x.copy())
                breaks = np.array([0.5 * (x_copy[-1] + x_copy[-3])])

            # Now try these breaks until a good fit including the break is
            # found. If none are found, it accepts that there wasn't a good
            # break and continues on.
            i = 0
            while True:
                self.fit = \
                    Lm_Seg(x, y, breaks[i])
                self.fit.fit_model(verbose=fit_verbose)

                if self.fit.params.size == 5:
                    # Success!
                    break
                i += 1
                if i >= breaks.size:
                    warnings.warn("No good break point found. Returned fit\
                                   does not include a break!")
                    break

            return self

        if not log_break:
            breaks = np.log10(breaks)

        # Fit the final model with whichever breaks were passed.
        self.fit = Lm_Seg(x, y, breaks)
        self.fit.fit_model(verbose=fit_verbose)
Exemple #29
0
# get the knots from the most difficult one
spl_0 = UnivariateSpline(hist_per_pe.bin_centers[220:511:1],
                         hist_per_pe.data[10][220:511:1],
                         s=0,
                         k=3)
'''
spl_0 = []
for i,pe in enumerate(pes):
    spl_0.append(UnivariateSpline(hist_per_pe.bin_centers[220:511:1], hist_per_pe.data[i][220:511:1], s=0,k=3))
    print(pe,len(spl_0[-1].get_knots()))
'''
#plt.figure(0)
#plt.step(hist_per_pe.bin_centers[220:511:1],hist_per_pe.data[-1][220:511:1])
#plt.plot(xs, spl_0(xs), 'k-', lw=1)

knt0 = spl_0.get_knots()
wei0 = spl_0.get_coeffs()
print(knt0)

#knt0 = hist_per_pe.bin_centers[221:510:1]
#print(knt0)
#print(5./0)
list_coef = []
for i, pe in enumerate(pes):
    if i > 99: continue
    #t = hist_per_pe.bin_centers[236:509:2]
    spl.append(
        LSQUnivariateSpline(hist_per_pe.bin_centers[220:511:1],
                            hist_per_pe.data[i][220:511:1], knt0[1:-1]))
    list_coef.append(list(spl[-1].get_coeffs()))