def spline_derivatives(landscape,n_bins=100): """ :param landscape: original landscape :param n_bins: number of bins for fitting :return: tuple of (spline fit to A_z, split fit to A_dot, fit to A_ddot """ z = landscape.z A_z = landscape.A_z sort_idx = np.argsort(z) z_sort = z[sort_idx] A_sort = A_z[sort_idx] knots = np.linspace(min(z_sort),max(z_sort),endpoint=True,num=n_bins) spline_A_z = LSQUnivariateSpline(x=z_sort,y=A_sort,k=3,t=knots[1:-1]) A_dot_spline_z = spline_A_z.derivative(1)(z) A_ddot_spline_z = spline_A_z.derivative(2)(z) return spline_A_z,A_dot_spline_z,A_ddot_spline_z
def spline_derivatives(landscape, n_bins=100): """ :param landscape: original landscape :param n_bins: number of bins for fitting :return: tuple of (spline fit to A_z, split fit to A_dot, fit to A_ddot """ z = landscape.z A_z = landscape.A_z sort_idx = np.argsort(z) z_sort = z[sort_idx] A_sort = A_z[sort_idx] knots = np.linspace(min(z_sort), max(z_sort), endpoint=True, num=n_bins) spline_A_z = LSQUnivariateSpline(x=z_sort, y=A_sort, k=3, t=knots[1:-1]) A_dot_spline_z = spline_A_z.derivative(1)(z) A_ddot_spline_z = spline_A_z.derivative(2)(z) return spline_A_z, A_dot_spline_z, A_ddot_spline_z
class dataSmoother: def __init__(self, file, knots=[]): self.filename = file with open(file, "r") as f: inData = genfromtxt(f, float, comments='#').T self.rho = inData[0] self.orig_data = inData[1] self.spline = LSQUnivariateSpline(self.rho, self.orig_data, knots, k=5) def plot_orig(self): fig = self._plot_base() fig.scatter(self.rho, self.orig_data, color='red') plt.show() return fig def plot(self): fig = self._plot_base() fig.scatter(self.rho, self.orig_data, color='red') fig.plot(self.rho, self.spline(self.rho), color='black') fig.scatter(self.rho, self.spline(self.rho), color='green', marker="x") fig.legend(["Spline", "Original", "Cleaned"]) plt.show() return fig def save(self): answer = "" while answer not in ["y", "n"]: answer = input("OK to continue [Y/N]? ").lower() print("Backing up old file") os.rename(self.filename, self.filename + '.bak') contents = np.array([self.rho, self.spline(self.rho)]).T savetxt(self.filename + "_clean", contents, delimiter=' ') def get_spline(self): return self.spline def set_knots(self, knots): self.spline = LSQUnivariateSpline(self.rho, self.orig_data, knots, k=5) print("knots set") def _plot_base(self): plot = plt.figure() fig = plot.add_subplot(111) fig.set_xlabel(r'$\rho', fontsize=30) fig.set_ylabel(r'Data', fontsize=30) plt.xticks(fontsize=20) plt.yticks(fontsize=20) return fig def plot_derivative(self): fig = self._plot_base() fig.scatter(self.rho, self.spline.derivative()(self.rho), color="blue") return fig
def spline(bin_size, offsets): """compute piecewise polynomial splines of degree three""" x, y = zip(*offsets) xs = np.arange(x[0], x[-1], 120) # array of knots (with the start and end which is addes automatically 13 knots meanaing 12 pieces) t = [offsets[0][0] + bin_size, offsets[0][0] + bin_size * 2, offsets[0][0] + 3 * bin_size, offsets[0][0] + 4 * bin_size, offsets[0][0] + 5 * bin_size, offsets[0][0] + 6 * bin_size, offsets[0][0] + 7 * bin_size, offsets[0][0] + 8 * bin_size, offsets[0][0] + 9 * bin_size, offsets[0][0] + 10 * bin_size, offsets[0][0] + 11 * bin_size] # compute a spline polynomial of degree 3 over 5 equal pieces for the y points over steps of 1 sec on the x axis. try: spl = LSQUnivariateSpline(x, y, t, w=None, bbox=[None, None], k=3) except ValueError as e: logging.error("ERROR: LSQUnivariateSpline ValueError failed with error {} and params x {} y {} t {} ".format(e, x, y, t)) raise spl_deriv = spl.derivative(1) # derivative of degree one orig_curve = spl(xs) deriv_curve = spl_deriv(xs) return orig_curve, deriv_curve, xs
class Tracker: def __init__(self, lb=-10, ub=10, budget=100, adapt_options={}, n_knots=-1, K=1, lam=1E-2, gamma=1E-2): """ An APP Tracker object. Parameters: lb (float): the lower bound for the range of possible STLE locations. ub (float): the upper bound for the range of possible STLE locations. budget (int): the number of dictionary data points. adapt_options: dictionary of options to be passed to the nlopt optimizer. Options include 'xtol_rel', 'maxeval', 'maxtime', and 'algo'. n_knots (int): number of knots for adaptive pressure profile. Defaults to -1 which chooses knots adaptively. K (int): degree of adaptive spline fit. Defaults to K=1. lam (float): L2 penalization parameter. gamma (float): Adaptive pressure profile spline fit smoothness parameter. Returns: An object of the Tracker class. """ # fixed parameters of algorithm self.lb = lb self.ub = ub self.budget = budget self.n_knots = n_knots self.K = K self.lam = lam self.gamma = gamma self.i = 0 # internal counter for num processed timepoints # estimated properties self.est_adapt = None self.p_mean = None self.p_std = None self.x1_d = None self.p_d = None self.delta_d = None self.time_d = None #optimizer self.adapt = None algo = nlopt.GN_CRS2_LM self.optim_params_init = { 'maxeval': 100, 'xtol_rel': 1E-10, 'algo': algo, 'maxtime': -1 } self.adapt_opt = self.init_opt(adapt_options, 1) self.adapt_opt.set_lower_bounds(self.lb) self.adapt_opt.set_upper_bounds(self.ub) def make_adapt_design(self, x, update=False): return x.reshape(-1) def init_opt(self, new_options, npar): ret_options = self.optim_params_init.copy() for k, v in new_options.items(): ret_options[k] = v opt = nlopt.opt(ret_options['algo'], npar) opt.set_xtol_rel(ret_options['xtol_rel']) opt.set_maxeval(ret_options['maxeval']) opt.set_maxtime(ret_options['maxtime']) return opt def predict(self, x, p, base_method=None, std_x=True): """ Predict STLE location and update adaptive fit. Parameters: x (float): the streamwise locations of the pressure transducers. p (float): the measure pressures. base_method (function): a base method to produce a weak estimate. Defaults to pressure ratio method. Function takes two arguments x and p and produces a weak estimate (and a measurement of uncertainty). std_x (boolean): Should fitting be done using standardized streamwise locations. """ self.i = self.i + 1 x_mean = 0 x_std = 1 if std_x: x_mean = np.mean(x) x_std = np.std(x) x = (x - x_mean) / x_std if base_method is None: base_method = lambda x, p: self.pressure_ratio(x, p, pa=1) # run base learner x1_base, delta = base_method(x, p) # update adaptive fit self.update_adapt(x, p, x1_base, delta) #initial guess for adaptive optimization (x0) x0 = self.est_adapt if x0 is None or ~np.isfinite(x0): x0 = x1_base # adaptive estimate if self.adapt is not None: self.est_adapt, adapt_cov = self.fit_adapt(x, p, x0) else: self.est_adapt = x0 adapt_cov = float('inf') return { 'est': self.est_adapt * x_std + x_mean, 'est_cov': adapt_cov * x_std**2, 'base': x1_base * x_std + x_mean, 'delta': delta } def update_adapt(self, x, p, x1_base, delta): # basic sanity check of input estimates # don't update if base estimate doesn't exist if ~np.isfinite(x1_base): return ## DICTIONARY management # add to dictionary if self.x1_d is None: # need to create dictionary self.x1_d = x - x1_base self.p_d = p self.delta_d = np.repeat(delta, len(x)) #uncertainty of base self.time_d = np.repeat(self.i, len(x)) #time of point self.x1_base = np.repeat(x1_base, len(x)) #base estimate of point else: self.x1_d = np.append(self.x1_d, x - x1_base) self.p_d = np.append(self.p_d, p) self.delta_d = np.append(self.delta_d, np.repeat(delta, len(x))) self.time_d = np.append(self.time_d, np.repeat(self.i, len(x))) self.x1_base = np.append(self.x1_base, np.repeat(x1_base, len(x))) # if over budget prune if self.p_d.shape[0] > self.budget: N_rmv = len(self.x1_d) - self.budget dff = np.diff(self.x1_d) mdff = .5 * (dff[1:len(dff)] + dff[0:(len(dff) - 1)]) mdff = np.concatenate([[dff[0]], mdff, [dff[len(dff) - 1]]]) eps = self.delta_d / mdff # accuracy of base estimate / closeness to nearby points rmv = np.where( np.logical_or(eps >= -np.sort(-eps)[N_rmv - 1], ~np.isfinite(eps)))[0] if len(rmv) > N_rmv: rmv = np.random.choice(a=rmv, size=N_rmv, replace=False) # remove pts from dictionary and assoc. information self.x1_d = np.delete(self.x1_d, rmv, axis=0) self.p_d = np.delete(self.p_d, rmv, axis=0) self.delta_d = np.delete(self.delta_d, rmv, axis=0) self.time_d = np.delete(self.time_d, rmv, axis=0) self.x1_base = np.delete(self.x1_base, rmv, axis=0) # sort and remove non-unique values (more stable fitting) # remove non-unique from dict ux, iux = np.unique(self.x1_d, return_index=True) self.x1_d = ux self.p_d = self.p_d[iux] self.delta_d = self.delta_d[iux] self.time_d = self.time_d[iux] self.x1_base = self.x1_base[iux] # sort dict ordx = np.argsort(self.x1_d, axis=0).reshape(-1) self.x1_d = self.x1_d[ordx] self.p_d = self.p_d[ordx] self.delta_d = self.delta_d[ordx] self.time_d = self.time_d[ordx] self.x1_base = self.x1_base[ordx] ## ADAPTIVE PROFILE fitting # need more data than knots if len(self.x1_d) <= self.K: return # determine location of knots based on some heuristics if len(self.x1_d) > 5 and self.n_knots > 0: l3len = int(np.ceil(len(self.x1_d) // 3)) if self.n_knots is not None: qs = np.linspace(.25, .75, np.min([l3len, self.n_knots])) else: qs = np.linspace(.25, .75, l3len) kseq = np.quantile(self.x1_d, qs) self.kseq = kseq[1:(len(kseq) - 1)] else: self.kseq = [] self.x_design = self.make_adapt_design(self.x1_d, update=True) self.p_mean = np.mean(self.p_d) self.p_std = np.std(self.p_d) self.p_z = (self.p_d - self.p_mean) / self.p_std # fit model and its derivative #just make sure everything is monotonic (for stability) mono_fit = IsotonicRegression().fit(self.x_design, self.p_z) mono_p = mono_fit.predict(self.x_design) if self.n_knots < 0: def err_mod(mod): #RSS for fit model return np.sum((mod(self.x1_d) - self.p_z)**2) / len(self.p_z) smooth_spl = UnivariateSpline(self.x_design, mono_p, k=self.K, ext=0, s=len(self.p_z) * self.gamma) # some heuristics on what knots to keep knts = smooth_spl.get_knots() keep_knts = ~np.logical_or(knts < np.quantile(self.x1_d, .1), knts > np.quantile(self.x1_d, .9)) knts = knts[keep_knts] linear_mod = LSQUnivariateSpline(self.x_design, mono_p, k=self.K, ext=0, t=[]) self.adapt = LSQUnivariateSpline(self.x_design, mono_p, k=self.K, ext=0, t=knts) adapt_err = err_mod(self.adapt) lin_err = err_mod(linear_mod) #if its not twice as good as a simple linear model just use the latter if lin_err > 0 and adapt_err / lin_err > 0.5: self.adapt = linear_mod else: #n_knots > 0 self.adapt = LSQUnivariateSpline(self.x_design, mono_p, k=self.K, ext=0, t=self.kseq) self.adapt_deriv = self.adapt.derivative(1) def adapt_pred(self, x): x_basis = self.make_adapt_design(x) pred0 = self.adapt(x_basis) return pred0 def fit_adapt(self, x, p, x0): # set up optimization obj = lambda par, grad: self.adapt_sqe(x, p, par) self.adapt_opt.set_min_objective(obj) x0 = np.max([self.lb, x0]) x0 = np.min([self.ub, x0]) # run optimization x_hat = self.adapt_opt.optimize([x0])[0] # calculate fitting sensitivity drv1 = self.adapt_deriv(x - x_hat) d2 = np.mean(drv1**2) d2 += self.lam sig2 = self.adapt_sqe(x, p, [x_hat], pen=False) / (len(x) - 1) cov = .5 * sig2 / d2 return x_hat, cov def adapt_sqe(self, x, p, cent, pen=True): xc = x - cent ft = self.adapt_pred(xc).reshape(-1) p_z = (p - self.p_mean) / self.p_std val = np.mean((p_z - ft)**2) penalty = 0 if pen: penalty = self.lam * np.abs(cent[0])**2 return val + penalty # other helper functions def pressure_rise(self, x, p, pct=.5): try: pmin = np.min(p) delta = np.max(p) - pmin p_q = pmin + delta * pct i = np.min(np.where(np.diff((p >= p_q).astype(int)))[0]) + 1 a = (p[i] - p[i - 1]) / (x[i] - x[i - 1]) b = p[i] - a * x[i] x_q = (p_q - b) / a rat = np.max(p) / np.min(p) if rat < 2: x_q = float('Inf') return x_q, rat except Exception: return float('Inf'), float('Inf') def pressure_ratio(self, x, p, pa, ratio=2): p = p / pa p_q = ratio ii = np.where(np.diff((p >= p_q).astype(int)))[0] if len(ii) == 0: return float('inf'), float('inf'), i = np.min(ii) + 1 a = (p[i] - p[i - 1]) / (x[i] - x[i - 1]) b = p[i] - a * x[i] x_q = (p_q - b) / a delta = np.min(np.abs(x - x_q)) return x_q, delta