def estimate_inertia(records): """We assume no stiffness or damping for the equation of motion is: T = I*alpha where T is the sum of torques applied I is the moment of inertia alpha is the angular acceleration """ t = get_time_vector(records) torque = ( ((records.sensors.kollmorgen_actual_torque).astype(float) - 2**11) / 2**11 * KOLLMORGEN_MAX_TORQUE) # N-m angle = np.unwrap(((records.sensors.steer_encoder_count).astype(float) * 2 * np.pi / ENCODER_COUNT_PER_REV)) # radians dt = np.diff(t).mean() angle_d = sg(angle, 11, 3, deriv=1, delta=dt, mode='nearest') angle_dd = sg(angle, 11, 3, deriv=2, delta=dt, mode='nearest') color = sns.color_palette('Paired', 10) fig, ax = plt.subplots() ax.plot(t, angle, label='angle', color=color[0]) ax.plot(t, angle_d, label='angular rate', color=color[2]) ax.plot(t, angle_dd, label='angular accel', color=color[4]) ax.plot(t, torque, label='torque', color=color[6]) ret = np.linalg.lstsq(np.reshape(angle_dd, (-1, 1)), np.reshape(torque, (-1, 1))) inertia = np.squeeze(ret[0]) ax.plot(t, inertia * angle_dd, label='estimated torque', color=color[7]) ax.legend() plt.show() return inertia
def add_sg(df, group, lag, diff): p = 3 df['ISO_kWh_smooth_' + str(lag)] = 0.0 df['SG_kWh_smooth_' + str(lag)] = 0.0 df['DEG_C_kWh_smooth_' + str(lag)] = 0.0 for si in df[group].unique(): index = df[group] == si df.loc[index, 'ISO_kWh_smooth_' + str(lag)] = sg(df[index].ISO_kWh, lag, p) df.loc[index, 'SG_kWh_smooth_' + str(lag)] = sg(df[index].SG_kWh, lag, p) df.loc[index, 'DEG_C_kWh_smooth_' + str(lag)] = sg(df[index].DEG_C_kWh, lag, p) return df
def sg_filter_data(data, num_to_remove=3, window_length=7, polyorder=3, fit_type='spline'): """ Applies a Savitzky-Golay filter to the data which is used to remove outlier or noisy points from the data Parameters ---------- data : numpy, array array of loops num_to_remove : numpy, int sets the number of points to remove window_length : numpy, int sets the size of the window for the sg filter polyorder : numpy, int sets the order of the sg filter fit_type : string selection of type of function for interpolation Returns ------- cleaned_data : numpy array array of loops """ # reshapes the data such that it can run with different data sizes if data.ndim == 2: data = data.reshape( np.sqrt(data.shape[0]).astype(int), np.sqrt(data.shape[0]).astype(int), -1) data = np.expand_dims(data, axis=3) elif data.ndim == 3: data = np.expand_dims(data, axis=3) cleaned_data = np.copy(data) # creates a vector of the size of the data point_values = np.linspace(0, 1, data.shape[2]) # Loops around the x index for i in range(data.shape[0]): # Loops around the y index for j in range(data.shape[1]): # Loops around the number of cycles for k in range(data.shape[3]): sg_ = sg(data[i, j, :, k], window_length=window_length, polyorder=polyorder) diff = np.abs(data[i, j, :, k] - sg_) sort_ind = np.argsort(diff) remove = sort_ind[-1 * num_to_remove::].astype(int) cleaned_data[i, j, remove, k] = np.nan cleaned_data = clean_and_interpolate(cleaned_data, fit_type) return cleaned_data
def sg_smooth(data, winsize, order, deriv=0, rate=1): """ savitzky-golay 平滑 @param data 数据 @param winsize 窗口大小, 必须为奇数 @param order 多项式次数 @param deriv 噪声方差 @param rate 窗口 @return 平滑后的数据 winsize = np.int(winsize) order = np.int(order) data = np.array(data) if winsize % 2 != 1 or winsize < 1: raise ValueError("winsize必须为奇数且不小于1") if winsize < order+2: raise ValueError("winsize过小") if winsize > len(data): raise ValueError("winsize过大") order_range = range(order+1) half_window = (winsize-1) // 2 bco = np.mat([[k**i for i in order_range] \ for k in range(-half_window, half_window+1)]) mco = np.linalg.pinv(bco).A[deriv] * rate**deriv*factorial(deriv) firstvals = data[0] - np.abs(data[1:half_window+1][::-1] - data[0]) lastvals = data[-1] + np.abs(data[-half_window-1: -1][::-1] - data[-1]) data = np.concatenate((firstvals, data, lastvals)) return np.convolve(mco[::-1], data, mode='valid')""" return sg(data, winsize, order)
def bayesave(x, trialdim=0, timedim=1, method='mean', smoothtrials=19): ntrials = x.shape[trialdim] if method == 'mean': summary = x.mean(axis=trialdim) * (ntrials)**0.5 else: summary = np.median(x, axis=trialdim) * (ntrials)**0.5 ntime = x.shape[timedim] wts = 1 / np.var(x - summary, axis=timedim, keepdims=True) wts = sg(wts, smoothtrials, 3, axis=trialdim) # Smooth the variance normFactor = wts.sum() wts = wts.repeat(ntime, axis=timedim) ave = (x * wts).sum(axis=trialdim) / normFactor return ave
def spec_sad(gulp, window=7): """ Uses Savgol filter to smooth along the time axis Args: gulp: dynamic spectra to be smoothed window: number of point to smooth Returns: Dynamic Spectrum smoothed along the time axis """ data_type = gulp.dtype return sg(gulp, window, 2, axis=1).astype(data_type)
def add_sg(df): w = 11 p = 2 for si in df.site_id.unique(): index = df.site_id == si df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p) df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p) df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1) df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1) df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2) df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2)
def savgol_filter(bandpass, channel_bandwidth, frequency_window=15, sigma=6): """ Apply savgol filter to the data. See [Agarwal el al. 2020](https://arxiv.org/abs/2003.14272) for details. Args: bandpass (numpy.ndarray): bandpass of the data channel_bandwidth (float): channel bandwidth (MHz) frequency_window (float): frequency window (MHz) sigma (float): sigma value to apply cutoff on Returns: numpy.ndarray: mask for channels """ window = int( np.ceil(frequency_window / np.abs(channel_bandwidth)) // 2 * 2 + 1) y = sg(bandpass, window, 2) sub = bandpass - y sigma = sigma * np.std(sub) mask = (sub > sigma) | (sub < -sigma) return mask
def transform(self, X): w = 11 p = 2 if self._SGFilter: for si in X.site_id.unique(): index = X.site_id == si X.loc[index, 'air_smooth'] = sg(X[index].air_temperature, w, p) X.loc[index, 'dew_smooth'] = sg(X[index].dew_temperature, w, p) X.loc[index, 'air_diff'] = sg(X[index].air_temperature, w, p, 1) X.loc[index, 'dew_diff'] = sg(X[index].dew_temperature, w, p, 1) X.loc[index, 'air_diff2'] = sg(X[index].air_temperature, w, p, 2) X.loc[index, 'dew_diff2'] = sg(X[index].dew_temperature, w, p, 2) return X
def savgol_filter(bandpass, channel_bandwidth, frequency_window=15, sigma=6): """ Apply savgol filter to the data. See [Agarwal el al. 2020](https://arxiv.org/abs/2003.14272) for details. Args: bandpass (numpy.ndarray): bandpass of the data channel_bandwidth (float): channel bandwidth (MHz) frequency_window (float): frequency window (MHz) sigma (float): sigma value to apply cutoff on Returns: numpy.ndarray: mask for channels """ window = int(np.ceil(frequency_window / np.abs(channel_bandwidth)) // 2 * 2 + 1) if window < 41: logger.warning( "Window size is less than 41 channels. Setting it to 41 channels." ) window = 41 if window > len(bandpass): logger.warning( "Window size is larger than the number of channels. Setting it to total number of channels." ) nch = len(bandpass) if nch % 2: window = nch - 1 else: window = nch logger.debug(f"Window size for savgol filter is {window}.") y = sg(bandpass, window, 2) sub = bandpass - y sigma = sigma * np.std(sub) mask = (sub > sigma) | (sub < -sigma) return mask
def extract(**kwargs): """ Extract the efficiency from a list of S1D_ frames """ # Load parameters frames = np.array( ascii.read(kwargs['framelist'], format='no_header')['col1']) cal = kwargs['cal'] plotf = kwargs['plot'] save = bool(kwargs['save']) figg = plt.figure(figsize=(7, 9)) gs = gridspec.GridSpec(3, 1) axg = [] axg.append(figg.add_subplot(gs[0:2])) axg.append(figg.add_subplot(gs[2])) color = -1 targ = '' for f in frames: print "Processing", f + "..." # Load observed spectrum spec = EsprS1D(f) # Convert to electons spec.adu_to_electron() # Load extinction ext = ESOExt(cal + 'atmoexan.fits') # Correct observed spectrum for extinction la_silla_spec = np.interp(spec.wave, ext.wave, ext.la_silla) #spec.flux = spec.flux * 10 ** (0.4*la_silla_spec*(spec.airm-1)) spec.flux = spec.flux * 10**(0.4 * la_silla_spec * (spec.airm)) # Load pipeline efficiency try: eff = EsprEff(f[:-10] + '_0005.fits') pipe = True except: pipe = False # Load catalogue spectrum std = ESOStd(cal + 'f' + spec.targ.lower() + '.dat', area=spec.area, expt=spec.expt) # Bin spectra and sum counts bins = np.arange(378.0, 788.0, 1.6) * u.nm bin_spec = binspec(spec.wave, spec.flux.value, bins) bin_std = binspec(std.wave, std.flux, bins) # Smooth the efficiency curve eff_raw = bin_spec / bin_std eff_raw[np.isnan(eff_raw)] = np.median(eff_raw[~np.isnan(eff_raw)]) eff_smooth = sg(eff_raw, 75, 3) # Individual plot figi = plt.figure(figsize=(7, 12)) figi.suptitle( f.split('/')[1][:-10] + ', ' + spec.targ + ', ' + spec.mode) axi = [] axi.append(figi.add_subplot(211)) axi.append(figi.add_subplot(212)) axi[0].semilogy(bins, bin_spec, c='C0', label="ESPRESSO") axi[0].semilogy(bins, bin_std, c='C1', label="catalogue") axi[0].set_xlabel("Wavelength (nm)") axi[0].set_ylabel("Photons") axi[0].legend() if pipe: axi[1].plot(eff.wave, eff.eff, c='black', linestyle='--', label="DRS") axi[1].plot(bins, eff_smooth, c='C2', label="measured") axi[1].set_xlabel("Wavelength (nm)") axi[1].set_ylabel("Efficiency") axi[1].legend() if plotf == 'all': plt.draw() if save: figi.savefig(filename=f[:-10] + '_eff.pdf', format='pdf') # Save results hdu0 = fits.PrimaryHDU(header=spec.hdul[0].header) col0 = fits.Column(name='wave', format='D', array=bins) col1 = fits.Column(name='eff', format='D', array=eff_smooth) hdu1 = fits.BinTableHDU.from_columns([col0, col1]) hdul = fits.HDUList([hdu0, hdu1]) hdul.writeto(f[:-10] + '_eff.fits', overwrite=True) print "...saved plot/efficiencies as", f[:-10] + '_eff.pdf/fits' + "." if plotf != 'all': plt.close() # Compute averages if spec.targ != targ: avecomp = f != frames[0] if avecomp: eff_save = eff_stack avecolor = color avetarg = targ avetime = str(Time(np.average(midtime_arr), format='mjd').isot) aveiq = "%3.2f" % np.average(iq_arr) eff_stack = eff_smooth midtime_arr = [spec.midtime.mjd] binx_arr = [spec.binx] iq_arr = [spec.dimm] targ = spec.targ color += 1 else: eff_stack = np.vstack((eff_stack, eff_smooth)) midtime_arr = np.append(midtime_arr, spec.midtime.mjd) binx_arr = np.append(binx_arr, spec.binx) iq_arr = np.append(iq_arr, spec.dimm) avecomp = f == frames[-1] if avecomp: eff_save = eff_stack avecolor = color avetarg = targ avetime = str(Time(np.average(midtime_arr), format='mjd').isot) aveiq = "%3.2f" % np.average(iq_arr) # Global plot if avecomp: label = avetarg + ', ' + avetime[:10] + ', ' + avetime + ', IQ: ' + aveiq eff_ave = np.average(eff_save, axis=0) eff_std = np.std(eff_save, axis=0) if 'eff_ref' not in locals(): eff_ref = eff_ave axg[0].plot(bins, eff_ave, c='C' + str(avecolor), label=label) axg[0].fill_between(bins.value, eff_ave - eff_std, eff_ave + eff_std, facecolor='C' + str(avecolor), alpha=0.3) axg[1].plot(bins, eff_ave / eff_ref, c='C' + str(avecolor), label=label) axg[1].fill_between(bins.value, (eff_ave - eff_std) / eff_ref, (eff_ave + eff_std) / eff_ref, facecolor='C' + str(avecolor), alpha=0.3) # Save averages hdu0 = fits.PrimaryHDU(header=spec.hdul[0].header) hdu0.header['HIERARCH ESO AVE IQ'] = aveiq hdu0.header['HIERARCH ESO OBS TARG NAME'] = avetarg hdu0.header['HIERARCH ESO MIDTIME'] = avetime col0 = fits.Column(name='wave', format='D', array=bins) col1 = fits.Column(name='eff_ave', format='D', array=eff_ave) col2 = fits.Column(name='eff_std', format='D', array=eff_std) hdu1 = fits.BinTableHDU.from_columns([col0, col1, col2]) hdul = fits.HDUList([hdu0, hdu1]) hdul.writeto(kwargs['framelist'][:-4] + '_' + avetime[:10] + '.fits', overwrite=True) print "...saved average efficiencies as", \ kwargs['framelist'][:-4]+'_'+avetime[:10]+'_eff.fits'+"." #axg[0].plot(bins, eff_smooth, c='C'+str(color), linestyle=':') if len(np.unique(binx_arr)) == 1: figg.suptitle(str(spec.binx) + 'x' + str(spec.biny)) axg[0].set_ylabel("Efficiency") axg[0].legend() axg[1].set_xlabel("Wavelength (nm)") axg[1].set_ylabel("Normalized to reference") if plotf != 'no': plt.show() if save: figg.savefig(kwargs['framelist'][:-4] + '.pdf', format='pdf') plt.close()
def smooth(x, fillen=27, polyord=3, axis=-1): y = sg(np.flip(sg(x, fillen, polyord, axis=axis), axis), fillen, polyord, axis=axis) return np.flip(y, axis)
x2 = evokeds[2] bstart, bstop = x2.time_as_index([-0.25, 0.]) bmean = x2.data[:, bstart:bstop].mean(axis=1) bstd = x2.data[:, bstart:bstop].std(axis=1) x2.data = (x2.data.T - bmean).T x2.data = (x2.data.T / bstd).T coh20 = pca.transform(x2.data.T) else: coh07 = pca.transform(evokeds[0].data.T) coh14 = pca.transform(evokeds[1].data.T) coh20 = pca.transform(evokeds[2].data.T) if combinecomps: filtlen = 31 filtord = 2 coh07summary[k, :] = sg((coh07**2.).sum(axis=1)**0.5 / normfac, filtlen, filtord) coh14summary[k, :] = sg((coh14**2.).sum(axis=1)**0.5 / normfac, filtlen, filtord) coh20summary[k, :] = sg((coh20**2.).sum(axis=1)**0.5 / normfac, filtlen, filtord) else: coh07summary[k, :] = coh07[:, 0] / normfac coh14summary[k, :] = coh14[:, 0] / normfac coh20summary[k, :] = coh20[:, 0] / normfac # Z-score results once again bmean = coh07summary[:, t < 0].mean(axis=1) bstd = coh07summary[:, t < 0].std(axis=1) coh07summary = (coh07summary.T - bmean).T coh07summary = (coh07summary.T / bstd).T