def estimate_inertia(records):
"""We assume no stiffness or damping for the equation of motion is:
T = I*alpha
where T is the sum of torques applied
I is the moment of inertia
alpha is the angular acceleration
"""
t = get_time_vector(records)
torque = (
((records.sensors.kollmorgen_actual_torque).astype(float) - 2**11) /
2**11 * KOLLMORGEN_MAX_TORQUE) # N-m
angle = np.unwrap(((records.sensors.steer_encoder_count).astype(float) *
2 * np.pi / ENCODER_COUNT_PER_REV)) # radians
dt = np.diff(t).mean()
angle_d = sg(angle, 11, 3, deriv=1, delta=dt, mode='nearest')
angle_dd = sg(angle, 11, 3, deriv=2, delta=dt, mode='nearest')
color = sns.color_palette('Paired', 10)
fig, ax = plt.subplots()
ax.plot(t, angle, label='angle', color=color[0])
ax.plot(t, angle_d, label='angular rate', color=color[2])
ax.plot(t, angle_dd, label='angular accel', color=color[4])
ax.plot(t, torque, label='torque', color=color[6])
ret = np.linalg.lstsq(np.reshape(angle_dd, (-1, 1)),
np.reshape(torque, (-1, 1)))
inertia = np.squeeze(ret[0])
ax.plot(t, inertia * angle_dd, label='estimated torque', color=color[7])
ax.legend()
plt.show()
return inertia
def add_sg(df, group, lag, diff):
p = 3
df['ISO_kWh_smooth_' + str(lag)] = 0.0
df['SG_kWh_smooth_' + str(lag)] = 0.0
df['DEG_C_kWh_smooth_' + str(lag)] = 0.0
for si in df[group].unique():
index = df[group] == si
df.loc[index,
'ISO_kWh_smooth_' + str(lag)] = sg(df[index].ISO_kWh, lag, p)
df.loc[index,
'SG_kWh_smooth_' + str(lag)] = sg(df[index].SG_kWh, lag, p)
df.loc[index,
'DEG_C_kWh_smooth_' + str(lag)] = sg(df[index].DEG_C_kWh, lag,
p)
return df
def sg_filter_data(data,
num_to_remove=3,
window_length=7,
polyorder=3,
fit_type='spline'):
"""
Applies a Savitzky-Golay filter to the data which is used to remove outlier or noisy points from the data
Parameters
----------
data : numpy, array
array of loops
num_to_remove : numpy, int
sets the number of points to remove
window_length : numpy, int
sets the size of the window for the sg filter
polyorder : numpy, int
sets the order of the sg filter
fit_type : string
selection of type of function for interpolation
Returns
-------
cleaned_data : numpy array
array of loops
"""
# reshapes the data such that it can run with different data sizes
if data.ndim == 2:
data = data.reshape(
np.sqrt(data.shape[0]).astype(int),
np.sqrt(data.shape[0]).astype(int), -1)
data = np.expand_dims(data, axis=3)
elif data.ndim == 3:
data = np.expand_dims(data, axis=3)
cleaned_data = np.copy(data)
# creates a vector of the size of the data
point_values = np.linspace(0, 1, data.shape[2])
# Loops around the x index
for i in range(data.shape[0]):
# Loops around the y index
for j in range(data.shape[1]):
# Loops around the number of cycles
for k in range(data.shape[3]):
sg_ = sg(data[i, j, :, k],
window_length=window_length,
polyorder=polyorder)
diff = np.abs(data[i, j, :, k] - sg_)
sort_ind = np.argsort(diff)
remove = sort_ind[-1 * num_to_remove::].astype(int)
cleaned_data[i, j, remove, k] = np.nan
cleaned_data = clean_and_interpolate(cleaned_data, fit_type)
return cleaned_data
def sg_smooth(data, winsize, order, deriv=0, rate=1):
"""
savitzky-golay 平滑
@param data 数据
@param winsize 窗口大小, 必须为奇数
@param order 多项式次数
@param deriv 噪声方差
@param rate 窗口
@return 平滑后的数据
winsize = np.int(winsize)
order = np.int(order)
data = np.array(data)
if winsize % 2 != 1 or winsize < 1:
raise ValueError("winsize必须为奇数且不小于1")
if winsize < order+2:
raise ValueError("winsize过小")
if winsize > len(data):
raise ValueError("winsize过大")
order_range = range(order+1)
half_window = (winsize-1) // 2
bco = np.mat([[k**i for i in order_range] \
for k in range(-half_window, half_window+1)])
mco = np.linalg.pinv(bco).A[deriv] * rate**deriv*factorial(deriv)
firstvals = data[0] - np.abs(data[1:half_window+1][::-1] - data[0])
lastvals = data[-1] + np.abs(data[-half_window-1: -1][::-1] - data[-1])
data = np.concatenate((firstvals, data, lastvals))
return np.convolve(mco[::-1], data, mode='valid')"""
return sg(data, winsize, order)
def bayesave(x, trialdim=0, timedim=1, method='mean', smoothtrials=19):
ntrials = x.shape[trialdim]
if method == 'mean':
summary = x.mean(axis=trialdim) * (ntrials)**0.5
else:
summary = np.median(x, axis=trialdim) * (ntrials)**0.5
ntime = x.shape[timedim]
wts = 1 / np.var(x - summary, axis=timedim, keepdims=True)
wts = sg(wts, smoothtrials, 3, axis=trialdim) # Smooth the variance
normFactor = wts.sum()
wts = wts.repeat(ntime, axis=timedim)
ave = (x * wts).sum(axis=trialdim) / normFactor
return ave
def spec_sad(gulp, window=7):
"""
Uses Savgol filter to smooth along the time axis
Args:
gulp: dynamic spectra to be smoothed
window: number of point to smooth
Returns:
Dynamic Spectrum smoothed along the time axis
"""
data_type = gulp.dtype
return sg(gulp, window, 2, axis=1).astype(data_type)
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique():
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2)
def savgol_filter(bandpass, channel_bandwidth, frequency_window=15, sigma=6):
"""
Apply savgol filter to the data. See [Agarwal el al. 2020](https://arxiv.org/abs/2003.14272) for details.
Args:
bandpass (numpy.ndarray): bandpass of the data
channel_bandwidth (float): channel bandwidth (MHz)
frequency_window (float): frequency window (MHz)
sigma (float): sigma value to apply cutoff on
Returns:
numpy.ndarray: mask for channels
"""
window = int(
np.ceil(frequency_window / np.abs(channel_bandwidth)) // 2 * 2 + 1)
y = sg(bandpass, window, 2)
sub = bandpass - y
sigma = sigma * np.std(sub)
mask = (sub > sigma) | (sub < -sigma)
return mask
def transform(self, X):
w = 11
p = 2
if self._SGFilter:
for si in X.site_id.unique():
index = X.site_id == si
X.loc[index, 'air_smooth'] = sg(X[index].air_temperature, w, p)
X.loc[index, 'dew_smooth'] = sg(X[index].dew_temperature, w, p)
X.loc[index, 'air_diff'] = sg(X[index].air_temperature, w, p,
1)
X.loc[index, 'dew_diff'] = sg(X[index].dew_temperature, w, p,
1)
X.loc[index, 'air_diff2'] = sg(X[index].air_temperature, w, p,
2)
X.loc[index, 'dew_diff2'] = sg(X[index].dew_temperature, w, p,
2)
return X
def savgol_filter(bandpass, channel_bandwidth, frequency_window=15, sigma=6):
"""
Apply savgol filter to the data. See [Agarwal el al. 2020](https://arxiv.org/abs/2003.14272) for details.
Args:
bandpass (numpy.ndarray): bandpass of the data
channel_bandwidth (float): channel bandwidth (MHz)
frequency_window (float): frequency window (MHz)
sigma (float): sigma value to apply cutoff on
Returns:
numpy.ndarray: mask for channels
"""
window = int(np.ceil(frequency_window / np.abs(channel_bandwidth)) // 2 * 2 + 1)
if window < 41:
logger.warning(
"Window size is less than 41 channels. Setting it to 41 channels."
)
window = 41
if window > len(bandpass):
logger.warning(
"Window size is larger than the number of channels. Setting it to total number of channels."
)
nch = len(bandpass)
if nch % 2:
window = nch - 1
else:
window = nch
logger.debug(f"Window size for savgol filter is {window}.")
y = sg(bandpass, window, 2)
sub = bandpass - y
sigma = sigma * np.std(sub)
mask = (sub > sigma) | (sub < -sigma)
return mask
def extract(**kwargs):
""" Extract the efficiency from a list of S1D_ frames """
# Load parameters
frames = np.array(
ascii.read(kwargs['framelist'], format='no_header')['col1'])
cal = kwargs['cal']
plotf = kwargs['plot']
save = bool(kwargs['save'])
figg = plt.figure(figsize=(7, 9))
gs = gridspec.GridSpec(3, 1)
axg = []
axg.append(figg.add_subplot(gs[0:2]))
axg.append(figg.add_subplot(gs[2]))
color = -1
targ = ''
for f in frames:
print "Processing", f + "..."
# Load observed spectrum
spec = EsprS1D(f)
# Convert to electons
spec.adu_to_electron()
# Load extinction
ext = ESOExt(cal + 'atmoexan.fits')
# Correct observed spectrum for extinction
la_silla_spec = np.interp(spec.wave, ext.wave, ext.la_silla)
#spec.flux = spec.flux * 10 ** (0.4*la_silla_spec*(spec.airm-1))
spec.flux = spec.flux * 10**(0.4 * la_silla_spec * (spec.airm))
# Load pipeline efficiency
try:
eff = EsprEff(f[:-10] + '_0005.fits')
pipe = True
except:
pipe = False
# Load catalogue spectrum
std = ESOStd(cal + 'f' + spec.targ.lower() + '.dat',
area=spec.area,
expt=spec.expt)
# Bin spectra and sum counts
bins = np.arange(378.0, 788.0, 1.6) * u.nm
bin_spec = binspec(spec.wave, spec.flux.value, bins)
bin_std = binspec(std.wave, std.flux, bins)
# Smooth the efficiency curve
eff_raw = bin_spec / bin_std
eff_raw[np.isnan(eff_raw)] = np.median(eff_raw[~np.isnan(eff_raw)])
eff_smooth = sg(eff_raw, 75, 3)
# Individual plot
figi = plt.figure(figsize=(7, 12))
figi.suptitle(
f.split('/')[1][:-10] + ', ' + spec.targ + ', ' + spec.mode)
axi = []
axi.append(figi.add_subplot(211))
axi.append(figi.add_subplot(212))
axi[0].semilogy(bins, bin_spec, c='C0', label="ESPRESSO")
axi[0].semilogy(bins, bin_std, c='C1', label="catalogue")
axi[0].set_xlabel("Wavelength (nm)")
axi[0].set_ylabel("Photons")
axi[0].legend()
if pipe:
axi[1].plot(eff.wave,
eff.eff,
c='black',
linestyle='--',
label="DRS")
axi[1].plot(bins, eff_smooth, c='C2', label="measured")
axi[1].set_xlabel("Wavelength (nm)")
axi[1].set_ylabel("Efficiency")
axi[1].legend()
if plotf == 'all':
plt.draw()
if save:
figi.savefig(filename=f[:-10] + '_eff.pdf', format='pdf')
# Save results
hdu0 = fits.PrimaryHDU(header=spec.hdul[0].header)
col0 = fits.Column(name='wave', format='D', array=bins)
col1 = fits.Column(name='eff', format='D', array=eff_smooth)
hdu1 = fits.BinTableHDU.from_columns([col0, col1])
hdul = fits.HDUList([hdu0, hdu1])
hdul.writeto(f[:-10] + '_eff.fits', overwrite=True)
print "...saved plot/efficiencies as", f[:-10] + '_eff.pdf/fits' + "."
if plotf != 'all':
plt.close()
# Compute averages
if spec.targ != targ:
avecomp = f != frames[0]
if avecomp:
eff_save = eff_stack
avecolor = color
avetarg = targ
avetime = str(Time(np.average(midtime_arr), format='mjd').isot)
aveiq = "%3.2f" % np.average(iq_arr)
eff_stack = eff_smooth
midtime_arr = [spec.midtime.mjd]
binx_arr = [spec.binx]
iq_arr = [spec.dimm]
targ = spec.targ
color += 1
else:
eff_stack = np.vstack((eff_stack, eff_smooth))
midtime_arr = np.append(midtime_arr, spec.midtime.mjd)
binx_arr = np.append(binx_arr, spec.binx)
iq_arr = np.append(iq_arr, spec.dimm)
avecomp = f == frames[-1]
if avecomp:
eff_save = eff_stack
avecolor = color
avetarg = targ
avetime = str(Time(np.average(midtime_arr), format='mjd').isot)
aveiq = "%3.2f" % np.average(iq_arr)
# Global plot
if avecomp:
label = avetarg + ', ' + avetime[:10] + ', ' + avetime + ', IQ: ' + aveiq
eff_ave = np.average(eff_save, axis=0)
eff_std = np.std(eff_save, axis=0)
if 'eff_ref' not in locals():
eff_ref = eff_ave
axg[0].plot(bins, eff_ave, c='C' + str(avecolor), label=label)
axg[0].fill_between(bins.value,
eff_ave - eff_std,
eff_ave + eff_std,
facecolor='C' + str(avecolor),
alpha=0.3)
axg[1].plot(bins,
eff_ave / eff_ref,
c='C' + str(avecolor),
label=label)
axg[1].fill_between(bins.value, (eff_ave - eff_std) / eff_ref,
(eff_ave + eff_std) / eff_ref,
facecolor='C' + str(avecolor),
alpha=0.3)
# Save averages
hdu0 = fits.PrimaryHDU(header=spec.hdul[0].header)
hdu0.header['HIERARCH ESO AVE IQ'] = aveiq
hdu0.header['HIERARCH ESO OBS TARG NAME'] = avetarg
hdu0.header['HIERARCH ESO MIDTIME'] = avetime
col0 = fits.Column(name='wave', format='D', array=bins)
col1 = fits.Column(name='eff_ave', format='D', array=eff_ave)
col2 = fits.Column(name='eff_std', format='D', array=eff_std)
hdu1 = fits.BinTableHDU.from_columns([col0, col1, col2])
hdul = fits.HDUList([hdu0, hdu1])
hdul.writeto(kwargs['framelist'][:-4] + '_' + avetime[:10] +
'.fits',
overwrite=True)
print "...saved average efficiencies as", \
kwargs['framelist'][:-4]+'_'+avetime[:10]+'_eff.fits'+"."
#axg[0].plot(bins, eff_smooth, c='C'+str(color), linestyle=':')
if len(np.unique(binx_arr)) == 1:
figg.suptitle(str(spec.binx) + 'x' + str(spec.biny))
axg[0].set_ylabel("Efficiency")
axg[0].legend()
axg[1].set_xlabel("Wavelength (nm)")
axg[1].set_ylabel("Normalized to reference")
if plotf != 'no':
plt.show()
if save:
figg.savefig(kwargs['framelist'][:-4] + '.pdf', format='pdf')
plt.close()
def smooth(x, fillen=27, polyord=3, axis=-1):
y = sg(np.flip(sg(x, fillen, polyord, axis=axis), axis),
fillen,
polyord,
axis=axis)
return np.flip(y, axis)
x2 = evokeds[2]
bstart, bstop = x2.time_as_index([-0.25, 0.])
bmean = x2.data[:, bstart:bstop].mean(axis=1)
bstd = x2.data[:, bstart:bstop].std(axis=1)
x2.data = (x2.data.T - bmean).T
x2.data = (x2.data.T / bstd).T
coh20 = pca.transform(x2.data.T)
else:
coh07 = pca.transform(evokeds[0].data.T)
coh14 = pca.transform(evokeds[1].data.T)
coh20 = pca.transform(evokeds[2].data.T)
if combinecomps:
filtlen = 31
filtord = 2
coh07summary[k, :] = sg((coh07**2.).sum(axis=1)**0.5 / normfac,
filtlen, filtord)
coh14summary[k, :] = sg((coh14**2.).sum(axis=1)**0.5 / normfac,
filtlen, filtord)
coh20summary[k, :] = sg((coh20**2.).sum(axis=1)**0.5 / normfac,
filtlen, filtord)
else:
coh07summary[k, :] = coh07[:, 0] / normfac
coh14summary[k, :] = coh14[:, 0] / normfac
coh20summary[k, :] = coh20[:, 0] / normfac
# Z-score results once again
bmean = coh07summary[:, t < 0].mean(axis=1)
bstd = coh07summary[:, t < 0].std(axis=1)
coh07summary = (coh07summary.T - bmean).T
coh07summary = (coh07summary.T / bstd).T
