def __init__(self, trajff, smooth=None):
super().__init__(trajff, smooth)
self.slat = csaps.CubicSmoothingSpline(
xdata=trajff.timeAtServer.values,
ydata=trajff.nnpredlatitude.values,
smooth=smooth).spline
self.slon = csaps.CubicSmoothingSpline(
xdata=trajff.timeAtServer.values,
ydata=trajff.nnpredlongitude.values,
smooth=smooth).spline
def stdFiltIt(wl, arr_1d, weights_1d, sdms, smo, plot_q):
if plot_q:
plt.figure(figsize=(10, 5))
plt.title('Filter plot')
plt.plot(wl, arr_1d, '.k')
fit = arr_1d + np.nan
for sdm in sdms:
gi = np.logical_and(np.isfinite(weights_1d), np.isfinite(arr_1d))
fit = csaps.CubicSmoothingSpline(wl[gi], arr_1d[gi], smooth=smo)(wl)
dy_sd = np.std(arr_1d[gi] - fit[gi]) * sdm
gi = np.logical_and(np.abs(arr_1d - fit) <= dy_sd, gi)
if plot_q:
plt.plot(wl[np.logical_not(gi)], arr_1d[np.logical_not(gi)], 'xr')
plt.plot(wl, fit + dy_sd, ':g')
plt.plot(wl, fit - dy_sd, ':r')
plt.grid(True)
arr_1d[np.logical_not(gi)] = np.nan
if plot_q:
plt.show()
return arr_1d, gi
def test_auto_smooth(univariate_data):
x, y, xi, yi_expected, *_, smooth_expected = univariate_data
s = csaps.CubicSmoothingSpline(x, y, smooth=None)
yi = s(xi)
assert s.smooth == pytest.approx(smooth_expected)
assert yi == pytest.approx(yi_expected)
def test_univariate_integrate(univariate_data):
x = univariate_data.x
y = univariate_data.y
integral_expected = univariate_data.integral
spline = csaps.CubicSmoothingSpline(x, y, smooth=None).spline
integral = spline.integrate(x[0], x[-1])
assert integral == pytest.approx(integral_expected)
def test_weighted(w, yid):
x = [1., 2., 4., 6.]
y = [2., 4., 5., 7.]
xi = np.linspace(1., 6., 10)
sp = csaps.CubicSmoothingSpline(x, y, weights=w)
yi = sp(xi)
np.testing.assert_allclose(yi, yid)
def __init__(self,
curve: 'Curve',
smooth: ty.Optional[float] = 1.0,
weights: ty.Optional[np.ndarray] = None):
super().__init__(curve)
self.csaps = csaps.CubicSmoothingSpline(curve.t,
curve.data,
weights=weights,
smooth=smooth,
axis=0)
def test_univariate_antiderivative(univariate_data):
x = univariate_data.x
y = univariate_data.y
xi = univariate_data.xi
yi_ad1_expected = univariate_data.yi_ad1
spline = csaps.CubicSmoothingSpline(x, y, smooth=None).spline
spline_ad1: csaps.SplinePPForm = spline.antiderivative(nu=1)
yi_ad1 = spline_ad1(xi)
assert spline_ad1.order == 5
assert yi_ad1 == pytest.approx(yi_ad1_expected)
def barosmooth(trajff,smooth):
'''fit a spline with x=timeAtServer and y=baroAltitude'''
speedmax = 50.8 # 10000ft/min
dd=precompute_diffalt(trajff.baroAltitude.values,trajff.timeAtServer.values,speedmax)
if np.sum(dd)>0:# count edges number, if "full" graph we can keep it all without computing longest path
longest_path = smooth.get_gtlongest(dd)
keep = np.array([i in longest_path for i in range(dd.shape[-1])])
else:
keep = np.array([True]*dd.shape[-1])
t = trajff.timeAtServer.values[keep]
h = trajff.baroAltitude.values[keep]
smo = csaps.CubicSmoothingSpline(xdata=t,ydata=h,smooth=smooth).spline
return smo
def normalize_region(wl, flux, ferr, sds, plot_q=True, ss_smo=1e-3):
__, gi = stdFiltIt(wl, np.copy(flux),
np.ones_like(flux),
sds, smo=ss_smo, plot_q=plot_q)
cont_fit = csaps.CubicSmoothingSpline(wl[gi], flux[gi], smooth=ss_smo)(wl)
if plot_q:
plt.figure(figsize=(10, 5))
plt.plot(wl, flux, '.-k')
plt.plot(wl, cont_fit, '-r')
plt.title('Fit')
plt.xlabel('time')
plt.ylabel('Flux')
plt.grid(1)
plt.show()
#####################################
# Normalize continuum
flux_norm = flux / cont_fit
ferr_norm = ferr / cont_fit
if plot_q:
plt.figure(figsize=(10, 5))
#plt.plot(wl, flux_norm, '.-k')
plt.plot(wl, flux_norm, '-', c=[0.5, 0.5, 0.5], linewidth=1, label='Flux')
if 1:
plt.fill_between(wl, flux_norm - ferr_norm, flux_norm + ferr_norm,
step='mid',
color=[0.85, 0.85, 0.85], label='Flux Error')
plt.axhline(1.0, color='m', linewidth=1, label='Continuum')
plt.title('Normalized')
plt.xlabel('time [day]')
plt.ylabel('Flux [ppt]')
plt.grid(1)
plt.show()
return cont_fit, flux_norm, ferr_norm
def test_cubic_bc_natural():
np.random.seed(1234)
x = np.linspace(0, 5, 20)
xi = np.linspace(0, 5, 100)
y = np.sin(x) + np.random.randn(x.size) * 0.3
cs = CubicSpline(x, y, bc_type='natural')
ss = csaps.CubicSmoothingSpline(x, y, smooth=1.0)
y_cs = cs(xi)
y_ss = ss(xi)
assert cs.c == pytest.approx(ss.spline.c)
assert y_cs == pytest.approx(y_ss)
def test_zero_smooth():
x = [1., 2., 4., 6.]
y = [2., 4., 5., 7.]
sp = csaps.CubicSmoothingSpline(x, y, smooth=0.)
assert sp.smooth == pytest.approx(0.)
ys = sp(x)
assert ys == pytest.approx([
2.440677966101695, 3.355932203389830, 5.186440677966102,
7.016949152542373
])
def test_evaluate_nu_extrapolate(nu, extrapolate):
x = [1, 2, 3, 4]
xi = [0, 1, 2, 3, 4, 5]
y = [1, 2, 3, 4]
cs = CubicSpline(x, y)
y_cs = cs(xi, nu=nu, extrapolate=extrapolate)
ss = csaps.CubicSmoothingSpline(x, y, smooth=1.0)
y_ss = ss(xi, nu=nu, extrapolate=extrapolate)
np.testing.assert_allclose(y_ss,
y_cs,
rtol=1e-05,
atol=1e-08,
equal_nan=True)
def bandwidth(mag,freq): """ Takes magnitude output from lsim, converts to db and returns the first instance of frequency array at which the magnitude drops by 3db from the first entry of the array """ mag = 20 * np.log10(mag) mag0 = mag[0] mag1 = mag0-3 arr = np.linspace(0,20,200) spl = csaps.CubicSmoothingSpline(freq,mag) ynew = spl(arr) inx = (np.argmax(ynew-mag1<0)) #print(mag1,mag,freq,inx) return arr[inx]
def test_univariate_derivative(univariate_data):
x = univariate_data.x
y = univariate_data.y
xi = univariate_data.xi
yi_d1_expected = univariate_data.yi_d1
yi_d2_expected = univariate_data.yi_d2
spline = csaps.CubicSmoothingSpline(x, y, smooth=None).spline
spline_d1: csaps.SplinePPForm = spline.derivative(nu=1)
spline_d2: csaps.SplinePPForm = spline.derivative(nu=2)
yi_d1 = spline_d1(xi)
yi_d2 = spline_d2(xi)
assert spline_d1.order == 3
assert spline_d2.order == 2
assert yi_d1 == pytest.approx(yi_d1_expected)
assert yi_d2 == pytest.approx(yi_d2_expected)
def test_axis(shape, axis):
y = np.arange(int(np.prod(shape))).reshape(shape)
x = np.arange(np.array(y).shape[axis])
s = csaps.CubicSmoothingSpline(x, y, axis=axis)
ys = s(x)
np.testing.assert_allclose(ys, y)
ss = s.spline
axis = len(shape) + axis if axis < 0 else axis
ndim = int(np.prod(shape)) // shape[axis]
order = 2 if len(x) < 3 else 4
pieces = len(x) - 1
coeffs_shape = (order, pieces) + shape[:axis] + shape[axis + 1:]
assert ss.breaks == pytest.approx(x)
assert ss.coeffs.shape == coeffs_shape
assert ss.axis == axis
assert ss.order == order
assert ss.pieces == pieces
assert ss.ndim == ndim
assert ss.shape == shape
def calibrate(vbs, res_inds, det_inds, skip=[], name='./data/cond', both=True, det_ind=2,\
shift=1, r_set=None, preamp=None, r_wires=215/2, coef=None,\
r0=None, vmax=1e-3, sg_rpar=None, sg_det=None, sg_power=(9, 1), cs_smooth=None):
"""
ADD AUTOEXCLUSION OF BAD CURVES
Provides calibration data and calibration function
DC measurement in 3 probe geometry.
If 1 terminal of sample is grounded (DC), r_wires must be 0
"""
preamp = varz.preamp if not preamp else preamp
r_set = varz.r_set if not r_set else r_set
r0 = varz.r0 if not r0 else r0
coef = varz.coef if not coef else coef
r_set = np.atleast_1d(r_set)
if len(r_set) == 1:
r_set = r_set * np.ones(len(res_inds))
# Resistance vs Vtg
res = []
for j, ind in enumerate(res_inds):
vb = vbs[j]
df = pd.read_csv(name + str(ind) + '.dat', sep=' ', header=None)
if both:
df.loc[1::2, 2::] = df.loc[1::2, :2:-1].values
for k in range(len(df)):
v = np.array(df.loc[k, 2:].values / preamp, dtype='float64')
# Shift typical occurs even in DC measurements
v = pd.Series(v).shift(-shift)
# Subtract preamp offset
v -= v[np.abs(vb).argmin()]
i = (vb * coef - v) / r_set[j]
inds = np.abs(v) < vmax
r = np.nan
if len(i[inds]) > 3:
r = np.polyfit(i[inds], v[inds], 1)[0]
res.append(r)
res = np.array(res)
# Detector vs Vtg
det = pd.DataFrame()
det_inds = set(det_inds) - set(skip)
for ind in det_inds:
df = pd.read_csv(name + str(ind) + '.dat', sep=' ', header=None)
df.set_index(1, inplace=True)
det = pd.concat([det, df[det_ind]], axis=1, ignore_index=True)
# Always start measurement from smallest resistance (largest detector signal)
# if det.index[0] < det.index[-1]:
# det = det[::-1]
# This is sometimes wrong
vtg = det.index
det = det.mean(axis=1)
# Subtract wires (3-probe measurements)
r_sample = max(res)
with warnings.catch_warnings():
warnings.simplefilter(action='ignore')
r_tr = res * r_sample / (r_sample - res)
rpar = 1 / (1 / r_tr + 1 / (r_sample - r_wires) + 2 / r0)
calib_tg = pd.DataFrame({'vtg':vtg, 'res':res, 'rpar':rpar, 'det':det,\
'power':det2power(det)})
if calib_tg['res'].isna().any():
print("Following transistor gate values are skipped due to vmax threshold:\n[",\
end='')
to_print = calib_tg[calib_tg['res'].isna()]['vtg'].values
[print('{:.4g}'.format(value), end=' ') for value in to_print]
print("]")
calib_tg.dropna(inplace=True)
# # subtract offset
# fit_inds = calib_tg['rpar'] < 300
# if len(calib_tg.loc[fit_inds, 'rpar']) > 2:
# _, P0 = np.polyfit(calib_tg.loc[fit_inds, 'rpar'], calib_tg.loc[fit_inds, 'power'], 1)
# calib_tg['power'] -= P0
fig, ax = plt.subplots(1, 3, figsize=varz.figsize13)
ax[0].plot(calib_tg['vtg'], calib_tg['rpar'], '.')
ax[1].plot(calib_tg['vtg'], calib_tg['det'], '.')
ax[2].plot(calib_tg['rpar'], calib_tg['power'], '.', ms=3)
# savgol_filter on rpar vs Vtg
if sg_rpar:
calib_tg.sort_values(by='vtg', inplace=True)
vtg_reg = np.arange(calib_tg['vtg'].min(), calib_tg['vtg'].max(),\
calib_tg['vtg'].diff().min())
rpar_smooth = savgol_filter(np.interp(vtg_reg, calib_tg['vtg'],\
calib_tg['rpar']), *sg_rpar)
calib_tg['rpar'] = np.interp(calib_tg['vtg'], vtg_reg, rpar_smooth)
ax[0].plot(calib_tg['vtg'], calib_tg['rpar'])
# savgol_filter on det vs Vtg
if sg_det:
calib_tg.sort_values(by='vtg', inplace=True)
vtg_reg = np.arange(calib_tg['vtg'].min(), calib_tg['vtg'].max(),\
calib_tg['vtg'].diff().min())
det_smooth = savgol_filter(np.interp(vtg_reg, calib_tg['vtg'],\
calib_tg['det']), *sg_det)
calib_tg['det'] = np.interp(calib_tg['vtg'], vtg_reg, det_smooth)
ax[1].plot(calib_tg['vtg'], calib_tg['det'])
if sg_det or sg_rpar:
calib_tg['power'] = det2power(calib_tg['det'])
ax[2].plot(calib_tg['rpar'], calib_tg['power'], '.', markersize=3)
# savgol_filter on power vs rpar or csaps smoothingspline
calib_tg.sort_values(by='rpar', inplace=True)
rpar_reg = np.linspace(calib_tg['rpar'].min(), calib_tg['rpar'].max(), 500)
if not cs_smooth:
power = np.interp(rpar_reg, calib_tg['rpar'], calib_tg['power'])
smoothed_power = savgol_filter(power, *sg_power)
calib = sp.interpolate.interp1d(rpar_reg,
smoothed_power,
fill_value='extrapolate')
else:
calib = csaps.CubicSmoothingSpline(calib_tg['rpar'], calib_tg['power'],\
smooth=cs_smooth)
calib_data = calib_tg.loc[calib_tg['rpar'] > 0, :]
ax[2].plot(rpar_reg, calib(rpar_reg))
ax[0].set_xlabel(get_label('Vtg'))
ax[0].set_ylabel(get_label('rpar'))
ax[1].set_xlabel(get_label('Vtg'))
ax[1].set_ylabel(get_label('Vdet'))
ax[2].set_xlabel(get_label('rpar'))
ax[2].set_ylabel(get_label('P'))
return calib_data, calib
def filt_ssfit(time, flux, weights, gi_mask, smo, sds, return_filt_index=False, plot_q=False):
# from csaps import UnivariateCubicSmoothingSpline as ss
flux_fit = csaps.CubicSmoothingSpline(time[gi_mask], flux[gi_mask], smooth=smo, weights=weights[gi_mask])(time)
df = flux - flux_fit
dff = np.copy(df)
# for sd in sds:
# dff = sigma_clip(dff, sigma=sd, maxiters=3)
# __, gi, __ = np.intersect1d(df, dff, return_indices=1)
# gi = np.sort(gi)
# bi = np.ones(len(time), dtype=bool)
# bi[gi] = False
# gi = ~bi
gi = np.copy(gi_mask)
gi_cnt_0 = np.sum(gi)
for sdm in sds:
sd = np.std(dff[gi])
gi_new = ((dff > -sd * sdm) & (dff < sd * sdm))
if np.sum(gi_new) < gi_cnt_0 * 0.5:
print('Filter might be too strong. Attempted to filter >50% of the points in a section. Not applying this filter.')
break
gi = gi & gi_new
# Apply mask
#gi = gi & gi_mask
bi = ~gi
cf = csaps.CubicSmoothingSpline(time[gi], flux[gi], smooth=smo, weights=weights[gi])(time)
if plot_q:
# Show the filter plot
plt.figure(figsize=(10, 7))
plt.title('Filter Stage (single)')
plt.plot(time, flux, '.k')
plt.plot(time[bi], flux[bi], 'xr')
plt.plot(time, cf, '-m')
plt.show()
plt.pause(0.1)
if return_filt_index:
return cf, bi
else:
return cf
def meas_sig_old(time, flux, weights, plot_q=False):
from astropy.stats import sigma_clip
df = np.diff(flux)
x = np.copy(time[:-1]) #np.arange(len(df))
dff1 = csaps.CubicSmoothingSpline(x, df, smooth=0.1, weights=weights[:-1],)(x)
# print('dff1', dff1)
dfn = df - dff1
dfnf = np.copy(dfn)
# plt.figure(figsize=(10, 4))
# plt.plot(x, df, '.k', label='Orig dF')
# plt.plot(x, dff1, '-c', label='dF fit 1')
# plt.figure(figsize=(10, 4))
# plt.plot(x, dfn, '.k', label='Orig dF')
for sd in [5]:
dfnf = sigma_clip(dfnf, sigma=sd, maxiters=2)
__, gi, __ = np.intersect1d(dfn, dfnf, return_indices=1)
gi = np.sort(gi)
# plt.figure(figsize=(10, 4))
# plt.plot(x[gi], dfn[gi], '.k', label='Orig dF')
# try:
dff = csaps.CubicSmoothingSpline(x[gi], df[gi], smooth=0.1, weights=weights[:-1][gi],)(x)
# except:
# print('FAIL', len(x), len(gi))
# #FAIL 271 270 36338 -36608
# #FAIL 271 270 36338 -36608
# return
dfn = df - dff
if plot_q:
plt.figure(figsize=(10, 4))
plt.plot(x, df, '.k', label='Orig dF')
plt.plot(x, dff1, '-c', label='dF fit 1')
plt.plot(x, dff, '-m', label='dF fit 2')
plt.legend()
for sd in [5, 3]:
dfn = sigma_clip(dfn, sigma=sd, maxiters=5)
if plot_q:
plt.figure(figsize=(10, 7))
plt.title('RMS = ' + str(np.std(dfn)))
plt.plot(x, dfn, '.k')
return np.std(dfn)
def test_vectorize(y):
x = np.arange(np.array(y).shape[-1])
ys = csaps.CubicSmoothingSpline(x, y)(x)
np.testing.assert_allclose(ys, y)
def test_invalid_data(x, y, w):
with pytest.raises(ValueError):
csaps.CubicSmoothingSpline(x, y, weights=w)
def test_npoints(x, y, xi, yid):
sp = csaps.CubicSmoothingSpline(x, y)
yi = sp(xi)
np.testing.assert_allclose(yi, yid)
def fitting(x, y, target_h, ratio_w, ratio_h):
out_x = []
out_y = []
count = 0
x_size = p.x_size / ratio_w
y_size = p.y_size / ratio_h
for x_batch, y_batch in zip(x, y):
predict_x_batch = []
predict_y_batch = []
for i, j in zip(x_batch, y_batch):
min_y = min(j)
max_y = max(j)
temp_x = []
temp_y = []
jj = []
pre = -100
for temp in j[::-1]:
if temp > pre:
jj.append(temp)
pre = temp
else:
jj.append(pre + 0.00001)
pre = pre + 0.00001
sp = csaps.CubicSmoothingSpline(jj, i[::-1], smooth=0.0001)
last = 0
last_second = 0
last_y = 0
last_second_y = 0
for h in target_h[count]:
temp_y.append(h)
if h < min_y:
temp_x.append(-2)
elif min_y <= h and h <= max_y:
temp_x.append(sp([h])[0])
last = temp_x[-1]
last_y = temp_y[-1]
if len(temp_x) < 2:
last_second = temp_x[-1]
last_second_y = temp_y[-1]
else:
last_second = temp_x[-2]
last_second_y = temp_y[-2]
else:
if last < last_second:
l = int(last_second - float(-last_second_y + h) *
abs(last_second - last) /
abs(last_second_y + 0.0001 - last_y))
if l > x_size or l < 0:
temp_x.append(-2)
else:
temp_x.append(l)
else:
l = int(last_second + float(-last_second_y + h) *
abs(last_second - last) /
abs(last_second_y + 0.0001 - last_y))
if l > x_size or l < 0:
temp_x.append(-2)
else:
temp_x.append(l)
predict_x_batch.append(temp_x)
predict_y_batch.append(temp_y)
out_x.append(predict_x_batch)
out_y.append(predict_y_batch)
count += 1
return out_x, out_y
def test_big_vectorized():
x = np.linspace(0, 10000, 10000)
y = np.random.rand(1000, 10000)
xi = np.linspace(0, 10000, 20000)
csaps.CubicSmoothingSpline(x, y)(xi)
