def test_axis(shape, axis):
y = np.arange(int(np.prod(shape))).reshape(shape)
x = np.arange(np.array(y).shape[axis])
ys = csaps.UnivariateCubicSmoothingSpline(x, y, axis=axis)(x)
np.testing.assert_allclose(ys, y)
def stdFiltIt(wl, arr_1d, weights_1d, sdms, smo, plot_q):
if plot_q:
plt.figure(figsize=(10, 5))
plt.title('Filter plot')
plt.plot(wl, arr_1d, '.k')
fit = arr_1d + np.nan
for sdm in sdms:
gi = np.logical_and(np.isfinite(weights_1d), np.isfinite(arr_1d))
fit = csaps.UnivariateCubicSmoothingSpline(wl[gi],
arr_1d[gi],
smooth=smo)(wl)
dy_sd = np.std(arr_1d[gi] - fit[gi]) * sdm
gi = np.logical_and(np.abs(arr_1d - fit) <= dy_sd, gi)
if plot_q:
plt.plot(wl[np.logical_not(gi)], arr_1d[np.logical_not(gi)], 'xr')
plt.plot(wl, fit + dy_sd, ':g')
plt.plot(wl, fit - dy_sd, ':r')
plt.grid(True)
arr_1d[np.logical_not(gi)] = np.nan
if plot_q:
plt.show()
return arr_1d, gi
def firm_behaviour(price, individual_firm_costs):
fig = plt.figure()
ax = fig.add_axes([0, 0, 1, 1])
output = individual_firm_costs.column("Output")[1:]
mc = individual_firm_costs.column("Marginal Cost")[1:]
sp_mc = csaps.UnivariateCubicSmoothingSpline(output, mc, smooth=0.85)
output_s = np.linspace(output.min(), output.max(), 150)
mc_s = sp_mc(output_s)
plt.plot(individual_firm_costs.column("Output")[1:],
individual_firm_costs.column("Average Variable Cost")[1:],
marker='o')
plt.plot(individual_firm_costs.column("Output")[1:],
individual_firm_costs.column("Average Total Cost")[1:],
marker='o')
plt.plot(output, mc, 's', color='tab:red')
plt.plot(output_s,
mc_s,
alpha=0.7,
lw=2,
label='_nolegend_',
color='tab:red')
plt.xlabel('Quantity')
plt.ylabel('Cost')
plt.title('Production')
plt.ylim(15, 105)
plt.axhline(y=price, color='k', linewidth=2)
plt.legend(
make_array("Average Variable Cost", "Average Total Cost",
"Marginal Cost", "Price"))
min_avc = min(individual_firm_costs.column("Average Variable Cost")[1:])
min_atc = min(individual_firm_costs.column("Average Total Cost")[1:])
output = max(
individual_firm_costs.where(
"Marginal Cost", are.below_or_equal_to(price)).column("Output"))
atc_at_output = individual_firm_costs.where(
"Output", output).column("Average Total Cost").item(0)
if price < min_avc:
print("No production")
elif price >= min_avc and price < min_atc:
print("Production at loss minimising quantity")
ax.add_patch(
patches.Rectangle((0, price),
output,
atc_at_output - price,
color='red',
alpha=.3,
zorder=1000))
elif price >= min_atc:
print("Production at a profit")
ax.add_patch(
patches.Rectangle((0, price),
output,
atc_at_output - price,
color='green',
alpha=.3,
zorder=1000))
def test_auto_smooth():
np.random.seed(1234)
x = np.linspace(0, 2 * np.pi, 21)
y = np.sin(x) + (np.random.rand(21) - .5) * .1
sp = csaps.UnivariateCubicSmoothingSpline(x, y)
xi = np.linspace(x[0], x[-1], 120)
yi = sp(xi)
assert isinstance(sp.spline, csaps.SplinePPForm)
np.testing.assert_almost_equal(sp.smooth, 0.996566686)
desired_yi = [
-0.0235609972734076, 0.0342554130011887, 0.0917604768962524,
0.148642848032251, 0.204591180029653, 0.259294126508924,
0.312440351240669, 0.363812477806949, 0.413516746584544,
0.461729017734914, 0.508625151419519, 0.554381007799819,
0.59917235322775, 0.643050468816528, 0.685696744725134,
0.726724183969348, 0.765745789564952, 0.802374564527724,
0.836223661222756, 0.866972935842014, 0.894473725242276,
0.918604542323229, 0.939243899984561, 0.956270311125961,
0.969562963467305, 0.979144106608908, 0.985355180101302,
0.988580811987222, 0.989205630309408, 0.987614263110598,
0.984190455154498, 0.979211074442117, 0.972745418402277,
0.964838935929978, 0.955537075920216, 0.944885287267991,
0.932927490881626, 0.919589386774692, 0.904596282338286,
0.88765407387356, 0.868468657681665, 0.846745930063754,
0.822194253635299, 0.794653654806466, 0.76415937272086,
0.73076244387651, 0.694513904771452, 0.655464791903716,
0.613667457460308, 0.569225401037061, 0.522308564522944,
0.473091330257213, 0.421748080579123, 0.368453197827928,
0.313386409949078, 0.256884901954999, 0.199465270307742,
0.141653871417018, 0.0839770616925367, 0.0269611975440111,
-0.0288790108903068, -0.0832967177157068, -0.136312941281018,
-0.187960346206531, -0.238271597112534, -0.287279358619317,
-0.335016296758048, -0.381515103495269, -0.426808493559216,
-0.47092918245087, -0.513909885671217, -0.555783318721242,
-0.596577904130849, -0.636257828862702, -0.674737831212602,
-0.711931377484918, -0.74775193398402, -0.782112967014277,
-0.814916147256374, -0.845917383266534, -0.874774272419685,
-0.901142570534573, -0.92467803342995, -0.945036416924564,
-0.961906593475871, -0.975319318974765, -0.985507042193372,
-0.99270481875293, -0.997147704274675, -0.999070754379844,
-0.998693087710525, -0.996095112164348, -0.991285814362011,
-0.984273590665723, -0.975066837437697, -0.963673951040142,
-0.95006546257481, -0.933932055202886, -0.914838983410284,
-0.892350910038224, -0.86603249792793, -0.835448409920623,
-0.800282146364173, -0.76096707154776, -0.718228237424455,
-0.672791349033563, -0.625382111414392, -0.576726229606248,
-0.527490657611685, -0.478024578050658, -0.428570228926154,
-0.379369767649891, -0.330665351633583, -0.282699138288946,
-0.23565484652653, -0.189444552928148, -0.143901464676976,
-0.098858780436212, -0.0541496988690512, -0.00960741863869263,
]
np.testing.assert_allclose(yi, desired_yi)
def test_splineppform(y, order, pieces, ndim):
x = np.arange(np.array(y).shape[-1])
y = np.array(y)
s = csaps.UnivariateCubicSmoothingSpline(x, y).spline
assert s.order == order
assert s.pieces == pieces
assert s.ndim == ndim
def test_weighted(w, yid):
x = [1., 2., 4., 6.]
y = [2., 4., 5., 7.]
xi = np.linspace(1., 6., 10)
sp = csaps.UnivariateCubicSmoothingSpline(x, y, weights=w)
yi = sp(xi)
np.testing.assert_allclose(yi, yid)
def test_univariate_two_points():
x = [1., 2.]
y = [3., 4.]
sp = csaps.UnivariateCubicSmoothingSpline(x, y)
xi = [1., 1.5, 2.]
yi = sp(xi)
np.testing.assert_almost_equal(sp.smooth, 1.)
np.testing.assert_allclose(yi, [3., 3.5, 4.])
def csaps1D(self, x, y, p, xx=None, w=1.0, dtype=complex):
if np.isscalar(w):
w = w * np.ones(len(x))
print np.shape(x), np.shape(y), np.shape(w)
sp = csobj.UnivariateCubicSmoothingSpline(x, y, w, p)
if xx is None:
xx = x
return sp(xx)
def test_zero_smooth():
x = [1., 2., 4., 6.]
y = [2., 4., 5., 7.]
sp = csaps.UnivariateCubicSmoothingSpline(x, y, smooth=0.)
assert sp.smooth == pytest.approx(0.)
ys = sp(x)
assert ys == pytest.approx([2.440677966101695,
3.355932203389830,
5.186440677966102,
7.016949152542373])
def test_univariate_four_points():
x = [1., 2., 4., 6.]
y = [2., 4., 5., 7.]
sp = csaps.UnivariateCubicSmoothingSpline(x, y)
xi = np.linspace(1., 6., 10)
yi = sp(xi)
desired_yi = [
2.2579392157892, 3.0231172855707, 3.6937304019483,
4.21971044584031, 4.65026761247821, 5.04804510368134,
5.47288175793241, 5.94265482897362, 6.44293945952166,
6.95847986982311
]
np.testing.assert_allclose(yi, desired_yi)
def test_vectorize(y):
x = np.arange(np.array(y).shape[-1])
ys = csaps.UnivariateCubicSmoothingSpline(x, y)(x)
np.testing.assert_allclose(ys, y)
def find_phase_offset(file_names: list, TEST=False):
wavelength = np.array([])
power_data = np.zeros((5001, 4))
for i in range(len(file_names)):
file_data = np.load(file_names[i], mmap_mode='r')
wavelength = np.array(file_data['wavelength']) * 1e-9
power_data[:, i] = 20 * np.log10(
np.array(file_data['power']).clip(min=0.0000001))
device_results = DeviceResult('Device_21', wavelength, power_data)
power_norm = copy.deepcopy(device_results.power)
plt.figure()
plt.suptitle('Device ' + device_results.filename)
for i in range(0, NUM_PORTS):
plt.subplot(NUM_PORTS, 1, i + 1)
amplitude = device_results.power[:, i]
plt.plot(device_results.wavelength * 1e9, amplitude, label='Measured')
p = np.polyfit(
device_results.wavelength - np.mean(device_results.wavelength),
amplitude, device_results.POLY_ORDER)
amplitude_baseline = np.polyval(
p, device_results.wavelength - np.mean(device_results.wavelength))
amplitude_corrected = amplitude - amplitude_baseline
amplitude_corrected = amplitude_corrected + max(
amplitude_baseline) - max(amplitude)
plt.plot(device_results.wavelength * 1e9,
amplitude_corrected,
label='GC removed')
plt.legend()
plt.xlim((1560, 1620))
power_norm[:, i] = amplitude_corrected
if TEST:
plt.show()
powers = 10**(power_norm / 20)
plt.figure()
plt.suptitle('Device ' + device_results.filename)
device_results.powerLinearNorm = copy.deepcopy(powers)
for i in range(0, NUM_PORTS):
x = device_results.wavelength
# power_negative = -one.powerLinearNorm[:, i]
# offset = -np.amin(power_negative)
# power_negative += offset
# lower_peaks = find_peaks(power_negative, height=height, distance=distance, width=width)[0]
# lower_fit = interp1d(x[lower_peaks], power_negative[lower_peaks], kind='cubic', fill_value='extrapolate')
# bottom_baseline = lower_fit(x)
# power_negative /= bottom_baseline
# power = np.array(-(power_negative - 1))
power = device_results.powerLinearNorm[:, i]
top_pkidx = find_peaks(power,
height=height,
distance=distance,
width=width)[0]
p = interp1d(x[top_pkidx],
power[top_pkidx],
kind='cubic',
fill_value='extrapolate')
top_baseline = p(x)
plt.subplot(NUM_PORTS, 1, i + 1)
plt.title('Port ' + str(i + 1))
plt.plot(x[top_pkidx] * 1e9, power[top_pkidx], "x")
plt.plot(x * 1e9, power)
plt.plot(x * 1e9, top_baseline)
plt.xlim((1560, 1620))
device_results.powerLinearNorm[:, i] = power / top_baseline
if TEST:
plt.show()
# print("Device \'" + one.filename + "\': Converting from wavelength to frequency")
# Slice off the bottom of the array which has wild tails due to spline fitting
device_results.frequency = np.flip(freq(device_results.wavelength) / 1e12)
device_results.powerLinearNormByFreq = np.flipud(
device_results.powerLinearNorm)
print(device_results.frequency.shape,
device_results.powerLinearNormByFreq.shape)
device_results.frequency = device_results.frequency[50:]
device_results.powerLinearNormByFreq = device_results.powerLinearNormByFreq[
50:, :]
print(device_results.frequency.shape,
device_results.powerLinearNormByFreq.shape)
# print("\n")
# print("Data Preview:")
# print("=============")
# print("Frequency array:", one.frequency)
# print("Power array:", one.powerLinearNormByFreq)
plt.figure()
for i in range(0, NUM_PORTS):
plt.subplot(NUM_PORTS, 1, i + 1)
plt.title('Port ' + str(i + 1))
plt.plot(device_results.frequency,
device_results.powerLinearNormByFreq[:, i])
if TEST:
plt.show()
# print("Smoothing:", smoothing)
device_results.sp = []
device_results.sp.append(
csaps.UnivariateCubicSmoothingSpline(
device_results.frequency,
device_results.powerLinearNormByFreq[:, 0],
smooth=smoothing))
device_results.sp.append(
csaps.UnivariateCubicSmoothingSpline(
device_results.frequency,
device_results.powerLinearNormByFreq[:, 1],
smooth=smoothing))
device_results.sp.append(
csaps.UnivariateCubicSmoothingSpline(
device_results.frequency,
device_results.powerLinearNormByFreq[:, 2],
smooth=smoothing))
device_results.sp.append(
csaps.UnivariateCubicSmoothingSpline(
device_results.frequency,
device_results.powerLinearNormByFreq[:, 3],
smooth=smoothing))
N = 10000
device_results.freqHighSamp = np.linspace(min(device_results.frequency),
max(device_results.frequency),
num=N)
powerHighSamp0 = device_results.sp[0](device_results.freqHighSamp)
powerHighSamp0 = powerHighSamp0 / max(powerHighSamp0)
powerHighSamp1 = device_results.sp[1](device_results.freqHighSamp)
powerHighSamp1 = powerHighSamp1 / max(powerHighSamp1)
powerHighSamp2 = device_results.sp[2](device_results.freqHighSamp)
powerHighSamp2 = powerHighSamp2 / max(powerHighSamp2)
powerHighSamp3 = device_results.sp[3](device_results.freqHighSamp)
powerHighSamp3 = powerHighSamp3 / max(powerHighSamp3)
device_results.powerHighSampNorm = np.stack(
(powerHighSamp0, powerHighSamp1, powerHighSamp2, powerHighSamp3),
axis=1)
plt.figure()
plt.suptitle(device_results.filename)
for i in range(0, NUM_PORTS):
plt.subplot(NUM_PORTS, 1, i + 1)
plt.title('Port ' + str(i + 1))
plt.plot(device_results.frequency,
device_results.powerLinearNormByFreq[:, i],
label='Normalized')
plt.plot(device_results.freqHighSamp,
device_results.powerHighSampNorm[:, i],
label='Fit to Spline')
plt.legend()
if TEST:
plt.show()
cosine_fit, cosine_covariance = curve_fit(
cos_squared, device_results.freqHighSamp,
device_results.powerHighSampNorm[:, i])
plt.plot(device_results.freqHighSamp, device_results.powerHighSampNorm[:,
i])
plt.plot(device_results.freqHighSamp,
cos_squared(device_results.freqHighSamp, *cosine_fit))
if TEST:
plt.show()
# print("====================")
# print(one.filename)
device_results.global_result = [None] * NUM_PORTS
device_results.local_result = [None] * NUM_PORTS
for port in range(NUM_PORTS):
Jmod = lambda x: J(device_results.freqHighSamp, device_results.
powerHighSampNorm[:, port], x)
bounds = [(0, 40), (0, 2 * np.pi)]
# Global optimization
device_results.global_result[port] = differential_evolution(
Jmod, bounds)
# Convex optimization
device_results.local_result[port] = minimize(
Jmod, device_results.global_result[port].x, method='nelder-mead')
# Verbose
# print('Port ' + str(port+1))
# print('Global:', one.global_result[port].x, one.global_result[port].fun)
# print('Local:', one.local_result[port].x, one.local_result[port].fun)
plt.figure()
plt.suptitle(device_results.filename)
for port in range(NUM_PORTS):
plt.subplot(NUM_PORTS, 1, port + 1)
device_results.plotPowerLinearNormByFreq(port)
device_results.plotPowerHighSampNormByFreq(port)
device_results.plotGlobalOptimizationCurve(port)
device_results.plotLocalOptimizationCurve(port)
plt.xlabel("Frequency (THz)")
plt.ylabel("Amplitude (arbitrary units)")
plt.xlim((freq(1620 * 1e-9) * 1e-12, freq(1560 * 1e-9) * 1e-12))
plt.title("Power")
if TEST:
plt.show()
phasediff = device_results.phase(device_results.frequency,
0) - device_results.phase(
device_results.frequency, 2)
middleindex = min(range(len(phasediff)),
key=lambda i: abs(phasediff[i] - (np.pi / 2)))
print("CROSSOVER FREQUENCY: ", device_results.frequency[middleindex],
"THz")
c0 = 299792458 # m/s
crossover_wavelength = c0 / (device_results.frequency[middleindex] * 1e12)
print("CROSSOVER WAVELENGTH:", crossover_wavelength * 1e9, "nm")
middlefreq = device_results.frequency[middleindex]
freq_array = [
min(device_results.frequency), middlefreq,
max(device_results.frequency)
]
fig = plt.figure(constrained_layout=True,
figsize=(18, 16),
dpi=80,
facecolor='w',
edgecolor='k')
gs = fig.add_gridspec(3, 3)
ax = fig.add_subplot(gs[0, :])
for i in range(NUM_PORTS):
device_results.plotPowerLinearNormByFreq(i)
for i in range(3):
plt.axvline(freq_array[i], ls='--', color='0.0')
plt.text(freq_array[i] - 0.02, -0.20, chr(i + 97) + ".")
plt.ylabel("Power (a.u.)")
plt.grid(True)
ax = fig.add_subplot(gs[1, :])
device_results.plotPhase()
for i in range(3):
plt.axvline(freq_array[i], ls='--', color='0.0')
plt.ylabel("Phase (rad)")
plt.xlabel("Frequency (THz)")
plt.grid(True)
mini = fig.add_subplot(gs[2, 0])
device_results.plotPolarPhaseAtFreq(freq_array[0], mini, True)
plt.title("a.")
mini = fig.add_subplot(gs[2, 1])
device_results.plotPolarPhaseAtFreq(freq_array[1], mini, True)
plt.title("b.")
mini = fig.add_subplot(gs[2, 2])
device_results.plotPolarPhaseAtFreq(freq_array[2], mini, True)
plt.title("c.")
plt.show()
def test_invalid_data(x, y, w):
with pytest.raises(ValueError):
csaps.UnivariateCubicSmoothingSpline(x, y, weights=w)
def hintergrund_fit_korrektur(path,
kind='both',
cutoff=1010,
sel_area=(1010, 'end'),
peak_widths=(1, 5),
butter=(3, 0.05)):
"""
Spline fit as the approximation of the background radiation in the plasma
is substraced from the spectrums
"""
sapa = r'V:\A11\2019\DIPLOM\BALHORN\Bilder\Splinefit\SplineFitTitanium.png'
name = path.split(os.sep)[-1]
name2 = path.split(os.sep)[-1].replace('.csv', '')
sapa = path.replace(name, f'\Spline_Korrektur\\{name2}')
print(sapa)
if kind == 'linear Interpolation' or kind == 'both':
lin_int_df_CCD = []
if kind == 'Spline' or kind == 'both':
spline_df_CCD = []
df = read_from_csv(path, cutoff=cutoff, sel_area=sel_area)
dfs = df_cut_wo_exc(df, padding=0)
for ix in [3]: # range(len(dfs)):
count = 0
print(f'Section {ix} von {len(dfs)}')
columns = list(dfs[ix].columns.values)
if kind == 'linear Interpolation' or kind == 'both':
lin_int_df = []
if kind == 'Spline' or kind == 'both':
spline_df = []
for col in columns:
ydata = dfs[ix].loc[:, col].values
xdata = dfs[ix].loc[:, col].index.values
b, a = signal.butter(butter[0], butter[1])
filtered = signal.filtfilt(b, a, ydata * (-1))
peak_indicies = signal.find_peaks_cwt(filtered,
peak_widths,
min_length=1)
new_peaks = []
win = 20
for ind in peak_indicies:
if ind < win:
win = ind
if len(ydata) - ind < win:
win = len(ydata) - ind
sl = ydata[ind - win:ind + win]
new_peaks += [np.argmin(sl) + ind - win]
new_peaks = list(dict.fromkeys(sorted(new_peaks)))
xd = xdata.copy()
if kind == 'linear Interpolation' or kind == 'both':
lin_int_df += [
pd.DataFrame(zip(
xdata, ydata -
np.interp(xd, xdata[new_peaks], ydata[new_peaks])),
columns=['WL', col]).set_index('WL')
]
if kind == 'Spline' or kind == 'both':
sp = csaps.UnivariateCubicSmoothingSpline(xdata[new_peaks],
ydata[new_peaks],
smooth=0.45)
spline_df += [
pd.DataFrame(zip(xdata, ydata - sp(xd)),
columns=['WL', col]).set_index('WL')
]
if count == 0:
fig, ax = plt.subplots(nrows=2,
ncols=1,
figsize=(5, 4),
sharex=True)
ax[0].set_title('Titanium Spectrum')
ax[0].plot(xdata, ydata, color='navy', label='Data')
ax[0].plot(xdata[new_peaks],
ydata[new_peaks],
color='orange',
ls='none',
marker='o',
label='Minima')
ax[0].plot(xdata, sp(xdata), color='red', label='Spline')
#ax[0].set_title('Spline-Fit')
ax[0].set_ylabel('Intensity I [a.u.]')
ax[0].grid()
ax[0].legend()
ax[1].plot(xdata,
ydata - sp(xd),
color='navy',
label='corrected Data')
#ax[1].set_title('Korrected Data')
ax[1].set_ylabel('Intensity I [a.u.]')
ax[1].set_xlabel('Wavelenght $\lambda$ [nm]')
ax[1].grid()
ax[1].legend()
plt.tight_layout()
plt.savefig(sapa, dpi=600)
plt.show()
else:
print(
'Zum Hintergrundfit sind nur "linear Interpolation", "Spline" und "both" implementiert'
)
count += 1
if kind == 'linear Interpolation' or kind == 'both':
lin_int_df = pd.concat(lin_int_df, axis=1)
lin_int_df_CCD += [lin_int_df]
if kind == 'Spline' or kind == 'both':
spline_df = pd.concat(spline_df, axis=1)
spline_df_CCD += [spline_df]
# if kind=='linear Interpolation' or kind=='both':
# lin_int_res = pd.concat(lin_int_df_CCD,axis=0)
# lin_int_res.to_csv(sapa + '_lin_int_corr.csv', sep=';', float_format='%.10f')
# if kind=='Spline' or kind=='both':
# spline_res = pd.concat(spline_df_CCD, axis=0)
# spline_res.to_csv(sapa + '_spline_corr.csv', sep=';', float_format='%.10f')
return
def meas_sig_old(time, flux, weights, plot_q=False):
from astropy.stats import sigma_clip
df = np.diff(flux)
x = np.copy(time[:-1]) #np.arange(len(df))
dff1 = csaps.UnivariateCubicSmoothingSpline(
x,
df,
smooth=0.1,
weights=weights[:-1],
)(x)
# print('dff1', dff1)
dfn = df - dff1
dfnf = np.copy(dfn)
# plt.figure(figsize=(10, 4))
# plt.plot(x, df, '.k', label='Orig dF')
# plt.plot(x, dff1, '-c', label='dF fit 1')
# plt.figure(figsize=(10, 4))
# plt.plot(x, dfn, '.k', label='Orig dF')
for sd in [5]:
dfnf = sigma_clip(dfnf, sigma=sd, maxiters=2)
__, gi, __ = np.intersect1d(dfn, dfnf, return_indices=1)
gi = np.sort(gi)
# plt.figure(figsize=(10, 4))
# plt.plot(x[gi], dfn[gi], '.k', label='Orig dF')
# try:
dff = csaps.UnivariateCubicSmoothingSpline(
x[gi],
df[gi],
smooth=0.1,
weights=weights[:-1][gi],
)(x)
# except:
# print('FAIL', len(x), len(gi))
# #FAIL 271 270 36338 -36608
# #FAIL 271 270 36338 -36608
# return
dfn = df - dff
if plot_q:
plt.figure(figsize=(10, 4))
plt.plot(x, df, '.k', label='Orig dF')
plt.plot(x, dff1, '-c', label='dF fit 1')
plt.plot(x, dff, '-m', label='dF fit 2')
plt.legend()
for sd in [5, 3]:
dfn = sigma_clip(dfn, sigma=sd, maxiters=5)
if plot_q:
plt.figure(figsize=(10, 7))
plt.title('RMS = ' + str(np.std(dfn)))
plt.plot(x, dfn, '.k')
return np.std(dfn)
def normalize_region(wl, flux, ferr, sds, plot_q=True, ss_smo=1e-3):
__, gi = stdFiltIt(wl,
np.copy(flux),
np.ones_like(flux),
sds,
smo=ss_smo,
plot_q=plot_q)
cont_fit = csaps.UnivariateCubicSmoothingSpline(wl[gi],
flux[gi],
smooth=ss_smo)(wl)
if plot_q:
plt.figure(figsize=(10, 5))
plt.plot(wl, flux, '.-k')
plt.plot(wl, cont_fit, '-r')
plt.title('Fit')
plt.xlabel('time')
plt.ylabel('Flux')
plt.grid(1)
plt.show()
#####################################
# Normalize continuum
flux_norm = flux / cont_fit
ferr_norm = ferr / cont_fit
if plot_q:
plt.figure(figsize=(10, 5))
#plt.plot(wl, flux_norm, '.-k')
plt.plot(wl,
flux_norm,
'-',
c=[0.5, 0.5, 0.5],
linewidth=1,
label='Flux')
if 1:
plt.fill_between(wl,
flux_norm - ferr_norm,
flux_norm + ferr_norm,
step='mid',
color=[0.85, 0.85, 0.85],
label='Flux Error')
plt.axhline(1.0, color='m', linewidth=1, label='Continuum')
plt.title('Normalized')
plt.xlabel('time [day]')
plt.ylabel('Flux [ppt]')
plt.grid(1)
plt.show()
return cont_fit, flux_norm, ferr_norm
from PIL import Image import numpy as np import requests from io import BytesIO import matplotlib.pyplot as plt url = "https://images-na.ssl-images-amazon.com/images/I/61MZ-5YHutL._SX425_.jpg" response = requests.get(url, stream=True) img = Image.open(response.raw) img = np.array(img) plt.figure() plt.imshow(img) plt.figure() # print(img.shape) ################################################################################ ### YOUR CODE GOES BELOW ### imgselected = img[10:100, 5, :] toplot = imgselected[:, 0] x = np.arange(90) sp = csaps.UnivariateCubicSmoothingSpline(x, toplot, smooth=0.9) xs = np.linspace(x[0], x[-1], 1000) ys = sp(xs) plt.figure(figsize=(10,6)) plt.plot(toplot, '-r', label='Data from the image') plt.plot(xs, ys, '-g', label='Smoothing spline fit') plt.legend()
def create(self, hours_ahead, test_size=0.1):
'''cleans and reshapes data into usable machine learning matricies X.
Parameters
----------
hours_ahead : int
the value of how many hours ahead we are trying to predict. The value must be an int between 4 and 8 (including both 4 and 8).
test_size : float, optional (default = 0.1)
the value of what percentage of X and y should be the saved for the testing sets. The value should be between 0 and 1 (not including 0).
'''
if not isinstance(hours_ahead, int):
print(
'hours_ahead needs to be an integer!\n Trying to make int...')
try:
hours_ahead = int(hours_ahead)
print('hours_ahead = %s' % (hours_ahead))
except:
print('Failed!')
import sys
sys.exit(1)
if not isinstance(test_size, float):
print(
'test_size needs to be a float between 0 and 1 (not including 0)'
)
print('setting to default of test_size = 0.1')
test_size = 0.1
if hours_ahead > 8:
print('Can\'t be greater than 8, so resetting hours_ahead to 8')
hours_ahead = 8
elif hours_ahead < 4:
print('Can\'t be less than 4, so resetting hours_ahead to 4')
hours_ahead = 4
self.n_temp_predictors = (24 - hours_ahead) * 10
self.n_predictors = self.n_temp_predictors + 4
self.hours_ahead = hours_ahead
values_ahead = hours_ahead * 10
data = self.df['OutTemp']
time = self.df['UT']
date = self.df['Date']
length = len(data)
X = []
y = []
y_date = []
pbar = ProgressBar()
for i in pbar(range(0,
length - self.n_temp_predictors - values_ahead)):
x = []
t = []
d = []
labels = []
if 3 <= self.df.hour[i + self.n_temp_predictors + values_ahead -
1] <= 7:
temp_hour = self.df.hour[i + self.n_temp_predictors +
values_ahead - 1]
for j in range(0, self.n_temp_predictors):
string = date[i + j] + ' ' + time[i + j]
date1 = datetime.strptime(string, '%Y-%m-%d %H:%M')
d.append(date1)
x.append(data[i + j])
labels.append('%s_hr_ago' % ((j + 1) * 6 / 60))
import csaps
t = np.array(range(len(x))) * 6
sp0 = csaps.UnivariateCubicSmoothingSpline(t, x, smooth=0.01)
xs0 = np.linspace(t[0], t[-1], self.n_temp_predictors)
x = sp0(xs0)
y_temp = data[i + self.n_temp_predictors + values_ahead - 1]
date_x_prob = np.sum(np.diff(d) != timedelta(minutes=6))
string = date[i + self.n_temp_predictors + values_ahead -
1] + ' ' + time[i + self.n_temp_predictors +
values_ahead - 1]
date_y = datetime.strptime(string, '%Y-%m-%d %H:%M')
date_y_prob = np.sum(
np.diff([d[-1], date_y]) != timedelta(
minutes=values_ahead * 6))
x = np.append(
x, self.df.month[i + self.n_temp_predictors +
values_ahead - 1])
x = np.append(
x,
self.df.day[i + self.n_temp_predictors + values_ahead - 1])
x = np.append(
x, self.df.hour[i + self.n_temp_predictors + values_ahead -
1])
x = np.append(
x, self.df.minute[i + self.n_temp_predictors +
values_ahead - 1])
labels.append('Month')
labels.append('Day')
labels.append('Hour')
labels.append('Minute')
if date_x_prob + date_y_prob == 0:
X.append(x)
y.append(y_temp)
y_date.append(date_y)
self.X = np.array(X)
self.y = np.array(y)
self.y_date = np.array(y_date)
print(self.X.shape)
if test_size * len(y) < 30:
test_size = 30
elif test_size * len(y) > 7200:
test_size = 7200
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=test_size, random_state=0, shuffle=False)
self.X_train = np.array(X_train)
self.X_test = np.array(X_test)
self.y_train = np.array(y_train)
self.y_test = np.array(y_test)
self.labels = np.array(labels)
def test_npoints(x, y, xi, yid):
sp = csaps.UnivariateCubicSmoothingSpline(x, y)
yi = sp(xi)
np.testing.assert_allclose(yi, yid)
def fitting(x, y, target_h, ratio_w, ratio_h):
out_x = []
out_y = []
count = 0
x_size = p.x_size / ratio_w
y_size = p.y_size / ratio_h
for x_batch, y_batch in zip(x, y):
predict_x_batch = []
predict_y_batch = []
for i, j in zip(x_batch, y_batch):
min_y = min(j)
max_y = max(j)
temp_x = []
temp_y = []
jj = []
pre = -100
for temp in j[::-1]:
if temp > pre:
jj.append(temp)
pre = temp
else:
jj.append(pre + 0.00001)
pre = pre + 0.00001
sp = csaps.UnivariateCubicSmoothingSpline(jj,
i[::-1],
smooth=0.0001)
last = 0
last_second = 0
last_y = 0
last_second_y = 0
for h in target_h[count]:
temp_y.append(h)
if h < min_y:
temp_x.append(-2)
elif min_y <= h and h <= max_y:
temp_x.append(sp([h])[0])
last = temp_x[-1]
last_y = temp_y[-1]
if len(temp_x) < 2:
last_second = temp_x[-1]
last_second_y = temp_y[-1]
else:
last_second = temp_x[-2]
last_second_y = temp_y[-2]
else:
if last < last_second:
l = int(last_second - float(-last_second_y + h) *
abs(last_second - last) /
abs(last_second_y + 0.0001 - last_y))
if l > x_size or l < 0:
temp_x.append(-2)
else:
temp_x.append(l)
else:
l = int(last_second + float(-last_second_y + h) *
abs(last_second - last) /
abs(last_second_y + 0.0001 - last_y))
if l > x_size or l < 0:
temp_x.append(-2)
else:
temp_x.append(l)
predict_x_batch.append(temp_x)
predict_y_batch.append(temp_y)
out_x.append(predict_x_batch)
out_y.append(predict_y_batch)
count += 1
return out_x, out_y
def test_big_vectorized():
x = np.linspace(0, 10000, 10000)
y = np.random.rand(1000, 10000)
xi = np.linspace(0, 10000, 20000)
csaps.UnivariateCubicSmoothingSpline(x, y)(xi)
def check_x(self, x, utc_dates):
'''this method will make sure to correctly interpolate/smooth the temperature data based off of the 'utc_dates' timestamps.
Parameters
----------
x : list
this should be a list of all the temperature values used as predictors. Each value should be 6 minutes appart and should be 160, 170, 180, 190, or 200 values in length.
utc_dates : list
this should be a list of the utc timestamp of the temperature value.
Returns
-------
list, list
Each list should be the lists are the same as the parameters but adjusted to have 6 minute intervals.
'''
if not isinstance(x, list) or not isinstance(utc_dates, list):
flag = True
try:
doesItWork = x[0]
except:
print('\'x\' must be a list.')
flag = False
try:
doesItWork = utc_dates[0]
except:
print('\'utc_dates\' must be a list.')
flag = False
if not flag:
return None, None
if len(x) != len(utc_dates):
print('\'x\' and \'utc_dates\' not the same length: %s and %s' %
(len(x), len(utc_dates)))
return None, None
if not isinstance(utc_dates[0], datetime):
print(
'\'utc_dates\' needs to be a list of datetime values. Currently, %s'
% (type(utc_dates[0])))
return None, None
if not isinstance(x[0], int):
print('\'x\' needs to be a list of int values. Currently, %s' %
(type(x[0])))
return None, None
dt = np.diff(utc_dates)
t = []
s = 0
for i in dt:
t.append(s)
s += i.seconds / 60
t.append(s)
sp0 = csaps.UnivariateCubicSmoothingSpline(t, x, smooth=0.01)
xs0 = np.array(range(int(np.ceil(t[-1] / 6)) + 1)) * 6
x_new = sp0(xs0)
new_utc_dates = []
for six in xs0:
temp_date = utc_dates[0] + timedelta(minutes=int(six))
new_utc_dates.append(temp_date)
return x_new, new_utc_dates
