def polinomial(i_registro):
# Espectro de entrada
x = np.copy(X)
y = np.copy(Y)
# Espectros de la base de datos RRUFF
xR = datos[i_registro][1]
yR = datos[i_registro][2]
# Empatar dominio de los espectros a comparar
xc, yc, xR, yR = fix_ind(x, y, xR, yR)
xc, yc, xR, yR = fix_ind(xc, yc, xR, yR)
# ============================================ Métodoc
# Eliminamos la tendencia del espectro de entrada
polynomial(yc, order=j + 1, plot=False)
# ============================================ Método
# =========================================== Filtro
# Filtramos el espectro
fs = 1000
fc = 50
order = 9
yf = lp(yc, fc, fs, order)
# Filtramos el espectro de la base de datos para tener una comparación en condiciones similares
yR = lp(yR, fc, fs, order)
# =========================================== Filtro
# Eliminamos los valores negativos del espectro llevando el mínimo a cero
yc = yf - np.amin(yf)
# Aplicamos un factor de relajación
yc = fac_re(yc, 1)
# Normalizamos los espectros
yc /= np.amax(yc)
yR /= np.amax(yR)
return mycorr(xc, yc, xR, yR)
def filtering(self, gsr_window, gsr_order, hr_window, hr_order):
# GSR
self.user_gsr = polynomial(self.user_gsr, order=gsr_order,
plot=False) # Dtrend
self.user_gsr = self.user_gsr.rolling(
window=gsr_window).mean().dropna().values # Moving Average Filter
plt.plot(self.user_gsr)
plt.show()
# Heart Rate
self.user_hr = self.user_hr.rolling(
window=hr_window).mean().dropna().values # Moving Average Filter
for i in range(20000, len(self.user_hr),
20000): # Apply dtrend filter every 20,000 data.
self.user_hr[i - 20000:i] = polynomial(self.user_hr[i - 20000:i],
order=hr_order,
plot=False) # Dtrend
if (i + 20000 > len(self.user_hr)):
self.user_hr[i:] = polynomial(self.user_hr[i:],
order=hr_order,
plot=False) # Dtrend
break
plt.plot(self.user_hr)
plt.show()
def test_polynomial_detrend_plotting(self):
"""
Tests the plotting of the polynomial detrend operation.
"""
tr = obspy.read()[0].filter("highpass", freq=2)
tr.data += 6000 + 4 * tr.times() ** 2 - 0.1 * tr.times() ** 3 - \
0.00001 * tr.times() ** 5
with ImageComparison(self.path_images, 'polynomial_detrend.png') as ic:
polynomial(tr.data, order=3, plot=ic.name)
def test_polynomial_detrend_plotting(self):
"""
Tests the plotting of the polynomial detrend operation.
"""
tr = obspy.read()[0].filter("highpass", freq=2)
tr.data += 6000 + 4 * tr.times() ** 2 - 0.1 * tr.times() ** 3 - \
0.00001 * tr.times() ** 5
with ImageComparison(self.path_images, 'polynomial_detrend.png') as ic:
polynomial(tr.data, order=3, plot=ic.name)
def poly_trend(X_inp):
X_inp_poly = []
for ele in X_inp:
X_temp = []
X_temp.extend(polynomial(ele[:len(ele) // 3], 6, plot=False))
X_temp.extend(
polynomial(ele[len(ele) // 3:2 * len(ele) // 3], 6, plot=False))
X_temp.extend(polynomial(ele[2 * len(ele) // 3:], 6, plot=False))
X_inp_poly.append(X_temp)
return np.array(X_inp_poly)
def finalfilter(st, f0, bazi, astime, aetime, rot, debug=False):
st2 = st.copy()
st2 = filtergauss(st2, f0)
for tr in st2:
polynomial(tr.data, 3)
if debug:
print(st2)
if rot:
st2.rotate('NE->RT', back_azimuth=bazi)
st2.trim(astime, aetime)
st2.taper(0.05)
st2.sort()
return st2
def detrend(df, cols, vehicle, device, settings):
detrended_df = pd.DataFrame()
fig, axn = plt.subplots(2, 1, figsize=(10, 10))
fig.suptitle("Experiment: {0}".format(vehicle), fontsize=18)
df_segments = df.groupby("SEGMENT_ID")
print(len(df_segments))
mean = dict()
for index, (_, segment) in enumerate(df_segments):
if settings["MEAN_SHIFT_TARGET_SEGMENT"] == index:
for col in cols:
if settings["MEAN_SHIFT_FROM_THE_END"]:
mean[col] = segment[col][-1800:].mean()
else:
mean[col] = segment[col][:1800].mean()
for _, segment in df_segments:
for col in cols:
polynomial(segment[col], order=settings["DETRENDING_ORDER"])
if settings["MEAN_SHIFT"]:
segment[col] += mean[col]
segment[col][segment[col] < 0] = 0
detrended_df = detrended_df.append(segment)
for ax, col in zip(axn, cols):
if settings["PLOT_ORIGINAL"]:
sns.lineplot(data=df[col], ax=ax, label="Original", ci=None)
if settings["PLOT_DETRENDED"]:
sns.lineplot(data=detrended_df[col],
ax=ax,
label="Detrended",
ci=None)
ax.legend(loc="best")
ax.set_title(col)
plt.show()
fig.savefig(
"../../../Google Drive/Academia/PhD Thesis/Modeling Outputs/{0}/{1} - Detrended PM and PN.jpg"
.format(settings[device]["OUTPUT_TYPE"], vehicle),
dpi=300,
quality=95,
bbox_inches="tight",
)
return detrended_df
def test_polynomial_detrend(self):
"""
Simple test removing polynomial detrends.
"""
coeffs = [(1, 2, 3), (2, -4), (-3, 2, -5, 15), (-10, 20, -1, 2, 15)]
data = np.linspace(-5, 5, 100)
for c in coeffs:
# Create data.
d = np.polyval(c, data)
original_ptp = np.ptp(d)
# Detrend with polynomial of same order.
detrended = polynomial(d, order=len(c) - 1)
# Make sure the maximum amplitude is reduced by some orders of
# magnitude. It should almost be reduced to zero as we detrend a
# polynomial with a polynomial...
self.assertLess(np.ptp(detrended) * 1E10, original_ptp)
def test_polynomial_detrend(self):
"""
Simple test removing polynomial detrends.
"""
coeffs = [(1, 2, 3), (2, -4), (-3, 2, -5, 15), (-10, 20, -1, 2, 15)]
data = np.linspace(-5, 5, 100)
for c in coeffs:
# Create data.
d = np.polyval(c, data)
original_ptp = np.ptp(d)
# Detrend with polynomial of same order.
detrended = polynomial(d, order=len(c) - 1)
# Make sure the maximum amplitude is reduced by some orders of
# magnitude. It should almost be reduced to zero as we detrend a
# polynomial with a polynomial...
self.assertLess(np.ptp(detrended) * 1E10, original_ptp)
# prev_ele = ele
# drop_terms.extend(['period','powerSetPoint','sigma','delay'])
drop_terms.extend(['delay', 'Unnamed: 0', 'delay_st'])
X_out = df_inp.drop(drop_terms, axis=1)
# X_out = np.array(X_out)
return np.array(X_out), np.array(y_out), np.array(y_out_st)
X_test, y_test, y_test_out = preprocess(test_df)
X_train, y_train, y_train_out = preprocess(train_df)
signal = X_train[2][:len(X_train[0]) // 3]
polynomial(signal, order=6, plot=True)
exit()
trend_X = []
for ele in X_train:
sg1 = ele[:len(ele) // 3]
sg2 = ele[len(ele) // 3:2 * len(ele) // 3]
sg3 = ele[2 * len(ele) // 3:]
polynomial(sg1, order=6)
polynomial(sg2, order=6)
polynomial(sg3, order=6)
new_ele = []
new_ele.extend(sg1)
new_ele.extend(sg2)
new_ele.extend(sg3)
new_ele.extend(ele - new_ele)
if not database.ready or not database.internet:
ser.write("2".encode());#Parar
break
if nframe == 0:
(x,y,z) = frame.shape
linea = int(180*x/270)
intervalo = int(40*y/480)
mitad=int(y/2)
#Calculamos los valores de la gráfica
h0 = frame[linea,:,0]/3 + frame[linea-1,:,0]/3 + frame[linea+1,:,0]/3
h1 = frame[linea,:,1] + frame[linea-1,:,1]/3 + frame[linea+1,:,1]/3
h2 = frame[linea,:,2] + frame[linea-1,:,2]/3 + frame[linea+1,:,2]/3
h = (h0/3+h1/3+h2/3)
v = polynomial(h,2)
#Si es el primer frame, calculamos la posición de las líneas de una forma distinta
if nframe == 0:
half = int(y/2)
maximo1 = np.argmax(v[0:half])
maximo2 = np.argmax(v[half:y]) + half
distanciaLineas = int(350*y/480)
if maximo2 - maximo1 < distanciaLineas-100:
unaLinea = True
if v[maximo1] > v[maximo2]:
maximo = maximo1
else:
maximo = maximo2
else:
distanciaLineasTotal = maximo2-maximo1
# Loading the Gtec file
f = h5py.File(filename, "r")
Samples = f["RawData"]["Samples"].value
# CAR = Samples.mean(axis=1)
# Samples = Samples - np.atleast_2d(CAR).T * np.ones([1, Samples.shape[1]])
fsample = 250.
samples = Samples.shape[0]
channels = Samples.shape[1]
time_point = 226
for ch in range(channels):
data = Samples[:, ch]
polynomial(data, order=3, plot=False)
Samples[:, ch] = data
sig = Samples.T
b, a = signal.butter(4, 45 / fsample, 'low', analog=True)
sig = signal.filtfilt(b, a, sig, axis=1)
b, a = signal.butter(4, 2 / fsample, 'high', analog=True)
sig = signal.filtfilt(b, a, sig, axis=1)
Q = 30.0 # Quality factor
w0 = 50 / (fsample / 2) # Normalized Frequency
b, a = signal.iirnotch(w0, Q)
sig = signal.filtfilt(b, a, sig)
Samples = sig.T
plt.figure()
plt.plot(Samples)
old_oinds = bins > np.min(newbins)
new_oinds = newbins < np.max(bins)
bins = np.concatenate((bins, newbins))
newdat = manifold[xpos][zpos][resp][1] #+ 1e-15 * manind
if manind != 0:
offset = np.mean(dat[old_oinds]) - np.mean(
newdat[new_oinds])
if np.isnan(offset):
offset = 0
newdat += offset
if detrend:
newdat = polynomial(newdat, order=order, plot=False)
#newdat = signal.detrend(newdat)
dat = np.concatenate((dat, newdat))
newerrs = manifold[xpos][zpos][resp][2]
errs = np.concatenate((errs, newerrs))
sort_inds = np.argsort(bins)
bins_sort = bins[sort_inds]
dat_sort = dat[sort_inds]
errs_sort = errs[sort_inds]
axarr[resp].plot(bins_sort, dat_sort, '.', markersize=2)
axarr[resp].set_ylabel('Force [N]', fontsize=10)
axarr[resp].set_ylim(plot_lim[0], plot_lim[1])
#pylab.savefig("/Users/sheilasagear/OneDrive/K2_Research/bls-ktransit/Pipeline/EPIC229227254-2sigclip/Period" + str(p) + "Radius" + str(r) + "/folded_pltPer" + str(p) + "Rad" + str(r) + 'BLSoverlay.png')
#pylab.show()
##########################
#Detrending Flux and Levenberg-Marquardt Fitting:
#if SDE>6
##########################
#start values are correct values
if True: #SDE >= 6:
#polynomial detrending
fluxDetrend = polynomial(flux, order=5)
#untrendy
#fluxDetrend, ferr = untrendy.untrend(time, merged_flux)
fitT = FitTransit()
fitT.add_guess_star(rho=1.5)
fitT.add_guess_planet(period=1.0, impact=0.0, T0=3.0,
rprs=.2) #need a guess rprs
fitT.add_data(time=time, flux=fluxDetrend)
vary_star = [
'rho'
] # not sure how to avoid free stellar parameters? ideally would not vary star at all
vary_planet = (['period', 'rprs'])
def _run_interface(self, runtime):
print("Linear detrending")
print("=================")
# Output from previous preprocessing step
ref_path = self.inputs.in_file
# Load data
dataimg = nib.load(ref_path)
data = dataimg.get_data()
tp = data.shape[3]
# GLM: regress out nuisance covariates
new_data_det = data.copy()
gm = nib.load(self.inputs.gm_file[0]).get_data().astype(np.uint32)
from scipy import signal
for index, value in np.ndenumerate(gm):
if value == 0:
continue
Ydet = signal.detrend(data[index[0], index[1],
index[2], :].reshape(tp, 1),
axis=0)
new_data_det[index[0], index[1], index[2], :] = Ydet[:, 0]
img = nib.Nifti1Image(new_data_det, dataimg.get_affine(),
dataimg.get_header())
nib.save(img, os.path.abspath("fMRI_detrending.nii.gz"))
if self.inputs.mode == "quadratic":
print("Quadratic detrending")
print("=================")
from obspy.signal.detrend import polynomial
# GLM: regress out nuisance covariates
new_data_det2 = new_data_det.copy()
for index, value in np.ndenumerate(gm):
if value == 0:
continue
Ydet = polynomial(new_data_det2[index[0], index[1],
index[2], :],
order=2)
img = nib.Nifti1Image(new_data_det2, dataimg.get_affine(),
dataimg.get_header())
nib.save(img, os.path.abspath("fMRI_detrending.nii.gz"))
if self.inputs.mode == "cubic":
print("Cubic-spline detrending")
print("=================")
from obspy.signal.detrend import spline
# GLM: regress out nuisance covariates
new_data_det2 = new_data_det.copy()
for index, value in np.ndenumerate(gm):
if value == 0:
continue
Ydet = spline(new_data_det2[index[0], index[1], index[2], :],
order=3)
img = nib.Nifti1Image(new_data_det2, dataimg.get_affine(),
dataimg.get_header())
nib.save(img, os.path.abspath("fMRI_detrending.nii.gz"))
print("[ DONE ]")
return runtime
# print "nsample", hdr_input.nSamples
# print "endsample", endsample
# print "begsample", begsample
dat_input = ft_input.getData([begsample, endsample])
# t = np.arange(sample_rate)
# f = 440
# signal = np.sin(t*f/sample_rate)*256
# signal = np.zeros([sample_rate, 1])
tmp = []
for ch in range(nInputs):
channel = np.zeros(300)
original = copy(dat_input[:, ch])
polynomial(original, order=10, plot=False)
# One
channel.append(np.ones(1) * scaling / 250.)
# Spectrum
fourier = np.fft.rfft(original, 250)[f_min:f_max]
channel.append(fourier)
# Positions
mask = np.zeros(30)
mask[(positions[ch][0] - 1):positions[ch][0]] = scaling / 250
mask[(10 + positions[ch][1] - 1):(10 +
positions[ch][1])] = scaling / 250
mask[(20 + positions[ch][2] - 1):(20 +
positions[ch][2])] = scaling / 250
channel.append(mask)
import matplotlib.pyplot as plt
import scipy.io
import numpy as np
import obspy
from obspy.signal.detrend import polynomial
import os
os.chdir("C:\Work Related\Projects\Sunil\Hackathon")
ecg=scipy.io.loadmat('100m.mat')['val']
#ecg=np.reshape(ecg,len(ecg))
plt.plot(ecg)
ecgdt = polynomial(ecg[1], order=12)
import pywt
from statsmodels.tsa.arima_model import ARIMA
model = ARIMA(signal, order=(2, 1, 2))
from statsmodels.tsa.ar_model import AR
x = ar_mod.fit(5)
x.fittedvalues
from sklearn import svm
from sklearn import svm
X = [[0, 0], [1, 1]]
In[117]: y = [0, 1]
clf = svm.SVC()
clf.fit(X, y)
plt.xlabel('Corrimiento Raman [cm⁻¹]')
plt.ylabel('Intensidad [U.A.]')
plt.legend()
for n in range(3):
grado_ini = grad[n]
grado_ini += 1
registro = imax[n]
corr = max[n]
nombre = datos[registro][0]
x = np.copy(X)
y = np.copy(Y)
xR = datos[registro][1]
yR = datos[registro][2]
xc, yc, xR, yR = fix_ind(x, y, xR, yR)
xc, yc, xR, yR = fix_ind(xc, yc, xR, yR)
polynomial(yc, order=grado_ini, plot=False)
fs = 1000
fc = 50
order = 9
yf = lp(yc, fc, fs, order)
yR = lp(yR, fc, fs, order)
yc = yf - np.amin(yf)
yc = fac_re(yc, 1)
yc /= np.amax(yc)
yR /= np.amax(yR)
plt.subplot(2, 2, n + 2)
plt.plot(xc, yc, label='Espectro corregido')
plt.plot(xR, yR, '.', label='Muestra: ' + nombre, markersize=1)
plt.xlabel('Corrimiento Raman [cm⁻¹]')
plt.title("Grado del polinomio: %1d" % grado_ini +
"\nCoeficiente de correlación: %.4f" % corr)
def main(obj):
t_period = obj.period
t_sig = obj.time
try:
ch = int(obj.ch)
sig = obj.data[ch, :]
except:
sig = obj.data
# detrending signal
de_sig = np.copy(sig)
# detrended by polynominal function
de_sig = detrend.polynomial(de_sig, order=3)
# conditioning process
condi_sig = cond_reshape(t_sig, de_sig, t_period)
# normalized time
nt = condi_sig['nt']
# data,time
d = condi_sig['d']
t = condi_sig['t']
# DATA FITTING
gn = condi_sig['gn']
coeff = np.zeros((gn, 2))
sig_fit = np.copy(d)
for i in range(0, gn):
# calculate diagnostic slowing-down time and source
coeff[i,:], pcov = \
curve_fit(
model_eq, nt[i,:], d[i,:], p0=[1e13,10]
)
# get fitting result
sig_fit[i,:] = \
model_eq(
nt[i,:], coeff[i,0], coeff[i,1]
)
# Calculate the conditional averaged slowing-down time and source
condi_coeff, pcov = \
curve_fit(
model_eq, nt[0,:], np.mean(d,axis=0), p0=[1e13,10]
)
# Get fitting result
condi_sig_fit = \
model_eq(nt[0,:], condi_coeff[0], condi_coeff[1])
obj.ddata = d
obj.ddatafit = sig_fit
obj.t_ddata = t
obj.conddata = np.mean(sig_fit, axis=0)
obj.condfit = condi_sig_fit
obj.ntime = nt
obj.S = coeff[:, 0]
obj.Tau = coeff[:, 1]
obj.condS = condi_coeff[0]
obj.condTau = condi_coeff[1]
obj.gn = gn
return obj
