def idwtmat(mmix_wt, n_coefs_approx):
"""
Invert wavelet transform data with order 2 Daubechies wavelet.
Parameters
----------
mmix_wt : {S, 2C} :obj:`numpy.ndarray`
Wavelet-transformed data. Approximation and detail coefficients are
horizontally concatenated for each row in mmix.
n_coefs_approx : :obj:`int`
The number of approximation coefficients from the wavelet
transformation. Used to split the wavelet-transformed data into
approximation and detail coefficients.
Returns
-------
mmix_iwt : {S, T} :obj:`numpy.ndarray`
Inverse wavelet-transformed data.
"""
n_samp = mmix_wt.shape[0]
coefs_approx = mmix_wt[:, :n_coefs_approx]
coefs_detail = mmix_wt[:, n_coefs_approx:]
n_trs = len(pywt.idwt(coefs_approx[0, :], coefs_detail[0, :], 'db2'))
mmix_iwt = np.zeros((n_samp, n_trs))
for i_samp in range(n_samp):
mmix_iwt[i_samp] = pywt.idwt(coefs_approx[i_samp, :],
coefs_detail[i_samp, :], 'db2')
return mmix_iwt
def iswt(coefficients, wavelet):
"""
Input parameters:
coefficients
approx and detail coefficients, arranged in level value
exactly as output from swt:
e.g. [(cA1, cD1), (cA2, cD2), ..., (cAn, cDn)]
wavelet
Either the name of a wavelet or a Wavelet object
"""
output = coefficients[0][0].copy() # Avoid modification of input data
#num_levels, equivalent to the decomposition level, n
num_levels = len(coefficients)
for j in range(num_levels,0,-1):
step_size = int(math.pow(2, j-1))
last_index = step_size
_, cD = coefficients[num_levels - j]
for first in range(last_index): # 0 to last_index - 1
# Getting the indices that we will transform
indices = arange(first, len(cD), step_size)
# select the even indices
even_indices = indices[0::2]
# select the odd indices
odd_indices = indices[1::2]
# perform the inverse dwt on the selected indices,
# making sure to use periodic boundary conditions
x1 = pywt.idwt(output[even_indices], cD[even_indices],wavelet, 'per')
x2 = pywt.idwt(output[odd_indices], cD[odd_indices],wavelet, 'per')
# perform a circular shift right
x2 = roll(x2, 1)
# average and insert into the correct indices
output[indices] = (x1 + x2)/2.
return output
def iswt(coefficients, wavelet):
"""
Input parameters:
coefficients
approx and detail coefficients, arranged in level value
exactly as output from swt:
e.g. [(cA1, cD1), (cA2, cD2), ..., (cAn, cDn)]
wavelet
Either the name of a wavelet or a Wavelet object
"""
output = coefficients[0][0].copy() # Avoid modification of input data
#num_levels, equivalent to the decomposition level, n
num_levels = len(coefficients)
for j in range(num_levels, 0, -1):
step_size = int(math.pow(2, j - 1))
last_index = step_size
_, cD = coefficients[num_levels - j]
for first in range(last_index): # 0 to last_index - 1
# Getting the indices that we will transform
indices = arange(first, len(cD), step_size)
# select the even indices
even_indices = indices[0::2]
# select the odd indices
odd_indices = indices[1::2]
# perform the inverse dwt on the selected indices,
# making sure to use periodic boundary conditions
x1 = pywt.idwt(output[even_indices], cD[even_indices], wavelet,
'per')
x2 = pywt.idwt(output[odd_indices], cD[odd_indices], wavelet,
'per')
# perform a circular shift right
x2 = roll(x2, 1)
# average and insert into the correct indices
output[indices] = (x1 + x2) / 2.
return output
def func_2():
Fs = 5000
T = 1 / Fs
N = 2000
t = np.linspace(0, N * T, N)
y = np.sin(2 * np.pi * 10 * t)# + 0.1 * np.sin(2 * np.pi * 300 * t)
y[500:1000] += 0.1 * np.sin(2 * np.pi * 500 * t[500:1000])
[cA3, cD3, cD2, cD1] = pywt.wavedec(y, wavelet='db1', level=3)
A3 = pywt.idwt(cA=cA3, cD=None, wavelet='db1')
D3 = pywt.idwt(cA=None, cD=cD3, wavelet='db1')
D2 = pywt.idwt(cA=None, cD=cD2, wavelet='db1')
D1 = pywt.idwt(cA=None, cD=cD1, wavelet='db1')
plt.subplot(511)
plt.plot(y)
plt.subplot(512)
plt.plot(A3)
plt.subplot(513)
plt.plot(D1)
plt.subplot(514)
plt.plot(D2)
plt.subplot(515)
plt.plot(D3)
plt.show()
def rm_noise(signal, wave_name='bior2.6', level=8):
"""
小波去除高频噪声和基线漂移
:param signal: 1-D array, 信号
:param wave_name: str, eg.'bior2.6',小波函数名
:param level: int，eg.8 小波分解阶数
:return: 1-D array，去除高频噪声和基线漂移的信号
"""
# 小波分解
a_n = []
d_n = []
for i in range(level):
c_a, c_d = pywt.dwt(signal, wave_name)
a_n.append(c_a)
d_n.append(c_d)
signal = c_a
a_n[level - 1] = np.zeros(a_n[level - 1].shape) # 去除基线漂移
d_n[0] = np.zeros(d_n[0].shape) # 去除高频噪声
d_n[1] = np.zeros(d_n[1].shape) # 去除高频噪声
# 小波重构
wave_upper = a_n
wave_lower = pywt.idwt(wave_upper[level - 1], d_n[level - 1], wave_name)
wave_upper = wave_lower
for i in reversed(range(level - 1)):
wave_lower = pywt.idwt(wave_upper[:len(d_n[i])], d_n[i], wave_name)
wave_upper = wave_lower
return wave_upper
def test_idwt_axis_arg():
x = [[3, 7, 1, 1], [-2, 5, 4, 6]]
cA, cD = pywt.dwt(x, "db2", axis=1)
x_ = pywt.idwt(cA, cD, "db2", axis=-1)
x = pywt.idwt(cA, cD, "db2", axis=1)
assert_allclose(x_, x)
def test_idwt_axis_arg():
x = [[3, 7, 1, 1], [-2, 5, 4, 6]]
cA, cD = pywt.dwt(x, 'db2', axis=1)
x_ = pywt.idwt(cA, cD, 'db2', axis=-1)
x = pywt.idwt(cA, cD, 'db2', axis=1)
assert_allclose(x_, x)
def test_idwt_single_axis():
x = [[3, 7, 1, 1], [-2, 5, 4, 6]]
cA, cD = pywt.dwt(x, 'db2', axis=-1)
x0 = pywt.idwt(cA[0], cD[0], 'db2', axis=-1)
x1 = pywt.idwt(cA[1], cD[1], 'db2', axis=-1)
assert_allclose(x[0], x0)
assert_allclose(x[1], x1)
def test_idwt_single_axis():
x = [[3, 7, 1, 1], [-2, 5, 4, 6]]
cA, cD = pywt.dwt(x, "db2", axis=-1)
x0 = pywt.idwt(cA[0], cD[0], "db2", axis=-1)
x1 = pywt.idwt(cA[1], cD[1], "db2", axis=-1)
assert_allclose(x[0], x0)
assert_allclose(x[1], x1)
def test_idwt_none_input():
# None input equals arrays of zeros of the right length
res1 = pywt.idwt([1, 2, 0, 1], None, 'db2', 'symmetric')
res2 = pywt.idwt([1, 2, 0, 1], [0, 0, 0, 0], 'db2', 'symmetric')
assert_allclose(res1, res2, rtol=1e-15, atol=1e-15)
res1 = pywt.idwt(None, [1, 2, 0, 1], 'db2', 'symmetric')
res2 = pywt.idwt([0, 0, 0, 0], [1, 2, 0, 1], 'db2', 'symmetric')
assert_allclose(res1, res2, rtol=1e-15, atol=1e-15)
# Only one argument at a time can be None
assert_raises(ValueError, pywt.idwt, None, None, 'db2', 'symmetric')
def test_idwt_single_axis():
x = [[3, 7, 1, 1], [-2, 5, 4, 6]]
x = np.asarray(x)
x = x + 1j * x # test with complex data
cA, cD = pywt.dwt(x, 'db2', axis=-1)
x0 = pywt.idwt(cA[0], cD[0], 'db2', axis=-1)
x1 = pywt.idwt(cA[1], cD[1], 'db2', axis=-1)
assert_allclose(x[0], x0)
assert_allclose(x[1], x1)
def test_idwt_mixed_complex_dtype():
x = np.arange(8).astype(float)
x = x + 1j * x[::-1]
cA, cD = pywt.dwt(x, 'db2')
x_roundtrip = pywt.idwt(cA, cD, 'db2')
assert_allclose(x_roundtrip, x, rtol=1e-10)
# mismatched dtypes OK
x_roundtrip2 = pywt.idwt(cA.astype(np.complex128), cD.astype(np.complex64),
'db2')
assert_allclose(x_roundtrip2, x, rtol=1e-7, atol=1e-7)
assert_(x_roundtrip2.dtype == np.complex128)
def yidwtfun(L, H, wname):
# Ls = np.shape(L)
# Hs = np.shape(H)
matSize = np.shape(L)
testData = pywt.idwt(L[0, :, 0], H[0, :, 0], wname)
data = np.zeros((matSize[0], testData.shape[0], matSize[2]), dtype='float64')
for i in range(0, matSize[0]):
for j in range(0, matSize[2]):
L1 = L[i, :, j]
H1 = H[i, :, j]
line = pywt.idwt(L1, H1, wname)
data[i, :, j] = line;
return data
def zidwtfun(L, H, wname):
# Ls = np.shape(L)
# Hs = np.shape(H)
matSize = np.shape(L)
testData = pywt.idwt(L[0, 0, :], H[0, 0, :], wname)
data = np.zeros((matSize[0], matSize[1], testData.shape[0]), dtype="float64")
for i in range(0, matSize[0]):
for j in range(0, matSize[1]):
L1 = L[i, j, :]
H1 = H[i, j, :]
line = pywt.idwt(L1, H1, wname)
data[i, j, :] = line;
return data
def test_idwt_single_axis():
x = [[3, 7, 1, 1],
[-2, 5, 4, 6]]
x = np.asarray(x)
x = x + 1j*x # test with complex data
cA, cD = pywt.dwt(x, 'db2', axis=-1)
x0 = pywt.idwt(cA[0], cD[0], 'db2', axis=-1)
x1 = pywt.idwt(cA[1], cD[1], 'db2', axis=-1)
assert_allclose(x[0], x0)
assert_allclose(x[1], x1)
def test_idwt_mixed_complex_dtype():
x = np.arange(8).astype(float)
x = x + 1j*x[::-1]
cA, cD = pywt.dwt(x, 'db2')
x_roundtrip = pywt.idwt(cA, cD, 'db2')
assert_allclose(x_roundtrip, x, rtol=1e-10)
# mismatched dtypes OK
x_roundtrip2 = pywt.idwt(cA.astype(np.complex128), cD.astype(np.complex64),
'db2')
assert_allclose(x_roundtrip2, x, rtol=1e-7, atol=1e-7)
assert_(x_roundtrip2.dtype == np.complex128)
def construct_Winv(dout=64,wave_name='db2',channels=1):
c1,c2 = pywt.dwt(np.ones(dout),wave_name)
d = 2*c1.size
X = np.identity(d)
d2 = int(np.ceil(d/2))
c3 = pywt.idwt(np.ones(d2),np.ones(d2),wave_name)
W = np.zeros((c3.size,d)) # zeros(d,d+1); for db2
for i in range(d):
a = pywt.idwt(X[:d2,i],X[d2:,i],wave_name)
W[:,i] = a
W_lin = np.kron(W,W)
W_lin = np.kron(np.identity(channels),W_lin)
return W_lin
def individual_model_baseline(trainingIteration,plot=False,printModelLoss=True):
predictions_multivariate = defaultdict(list)
predictions_single = defaultdict(list)
for key, value in multivariate_models.items():
predictions_multivariate[key]=[]
for key, value in singlevariate_models.items():
predictions_single[key]=[]
X,Y,testData,testValue,scalers,cA_list,cD_list=dataSetup(multivariate=True,time_steps_back=time_steps_back,printDates=True)
X,Y,testData_single,testValue,scalers,cA_list,cD_list=dataSetup(time_steps_back=time_steps_back)
actual=[]
for i in range(1,63):
actual.append(testValue)
for key, value in multivariate_models.items():
model=value
value.load_weights("weights/"+key+"_"+str(trainingIteration)+'.h5')
y=scalers.inverse_transform(model.predict(testData))
reconstruct=pywt.idwt(np.reshape(y, (y.shape[0])), cD_list[-1], 'haar')
predictions_multivariate[key].append(reconstruct[0])
for key, value in singlevariate_models.items():
model=value
value.load_weights("weights/"+key+"_"+str(trainingIteration)+'.h5')
y=scalers.inverse_transform(model.predict(testData_single))
reconstruct=pywt.idwt(np.reshape(y, (y.shape[0])), cD_list[-1], 'haar')
predictions_single[key].append(reconstruct[0])
X,Y,testData,testValue,scalers,cA_list,cD_list=dataSetup(multivariate=True,testDate=i,time_steps_back=time_steps_back)
X,Y,testData_single,testValue,scalers,cA_list,cD_list=dataSetup(time_steps_back=time_steps_back,testDate=i)
if(printModelLoss,actual):
printLosses(predictions_single,actual)
printLosses(predictions_multivariate,actual,printPersistance=True)
if(plot):
figure = plt.figure()
tick_plot = figure.add_subplot(1, 1, 1)
tick_plot.plot(np.arange(0,len(actual)), actual, marker='o', label='actual')
for key, value in predictions_multivariate.items():
tick_plot.plot(np.arange(0,len(actual)), value, marker='o', label=key)
for key, value in predictions_single.items():
tick_plot.plot(np.arange(0,len(actual)), value, marker='o', label=key)
plt.legend(loc='upper left')
plt.show()
def test_pywt(selenium):
import numpy as np
import pywt
def checkit(a, v):
assert (np.rint(a) == v).all()
x = [3, 7, 1, 1, -2, 5, 4, 6]
cA, cD = pywt.dwt(x, "db2")
w = pywt.Wavelet("sym3")
checkit(pywt.idwt(cA, cD, "db2"), x)
cA, cD = pywt.dwt(x, wavelet=w, mode="periodization")
checkit(pywt.idwt(cA, cD, "sym3", "symmetric"), [1, 1, -2, 5])
checkit(pywt.idwt(cA, cD, "sym3", "periodization"), x)
def remove_baseline(signal, wavelet='bior2.6', is_plot=False):
"""
Removal of baseline wandering using wavelet
wavelet: bior2.6
level: 8
"""
A8, D8, D7, D6, D5, D4, D3, D2, D1 = pywt.wavedec(
signal, wavelet=pywt.Wavelet(wavelet), level=8)
A8 = np.zeros_like(A8[0]) # low frequency info
RA7 = pywt.idwt(A8, D8[0], wavelet)
RA6 = pywt.idwt(RA7[0:len(D7[0])], D7[0], wavelet)
RA5 = pywt.idwt(RA6[0:len(D6[0])], D6[0], wavelet)
RA4 = pywt.idwt(RA5[0:len(D5[0])], D5[0], wavelet)
RA3 = pywt.idwt(RA4[0:len(D4[0])], D4[0], wavelet)
RA2 = pywt.idwt(RA3[0:len(D3[0])], D3[0], wavelet)
D2 = np.zeros_like(D2[0]) # high frequency noise
RA1 = pywt.idwt(RA2[0:len(D2)], D2, wavelet)
D1 = np.zeros_like(D1[0]) # high frequency noise
DenoisingSignal = pywt.idwt(RA1[0:len(D1)], D1, wavelet)
if is_plot:
plt.plot(signal[0], 'b')
plt.plot(DenoisingSignal, 'g')
plt.show()
return DenoisingSignal
def test_dwt_idwt_basic():
x = [3, 7, 1, 1, -2, 5, 4, 6]
cA, cD = pywt.dwt(x, "db2")
cA_expect = [5.65685425, 7.39923721, 0.22414387, 3.33677403, 7.77817459]
cD_expect = [-2.44948974, -1.60368225, -4.44140056, -0.41361256, 1.22474487]
assert_allclose(cA, cA_expect)
assert_allclose(cD, cD_expect)
x_roundtrip = pywt.idwt(cA, cD, "db2")
assert_allclose(x_roundtrip, x, rtol=1e-10)
# mismatched dtypes OK
x_roundtrip2 = pywt.idwt(cA.astype(np.float64), cD.astype(np.float32), "db2")
assert_allclose(x_roundtrip2, x, rtol=1e-7, atol=1e-7)
assert_(x_roundtrip.dtype == np.float64)
def check_reconstruction(pmode, mmode, wavelet, dtype):
data_size = list(range(
2, 40)) + [100, 200, 500, 1000, 2000, 10000, 50000, 100000]
np.random.seed(12345)
# TODO: smoke testing - more failures for different seeds
if dtype == np.float32:
# was 3e-7 has to be lowered as db21, db29, db33, db35, coif14, coif16 were failing
epsilon = 6e-7
else:
epsilon = 5e-11
for N in data_size:
data = np.asarray(np.random.random(N), dtype)
# compute dwt coefficients
pa, pd = pywt.dwt(data, wavelet, pmode)
# compute reconstruction
rec = pywt.idwt(pa, pd, wavelet, pmode)
if len(data) % 2:
rec = rec[:len(data)]
rms_rec = np.sqrt(np.mean((data - rec)**2))
msg = ('[RMS_REC > EPSILON] for Mode: %s, Wavelet: %s, '
'Length: %d, rms=%.3g' % (pmode, wavelet, len(data), rms_rec))
assert_(rms_rec < epsilon, msg=msg)
def check_reconstruction(pmode, mmode, wavelet, dtype):
data_size = list(range(2, 40)) + [100, 200, 500, 1000, 2000, 10000,
50000, 100000]
np.random.seed(12345)
# TODO: smoke testing - more failures for different seeds
if dtype == np.float32:
# was 3e-7 has to be lowered as db21, db29, db33, db35, coif14, coif16 were failing
epsilon = 6e-7
else:
epsilon = 5e-11
for N in data_size:
data = np.asarray(np.random.random(N), dtype)
# compute dwt coefficients
pa, pd = pywt.dwt(data, wavelet, pmode)
# compute reconstruction
rec = pywt.idwt(pa, pd, wavelet, pmode)
if len(data) % 2:
rec = rec[:len(data)]
rms_rec = np.sqrt(np.mean((data-rec)**2))
msg = ('[RMS_REC > EPSILON] for Mode: %s, Wavelet: %s, '
'Length: %d, rms=%.3g' % (pmode, wavelet, len(data), rms_rec))
assert_(rms_rec < epsilon, msg=msg)
def check_reconstruction(pmode, mmode, wavelet, dtype):
data_size = list(range(2, 40)) + [100, 200, 500, 1000, 2000, 10000,
50000, 100000]
np.random.seed(12345)
#TODO: smoke testing - more failures for different seeds
if dtype == np.float32:
epsilon = 3e-7
else:
#FIXME: limit was 5e-11, but gave failures. Investigate
epsilon = 1e-8
for N in data_size:
data = np.asarray(np.random.random(N), dtype)
# compute dwt coefficients
pa, pd = pywt.dwt(data, wavelet, pmode)
# compute reconstruction
rec = pywt.idwt(pa, pd, wavelet, pmode)
if len(data) % 2:
rec = rec[:len(data)]
rms_rec = np.sqrt(np.mean((data-rec)**2))
msg = ('[RMS_REC > EPSILON] for Mode: %s, Wavelet: %s, '
'Length: %d, rms=%.3g' % (pmode, wavelet, len(data), rms_rec))
assert_(rms_rec < epsilon, msg=msg)
def remove_noise(noisy_signal):
denoised = noisy_signal
for i in range(5):
denoised = pywt.dwt(denoised,'sym8')[0]
for i in range(5):
denoised = pywt.idwt(denoised,remove_signal(denoised),wavelet='sym8')
return denoised
def test1(arr):
aa1 =[1,2, 3, 4]
aa1dwt = pywt.dwt(aa1, 'haar')
aa1re = pywt.idwt(aa1dwt[0], aa1dwt[1], 'haar')
print aa1
print aa1dwt
print aa1re
def denoise(self, series, configs):
#perform normalization
normalized = (series - series.mean()) / (series.max() - series.min())
#WaveShrink
cA, cD = pywt.dwt(normalized,
configs['preprocessing']['denoise']['wavelet'])
#threshold selection
thr = np.std(normalized) / 23
cA_shrinked = pywt.threshold(
cA, thr, mode=configs['preprocessing']['denoise']['thr_mode'])
cD_shrinked = pywt.threshold(
cD, thr, mode=configs['preprocessing']['denoise']['thr_mode'])
#reconstructs data from the given shrinked coefficients
denoised = pywt.idwt(cA_shrinked, cD_shrinked,
configs['preprocessing']['denoise']['wavelet'])
if len(denoised) > len(normalized):
denoised = denoised[:-1]
# what???!!!
if len(denoised.shape) > 1:
denoised = [x[0] for x in denoised]
self.normalized = normalized
self.denoised = denoised
def __remove_noise(self, data):
""" Remove noise from data using DWT and IDWT transform with threshold in between """
denoised_data = []
for feature in data.T:
var = np.var(feature)
n = float(len(feature))
threshold = np.sqrt(2 * var * np.log10(n) / n)
ca, cd = pywt.dwt(feature, 'db1')
for i in range(len(ca)):
if np.abs(ca[i]) >= threshold:
ca[i] = np.sign(ca[i]) * (np.absolute(ca[i]) - threshold)
else:
ca[i] = 0
for i in range(len(cd)):
if np.abs(cd[i]) >= threshold:
cd[i] = np.sign(cd[i]) * (np.absolute(cd[i]) - threshold)
else:
cd[i] = 0
new_data = pywt.idwt(ca, cd, 'db1')
denoised_data.append(new_data)
denoised_data = np.array(denoised_data).T
denoised_data = np.delete(denoised_data, -1, axis=0)
return denoised_data
def denoise(sig):
## keep [2,7]
sA, sD = [], []
cA = sig
for i in range(0, 7):
cA, cD = pywt.dwt(cA, 'coif2')
sD.append(cD)
## filter
#for i in range(0,len(sD)):
#print 'sD',i,'len:',len(sD[i])
for i in range(0, 2):
sD[i] = [0] * len(sD[i])
cA = [0] * len(cA)
## idwt
for i in range(6, -1, -1):
# padding with zeros
pdzeros = len(cA) - len(sD[i])
#print pdzeros
if pdzeros > 0:
#print 'sD[i]:',sD[i].shape
zstk = np.zeros((pdzeros, ))
#print 'zstk:',zstk.shape
sD[i] = np.hstack((sD[i], zstk))
#print 'sD[i]:',sD[i].shape
elif pdzeros < 0:
zstk = np.zeros((-pdzeros, ))
np.hstack((cA, zstk))
#print 'len cA:',len(cA)
#print 'len sD[',i,']:',len(sD[i])
cA = pywt.idwt(cA, sD[i], 'coif2')
sD.append(cD)
return cA
def collapse(triangle,trind,wavelet):
low=get_band(triangle,trind,trind.shape[0]-1)
for band in range(1,trind.shape[0]):
high=get_band(triangle,trind,trind.shape[0]-band-1)
low=low[:high.shape[0]]
low=pywt.idwt(low,high,wavelet)
return low
def waverec(coeffs_list, wavelet, mode='sym'):
a, ds = coeffs_list[0], coeffs_list[1:]
for d in ds:
a = pywt.idwt(a, d, wavelet, mode, 1)
return a
def wave(arr):
return_ = np.diff(arr)
# return_=return_[1:]
# close2 / close
# print(pywt.wavelist())
#Do the Wavelet Transform for the return and inverse transformation
#return_ is a dataframe series
method = 'haar'
mode_ = "soft"
(ca, cd) = pywt.dwt(return_, method)
cat = pywt.threshold(ca, 0.3 * np.std(ca), mode=mode_)
cdt = pywt.threshold(cd, 0.3 * np.std(cd), mode=mode_)
tx = pywt.idwt(cat, cdt, method, "smooth")
# tx=pd.DataFrame(tx,index=return_.index)
#Get back to the Stock price using denoising wavelet transform
start_price = arr[0]
# txx=tx.iloc[:,0]
# txx=np.exp(tx)
# txx=np.array(txx)
temp = np.array([start_price])
np.hstack((temp, tx))
txx = np.cumsum(tx)
# txx = pd.Series(txx, index=arr.index)
# txx=pd.DataFrame(txx,index=return_.index)
return txx
def my_waverec(data, wavelet, mode='sym', level=None):
"""
Multilevel 1D Discrete Wavelet Transform of data.
Returns coefficients list - [cAn, cDn, cDn-1, ..., cD2, cD1]
data - input data
wavelet - wavelet to use (Wavelet object or name string)
mode - signal extension mode, see MODES
level - decomposition level. If level is None then it will be
calculated using `dwt_max_level` function.
"""
if not isinstance(wavelet, pywt.Wavelet):
wavelet = pywt.Wavelet(wavelet)
if level is None:
level = pywt.dwt_max_level(len(data), wavelet.dec_len)
elif level < 0:
raise ValueError("Level value of %d is too low . Minimum level is 0." % level)
a = data[0:2]
d = data[2:4]
for i in xrange(int(level)):
offs = pow(2,i+2)
a = pywt.idwt(a, d, wavelet, mode)
d = data[offs:2*offs]
return a
def func_dwt(Fs, T, N):
t = np.linspace(0, N * T, N)
y = np.sin(2 * np.pi * 10 * t) + 0.1 * np.sin(2 * np.pi * 300 * t)
(cA, cD) = pywt.dwt(y, 'db1')
yy = pywt.idwt(cA, cD, 'db1')
print(np.sum(np.abs(yy - y)) / N)
def denoise_ecg(ecg_segment):
"""
Remove baseline drafts from ECG signal.
:param ecg_segment: ecg record, a numpy array.
:return: denoised ecg record, a numpy array.
Example:
denoising_ecg = denoise_ecg(raw_ecg)
"""
denoising_wd_level = 6
denoising_wd_wavelet = "db6"
coffes_set = []
cA_signal = np.reshape(ecg_segment, len(ecg_segment))
for index_dec in range(denoising_wd_level):
cA, cD = pywt.dwt(cA_signal, denoising_wd_wavelet)
coffes_set.append(cD)
cA_signal = cA
coffes_set.append(cA_signal)
coffes_set[denoising_wd_level] = np.zeros(
len(coffes_set[denoising_wd_level]))
cA_signal = coffes_set[denoising_wd_level]
for index_dec in range(denoising_wd_level):
cD_signal = coffes_set[denoising_wd_level - 1 - index_dec]
if len(cD_signal) != len(cA_signal):
cA_signal = np.delete(cA_signal, len(cA_signal) - 1, axis=0)
cA_signal = pywt.idwt(cA_signal, cD_signal, denoising_wd_wavelet)
cA_signal = np.reshape(cA_signal, (len(ecg_segment), 1))
return cA_signal
def update_dwt(coeffs, wavelet, mode='sym'):
new_coeffs = [0] * len(coeffs)
resized_coeffs = []
# parse coeffs into approximation and details
a, ds = coeffs[0], coeffs[1:]
for d in ds:
if isinstance(d, collections.Iterable):
for index in range(0, len(d)):
try:
d[index] = float(d[index])
except Exception:
d[index] = float(d[index].split('e')[0])
d_copy = copy([float(e) for e in d if e != ''])
# d_copy = copy([float(e) for e in d])
if len(a) != len(d):
d_copy.resize(len(a), refcheck = False)
a = pywt.idwt(a, d_copy, wavelet, mode, 1)
resized_coeffs.append(d_copy)
new_coeffs[0] = coeffs[0]
new_coeffs[1:] = resized_coeffs
return new_coeffs
def reconstructWPT(self,new_wp,wavelet,listleaves):
""" Create a new wavelet packet tree by copying in the data for the leaves and then performing
the idwt up the tree to the root.
Assumes that listleaves is top-to-bottom, so just reverses it.
"""
# Sort the list of leaves into order bottom-to-top, left-to-right
working = listleaves.copy()
working = working[-1::-1]
level = int(np.floor(np.log2(working[0] + 1)))
while level > 0:
first = 2 ** level - 1
while working[0] >= first:
# Note that it assumes that the whole list is backwards
parent = (working[0] - 1) // 2
p = self.ConvertWaveletNodeName(parent)
new_wp[p].data = pywt.idwt(new_wp[self.ConvertWaveletNodeName(working[1])].data,new_wp[self.ConvertWaveletNodeName(working[0])].data, wavelet)[:len(new_wp[p].data)]
# Delete these two nodes from working
working = np.delete(working, 1)
working = np.delete(working, 0)
# Insert parent into list of nodes at the next level
ins = np.where(working > parent)
if len(ins[0]) > 0:
ins = ins[0][-1] + 1
else:
ins = 0
working = np.insert(working, ins, parent)
level = int(np.floor(np.log2(working[0] + 1)))
return new_wp
def denoise_signal(self, signal, wavelet="db4"):
"""Denoise filtered spectrum using wavelet transform.
The following is performed:
1. Obtain detail coefficients (cD) from discrete wavelet transform
2. Determine middle point of cD (the median is the convention)
3. Calculate sigma and the threshold level as described
in the thesis on pp. 19-20
4. Apply hard thresholding as described in the thesis on p. 19
5. Perform inverse discrete wavelet transform to obtain
the denoised spectrum
"""
_, cD = pywt.dwt(signal, wavelet, mode="per")
# Center point; take to be median
median = np.median(cD)
# MAD calculation
mad = np.mean(np.abs(np.abs(cD) - np.median(cD)))
# Sigma calculation
sigma = 1.0 / 0.6745 * mad
# Threshold calculation (log10 works better than log (base e))
td = sigma * np.sqrt(2.0 * np.log10(len(cD)))
# Apply Threshold
cD_t = pywt.threshold(cD, td, mode='hard', substitute=0)
# Reconstruct signal (denoised)
reconstructed = pywt.idwt(cA=None, cD=cD_t, wavelet="db4", mode='per')
return reconstructed
def check_reconstruction(pmode, mmode, wavelet, dtype):
data_size = list(range(
2, 40)) + [100, 200, 500, 1000, 2000, 10000, 50000, 100000]
np.random.seed(12345)
#TODO: smoke testing - more failures for different seeds
if dtype == np.float32:
epsilon = 3e-7
else:
#FIXME: limit was 5e-11, but gave failures. Investigate
epsilon = 1e-8
for N in data_size:
data = np.asarray(np.random.random(N), dtype)
# compute dwt coefficients
pa, pd = pywt.dwt(data, wavelet, pmode)
# compute reconstruction
rec = pywt.idwt(pa, pd, wavelet, pmode)
if len(data) % 2:
rec = rec[:len(data)]
rms_rec = np.sqrt(np.mean((data - rec)**2))
msg = ('[RMS_REC > EPSILON] for Mode: %s, Wavelet: %s, '
'Length: %d, rms=%.3g' % (pmode, wavelet, len(data), rms_rec))
assert_(rms_rec < epsilon, msg=msg)
def test_dwt_idwt_allmodes():
# Test that :func:`dwt` and :func:`idwt` can be performed using every mode
x = [1, 2, 1, 5, -1, 8, 4, 6]
dwt_result_modes = {
'zero':
([-0.03467518, 1.73309178, 3.40612438, 6.32928585, 6.95094948],
[-0.12940952, -2.15599552, -5.95034847, -1.21545369, -1.8625013]),
'constant':
([1.28480404, 1.73309178, 3.40612438, 6.32928585, 7.51935555],
[-0.48296291, -2.15599552, -5.95034847, -1.21545369, 0.25881905]),
'symmetric':
([1.76776695, 1.73309178, 3.40612438, 6.32928585, 7.77817459],
[-0.61237244, -2.15599552, -5.95034847, -1.21545369, 1.22474487]),
'periodic':
([6.9162743, 1.73309178, 3.40612438, 6.32928585, 6.9162743],
[-1.99191082, -2.15599552, -5.95034847, -1.21545369, -1.99191082]),
'smooth':
([-0.51763809, 1.73309178, 3.40612438, 6.32928585,
7.45000519], [0, -2.15599552, -5.95034847, -1.21545369, 0]),
'periodization': ([4.053172, 3.05257099, 2.85381112, 8.42522221],
[0.18946869, 4.18258152, 4.33737503, 2.60428326])
}
for mode in pywt.Modes.modes:
cA, cD = pywt.dwt(x, 'db2', mode)
assert_allclose(cA, dwt_result_modes[mode][0], rtol=1e-7, atol=1e-8)
assert_allclose(cD, dwt_result_modes[mode][1], rtol=1e-7, atol=1e-8)
assert_allclose(pywt.idwt(cA, cD, 'db2', mode), x, rtol=1e-10)
def test_dwt_idwt_partial_complex():
x = np.asarray([3, 7, 1, 1, -2, 5, 4, 6])
x = x + 0.5j*x
cA, cD = pywt.dwt(x, 'haar')
cA_rec_expect = np.array([5.0+2.5j, 5.0+2.5j, 1.0+0.5j, 1.0+0.5j,
1.5+0.75j, 1.5+0.75j, 5.0+2.5j, 5.0+2.5j])
cA_rec = pywt.idwt(cA, None, 'haar')
assert_allclose(cA_rec, cA_rec_expect)
cD_rec_expect = np.array([-2.0-1.0j, 2.0+1.0j, 0.0+0.0j, 0.0+0.0j,
-3.5-1.75j, 3.5+1.75j, -1.0-0.5j, 1.0+0.5j])
cD_rec = pywt.idwt(None, cD, 'haar')
assert_allclose(cD_rec, cD_rec_expect)
assert_allclose(cA_rec + cD_rec, x)
def test_perfect_reconstruction(families, wavelets, modes, epsilon, dtype):
for wavelet in wavelets:
for pmode, mmode in modes:
print "Wavelet: %-8s Mode: %s" % (wavelet, pmode),
data_size = range(
2, 40) + [100, 200, 500, 1000, 2000, 10000, 50000, 100000]
ok, over = 0, 0
for N in data_size:
data = numpy.asarray(numpy.random.random(N), dtype)
# compute dwt coefficients
pa, pd = pywt.dwt(data, wavelet, pmode)
# compute reconstruction
rec = pywt.idwt(pa, pd, wavelet, pmode)
if len(data) % 2:
rec = rec[:len(data)]
rms_rec = rms(data, rec)
if rms_rec > epsilon:
if not over:
print
print '[RMS_REC > EPSILON] for Mode: %s, Wavelet: %s, ' \
'Length: %d, rms=%.3g' % (
pmode, wavelet, len(data), rms_rec)
over += 1
else:
ok += 1
if not over:
print "- RMSE for all %d cases was under %s" % (len(data_size),
epsilon)
def calc_baseline(signal):
"""
Calculate the baseline of signal.
Args:
signal (numpy 1d array): signal whose baseline should be calculated
Returns:
baseline (numpy 1d array with same size as signal): baseline of the signal
"""
ssds = np.zeros((3))
cur_lp = np.copy(signal)
iterations = 0
while True:
# Decompose 1 level
lp, hp = pywt.dwt(cur_lp, "db4")
# Shift and calculate the energy of detail/high pass coefficient
ssds = np.concatenate(([np.sum(hp ** 2)], ssds[:-1]))
# Check if we are in the local minimum of energy function of high-pass signal
if ssds[2] > ssds[1] and ssds[1] < ssds[0]:
break
cur_lp = lp[:]
iterations += 1
# Reconstruct the baseline from this level low pass signal up to the original length
baseline = cur_lp[:]
for _ in range(iterations):
baseline = pywt.idwt(baseline, np.zeros((len(baseline))), "db4")
return baseline[: len(signal)]
def test_dwt_idwt_allmodes():
# Test that :func:`dwt` and :func:`idwt` can be performed using every mode
x = [1, 2, 1, 5, -1, 8, 4, 6]
dwt_result_modes = {
'zero': ([-0.03467518, 1.73309178, 3.40612438, 6.32928585, 6.95094948],
[-0.12940952, -2.15599552, -5.95034847, -1.21545369,
-1.8625013]),
'constant': ([1.28480404, 1.73309178, 3.40612438, 6.32928585,
7.51935555],
[-0.48296291, -2.15599552, -5.95034847, -1.21545369,
0.25881905]),
'symmetric': ([1.76776695, 1.73309178, 3.40612438, 6.32928585,
7.77817459],
[-0.61237244, -2.15599552, -5.95034847, -1.21545369,
1.22474487]),
'reflect': ([2.12132034, 1.73309178, 3.40612438, 6.32928585,
6.81224877],
[-0.70710678, -2.15599552, -5.95034847, -1.21545369,
-2.38013939]),
'periodic': ([6.9162743, 1.73309178, 3.40612438, 6.32928585,
6.9162743],
[-1.99191082, -2.15599552, -5.95034847, -1.21545369,
-1.99191082]),
'smooth': ([-0.51763809, 1.73309178, 3.40612438, 6.32928585,
7.45000519],
[0, -2.15599552, -5.95034847, -1.21545369, 0]),
'periodization': ([4.053172, 3.05257099, 2.85381112, 8.42522221],
[0.18946869, 4.18258152, 4.33737503, 2.60428326])
}
for mode in pywt.Modes.modes:
cA, cD = pywt.dwt(x, 'db2', mode)
assert_allclose(cA, dwt_result_modes[mode][0], rtol=1e-7, atol=1e-8)
assert_allclose(cD, dwt_result_modes[mode][1], rtol=1e-7, atol=1e-8)
assert_allclose(pywt.idwt(cA, cD, 'db2', mode), x, rtol=1e-10)
def wavelet_denoise(self, wavelet='db1', plot='Y', level=1):
### only built for Level 1 decomp
self.Xtrans = self.XData.transpose()
self.Xnormtrans = []
self.wavecoef = []
self.thresh_wavecoef = []
self.denoisedXtrans = []
for i in range(len(self.Xtrans)):
self.maxwave = pwt.dwt_max_level(len(self.Xtrans[i]), wavelet)
if level != 'Auto':
self.level = level
self.wavecoef.append(
pwt.wavedec(self.Xtrans[i], wavelet, level=self.level))
self.thresh_A = (np.average(np.abs(
self.wavecoef[i][0]))) - (np.std(np.abs(self.wavecoef[i][0])))
self.thresh_D = (np.average(np.abs(
self.wavecoef[i][1]))) - (np.std(np.abs(self.wavecoef[i][1])))
self.thresh_wavecoef.append(
(pwt.threshold(self.wavecoef[i][0], self.thresh_A),
(pwt.threshold(self.wavecoef[i][1], self.thresh_D))))
self.denoisedXtrans.append(
pwt.idwt(self.thresh_wavecoef[-1][0],
self.thresh_wavecoef[-1][1], wavelet))
self.XDataNT = np.asarray(self.denoisedXtrans)
self.XData = self.XDataNT.transpose()
if plot != 'N':
plt.plot(self.Xtrans[plot])
plt.plot(self.denoisedXtrans[plot])
plt.show()
def iwp(Nodes, wavelet="db4", mode=pywt.MODES.ppd):
'''
Returns the inverse 1D discrete wavelet packet transformation for the given
list containing the nodes of the 1D discrete wavelet packet transformation.
@param Nodes: List containing the nodes of the 1D discrete wavelet packet
transformation.
@param wavelet: Wavelet to use in the transform.
This must be a name of the wavelet from the wavelist() list.
@param mode: Signal extension mode to deal with the border distortion problem.
The default mode is periodic-padding.
@return: The inverse 1D discrete wavelet packet transformation for the given
list containing the nodes of the 1D discrete wavelet packet transformation.
'''
heapq.heapify(Nodes)
while len(Nodes) != 1:
Node1 = heapq.heappop(Nodes)
Node2 = heapq.heappop(Nodes)
S = pywt.idwt(Node1.C,
Node2.C,
wavelet=wavelet,
mode=mode,
correct_size=True)
Merged = node.Node(S, (Node1.level - 1), (Node1.index / 2))
heapq.heappush(Nodes, Merged)
return Nodes[0].C
def __apply_wavelet_transform(self, x):
(ca, cd) = pywt.dwt(x, "haar")
cat = pywt.threshold(ca, np.std(ca), mode="soft")
cdt = pywt.threshold(cd, np.std(cd), mode="soft")
tx = pywt.idwt(cat, cdt, "haar")
return tx
def w2d(imArray,imagePath):
mode='db1'
(ca, cd) = pywt.dwt(imArray,mode)
cat = pywt.threshold(ca, np.std(ca)/250)
cdt = pywt.threshold(cd, np.std(cd)/250)
ts_rec = pywt.idwt(cat, cdt, mode)
return np.array(cat)
def test_perfect_reconstruction(families, wavelets, modes, epsilon, dtype):
for wavelet in wavelets:
for pmode, mmode in modes:
print "Wavelet: %-8s Mode: %s" % (wavelet, pmode),
data_size = range(2, 40) + [100, 200, 500, 1000, 2000, 10000, 50000, 100000]
ok, over = 0, 0
for N in data_size:
data = numpy.asarray(numpy.random.random(N), dtype)
# compute dwt coefficients
pa, pd = pywt.dwt(data, wavelet, pmode)
# compute reconstruction
rec = pywt.idwt(pa, pd, wavelet, pmode)
if len(data) % 2:
rec = rec[: len(data)]
rms_rec = rms(data, rec)
if rms_rec > epsilon:
if not over:
print
print "[RMS_REC > EPSILON] for Mode: %s, Wavelet: %s, " "Length: %d, rms=%.3g" % (
pmode,
wavelet,
len(data),
rms_rec,
)
over += 1
else:
ok += 1
if not over:
print "- RMSE for all %d cases was under %s" % (len(data_size), epsilon)
def test_default_mode():
# The default mode should be 'symmetric'
x = [1, 2, 1, 5, -1, 8, 4, 6]
cA, cD = pywt.dwt(x, 'db2')
cA2, cD2 = pywt.dwt(x, 'db2', mode='symmetric')
assert_allclose(cA, cA2)
assert_allclose(cD, cD2)
assert_allclose(pywt.idwt(cA, cD, 'db2'), x)
def wlFlag(data, threshold, wavelet='haar', mode='cpd'):
""" Wavelet flagging test"""
(cA, cD) = pywt.dwt(data, wavelet=wavelet, mode=mode)
flags = np.abs(cD) >= threshold
cD[flags] = 0
return pywt.idwt(cA, cD, wavelet=wavelet, mode=mode), flags
def test_dwt_idwt_dtypes():
wavelet = pywt.Wavelet('haar')
for dt_in, dt_out in zip(dtypes_in, dtypes_out):
x = np.ones(4, dtype=dt_in)
errmsg = "wrong dtype returned for {0} input".format(dt_in)
cA, cD = pywt.dwt(x, wavelet)
assert_(cA.dtype == cD.dtype == dt_out, "dwt: " + errmsg)
x_roundtrip = pywt.idwt(cA, cD, wavelet)
assert_(x_roundtrip.dtype == dt_out, "idwt: " + errmsg)
def test_dwt_idwt_basic():
x = [3, 7, 1, 1, -2, 5, 4, 6]
cA, cD = pywt.dwt(x, 'db2')
cA_expect = [5.65685425, 7.39923721, 0.22414387, 3.33677403, 7.77817459]
cD_expect = [-2.44948974, -1.60368225, -4.44140056, -0.41361256,
1.22474487]
assert_allclose(cA, cA_expect)
assert_allclose(cD, cD_expect)
x_roundtrip = pywt.idwt(cA, cD, 'db2')
assert_allclose(x_roundtrip, x, rtol=1e-10)
def func_1():
Fs = 1000
T = 1 / Fs
N = 500
t = np.linspace(0, N * T, N)
y = np.sin(2 * np.pi * 10 * t) + 0.1 * np.sin(2 * np.pi * 300 * t)
(cA, cD) = pywt.dwt(y, 'db1')
A = pywt.idwt(cA=cA, cD=None, wavelet='db1')
D = pywt.idwt(cA=None, cD=cD, wavelet='db1')
plt.subplot(311)
plt.plot(y)
plt.subplot(312)
plt.plot(A)
plt.subplot(313)
plt.plot(D)
plt.show()
def test_idwt_correct_size_kw():
res = pywt.idwt([1, 2, 3, 4, 5], [1, 2, 3, 4], 'db2', 'sym',
correct_size=True)
expected = [1.76776695, 0.61237244, 3.18198052, 0.61237244, 4.59619408,
0.61237244]
assert_allclose(res, expected)
assert_raises(ValueError, pywt.idwt,
[1, 2, 3, 4, 5], [1, 2, 3, 4], 'db2', 'sym')
assert_raises(ValueError, pywt.idwt, [1, 2, 3, 4], [1, 2, 3, 4, 5], 'db2',
'sym', correct_size=True)
def test_dwt_idwt_basic_complex():
x = np.asarray([3, 7, 1, 1, -2, 5, 4, 6])
x = x + 0.5j * x
cA, cD = pywt.dwt(x, "db2")
cA_expect = np.asarray([5.65685425, 7.39923721, 0.22414387, 3.33677403, 7.77817459])
cA_expect = cA_expect + 0.5j * cA_expect
cD_expect = np.asarray([-2.44948974, -1.60368225, -4.44140056, -0.41361256, 1.22474487])
cD_expect = cD_expect + 0.5j * cD_expect
assert_allclose(cA, cA_expect)
assert_allclose(cD, cD_expect)
x_roundtrip = pywt.idwt(cA, cD, "db2")
assert_allclose(x_roundtrip, x, rtol=1e-10)
def grab_adc16(roach, arr, filter=False):
""" Run subprocess to call adc16_dump_chans script """
sp = subprocess.Popen(["adc16_dump_chans.rb", "-l", str(n_samples), roach], stdout=subprocess.PIPE)
out, err = sp.communicate()
data = np.fromstring(out, sep=' ', dtype='int')
#print data.shape
if filter:
# Filter out low frequency stuff
ca, cd = dwt(data, 'haar')
cd = np.zeros_like(cd)
data = idwt(ca,cd,'haar').astype('int')
for i in range(len(arr)):
arr[i] = data[i]
return
def waverec(coeffs, wavelet, mode='sym'):
if not isinstance(coeffs, (list, tuple)):
raise ValueError("Expected sequence of coefficient arrays.")
if len(coeffs) < 2:
raise ValueError( "Coefficient list too short (minimum 2 arrays required).")
a, ds = coeffs[0], coeffs[1:]
for d in ds:
a = idwt(a, d, wavelet, mode, 1)
return a
