def test_eemd_simpleRun(self):
T = np.linspace(0, 1, 100)
S = np.sin(2*np.pi*T)
eemd = EEMD(trials=10, max_imf=1)
eemd.EMD.FIXE_H = 5
eemd.eemd(S)
def test_eemd_simpleRun():
T = np.linspace(0, 1, 100)
S = np.sin(2*np.pi*T)
config = {"processes": 1}
eemd = EEMD(trials=10, max_imf=1, **config)
eemd.EMD.FIXE_H = 5
eemd.eemd(S)
def extract_imfs(self, signal):
eemd = EEMD()
eemd.eemd(signal)
res = eemd.get_imfs_and_residue()
imfs = list(res)[0]
# drop the noise/ high frequency imfs, may incur index error due to the signal is too simple
imfs_left = imfs[-1:]
return imfs_left # np.array type
def main():
#load data
e = exchange.Exchange('../lib/binance.db')
start = int(datetime.datetime(2018, 7, 1).timestamp() * 1000)
#end = int(datetime.datetime(2018, 8, 1).timestamp() * 1000)
end = int(datetime.datetime(2019, 7, 1).timestamp() * 1000)
print('Loading order data...')
number_of_orders, prices = e.get_total_orders_ts('BTCUSDT',
24 * 60 * 60 * 1000,
start, end) #hourly data
print('done')
buy_orders = np.array([b for s, b in number_of_orders])
sell_orders = np.array([s for s, b in number_of_orders])
t = [i for i, o in enumerate(buy_orders)]
eemd = EEMD()
emd = eemd.EMD
eIMFs_buys = eemd.eemd(buy_orders)
eIMFs_sells = eemd.eemd(sell_orders)
if eIMFs_buys.shape[0] != eIMFs_sells.shape[0]:
print('size mismatch')
n = min(eIMFs_buys.shape[0], eIMFs_sells.shape[0])
fig, axs = plt.subplots(3, figsize=(12, 9), sharex=True)
axs[0].plot(prices)
axs[0].set_ylabel('Price')
buys = np.sum(eIMFs_buys[1:-2], axis=0)
sells = np.sum(eIMFs_sells[1:-2], axis=0)
axs[1].plot(buys, color='g')
axs[1].plot(sells, color='r')
axs[2].plot(calculate_returns(prices), color='b')
ax3 = axs[2].twinx()
ax3.plot((0, len(prices)), (0.5, 0.5))
ax3.plot(buys - sells)
plt.xlabel("Time /days")
#plt.tight_layout()
plt.show()
def test_eemd_simpleRun(self):
T = np.linspace(0, 1, 100)
S = np.sin(2 * np.pi * T)
config = {"processes": 1}
eemd = EEMD(trials=10, max_imf=1, **config)
eemd.EMD.FIXE_H = 5
eemd.eemd(S)
self.assertTrue('pool' in eemd.__dict__)
def nonlinear_trend_test(trend):
#Use surrogate method to test for EEMD trend significance
t = len(trend)
ts = trend.reshape(1, -1) #reshape into 2d array for surrogate methods
ts_sur = timeseries.surrogates.Surrogates(
ts) #generate an instance of surrogate class
sur_list = [
ts
] #original time series is the first item in the surrogate list
# Assign EEMD to `eemd` variable
eemd = EEMD()
eemd.noise_seed(12345)
#detect extrema using parabolic method
emd = eemd.EMD
emd.extrema_detection = "parabol"
for i in range(19): #0.05 significance level for one-sided test
sur = ts_sur.refined_AAFT_surrogates(
ts, 100
) #Return surrogates using the iteratively refined amplitude adjusted Fourier transform method.
#detrend surrogate
eIMFs_n = eemd.eemd(sur.flatten(), np.arange(t))
sur_trend = eIMFs_n[-1]
sur_list.append(sur_trend)
ta_list = [timeAsymmetry(ts.flatten())
for ts in sur_list] #test statistic: time asymmetry
return np.array(ta_list)
def rolling_EEMD(sym, column, win, per, perend):
datablock = get_data_for_EEMD(sym, column, per, perend)
winlength = int(win * 24 * 12)
t = np.linspace(0, winlength - 1, winlength)
eemd = EEMD()
emd = eemd.EMD
emd.extrema_detection = "parabol"
nIMF = []
last = []
dlast = []
timelist = []
timearray = datablock.index[(winlength):]
for i in tqdm(range(winlength, datablock.__len__(), 1)):
datawindow = datablock.iloc[i - winlength:i]
# Execute EEMD on window
maximf = 6
eIMFs = eemd.eemd(datawindow.tolist(), t, max_imf=maximf)
nIMFs = eIMFs.shape[0]
nIMF.append(nIMFs)
last.append(eIMFs[0:nIMFs, -1])
d = eIMFs[0:nIMFs, -1] - eIMFs[0:nIMFs, -2]
dlast.append(d)
timelist.append(datawindow.index[-1])
lastdf = pd.DataFrame(last, index=timearray)
dlastdf = pd.DataFrame(dlast, index=timearray)
return [lastdf, dlastdf]
def read_data():
"""
读取data下面所有文件
:param time_steps: 采样率，输入数据的长度，步长
:return:
"""
eemd = EEMD()
emd = eemd.EMD
emd.extrema_detection = "parabol"
for item in os.listdir('../data'):
# 每读取一个文件，保存一个
print("start...{%s}" % item)
# 获取文件名
short_name, _ = os.path.splitext(item)
time_series = read_matdata('../data/' + item)
# 将故障样本拆分为训练样本长度
input_length = time_series.shape[0] // TIME_PERIODS
# 三维矩阵，记录每个输入样本长度，每个分量，信号信息
x = np.zeros((input_length, IMF_LENGTH, TIME_PERIODS))
# 获取序列最大长度，去掉信号后面的余数
idx_last = -(time_series.shape[0] % TIME_PERIODS)
# 切片分割(input_length, time_steps)
clips = time_series[:idx_last].reshape(-1, TIME_PERIODS)
# 对每个样本做eemd处理
for i in range(clips.shape[0]):
eimfs = eemd.eemd(clips[i])
print("start emf...%s-%d" % (item, i))
print(eimfs)
x[i] = eimfs[0:IMF_LENGTH]
# 将每个输入样本拉平，存储
b = x.reshape(input_length, -1)
np.savetxt('../emd_data/' + short_name + '.txt', b)
print(x)
def main():
e = exchange.Exchange('../lib/binance.db')
start = datetime.datetime(2018, 8, 1)
end = datetime.datetime(2019, 8, 1)
params = parameter_estimation.calculate_parameters(start, end, 6, e)
eemd = EEMD()
emd = eemd.EMD
eIMFs_rate = eemd.eemd(np.array([p[1] for p in params]))
fig, axs = plt.subplots(eIMFs_rate.shape[0],
figsize=(12, eIMFs_rate.shape[0] * 3))
rate = np.zeros(eIMFs_rate.shape[1])
for ax, IMF in zip(axs, eIMFs_rate):
#ax.plot(IMF)
ax.plot(IMF)
analytic_signal = hilbert(IMF)
phase = np.unwrap(np.angle(analytic_signal))
freq = np.diff(phase) / (2 * np.pi * 0.25)
print(np.mean(1 / freq))
plt.tight_layout()
plt.show()
def FEI(_data, NFFT = 1024):
N = len(_data)
width = _data.shape[1]
n = NFFT
_data = (np.vstack((np.zeros((n,width)), zmean(_data),np.zeros((n-N%n,width)))))
# w = signal.windows.chebwin(n, 100).reshape((n,1))
# y = np.zeros_like(_data, dtype=complex)
# for ii in range(0, len(_data)-n, n//2):
# Y = _data[ii:ii+n,:]*w
# k = (1j*2*pi*fftfreq(len(Y), dt).reshape((n,1)))
# y[ii:ii+n,:] = (ifft(np.vstack((np.zeros((1,width)),fft(Y, axis=0)[1:]/(k[1:]))), axis=0))
# _datai = zmean(np.real(y))
_datai = zmean(integ(_data))
# return _datai[n:N+n]
_dataI = np.zeros_like(_datai)
for ii in range(0, len(_data)-n, n):
for jj in range(width):
F0 = featExt(_datai[ii:ii+n,jj])
eemd = EEMD(noise_width=0.15)
eIMFs = eemd.eemd(_datai[ii:ii+n,jj]).T
F = []
for kk in range(eIMFs.shape[1]):
F.append(featExt(eIMFs[:,kk]))
nn = norm(np.array(F)**2-F0, axis=1).argmin()
_dataI[ii:ii+n,jj] = eIMFs[:,nn]
return _dataI[n:N+n]
def Plot_IMF(obs, mod, site_id):
print('Process on IMF ' + 'No.' + str(site_id) + '!')
data = (obs.extractDatasites(lat=obs.lat, lon=obs.lat)).data[:, site_id]
time = mod.time[~data.mask]
y = (mod.extractDatasites(lat=obs.lat, lon=obs.lat)).data[:, site_id]
time_y = mod.time[~y.mask]
d_mod = []
d_mod.append(y)
fig1 = plt.figure(figsize=(4 * (len(d_mod) + 1), 8))
fig2 = plt.figure(figsize=(4 * (len(d_mod) + 1), 8))
fig3 = plt.figure(figsize=(5, 5))
fig4 = plt.figure(figsize=(5, 5))
fig1.subplots_adjust(wspace=0.5, hspace=0.3)
fig2.subplots_adjust(wspace=0.5, hspace=0.3)
eemd = EEMD(trials=5)
imfs = eemd.eemd(data.compressed())
fig3, freq = hht(data.compressed(), imfs, time, 1, fig3)
if len(imfs) >= 1:
fig1 = plot_imfs(data.compressed(),
imfs,
time_samples=time,
fig=fig1,
no=1,
m=len(d_mod))
fig2 = plot_frequency(data.compressed(),
freq.T,
time_samples=time,
fig=fig2,
no=1,
m=len(d_mod))
imfs2 = eemd.eemd(y.compressed())
fig4, freq2 = hht(y.compressed(), imfs2, time_y, 1, fig4)
if len(imfs) >= 1:
fig1 = plot_imfs(y.compressed(),
imfs2,
time_samples=time_y,
fig=fig1,
no=2,
m=len(d_mod))
fig2 = plot_frequency(y.compressed(),
freq2.T,
time_samples=time_y,
fig=fig2,
no=2,
m=len(d_mod))
def test_eemd_notParallel(self):
S = np.random.random(100)
eemd = EEMD(trials=5, max_imf=2, parallel=False)
eemd.EMD.FIXE_H = 2
eIMFs = eemd.eemd(S)
self.assertTrue(eIMFs.shape[0] > 0)
self.assertTrue(eIMFs.shape[1], len(S))
self.assertFalse('pool' in eemd.__dict__)
def Signal():
global E_imfNo
E_imfNo = np.zeros(50, dtype=np.int)
# EEMD options
max_imf = 7
"""
信号参数：
N:采样频率500Hz
tMin:采样开始时间
tMax:采样结束时间 2*np.pi
"""
N = 500
tMin, tMax = 0, 2 * np.pi
T = np.linspace(tMin, tMax, N)
# 信号S:是多个信号叠加信号
S = 3 * np.sin(4 * T) + 4 * np.cos(9 * T) + np.sin(8.11 * T + 1.2)
# EEMD计算
eemd = EEMD()
eemd.trials = 50
eemd.noise_seed(12345)
E_IMFs = eemd.eemd(S)
imfNo = E_IMFs.shape[0]
# Plot results in a grid
c = np.floor(np.sqrt(imfNo + 1))
r = np.ceil((imfNo + 1) / c)
plt.ioff()
plt.subplot(r, c, 1)
plt.plot(T, S, 'r')
plt.xlim((tMin, tMax))
plt.title("Original signal")
i = 1
for imf in E_IMFs:
plt.subplot(len(E_IMFs), 1, i)
plt.plot(imf)
i += 1
# for num in range(imfNo):
# plt.subplot(r, c, num + 2)
# plt.plot(T, E_IMFs[num], 'g')
# plt.xlim((tMin, tMax))
# plt.title("Imf " + str(num + 1))
plt.text(0,
0,
str(format(i, '.4f')),
style='italic',
ha='center',
wrap=True)
plt.save("haha.jpg")
def hht(data, time, freqsol=33, timesol=50):
# freqsol give frequency - axis resolution for hilbert - spectrum
# timesol give time - axis resolution for hilbert - spectrum
t0 = time[0]
t1 = time[-1]
dt = (t1 - t0) / (len(time) - 1)
eemd = EEMD()
imfs = eemd.eemd(data)
freq, amp = FAhilbert(imfs, dt)
# fw0 = np.min(np.min(freq)) # maximum frequency
# fw1 = np.max(np.max(freq)) # maximum frequency
# if fw0 <= 0:
# fw0 = np.min(np.min(freq[freq > 0])) # only consider positive frequency
# fw = fw1-fw0
tw = t1 - t0
bins = np.linspace(0, 12, freqsol) #np.logspace(0, 10, freqsol, base=2.0)
p = np.digitize(freq, 2**bins)
t = np.ceil((timesol - 1) * (time - t0) / tw)
t = t.astype(int)
hilbert_spectrum = np.zeros([timesol, freqsol])
for i in range(len(time)):
for j in range(imfs.shape[0] - 1):
if p[i, j] >= 0 and p[i, j] < freqsol:
hilbert_spectrum[t[i], p[i, j]] += amp[i, j]
hilbert_spectrum = abs(hilbert_spectrum)
fig1 = plt.figure(figsize=(5, 5))
plot_imfs(data, imfs, time_samples=time, fig=fig1)
fig2 = plt.figure(figsize=(5, 5))
plot_frequency(data, freq.T, time_samples=time, fig=fig2)
fig0 = plt.figure(figsize=(5, 5))
ax = plt.gca()
c = ax.contourf(
np.linspace(t0, t1, timesol), bins, hilbert_spectrum.T
) #, colors=('whites','lategray','navy','darkgreen','gold','red')
ax.invert_yaxis()
ax.set_yticks(np.linspace(1, 11, 11))
Yticks = [float(math.pow(2, p))
for p in np.linspace(1, 11, 11)] # make 2^periods
ax.set_yticklabels(Yticks)
ax.set_xlabel('Time', fontsize=8)
ax.set_ylabel('Period', fontsize=8)
position = fig0.add_axes([0.2, 0.05, 0.6, 0.01])
cbar = plt.colorbar(c, cax=position, orientation='horizontal')
cbar.set_label('Power')
plt.show()
def x_eemd(x):
"""
eemd变换
:param x: 代表采样数据[m,n],m代表每个周期的样本，n代表采样点
:return: 返回每个样本IMF分量，[m,IMF,n]
"""
eemd = EEMD()
emd = eemd.EMD
emd.extrema_detection = "parabol"
for i in range(x.shape[0]):
eIMFs = eemd.eemd(x[i])
print(eIMFs)
def eemd(signal):
"""
eemd分解
:param signal:采样信号
:return:
"""
# Assign EEMD to `eemd` variable
eemd = EEMD()
# Say we want detect extrema using parabolic method
emd = eemd.EMD
emd.extrema_detection = "parabol"
# Execute EEMD on S
eIMFs = eemd.eemd(S)
return eIMFs
def main():
#load data
e = exchange.Exchange('../lib/binance.db')
start = int(datetime.datetime(2018, 7, 1).timestamp() * 1000)
end = int(datetime.datetime(2019, 7, 1).timestamp() * 1000)
print('Loading order data...')
number_of_orders, prices = e.get_total_orders_ts('BTCUSDT', 60 * 60 * 1000, start, end) #hourly data
print('done')
buy_orders = np.array([b for s, b in number_of_orders])
sell_orders = np.array([s for s, b in number_of_orders])
t = [i for i, o in enumerate(buy_orders)]
eemd = EEMD()
emd = eemd.EMD
eIMFs = eemd.eemd(buy_orders - sell_orders)
n = eIMFs.shape[0]
fig, axs = plt.subplots(n + 3, figsize=(12,9), sharex=True)
axs[0].plot(prices)
axs[0].set_ylabel('Price')
axs[1].plot(buy_orders - sell_orders, color='g')
axs[1].set_ylabel('Orders - (Buy - sell)')
axs[2].plot(calculate_returns(prices))
for i in range(n):
axs[i + 3].plot(eIMFs[i], color='g')
axs[i + 3].set_ylabel('eIMF ' + str(i + 1))
plt.xlabel("Time /days")
#plt.tight_layout()
plt.show()
def find_interval(signal, fs, imf_no=1):
"""
Compute the interval setting for the TIV.
Calculating a hilbert transform from the instrinct mode functions (imfs).
This is based on the hilbert-huang transform assuming non-stationarity of the given dataset.
The function returns a suggested interval (most powerful frequency) based on the weighted average.
The weights are the calculated instantaneous energies.
Parameters
----------
signal: 1d np.ndarray
a one dimensional array which contains the brightness temperature (perturbation) of one pixel over time.
fs: int
the fps which was used to record the imagery
imf_no: int (default 1)
The imf which will be used to calculate the interval on. IMF 1 is the one with the highest frequency.
Returns
-------
recommended_interval : float
The found most powerful interval in float. Needs rounding to the next int.
"""
eemd = EEMD()
imfs = eemd.eemd(signal)
imf = imfs[imf_no - 1, :]
sig = hilbert(imf)
energy = np.square(np.abs(sig))
phase = np.arctan2(sig.imag, sig.real)
omega = np.gradient(np.unwrap(phase))
omega = fs / (2 * math.pi) * omega
#omegaIdx = np.floor((omega-F[0])/FResol)+1;
#freqIdx = 1/omegaIdx;
insf = omega
inse = energy
rel_inse = inse / np.nanmax(inse)
insf_weigthed_mean = np.average(insf, weights=rel_inse)
insp = 1 / insf_weigthed_mean
recommended_interval = np.round(fs * insp, 1)
gc.collect()
return (recommended_interval)
def ee(data, drawflag):
data.shape = (len(data),)
x = np.linspace(1, len(data), len(data))
eemd = EEMD()
# Say we want detect extrema using parabolic method
emd = eemd.EMD
emd.extrema_detection = "parabol"
print(colored("decomposing - EEmd mode...", 'green'))
imfs = eemd.eemd(data, x)
if drawflag == 1:
size = imfs.shape
plt.figure(figsize=(20, 18))
for loop in range(1, size[0]+1):
plt.subplot(size[0], 1, loop)
plt.plot(x, imfs[loop-1])
plt.title(loop)
plt.show()
return imfs
def emdT(signal, t):
from PyEMD import EEMD
import numpy as np
import pylab as plt
# Define signal
# Assign EEMD to `eemd` variable
eemd = EEMD()
# Say we want detect extrema using parabolic method
emd = eemd.EMD
emd.extrema_detection = "simple"
# Execute EEMD on S
S = signal[0][:]
S = S[20000:25000]
t = t[20000:25000]
eIMFs = eemd.eemd(S, t)
nIMFs = eIMFs.shape[0]
# Plot results
plt.figure(figsize=(20, 12))
plt.subplot(nIMFs + 1, 1, 1)
plt.subplots_adjust(hspace=0.01)
plt.xticks([])
plt.plot(t, S, color='black')
for n in range(nIMFs):
if n < 7:
plt.subplot(nIMFs + 1, 1, n + 2)
plt.plot(t, eIMFs[n], color='black')
plt.ylabel("eIMF %i" % (n + 1))
plt.locator_params(axis='y', nbins=2)
plt.xticks([])
elif n == 7:
plt.subplot(nIMFs + 1, 1, n + 2)
plt.plot(t, eIMFs[n], color='black')
plt.ylabel("eIMF %i" % (n + 1))
plt.locator_params(axis='y', nbins=2)
plt.xlabel("Time [s]")
plt.show()
def prepare_data():
con = engine.connect()
# con.execute("truncate DOdataPrecictOnly")
# con.execute("INSERT INTO DOdataPrecictOnly(Date,`Dissolved Oxygen`) ( SELECT Date,`Dissolved Oxygen` FROM DOdata order by Date ASC)")
rv = con.execute("select `Dissolved Oxygen` from DOdataPrecictOnly")
do = []
for i in rv:
print(i)
do.append(i[0])
DO = []
for i in range(0, len(do)):
DO.append([do[i]])
scaler_DO = MinMaxScaler(feature_range=(0, 1))
DO = scaler_DO.fit_transform(DO)
# DO = isolutionforest(DO)
eemd = EEMD()
eemd.noise_seed(12345)
imfs = eemd.eemd(DO.reshape(-1), None, 8)
con.close()
return imfs, scaler_DO
# 设置plt格式
sns.set_style("white")
# 读取数据集导入数据
df = pd.read_excel('.\\data\\gas_load.xlsx')
# 异常数据直接剔除
df.gas_use[601] = (df.gas_use[600] + df.gas_use[602]) / 2
df.gas_use[614] = (df.gas_use[613] + df.gas_use[615]) / 2
from PyEMD import EEMD
eemd = EEMD()
emd = eemd.EMD
emd.extrema_detection = "parabol"
eIMFs = eemd.eemd(df.gas_use[0:850].values, np.array([i for i in range(850)]))
nIMFs = eIMFs.shape[0]
# plot results
plt.figure(figsize=(12, 30))
plt.subplot(nIMFs + 1, 1, 1)
plt.plot(np.array([i for i in range(850)]), df.gas_use[0:850].values, 'r')
for n in range(nIMFs):
plt.subplot(nIMFs + 1, 1, n + 2)
plt.plot(np.array([i for i in range(850)]), eIMFs[n], 'g')
plt.ylabel("eIMF %i" % (n + 1))
plt.locator_params(axis='y', nbins=5)
plt.xlabel("Time [s]")
plt.tight_layout()
sin = lambda x,p: np.sin(2*np.pi*x*t+p)
S = 3*sin(18,0.2)*(t-0.2)**2
S += 5*sin(11,2.7)
S += 3*sin(14,1.6)
S += 1*np.sin(4*2*np.pi*(t-0.8)**2)
S += t**2.1 -t
# Assign EEMD to `eemd` variable
eemd = EEMD()
# Say we want detect extrema using parabolic method
emd = eemd.EMD
emd.extrema_detection="parabol"
# Execute EEMD on S
eIMFs = eemd.eemd(S, t)
nIMFs = eIMFs.shape[0]
# Plot results
plt.figure(figsize=(12,9))
plt.subplot(nIMFs+1, 1, 1)
plt.plot(t, S, 'r')
for n in range(nIMFs):
plt.subplot(nIMFs+1, 1, n+2)
plt.plot(t, eIMFs[n], 'g')
plt.ylabel("eIMF %i" %(n+1))
plt.locator_params(axis='y', nbins=5)
plt.xlabel("Time [s]")
plt.tight_layout()
def plot_decomposer_imf(self):
d_obs = self.d_obs
d_mod = self.d_mod
d_t_obs = self.d_t_obs
scores = []
for j, site in enumerate(self.sitename):
if self.sitename.mask[j]:
continue
print('Process on Decomposer_IMF_' + site + '_No.' + str(j) + '!')
fig0 = plt.figure(figsize=(8, 4))
fig3 = plt.figure(figsize=(5, 5))
data = d_obs[j, :].compressed()
time = d_t_obs[~d_obs[j, :].mask]
eemd = EEMD(trials=5)
if len(data) > 0:
imfs = eemd.eemd(data)
# print('obs',imfs.shape)
if len(imfs) >= 1:
ax0 = fig0.add_subplot(1, 2, 1)
ax0.plot(time, (imfs[len(imfs) - 1]),
'k-',
label='Observed')
d_d_obs = np.asarray([
str(start_year + int(x) / 365) +
('0' + str(int(x) % 365 / 31 + 1) if
int(x) % 365 / 31 < 9 else str(int(x) % 365 / 31 + 1))
for x in time
])
ax0.xaxis.set_ticks([
time[0], time[2 * len(time) / 5],
time[4 * len(time) / 5]
])
ax0.set_xticklabels([
d_d_obs[0], d_d_obs[2 * len(d_d_obs) / 5],
d_d_obs[4 * len(d_d_obs) / 5]
])
## hht spectrum
if len(imfs) >= 1:
fig3, freq = hht(data, imfs, time, 1, fig3)
fig3.savefig(self.filedir + self.variable + '/' + site +
'_hht_IMF_observed' + self.variable + '.png',
bbox_inches='tight')
fig1 = plt.figure(figsize=(4 * (len(d_mod) + 1), 8))
fig2 = plt.figure(figsize=(4 * (len(d_mod) + 1), 8))
fig1.subplots_adjust(wspace=0.5, hspace=0.3)
fig2.subplots_adjust(wspace=0.5, hspace=0.3)
if len(data) > 0:
if len(imfs) >= 1:
fig1 = plot_imfs(data,
imfs,
time_samples=time,
fig=fig1,
no=1,
m=len(d_mod))
fig2 = plot_frequency(data,
freq.T,
time_samples=time,
fig=fig2,
no=1,
m=len(d_mod))
models1 = []
datamask = []
data1 = imfs[len(imfs) - 1]
for m in range(len(d_mod)):
## hht spectrum
eemd = EEMD(trials=5)
fig3 = plt.figure(figsize=(5, 5))
data2 = d_mod[m][j, :][~d_obs[j, :].mask]
imfs = eemd.eemd(data2.compressed())
# print('mod'+str(m), imfs.shape)
if len(imfs) >= 1:
fig3, freq = hht(data2.compressed(), imfs,
time[~data2.mask], 1, fig3)
fig3.savefig(self.filedir + self.variable + '/' + site +
'_hht_IMF_model' + str(m + 1) +
self.variable + '.png',
bbox_inches='tight')
if len(imfs) >= 1:
fig1 = plot_imfs(data2.compressed(),
imfs,
time_samples=time[~data2.mask],
fig=fig1,
no=m + 2,
m=len(d_mod))
fig2 = plot_frequency(data2.compressed(),
freq.T,
time_samples=time[~data2.mask],
fig=fig2,
no=m + 2,
m=len(d_mod))
ax0.plot(time[~data2.mask], (imfs[len(imfs) - 1]),
'-',
label='Model' + str(m + 1),
c=col[m])
models1.append(imfs[len(imfs) - 1])
datamask.append(data2)
ax0.set_xlabel('Time', fontsize=fontsize)
ax0.set_ylabel('' + self.variable + '(' + self.d_unit_obs +
')',
fontsize=fontsize)
ax0.yaxis.tick_right()
ax0.yaxis.set_label_position("right")
ax0.legend(bbox_to_anchor=(-0.05, 1),
shadow=False,
fontsize='medium')
plot_Taylor_graph(data1, models1, fig0, 122, datamask=datamask)
else:
print("'Data's length is too short !")
fig1.savefig(self.filedir + self.variable + '/' + site +
'_Decompose_IMF_' + self.variable + '.png',
bbox_inches='tight')
fig2.savefig(self.filedir + self.variable + '/' + site +
'_deviation_IMF_' + self.variable + '.png',
bbox_inches='tight')
fig0.subplots_adjust(left=0.1, hspace=0.25, wspace=0.55)
fig0.savefig(self.filedir + self.variable + '/' + site + '_' +
'IMF' + '_' + self.variable + '.png',
bbox_inches='tight')
plt.close('all')
return scores
skiprows=1,
unpack=True)
#noAct, control, piano, reading = loadtxt('20181102_4act.csv', delimiter=',', dtype='float', skiprows=1, unpack=True)
#S = control
t = np.arange(len(S))
# Assign EEMD to `eemd` variable
eemd = EEMD()
# Say we want detect extrema using parabolic method
emd = eemd.EMD
emd.extrema_detection = "parabol"
# Execute EEMD on S
#eIMFs = eemd.eemd(S, t)
eIMFs = eemd.eemd(S)
nIMFs = eIMFs.shape[0]
# Plot results
plt.figure(figsize=(12, 9))
plt.subplot(nIMFs + 1, 1, 1)
plt.plot(t, S, 'r')
nFiltered = int(0.8 * nIMFs)
filteredS = [sum(x) for x in zip(*eIMFs[range(nFiltered)])]
np.savetxt("filtered_Read.csv", S, delimiter=",")
for n in range(nIMFs):
plt.subplot(nIMFs + 1, 1, n + 2)
plt.plot(t, eIMFs[n], 'g')
plt.ylabel("eIMF %i" % (n + 1))
def protected_mask(path_nc4, path_protected_shp, lats, lons, mean_file):
#read data and variables
f = netCDF4.Dataset(path_nc4)
f.set_auto_mask(False) #missing value = -9999.0
LAI = f.variables['LAI']
time = f.variables['time']
dates = num2date(time[:], time.units)
latt, lonn = f.variables['lat'][:], f.variables['lon'][:]
#get actual lon/lat range
latbounds = list(latt[(latt <= lats[1]) & (latt >= lats[0])])
lonbounds = list(lonn[(lonn <= lons[1]) & (lonn >= lons[0])])
#create map polygon
grid, rows, cols = create_grid(lonbounds[0], latbounds[0], lonbounds[-1],
latbounds[-1])
#read proctected shape file
points_shape_map = gpd.read_file(path_protected_shp)
points_shape_map.to_crs(epsg=4326, inplace=True)
#identify grid cells in the shape file intersect with polygons
data = []
for index, protected in points_shape_map.iterrows():
for index2, cell in grid.iterrows():
if protected['geometry'].intersects(cell['geometry']):
data.append({'geometry': cell['geometry']})
df = gpd.GeoDataFrame(data, columns=['geometry'])
#and drop duplicates by convert to wkb
df["geometry"] = df["geometry"].apply(lambda geom: geom.wkb)
df = df.drop_duplicates(["geometry"])
# convert back to shapely geometry
df["geometry"] = df["geometry"].apply(lambda geom: shapely.wkb.loads(geom))
#map back to lat,lon list
#extract all coordinates from geometry column
g = [i for i in df.geometry]
#map all polygons to coordinates
all_coords = [mapping(item)["coordinates"] for item in g]
#loop through all coordinates to find corresponding lat, lon tuple
list_protected = []
for coords in all_coords:
idx_tup = coords[0][0]
idx_lat = np.where(latt == idx_tup[1])[0][0]
idx_lon = np.where(lonn == idx_tup[0])[0][0]
list_protected.append([idx_lat, idx_lon])
#get all monthly data, perform detrending using EEMD, and perform surrogate test for significance
# Assign EEMD to `eemd` variable
eemd = EEMD()
eemd.noise_seed(12345)
#detect extrema using parabolic method
emd = eemd.EMD
emd.extrema_detection = "parabol"
#initial result dataframe
lai_monthly_mkt, lai_monthly_cat = [], []
for i in range(12):
t = mean_file.shape[1] #get time-length, lat length, lon length
lai_cat = np.zeros(
(len(latt), len(lonn))) #initiate global mask for categorization
lai_mkt = np.zeros(
(len(latt),
len(lonn))) #initiate global mask for stipling significance
for coords in list_protected:
lat, lon = coords[0], coords[1]
lai_monthly = mean_file[
i, :, lat,
lon] #subset per month per lat lon, using the calculated mean_file
if lai_monthly.all() < 0: #missing data present
lai_cat[lat, lon] = 0
lai_mkt[lat, lon] = 0
else:
eIMFs_n = eemd.eemd(lai_monthly, np.arange(t))
trend = eIMFs_n[-1]
if check_increasing(trend):
lai_cat[lat, lon] = 1
#test for significance:
ta_list = nonlinear_trend_test(trend)
if np.argmax(ta_list) == 0:
lai_mkt[lat, lon] = 1
elif check_decreasing(trend):
lai_cat[lat, lon] = -1
#test for significance:
ta_list = nonlinear_trend_test(trend)
if np.argmin(ta_list) == 0:
lai_mkt[lat, lon] = -1
else:
lai_cat[
lat,
lon] = 5 #flag out inflection shape trend series if any
lai_monthly_mkt.append(lai_mkt)
lai_monthly_cat.append(lai_cat)
#convert the result list back to arrays
lai_monthly_mkt = np.array(lai_monthly_mkt)
lai_monthly_cat = np.array(lai_monthly_cat)
# subset the region of interest by lon lat indexes
# latitude lower and upper index
latli = np.argmin(np.abs(latt - lats[0]))
latui = np.argmin(np.abs(latt - lats[1]))
# longitude lower and upper index
lonli = np.argmin(np.abs(lonn - lons[0]))
lonui = np.argmin(np.abs(lonn - lons[1]))
lai_cat = lai_monthly_cat[:, latli:latui, lonli:lonui]
lai_mkt = lai_monthly_mkt[:, latli:latui, lonli:lonui]
# Write the array to disk
with open('cat_sg3_ken_sur.txt', 'w') as outfile:
outfile.write('# Array shape: {0}\n'.format(lai_cat.shape))
for data_slice in lai_cat:
np.savetxt(outfile, data_slice, fmt='%-7.2f')
outfile.write('# New slice\n')
# Write the array to disk
with open('mkt_sg3_ken_sur.txt', 'w') as outfile:
outfile.write('# Array shape: {0}\n'.format(lai_mkt.shape))
for data_slice in lai_mkt:
np.savetxt(outfile, data_slice, fmt='%-7.2f')
outfile.write('# New slice\n')
#print out any infletion point index
print(np.where(lai_cat == 5))
return lai_cat, lai_mkt, latbounds, lonbounds, '20 SG3 models mean'
#label_u=np.full((13,1),0)
#label=np.concatenate((label_c,label_u))
#------------------------------------------------------------------------------
#name of datasets
numberofcase = len(namets)
#load datasets
for i in range (0,numberofcase):
name = 'chatter_%d' %(i+1)
nameofdata = '%s_downsampled.mat' %(namets[i])
exec("%s = sio.loadmat(nameofdata)" % (name))
exec('%s = %s["tsDS"]' %(name,name))
#Ensemble Emprical Mode Decomposition (EEMD)
from PyEMD import EEMD
eemd = EEMD()
emd = eemd.EMD
emd.trials = 200 #default = 100
emd.noise_width = 0.2 #default = 0.05
#signal
S = chatter_1[:,1]
t = chatter_1[:,0]
eIMFs = eemd.eemd(S, t)
nIMFs = eIMFs.shape[0]
def main():
#load data
e = exchange.Exchange('../lib/binance.db')
start = int(datetime.datetime(2019, 6, 1).timestamp() * 1000)
end = int(datetime.datetime(2019, 7, 1).timestamp() * 1000)
#end = int(datetime.datetime(2019, 7, 1).timestamp() * 1000)
print('Loading order data...')
number_of_orders, prices = e.get_total_orders_ts('BTCUSDT', 60 * 1000, start, end) #hourly data
print('done')
buy_orders = np.array([b for s, b in number_of_orders])
sell_orders = np.array([s for s, b in number_of_orders])
t = [i for i, o in enumerate(buy_orders)]
eemd = EEMD()
emd = eemd.EMD
eIMFs_buys = eemd.eemd(buy_orders)
eIMFs_sells = eemd.eemd(sell_orders)
if eIMFs_buys.shape[0] != eIMFs_sells.shape[0]:
print('size mismatch')
n = min(eIMFs_buys.shape[0], eIMFs_sells.shape[0])
fig, axs = plt.subplots(n + 2, figsize=(12,9), sharex=True)
axs[0].plot(prices)
axs[0].set_ylabel('Price')
axs[1].plot(buy_orders, color='g')
axs[1].plot(sell_orders, color='r')
axs[1].set_ylabel('Orders')
for i in range(n):
axs[i + 2].plot(eIMFs_buys[i], color='g')
axs[i + 2].plot(eIMFs_sells[i], color='r')
axs[i + 2].set_ylabel('eIMF ' + str(i + 1))
plt.xlabel("Time /days")
plt.tight_layout()
plt.show()
def hht(data, time, freqsol=33, timesol=50):
"""
hht function for the Hilbert Huang Transform spectrum
Parameters
----------
data : array-like, shape (n_samples,)
The input signal.
time : array-like, shape (n_samples), optional
Time instants of the signal samples.
(defaults to `np.arange(1, len(signal))`)
-------
`matplotlib.figure.Figure`
The figure (new or existing) in which the hht spectrum is plotted.
example:
--------------------
.. sourcecode:: ipython
f = Dataset('./source/obs.nc')
# read one example data
fsh = f.variables['FSH']
time = f.variables['time']
one_site = np.ma.masked_invalid(fsh[0,:])
time = time[~one_site.mask]
data = one_site.compressed()
hht(data, time)
----------------
"""
# freqsol give frequency - axis resolution for hilbert - spectrum
# timesol give time - axis resolution for hilbert - spectrum
t0 = time[0]
t1 = time[-1]
dt = (t1 - t0) / (len(time) - 1)
eemd = EEMD()
imfs = eemd.eemd(data)
freq, amp = FAhilbert(imfs, dt)
# fw0 = np.min(np.min(freq)) # maximum frequency
# fw1 = np.max(np.max(freq)) # maximum frequency
# if fw0 <= 0:
# fw0 = np.min(np.min(freq[freq > 0])) # only consider positive frequency
# fw = fw1-fw0
tw = t1 - t0
bins = np.linspace(0, 12, freqsol) # np.logspace(0, 10, freqsol, base=2.0)
p = np.digitize(freq, 2**bins)
t = np.ceil((timesol - 1) * (time - t0) / tw)
t = t.astype(int)
hilbert_spectrum = np.zeros([timesol, freqsol])
for i in range(len(time)):
for j in range(imfs.shape[0] - 1):
if p[i, j] >= 0 and p[i, j] < freqsol:
hilbert_spectrum[t[i], p[i, j]] += amp[i, j]
hilbert_spectrum = abs(hilbert_spectrum)
fig1 = plt.figure(figsize=(5, 5))
plot_imfs(data, imfs, time_samples=time, fig=fig1)
fig2 = plt.figure(figsize=(5, 5))
plot_frequency(data, freq.T, time_samples=time, fig=fig2)
fig0 = plt.figure(figsize=(5, 5))
ax = plt.gca()
c = ax.contourf(
np.linspace(t0, t1, timesol), bins, hilbert_spectrum.T
) # , colors=('whites','lategray','navy','darkgreen','gold','red')
ax.invert_yaxis()
ax.set_yticks(np.linspace(1, 11, 11))
Yticks = [float(math.pow(2, p))
for p in np.linspace(1, 11, 11)] # make 2^periods
ax.set_yticklabels(Yticks)
ax.set_xlabel('Time', fontsize=8)
ax.set_ylabel('Period', fontsize=8)
position = fig3.add_axes([0.2, -0., 0.6, 0.01])
cbar = plt.colorbar(c, cax=position, orientation='horizontal')
cbar.set_label('Power')
plt.show()
values = dataset.values
groups = [0, 1, 2, 3]
# fig, axs = plt.subplots(1)
df = pd.DataFrame(dataset) # 整体数据的全部字典类型
do = df['Dissolved Oxygen'] # 返回溶解氧那一列，用字典的方式
DO = []
for i in range(0, len(do)):
DO.append([do[i]])
scaler_DO = MinMaxScaler(feature_range=(0, 1))
DO = scaler_DO.fit_transform(DO)
eemd = EEMD()
eemd.noise_seed(12345)
imfs = eemd.eemd(DO.reshape(-1), None, 8)
c = int(len(do) * .85)
lookback_window = 6
imfs_prediction = []
# i = 1
# for imf in imfs:
# plt.subplot(len(imfs), 1, i)
# plt.plot(imf)
# i += 1
#
# plt.savefig('res/result_imf.png')
# plt.show()
test = np.zeros([len(do) - c - lookback_window, 1])
def test_general_eemd(self):
S = np.random.random(100)
eemd = EEMD()
eemd.trials = 10
eemd.eemd(S)
