def test_stl_2(self):
"Tests a robust STL."
(co2_data, co2_results, parameters) = self.d
co2_fitted = stl(co2_data, robust=True, **parameters)
assert_almost_equal(co2_fitted.seasonal, co2_results[4], 6)
assert_almost_equal(co2_fitted.trend, co2_results[5], 6)
assert_almost_equal(co2_fitted.weights, co2_results[6], 6)
def calculate_resid(cN, layer="two"):
"""
消灭季节性特征
"""
# remove seasonality and trend
stl_params = dict(
np = 96, # period of season
ns = 95, # seasonal smoothing
nt = None, # trend smooting int((1.5*np)/(1-(1.5/ns)))
nl = None, # low-pass filter leat odd integer >= np
isdeg=1,
itdeg=1,
ildeg=1,
robust=True,
ni = 1,
no = 5)
tsts = pd.read_pickle(f"data/tsts/C{cN}-{layer}-layer.pl")
resid = pd.DataFrame(index=tsts.index)
print("Loaded!")
for col in ["ts", "non_ts", "T_ts", "C_ts", "influ_ts", "non_influ_ts",
"T_flu_ts", "C_flu_ts", "T_nonflu_ts", "C_nonflu_ts"]:
print(col)
tsts[col] = tsts[col].fillna(0)
resid[col] = stl(tsts[col].values, **stl_params).residuals
resid.to_pickle(f"data/tsts/resid_C{cN}-{layer}-layer.pl")
print("saved!")
def _detect_anomaly_for_one_window(x, period, max_anoms, alpha, direction,
e_values, window_start, breakout_kwargs):
# The core part of anomaly detection:
# 1. Use STL to perform seasonal decomposition.
# parameters are copied from R's stl()
stl_ret = stl(x,
np=period,
ns=len(x) * 10 + 1,
isdeg=0,
robust=True,
ni=1,
no=15)
# 2. Calculate residuals using seasonal from STL result and median as the trends.
seasons = stl_ret['seasonal']
if e_values: # store the expected values if e_value is set
trends = stl_ret['trend']
for i in range(0, len(x)):
if e_values[window_start + i] is None:
e_values[window_start + i] = floor(seasons[i] + trends[i])
if breakout_kwargs:
trends = _get_trends_by_breakout_detection(x, breakout_kwargs)
else:
trends = _get_trends_by_median(x)
residuals = [x[i] - seasons[i] - trends[i] for i in range(0, len(x))]
# 3. Use ESD to find out outliers from residuals. These outliers' corresponding values in x are the anomalies
max_anom_num = max(1, int(len(x) * max_anoms))
anom_index = _esd(residuals, max_anom_num, alpha, direction=direction)
ret = set()
for anom_i in anom_index:
ret.add(window_start + anom_i) # convert the index to the index in x
return ret
def test_stl_1(self):
"Tests a classic STL."
(co2_data, co2_results, parameters) = self.d
co2_fitted = stl(co2_data, robust=False, **parameters)
assert_almost_equal(co2_fitted.seasonal, co2_results[0], 6)
assert_almost_equal(co2_fitted.trend, co2_results[1], 6)
assert_almost_equal(co2_fitted.weights, co2_results[2], 6)
def run_maskedarray():
from pyloess.mpyloess import stl
res = stl(y)#, np=np, ns=ns, nt=nt, nl=nl, isdeg=isdeg, itdeg=itdeg, ildeg=ildeg, nsjump=nsjump, ntjump=ntjump, nljump=nljump, robust=robust, ni=ni, no=no)
figure()
plot(y, "k")
plot(res.trend, "b")
plot(res.seasonal, "g")
plot(res.residuals, "r")
figure()
plot(res.weights)
def run_ndarray():
from pyloess import stl
y = prd.compressed()
res = stl(y, np, ns, nt, nl, isdeg, itdeg, ildeg, nsjump, ntjump, nljump, robust, ni, no)
figure()
plot(y, "k")
plot(res.trend, "b")
plot(res.seasonal, "g")
plot(res.residuals, "r")
figure()
plot(res.weights)
def detect_anom_local(self, x, plot=False):
rate = 0.01
decomp = stl(x, np=self.period, no=self.no, nt=self.nt) # STL decomposition
seasonal = decomp['seasonal']
trend = decomp['trend']
# pick anomaly
median = np.median(x)
residual = x - seasonal - trend
ret = pyasl.generalizedESD(residual, int(x.shape[0] * self.esd_rate))
anom_ind = ret[1]
anom_val = np.array([x[k] for k in anom_ind])
if plot is True:
plot_verticle([x], anom_ind, anom_val)
return anom_ind
def _detect_anomaly_for_one_window(x, period, max_anoms, alpha, direction, e_values, window_start, breakout_kwargs):
# The core part of anomaly detection:
# 1. Use STL to perform seasonal decomposition.
# parameters are copied from R's stl()
stl_ret = stl(x, np=period, ns=len(x) * 10 + 1, isdeg=0, robust=True, ni=1, no=15)
# 2. Calculate residuals using seasonal from STL result and median as the trends.
seasons = stl_ret['seasonal']
if e_values: # store the expected values if e_value is set
trends = stl_ret['trend']
for i in range(0, len(x)):
if e_values[window_start + i] is None:
e_values[window_start + i] = floor(seasons[i] + trends[i])
if breakout_kwargs:
trends = _get_trends_by_breakout_detection(x, breakout_kwargs)
else:
trends = _get_trends_by_median(x)
residuals = [x[i] - seasons[i] - trends[i] for i in range(0, len(x))]
# 3. Use ESD to find out outliers from residuals. These outliers' corresponding values in x are the anomalies
max_anom_num = max(1, int(len(x) * max_anoms))
anom_index = _esd(residuals, max_anom_num, alpha, direction=direction)
ret = set()
for anom_i in anom_index:
ret.add(window_start + anom_i) # convert the index to the index in x
return ret
def run_maskedarray():
from pyloess.mpyloess import stl
res = stl(y)
stl_plot(y, res.outputs)
plt.show()
def run_ndarray():
from pyloess import stl
res = stl(y, np, ns, nt, nl, isdeg, itdeg, ildeg, nsjump, ntjump, nljump, robust, ni, no)
stl_plot(y, res)
plt.show()
stl_params = dict(
np=n_p, # period of season
ns=n_s, # seasonal smoothing
nt=None, # trend smooting int((1.5*np)/(1-(1.5/ns)))
nl=None, # low-pass filter leat odd integer >= np
isdeg=1,
itdeg=1,
ildeg=1,
robust=True,
ni=1,
no=5)
stl_res_dict = {}
for col in dfdrop:
stl_res_dict[col] = stl(dfdrop[col].values, **stl_params)
dftrend = pd.DataFrame(index=dfdrop.index)
dfseasonal = pd.DataFrame(index=dfdrop.index)
dfresiduals = pd.DataFrame(index=dfdrop.index)
for col in stl_res_dict.keys():
dftrend[col] = stl_res_dict[col].trend
dfseasonal[col] = stl_res_dict[col].seasonal
dfresiduals[col] = stl_res_dict[col].residuals
save_dir_file = os.path.join(save_dir, 'time_series/', graph_type,
period)
os.makedirs(save_dir_file, exist_ok=True)
'''
Created on 8/07/2015
@author: aagranonik
'''
import pyloess
import os
from numpy import fromiter
from numpy import bool_, float_
dfile = open(os.path.join('madeup_data'), 'r')
dfile.readline()
x = fromiter((float(v) for v in dfile.readline().rstrip().split()),
float_).reshape(-1,2)
dfile.readline()
y = fromiter((float(v) for v in dfile.readline().rstrip().split()),
float_)
print pyloess.stl(y)
'''
Created on 8/07/2015
@author: aagranonik
'''
import pyloess
import os
from numpy import fromiter
from numpy import bool_, float_
dfile = open(os.path.join('madeup_data'), 'r')
dfile.readline()
x = fromiter((float(v) for v in dfile.readline().rstrip().split()),
float_).reshape(-1, 2)
dfile.readline()
y = fromiter((float(v) for v in dfile.readline().rstrip().split()), float_)
print pyloess.stl(y)
