def create_moments(data):
"""
gets the moments he asks for (growth rate, std, autocorr)
"""
if data.ndim == 2:
df = pd.DataFrame(data).T
total_obs = data.shape[0]
diff = (data[:, 2:] - data[:, 1:-1]) / data[:, 1:-1]
growth = np.mean(diff, axis=1).mean()
filt_data = np.empty(data.shape)
for i in range(total_obs):
filt_data[i, :] = hpfilter(data[i,:], lamb=1600)[0]
std = filt_data.std(ddof=1, axis=1).mean()
filt_df = pd.DataFrame(filt_data).T
all_corrs = np.zeros(total_obs)
for i in range(total_obs):
all_corrs[i] = filt_df[i].autocorr()
autos = all_corrs.mean()
else:
diff = (data[2:] - data[1:-1]) / data[1:-1]
growth = np.mean(diff)
filt = hpfilter(np.log(data), lamb=1600)[0]
std = np.std(filt, ddof=1)
autos = np.corrcoef(filt[2:], filt[1:-1])[0, 1]
return [growth, std, autos]
def denoise(train, test, sensor_columns, lamb=1600):
for sensor in sensor_columns:
for engine_id in set(train.index):
_, train.loc[engine_id, sensor] = \
hp_filter.hpfilter(train.loc[engine_id, sensor], lamb)
for engine_id in set(test.index):
_, test.loc[engine_id, sensor] = \
hp_filter.hpfilter(test.loc[engine_id, sensor], lamb)
return train, test
def test_hpfilter_pandas():
dta = macrodata.load_pandas().data
index = date_range(start='1959-01-01', end='2009-10-01', freq='Q')
dta.index = index
cycle, trend = hpfilter(dta["realgdp"])
ndcycle, ndtrend = hpfilter(dta['realgdp'].values)
assert_equal(cycle.values, ndcycle)
assert_equal(cycle.index[0], datetime(1959, 3, 31))
assert_equal(cycle.index[-1], datetime(2009, 9, 30))
assert_equal(cycle.name, "realgdp_cycle")
def standarize(method, data, median, mad, hp_filter=True, index_f=None, window_f=None):
if method == 'asinh-hp' or method == 'asinh':
if method == 'asinh-hp' and hp_filter:
data = hp.hpfilter(data, 110930628906)[0]
median, mad = count_median_mad(data.loc[index_f - window_f:index_f])
data = (data - median) / mad
data = np.arcsinh(data)
elif method == 'hp' and hp_filter:
data = hp.hpfilter(data, 110930628906)[0]
median, mad = count_median_mad(data.loc[index_f - window_f:index_f])
data = (data - median) / mad
else:
data = (data - median) / mad
return data
def hp_filter(ts, min_periods=10, freq='M', lamb=None):
"""
HP滤波
Parameters:
-----------
ts: Series
window: int
滚动时间窗口
alpha: float:
hp滤波参数
"""
if lamb is None:
lamb = {'M': 129600, 'Q': 1600, 'Y': 6.25, 'D': 1600 * 60**4}[freq]
def _get_last_one(sub_ts, l):
sub_cy, sub_trend = hpfilter(sub_ts, l)
return sub_cy[-1]
ts_nonna = ts.dropna()
cycle, trend = hpfilter(ts_nonna, lamb)
roll_trend = ts_nonna.expanding(min_periods=min_periods).agg(_get_last_one,
l=lamb)
roll_trend.iloc[:min_periods] = cycle
return roll_trend.reindex(ts.index)
def plot_forecast(data, col, product, periods):
"""
Returns a plotly graph object
Parameters
-----------
data: DataFrame
A dataframe containing a series of which a trend is to be determined
col: str
A column name from which the trend is to be determined
product: str
A column name of series from which a trend is to be determined
periods: int
Forecast horizon in units of the series
Returns
--------
A plotly graph object
"""
price_cycle, price_trend = hpfilter(data[col])
predictions = arima_forecast(data, 'avg_per_kg', periods)
start = data.index[-1] + datetime.timedelta(days=1) # Prediction start date
prediction_index = pd.date_range(start, periods=periods, freq='B')
forecasts = pd.Series(predictions, index=prediction_index)
fig = go.Figure(
data=[
go.Scatter(
x=data.index,
y=data[col],
name='Average',
mode='lines'
),
go.Scatter(
x=data.index,
y=round(price_trend,2),
name='Trend',
mode='lines'
),
go.Scatter(
x=prediction_index,
y=predictions,
name='Forecast',
mode='lines'
)
],
layout=go.Layout(
title_text=f'Price of {product}',
title_x=0.5,
xaxis={'title': 'Date'},
yaxis={'title': 'R/kg'},
template='none'
)
)
return fig, forecasts
def run(self, data):
index, col, lamb = (self.cfg.get(p) for p in ["index", "col", "lamb"])
df = build_data(data, self.cfg)
cycle, trend = hpfilter(df, lamb=1600)
df = pd.concat([df, cycle, trend], axis=1, keys=[col, "cycle", "trend"])
df = df.reset_index()
return_data, _code = build_base_chart(
df.fillna(0), index, [col, "cycle", "trend"], agg="raw"
)
return return_data
def trend(s, lamb=1600):
"""|
| 引数:
| s: Seriesもしくは１列のDataFrameとし，行のラベルはDatetimeIndexとすること。
| lamb: 四半期用のデータでは通常の値（デフォルト：1600）
|
| 返り値:
| Hodrick-Prescott filterで計算したtrend（トレンド）のSeries
|
| 例: trend(df.loc[:,'gdp'])"""
from statsmodels.tsa.filters.hp_filter import hpfilter
return hpfilter(s.dropna(), lamb=lamb)[1]
def pca(used_macro_data, theta):
components = 6 # 取多少个主成分
index = used_macro_data.index
pca = PCA(n_components=components,svd_solver='auto')
explained_variance = pca.fit(used_macro_data).explained_variance_ratio_
weight = np.array(explained_variance) / sum(explained_variance)
pca_data = pca.fit_transform(used_macro_data)
pca_data = pd.DataFrame(pca_data, columns=['fac%d' % i for i in range(1, components + 1)], index=index)
pca_data = pca_data.diff() / pca_data.shift() # 计算变化率
pca_data.dropna(inplace=True)
# pca_data = pca_data.apply(lambda x: winsorize(x,limits = (0.02,0.02)))
pca_data[pca_data > 20] = 20
pca_data[pca_data < -20] = -20
pca_data = pca_data.apply(lambda x: hpfilter(x, theta)[1])
return (pca_data, weight)
def plot3(self):
df = pd.read_csv('data/macrodata.csv', index_col=0, parse_dates=True)
df.head()
ax = df['realgdp'].plot()
ax.autoscale(axis='x', tight=True)
ax.set(ylabel='REAL GDP')
# Tuple unpacking
gdp_cycle, gdp_trend = hpfilter(df['realgdp'], lamb=1600)
df['trend'] = gdp_trend
df[['trend', 'realgdp']].plot().autoscale(axis='x', tight=True)
df[['trend', 'realgdp'
]]['2000-03-31':].plot(figsize=(12, 8)).autoscale(axis='x',
tight=True)
plt.interactive(False)
plt.show()
def plot_denoising_process(signal):
fig = plt.figure()
signal = signal.reset_index(drop=True)
plt.ylim((signal.min(), signal.max()))
plt.plot(signal, label='Noisy signal')
plt.plot(signal.rolling(30).mean(), label='Rolling mean filter', c='green')
plt.plot(signal.ewm(alpha=0.05, adjust=True).mean(),
label='Exponential smoothing',
c='orange')
plt.plot(pd.Series(hp_filter.hpfilter(signal, 800)[1]),
label='Hodrick-Prescott filter',
c='red')
plt.legend()
plt.xlabel('time')
plt.ylabel(signal.name)
plt.title('Comparison of filtering methods')
plt.show()
def hp_filter(self, column, lamb=1600):
"""Hodrick Prescott filter, splits series into cyclic and trend components"""
cycle, trend = hpfilter(self.df[column], lamb=lamb)
self.df[f'hp_cycle_{column}'] = cycle
self.df[f'hp_trend_{column}'] = trend
def f(x):
_, trend = hp_filter.hpfilter(x, 129600)
return trend.iloc[-1]
import statsmodels.formula.api as smf
import os.path
# download data from FRED
CC = fred.get_series('CCSA')['1967-01-01':'2017-07-01'].asfreq(
freq='QS', method='ffill')['1968-01-01':'2017-07-01'] #continued claims
DPI = fred.get_series('DPI')['1947-01-01':'2017-07-01']
u = fred.get_series('UNRATE')['1968-01-01':'2017-07-01'].asfreq(
freq='QS') / 100
bPayments = fred.get_series('W825RC1Q027SBEA')['1968-01-01':'2017-07-01']
Pop = fred.get_series('CNP16OV').asfreq(
freq='QS')['1968-01-01':'2017-07-01'] * 1000
#smooth DPI
DPI_cyc, DPI_trend = hpfilter(np.log(DPI), lamb=1600)
DPI_trend = np.exp(DPI_trend['1968-01-01':'2017-07-01'])
# Calculation
b = (0.5 + 0.5 * bPayments / DPI_trend / (CC / Pop))**(1 - 0.151)
def AddRecBars(ax):
rec = fred.get_series('USREC')['1967-10-01':'2017-07-01'].asfreq(freq='QS')
peaks = rec.index[1:][np.where(rec.diff()[1:] == 1.0)]
troughs = rec.index[1:][np.where(rec.diff()[1:] == -1.0)]
y1, y2 = ax.get_ylim()
for pt in zip(peaks, troughs):
x = pt
ax.fill_between(x, y1=y1, y2=y2, alpha=.25, color='k')
def test_hpfilter():
# Test Hodrick-Prescott Filter. Results taken from Stata.
hpfilt_res = array([
[3.951191484487844718e+01, 2.670837085155121713e+03],
[8.008853245681075350e+01, 2.698712467543189177e+03],
[4.887545512195401898e+01, 2.726612544878045810e+03],
[3.059193256079834100e+01, 2.754612067439201837e+03],
[6.488266733421960453e+01, 2.782816332665780465e+03],
[2.304024204546703913e+01, 2.811349757954532834e+03],
[-1.355312369487364776e+00, 2.840377312369487299e+03],
[-6.746236512580753697e+01, 2.870078365125807522e+03],
[-8.136743836853429457e+01, 2.900631438368534418e+03],
[-6.016789026443257171e+01, 2.932172890264432681e+03],
[-4.636922433138215638e+01, 2.964788224331382025e+03],
[-2.069533915570400495e+01, 2.998525339155703932e+03],
[-2.162152558595607843e+00, 3.033403152558595593e+03],
[-4.718647774311648391e+00, 3.069427647774311481e+03],
[-1.355645669169007306e+01, 3.106603456691690099e+03],
[-4.436926204475639679e+01, 3.144932262044756499e+03],
[-4.332027378211660107e+01, 3.184407273782116590e+03],
[-4.454697106352068658e+01, 3.224993971063520803e+03],
[-2.629875787765286077e+01, 3.266630757877652741e+03],
[-4.426119635629265758e+01, 3.309228196356292756e+03],
[-1.443441190762496262e+01, 3.352680411907625057e+03],
[-2.026686669186437939e+01, 3.396853866691864368e+03],
[-1.913700136208899494e+01, 3.441606001362089046e+03],
[-5.482458977940950717e+01, 3.486781589779409387e+03],
[-1.596244517937793717e+01, 3.532213445179378141e+03],
[-1.374011542874541192e+01, 3.577700115428745448e+03],
[1.325482813403914406e+01, 3.623030171865960710e+03],
[5.603040174253828809e+01, 3.667983598257461836e+03],
[1.030743373627105939e+02, 3.712348662637289181e+03],
[7.217534795943993231e+01, 3.755948652040559864e+03],
[5.462972503693208637e+01, 3.798671274963067845e+03],
[4.407065050666142270e+01, 3.840449349493338559e+03],
[3.749016270204992907e+01, 3.881249837297949853e+03],
[-1.511244199923112319e+00, 3.921067244199923152e+03],
[-9.093507374079763395e+00, 3.959919507374079785e+03],
[-1.685361946760258434e+01, 3.997823619467602384e+03],
[2.822211031434289907e+01, 4.034790889685657021e+03],
[6.117590627896424849e+01, 4.070822093721035344e+03],
[5.433135391434370831e+01, 4.105935646085656117e+03],
[3.810480376716623141e+01, 4.140188196232833434e+03],
[7.042964928802848590e+01, 4.173670350711971878e+03],
[4.996346842507591646e+01, 4.206496531574924120e+03],
[4.455282059571254649e+01, 4.238825179404287155e+03],
[-7.584961950576143863e+00, 4.270845961950576566e+03],
[-4.620339247697120300e+01, 4.302776392476971523e+03],
[-7.054024364552969928e+01, 4.334829243645529459e+03],
[-6.492941099801464588e+01, 4.367188410998014660e+03],
[-1.433567024239555394e+02, 4.399993702423955256e+03],
[-5.932834493089012540e+01, 4.433344344930889747e+03],
[-6.842096758743628016e+01, 4.467249967587436004e+03],
[-6.774011924654860195e+01, 4.501683119246548813e+03],
[-9.030958565658056614e+01, 4.536573585656580690e+03],
[-4.603981499136807543e+01, 4.571808814991368308e+03],
[2.588118806672991923e+01, 4.607219811933269739e+03],
[3.489419371912299539e+01, 4.642608806280876706e+03],
[7.675179642495095322e+01, 4.677794203575049323e+03],
[1.635497817724171910e+02, 4.712616218227582976e+03],
[1.856079654765617306e+02, 4.746963034523438182e+03],
[1.254269446392718237e+02, 4.780825055360728584e+03],
[1.387413113837174024e+02, 4.814308688616282780e+03],
[6.201826599282230745e+01, 4.847598734007177882e+03],
[4.122129542972197669e+01, 4.880966704570278125e+03],
[-4.120287475842360436e+01, 4.914722874758424041e+03],
[-9.486328233441963675e+01, 4.949203282334419782e+03],
[-1.894232132641573116e+02, 4.984718213264157384e+03],
[-1.895766639620087517e+02, 5.021518663962008759e+03],
[-1.464092413342650616e+02, 5.059737241334265491e+03],
[-1.218770668721217589e+02, 5.099388066872122181e+03],
[-4.973075629078175552e+01, 5.140393756290781312e+03],
[-5.365375213897277717e+01, 5.182600752138972894e+03],
[-7.175241524251214287e+01, 5.225824415242512259e+03],
[-7.834757283225462743e+01, 5.269846572832254424e+03],
[-6.264220687943907251e+01, 5.314404206879438789e+03],
[-3.054332122210325906e+00, 5.359185332122210639e+03],
[4.808218808024685131e+01, 5.403838811919753425e+03],
[2.781399326736391231e+00, 5.448011600673263274e+03],
[-2.197570415173231595e+01, 5.491380704151732061e+03],
[1.509441335012807031e+02, 5.533624866498719712e+03],
[1.658909029574851957e+02, 5.574409097042514986e+03],
[2.027292548049981633e+02, 5.613492745195001589e+03],
[1.752101578176061594e+02, 5.650738842182393455e+03],
[1.452808749847536092e+02, 5.686137125015246056e+03],
[1.535481629475025329e+02, 5.719786837052497503e+03],
[1.376169777998875361e+02, 5.751878022200112355e+03],
[1.257703080340770612e+02, 5.782696691965922582e+03],
[-2.524186846895645431e+01, 5.812614868468956047e+03],
[-6.546618027042404719e+01, 5.842083180270424236e+03],
[1.192352023580315290e+01, 5.871536479764196883e+03],
[1.043482970188742911e+02, 5.901368702981125352e+03],
[2.581376184768396342e+01, 5.931981238152316109e+03],
[6.634330880534071184e+01, 5.963840691194659485e+03],
[-4.236780162594641297e+01, 5.997429801625946311e+03],
[-1.759397735321817891e+02, 6.033272773532181418e+03],
[-1.827933311233055065e+02, 6.071867331123305121e+03],
[-2.472312362505917918e+02, 6.113601236250591683e+03],
[-2.877470049336488955e+02, 6.158748004933649099e+03],
[-2.634066336693540507e+02, 6.207426633669354487e+03],
[-1.819572770763625158e+02, 6.259576277076362203e+03],
[-1.175034606274621183e+02, 6.314971460627461965e+03],
[-4.769898649718379602e+01, 6.373272986497183410e+03],
[1.419578280287896632e+01, 6.434068217197121157e+03],
[6.267929662760798237e+01, 6.496914703372392069e+03],
[6.196413196753746888e+01, 6.561378868032462378e+03],
[5.019769125317907310e+01, 6.627066308746821051e+03],
[4.665364933213822951e+01, 6.693621350667861407e+03],
[3.662430749527266016e+01, 6.760719692504727391e+03],
[7.545680850246480986e+01, 6.828066191497535328e+03],
[6.052940492147536133e+01, 6.895388595078524304e+03],
[6.029518881462354329e+01, 6.962461811185376064e+03],
[2.187042136652689805e+01, 7.029098578633473153e+03],
[2.380067926824722235e+01, 7.095149320731752596e+03],
[-7.119129802169481991e+00, 7.160478129802169860e+03],
[-3.194497359120850888e+01, 7.224963973591208742e+03],
[-1.897137038934124575e+01, 7.288481370389341464e+03],
[-1.832687287845146784e+01, 7.350884872878451461e+03],
[4.600482336597542599e+01, 7.412017176634024509e+03],
[2.489047706403016491e+01, 7.471709522935970199e+03],
[6.305909392127250612e+01, 7.529821906078727807e+03],
[4.585212309498183458e+01, 7.586229876905018500e+03],
[9.314260180878318351e+01, 7.640848398191216802e+03],
[1.129819097095369216e+02, 7.693621090290463144e+03],
[1.204662123176703972e+02, 7.744549787682329224e+03],
[1.336860614601246198e+02, 7.793706938539875409e+03],
[1.034567175813735957e+02, 7.841240282418626521e+03],
[1.403118873372050075e+02, 7.887381112662795204e+03],
[1.271726169351004501e+02, 7.932425383064899506e+03],
[8.271925765282139764e+01, 7.976756742347178260e+03],
[-3.197432211752584408e+01, 8.020838322117525422e+03],
[-1.150209535194062482e+02, 8.065184953519406008e+03],
[-1.064694837456772802e+02, 8.110291483745677397e+03],
[-1.190428718925368230e+02, 8.156580871892536379e+03],
[-1.353635336292991269e+02, 8.204409533629299403e+03],
[-9.644348283027102298e+01, 8.254059482830271008e+03],
[-6.143413116116607853e+01, 8.305728131161165948e+03],
[-3.019161311097923317e+01, 8.359552613110980019e+03],
[1.384333163552582846e+00, 8.415631666836447039e+03],
[-4.156016073666614830e+01, 8.474045160736666730e+03],
[-4.843882841860977351e+01, 8.534873828418609264e+03],
[-6.706442838867042155e+01, 8.598172428388670596e+03],
[-2.019644488579979225e+01, 8.663965444885800025e+03],
[-4.316446881084630149e+00, 8.732235446881084499e+03],
[4.435061943264736328e+01, 8.802952380567352520e+03],
[2.820550564155564643e+01, 8.876083494358445023e+03],
[5.155624419490777655e+01, 8.951623755805092514e+03],
[-4.318760899315748247e+00, 9.029585760899315574e+03],
[-6.534632828542271454e+01, 9.110014328285422380e+03],
[-7.226757738268497633e+01, 9.192951577382684263e+03],
[-9.412378615444868046e+01, 9.278398786154448317e+03],
[-1.191240653288368776e+02, 9.366312065328836979e+03],
[-4.953669826751865912e+01, 9.456588698267518339e+03],
[-6.017251579067487910e+01, 9.549051515790675694e+03],
[-5.103438828313483100e+01, 9.643492388283135369e+03],
[-7.343057830678117170e+01, 9.739665578306781754e+03],
[-2.774245193054957781e+01, 9.837293451930549054e+03],
[-3.380481112519191811e+00, 9.936052481112519672e+03],
[-2.672779877794346248e+01, 1.003560179877794326e+04],
[-3.217342505148371856e+01, 1.013559842505148299e+04],
[-4.140567518359966925e+01, 1.023568267518359971e+04],
[-6.687756033938057953e+00, 1.033547475603393832e+04],
[7.300600408459467872e+01, 1.043456899591540605e+04],
[6.862345670680042531e+01, 1.053255554329319966e+04],
[5.497882461487461114e+01, 1.062907017538512628e+04],
[9.612244093055960548e+01, 1.072379155906944106e+04],
[1.978212770103891671e+02, 1.081643272298961165e+04],
[1.362772276848754700e+02, 1.090676677231512440e+04],
[2.637635494867263333e+02, 1.099469045051327339e+04],
[1.876813256815166824e+02, 1.108018567431848351e+04],
[1.711447873158413131e+02, 1.116339921268415856e+04],
[5.257586460826678376e+01, 1.124459513539173349e+04],
[4.710652228531762375e+01, 1.132414447771468258e+04],
[-6.237613484241046535e+01, 1.140245113484241119e+04],
[-9.982044354035315337e+01, 1.147994844354035376e+04],
[-7.916275548997509759e+01, 1.155703075548997549e+04],
[-9.526003459472303803e+01, 1.163403003459472347e+04],
[-1.147987680369169539e+02, 1.171122876803691724e+04],
[-1.900259054765901965e+02, 1.178884990547659072e+04],
[-2.212256473439556430e+02, 1.186704464734395515e+04],
[-2.071394278781845060e+02, 1.194584542787818464e+04],
[-8.968541528904825100e+01, 1.202514641528904758e+04],
[-6.189531564415665343e+01, 1.210471231564415575e+04],
[-5.662878162551714922e+01, 1.218425178162551674e+04],
[-4.961678134413705266e+01, 1.226343478134413635e+04],
[-3.836288992144181975e+01, 1.234189588992144127e+04],
[-8.956671991456460091e+00, 1.241923867199145570e+04],
[3.907028461866866564e+01, 1.249504271538133071e+04],
[1.865299000184495526e+01, 1.256888200999815490e+04],
[4.279803532226833340e+01, 1.264035496467773191e+04],
[3.962735362631610769e+01, 1.270907164637368442e+04],
[1.412691291877854383e+02, 1.277466887081221466e+04],
[1.256537791844366438e+02, 1.283680822081556289e+04],
[7.067642758858892194e+01, 1.289523957241141034e+04],
[1.108876647603192396e+02, 1.294979133523968085e+04],
[9.956490829291760747e+01, 1.300033609170708223e+04],
[1.571612709880937473e+02, 1.304681572901190702e+04],
[2.318746375812715996e+02, 1.308923436241872878e+04],
[2.635546670125277160e+02, 1.312769433298747208e+04],
[2.044220965739259555e+02, 1.316244290342607383e+04],
[2.213739418903714977e+02, 1.319389205810962812e+04],
[1.020184547767112235e+02, 1.322258154522328914e+04],
[-1.072694716663390864e+02, 1.324918947166633916e+04],
[-3.490477058718843182e+02, 1.327445770587188417e+04],
[-3.975570728533530200e+02, 1.329906107285335383e+04],
[-3.331152428080622485e+02, 1.332345624280806260e+04]])
dta = macrodata.load_pandas().data['realgdp'].values
res = column_stack((hpfilter(dta, 1600)))
assert_almost_equal(res, hpfilt_res, 6)
import datetime as dt
from statsmodels.tsa.filters.hp_filter import hpfilter
theta = 0.35
delta = 0.02
start1 = dt.datetime(1947, 1, 1)
end1 = dt.datetime(2012, 4, 1)
output = asarray(DataReader('GDPC1', 'fred',
start=start1, end=end1)).squeeze()
differ = (output[1:] - output[:-1]) / output[:-1]
ave_trend = differ.mean()
filtered = hpfilter(np.log(output), lamb=1600)[0]
std_filt = filtered.std(ddof=1)
auto_corr_filt = np.corrcoef(filtered[1:], filtered[:-1], ddof=1)[0, 1]
T = 254
num_sims = 10000
g2 = np.array([0.01, -0.03])
g3 = np.array([0.02, 0.01, -0.03])
marcov2 = np.array([[0.9, 0.1],
[0.5, 0.5]]).cumsum(1)
marcov3 = np.array([[0.5, 0.45, 0.05],
[0.05, 0.85, 0.10],
[0.25, 0.25, 0.5]]).cumsum(1)
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from statsmodels.tsa.filters.hp_filter import hpfilter
df = pd.read_csv('macrodata.csv', index_col=0, parse_dates=True)
df['realgdp'].plot.line(figsize = (10,3), c = 'blue', lw = 5)
plt.show()
gdp_cycle,gdp_trend = hpfilter(df['realgdp'], lamb=1600)
gdp_trend.plot()
df['trend'] = gdp_trend
df[['trend','realgdp']]['2005-01-01':].plot(figsize = (12,5))
plt.show()
#ETS Models (Error-Trend-Seasonality)
airline = pd.read_csv('airline_passengers.csv', index_col="Month")
airline.dropna(inplace=True)
airline['6-month-SMA'] = airline['Thousands of Passengers'].rolling(window=6).mean()
df = df.fillna(method="bfill", axis=0)
df.isna().sum()
# In[9]:
df.tail()
# # Trend
#
#
# ## Detecting Trend Using a Hodrick-Prescott Filter
# In[10]:
#df = pd.read_excel('India_Exchange_Rate_Dataset.xls', parse_dates=True)
Gold_cycle, Gold_trend = hpfilter(df['Gold'], lamb=1600)
Gold_trend.plot(figsize=(15, 6)).autoscale(axis='x', tight=True)
# ## Detrending Using Differencing
# In[11]:
diff = df.Gold.diff()
plt.figure(figsize=(15, 6))
plt.plot(diff)
plt.title('Detrending using Differencing', fontsize=16)
plt.xlabel('Year')
plt.ylabel('Gold prices')
plt.show()
# ## Detrending Using a SciPy Signal
def dados_apresentaveis(x):
x = round(x)
x = "{:,}".format(x)
x = x.replace(',', '.')
return x
def to_zero(x):
if x < 0:
x = 0
return x
_, trend_newCases = hpfilter(total_de_casos_amazonas['newCases'])
trend_newCases = trend_newCases.apply(to_zero)
_, trend_newDeaths = hpfilter(total_de_casos_amazonas['newDeaths'])
trend_newDeaths = trend_newDeaths.apply(to_zero).round()
def show_figure1():
fig = make_subplots(subplot_titles=(
'10 Cidades com os Maiores Casos & Óbitos Registrados por COVID-19',
'Relação de Casos & Óbitos por 100k Habitantes por Estado'),
rows=1,
cols=2)
### Configuração de gráfico a esquerda
fig.add_trace(go.Scatter(
x=df_total_10_maiores_cidades['city'],
return rec_period
#
# plot real gdp
fig1,ax = plt.subplots(figsize=(10,6), dpi = 300)
ax.plot(rgdp/1e3)
ax.set_xlabel('time')
ax.set_ylabel('trillions of chained 2012 dollars')
ax.set_title('US Real GDP, Quarterly')
# shade recession periods
rec_dates = get_recession_start_end(rec)
for pair in rec_dates:
ax.axvspan(pair[0], pair[1], color='grey', alpha=0.3)
# apply hpfilter
cycle, trend = hpfilter(rgdp)
#
# plot real gdp with hpfilter
fig2,ax = plt.subplots(figsize=(10,6), dpi = 300)
ax.plot(rgdp/1e3)
ax.plot(trend/1e3, label = 'Hodrick–Prescott smoothing')
ax.set_xlabel('time')
ax.set_ylabel('trillions of chained 2012 dollars')
ax.set_title('US Real GDP, Quarterly')
ax.legend()
for pair in rec_dates:
ax.axvspan(pair[0], pair[1], color='grey', alpha=0.3)
#
# log deviations from trend
def append(self, actual, predicted):
_, actual_trend = hpfilter(actual, lamb=LAMB)
_, predicted_trend = hpfilter(predicted, lamb=LAMB)
for metric in self.metrics:
metric.append(actual_trend, predicted_trend)
def so_filter(series,
sd_method='rolling',
l=None,
k=0,
lamb=700,
center=True,
visualize=False,
hp_filter=True,
return_df=False,
d=1):
'''Takes in a pandas series and filters it to become stationary according to the process
outlined by Stockhammar & Oller (2007). Series passed should be the already differenced
series (close to stationary but with local trends and heteroscedasticity).
Params:
::series:: (pandas Series) input data.
::sd_method:: (str) either 'rolling' or 'GARCH'. Method for estimating local SD.
::l:: (int) the window size of rolling SD calculation
::k:: (int) the window size of mean filtering. If zero, no mean filter is performed.
::lamb:: (int) lambda value for hp_filter of SD. Does nothing if hp_filter == False.
::center:: (bool) whether to use centered windows for local mean and SD calculation.
Stockhammar & Oller use centered windows in their study.
::visualize:: (bool) whether to show plots showing filter process.
::hp_filter:: (bool) whether to apply Hodrick-Prescott filter to local SD.
::return_df:: (bool) if False, returns filtered series. If True, returns filtered series along
with columns containing local means and SDs used in filtering.
'''
series_mean = series.mean()
if k > 0:
rolling_means = series.rolling(window=k, center=center).mean()
mean_filt = series - rolling_means
rolling_means.name = 'ma'
else:
mean_filt = series - series_mean
# Get local stds using Stockhammer & Oller (2007) method
if sd_method == 'rolling':
stds = series.rolling(window=l, center=center).std(ddof=1).dropna()
# Get local stds using GARCH
if sd_method == 'GARCH':
arch = arch_model(
series,
mean='Zero',
vol='GARCH',
#dist='t'
rescale=True).fit()
stds = arch.conditional_volatility / arch.scale
print(arch.summary())
# Perform filtering
if hp_filter:
stds = pd.DataFrame(hpfilter(stds, lamb=lamb)).T.iloc[:, 1]
stds.name = 'sd'
filtered = series.std() * ((mean_filt) / stds) + series_mean
filtered = filtered.dropna()
filtered.name = 'filtered_series'
if visualize:
fig, (ax1, ax2) = plt.subplots(2, figsize=(11, 10), sharex=True)
series.plot(label=series.name, ax=ax1)
stds.plot(label='local SD', ax=ax1)
ax1.set_title('Series with local SD')
ax1.legend()
filtered.plot(label=series.name, ax=ax2)
ax2.set_title('Filtered Series')
ax2.legend()
plt.tight_layout()
plt.show()
results = pd.merge(filtered, stds, left_index=True, right_index=True)
if k > 0:
results = pd.merge(results,
rolling_means,
left_index=True,
right_index=True)
if return_df:
return results
else:
return filtered
A[i,j] = -2
k = 0
for lamda in range(lamda1, lamda2+1, step):
for i in range(T):
As = np.delete(A, i, axis=1)
la = np.dot(inv(np.eye(T-1) - lamda*(np.dot(As.T, As))), np.delete(ffr, i))
Gtk[i] = np.sum(la)
GCV[k] = (1/T + 2/lamda)*np.sum((ffr-Gtk)*(ffr-Gtk))
k += 1
# Value of lamda corresponding to min GCV value
lamdamin = lamda1 + np.argmin(GCV)*step
return lamdamin
# Reading in the Input File
input_name = 'INPUT.csv'
df = pd.read_csv(input_name, index_col=0,parse_dates=True)
#Determine optimum
t1=clock()
opt_lambda = optimum_lambda(df, 10000, 20000)
t2=clock()
print ('Time: {0} s'.format(t2-t1))
print(opt_lambda)
#Trend component for optimum lamda
_, trend = hpfilter(df[1:],opt_lambda)
def hodrick_prescott(self, series, lamb = 6.25):
df = series.to_frame()
cycle, trend = hpfilter(series, lamb= lamb)
df['cycle'], df['trend'] = cycle, trend
return df
import pandas as pd
from matplotlib import pyplot as plt
from statsmodels.tsa.filters.hp_filter import hpfilter
import Constants
statsDF = pd.read_html(Constants.readhtml(Constants.MACRODATA).text,
index_col=0,
parse_dates=True)
statsDF = statsDF[0]
statsDF.rename(columns={'Unnamed: 1': 'Date'}, inplace=True)
statsDF['Date'] = pd.to_datetime(statsDF['Date'])
statsDF.set_index('Date', inplace=True)
statsDF['realgdp']['2005':].plot(title='GDP vs time', color=(.76, .86, .21))
gdp_cycle, gdp_trend = hpfilter(statsDF['realgdp'], lamb=1600)
statsDF['GDPtrend'] = gdp_trend
statsDF['GDPtrend']['2005':].plot(label='GDP trend',
ls='--',
color=(.98, .12, .53))
plt.show()
import matplotlib.pyplot as plt
from statsmodels.tsa.filters import hp_filter
tabela = pd.read_csv("C:/tabela_meses_anomalia_nao_oma.csv")
tabela = pd.DataFrame(tabela)
datas = list(range(-2017, -1986,1)) + list(range(1986, 2018,1))
sup = 1.96/np.sqrt(372)
inf = (1.96/np.sqrt(372))*-1
plt.figure(figsize = (40,10))
auto5 = np.correlate(tabela['Media'], tabela["OMA"], mode = 'full')
auto5 = auto5/auto5.max()
auto5 = auto5[auto5.size//2:]
oma_media = hp_filter.hpfilter(auto5,129600)
plt.plot(auto5)
plt.plot(oma_media[1], label = "HP Filter", c = "grey", linewidth = 10)
plt.axhline(inf, color="grey", lw = 5, linestyle = "--")
plt.axhline(sup, color="grey", lw = 5, linestyle = "--")
plt.axhline(y=0, color="black", lw = 2)
st = plt.stem(auto5, linefmt = "-", markerfmt='none', use_line_collection = True)
plt.setp(st, color = "k")
plt.xticks(range(0,373,12), labels = list(range(0, 32, 1)), size = 30)
plt.yticks(np.arange(-1,1.1,0.5), size = 30)
plt.grid()
plt.legend(loc = 2, fontsize = 20)
plt.figure(figsize = (40,10))
def hodrick_prescott(X, lamda=6.5):
# Filter : Centers around zero
cycle, trend = hpfilter(X, lamda)
return cycle
print(scores)
print(np.count_nonzero(scores))
number_of_boxes_drawn.append(np.count_nonzero(scores))
print(number_of_boxes_drawn)
print('Count Boxes: Iteration %d: %.3f sec'%(frame_number, time.time()-start_time))
plt.plot(number_of_boxes_drawn)
plt.draw()
plt.pause(.001)
#smooth graph/remove noise with hp filter; store the smoothed values in "trend"
if frame_number > 50:
#cycle, trend = hp_filter.hpfilter(number_of_boxes_drawn, lamb=50)
#cycle, trend = hp_filter.hpfilter(number_of_boxes_drawn, lamb=5800)
cycle, trend = hp_filter.hpfilter(number_of_boxes_drawn, lamb=50000)
plt.plot(trend)
plt.draw()
plt.pause(.001)
plt.clf()
#how to find where there is a + slope change for a long time(add 50 frames gradient)
#find the max gradient
if frame_number > 50:
if not endpoint and ((trend[frame_number-1]-trend[frame_number-50]) > .1):
#print("in endpoint loop")
#print(trend[frame_number-1])
#append slope to endpoint list
endpoint.append(trend[frame_number-1])
endpoint_final_frame_number = frame_number - 50
count = 0
def tube_ready_not_ready_function():
#print('in function')
#path to all
path = Path('analysis')
#path for non live testing
#picture_path = Path('clotting_test_2')
#path for live test
picture_path = Path('cropped_pics')
#path_to_IMG = Path(/home/llu-2/Desktop/lluFolder/masterProgram/01_26_19/cropped)
classes = ['tube_ready', 'tube_not_ready']
full_clot_probability_array = []
frame_number = 1
full_path = Path("analysis/cropped_pics")
endpoint = []
frame_endpoint_time = []
endpoint_final_frame_number = []
#clotting_endpoint_time = []
#numpy rolling averages function
def movingaverage(values, window):
weights = np.repeat(1.0, window) / window
sma = np.convolve(values, weights, 'valid')
return sma
clotting_endpoint_time = 0
num_analysis_done = 0
seconds_array = []
endpoint_from_trend = 0
#make graph object
fig, ax = plt.subplots()
while True:
if num_analysis_done == len(os.listdir(path / picture_path)):
time.sleep(1)
if num_analysis_done == len(os.listdir(path / picture_path)):
time.sleep(1)
if num_analysis_done == len(os.listdir(path / picture_path)):
files = glob.glob(
'/home/chris/Desktop/Soluble_Fibrin/analysis/cropped_pics/*'
)
for f in files:
os.remove(f)
break
files = natsorted(os.listdir(path / picture_path))
filename = files[num_analysis_done]
img = open_image(path / picture_path / filename)
learn = load_learner(path, file='export_tube_not_ready_mar12.pkl')
pred_class, pred_idx, outputs = learn.predict(img)
#print(filename)
#pred_class2,pred_idx2,outputs2 = learn.predict(imgM2)
#this_image = os.path.join('/home/bha/Desktop/lluFolder/masterProgram/01_26_19/analysis/cropped_pics', filename)
this_image = os.path.join(
'/home/chris/Desktop/Soluble_Fibrin/analysis/cropped_pics',
filename)
image_read = cv2.imread(this_image)
resized_image = cv2.resize(image_read, (1800, 400))
#resized_image = cv2.resize(image_read, (1200, 250))
#cv2.imshow("Tube Ready / Not ready", resized_image)
#convert torch.Tensor to nparray
#print(type(outputs))
tensor_to_np = outputs.numpy()
#print(tensor_to_np)
#grab clot probability from each output
grab_clot_probability = 1 - tensor_to_np[0]
#print(grab_clot_probability)
#add each new clot probability to end of array
full_clot_probability_array.append(grab_clot_probability)
#print("probability array is")
#print(full_clot_probability_array)
#get seconds for each frame, extract first whole integer set before .extension in "image"
file = filename
seconds = None
position = file.index('.')
#gets filename position two before last(.) until last(.)
seconds = filename.split('.')
seconds = seconds[1] + '.' + seconds[2]
seconds = float(seconds)
seconds_array.append(seconds)
plt.title('Tube Ready / Not Ready')
#mngr = plt.get_current_fig_manager()
#mngr.window.SetPosition(0,0)
#plt.xticks(np.arange(min(seconds_array), max(seconds_array)+1, 1.0))
plt.plot(seconds_array, full_clot_probability_array)
plt.get_current_fig_manager().window.wm_geometry(
"+1300+0") # move the window
plt.draw()
plt.pause(.001)
#smooth graph/remove noise with hp filter; store the smoothed values in "trend", and plot on same graph
if frame_number > 1:
cycle, trend = hp_filter.hpfilter(full_clot_probability_array,
lamb=100000)
#print("trend is")
#print(trend[-1])
plt.plot(seconds_array, trend)
plt.draw()
plt.pause(.001)
plt.clf()
#numpy rolling average
if frame_number > 1:
clot_moving_average = movingaverage(full_clot_probability_array,
20)
plt.plot(clot_moving_average)
plt.draw()
#use trend to find endpoint
if frame_number > 1 and endpoint_from_trend == 0 and trend[-1] > .5:
endpoint_from_trend = seconds_array[-1]
if endpoint_from_trend != 0:
#print("endpoint frame is")
#print(endpoint_final_frame_number)
plt.axvline(x=endpoint_from_trend, color='r', linestyle='--')
#stdout = sys.stdout.write(str(endpoint_from_trend))
break
num_analysis_done = num_analysis_done + 1
frame_number = frame_number + 1
endpoint_from_trend = str(endpoint_from_trend)
sys.stderr.write(endpoint_from_trend)
from statsmodels.tsa.api import VAR
from statsmodels.tsa.filters.hp_filter import hpfilter
from statsmodels.sandbox.pca import Pca
from scipy import stats
from affine.model.affine import Affine
from affine.constructors.helper import pickle_file, success_mail, to_mth, gen_guesses, ap_constructor, pass_ols
########################################
# Get macro data #
########################################
mthdata = px.read_csv("./data/macro_data.csv", na_values="M", sep=";", index_col=0, parse_dates=True)
index = mthdata["Total_Nonfarm_employment_seas"].dropna().index
nonfarm = mthdata["Total_Nonfarm_employment_seas"].dropna()
tr_empl_gap, hp_ch = hpfilter(nonfarm, lamb=129600)
mthdata["tr_empl_gap"] = px.Series(tr_empl_gap, index=index)
mthdata["hp_ch"] = px.Series(hp_ch, index=index)
mthdata["tr_empl_gap_perc"] = mthdata["tr_empl_gap"] / mthdata["hp_ch"]
# output
output = mthdata.reindex(columns=["unemployment_seas", "indust_prod_seas", "help_wanted_index"]).dropna()
output["empl_gap"] = mthdata["tr_empl_gap_perc"]
# normalize each output value to zero mean and unit variance
for var in output.columns:
output[var + "_norm"] = (output[var] - output[var].mean()) / output[var].std()
output = output.filter(regex=".*_norm")
def calcaggstat(json, simT, drop, filename):
Kvec = np.array(json["output"]["Kvec"])
Zvec = np.array(json["output"]["Zvec"])
Yvec = np.array(json["output"]["Yvec"])
Ivec = np.array(json["output"]["Ivec"])
Nvec = np.array(json["output"]["Nvec"])
Cvec = np.array(json["output"]["Cvec"])
Kpvec = np.array(json["output"]["Kpvec"])
Xvec = np.array(json["output"]["Xvec"])
Kvec = Kvec[drop:simT + drop]
Zvec = Zvec[drop:simT + drop]
Yvec = Yvec[drop:simT + drop]
Ivec = Ivec[drop:simT + drop]
Nvec = Nvec[drop:simT + drop]
Cvec = Cvec[drop:simT + drop]
Kpvec = Kpvec[drop:simT + drop]
# BUG in the use of np.hstack and hpfilter???
y = np.log(np.hstack((Yvec, Cvec, Ivec, Nvec, Kvec, Zvec)))
# print(y.shape)
yd, yf = hpfilter(y, 100)
yd = np.reshape(yd, (simT, 6), 'F')
# yf = np.reshape(yf, (simT, 6), 'F')
yd = yd[8:simT - 8, :]
std0 = np.std(yd, axis=0)
print('Standard deviation')
# print('Y C I N K Z')
data1 = np.array([
std0[0] * 100, std0[1] / std0[0], std0[2] / std0[0], std0[3] / std0[0],
std0[4] / std0[0], std0[5] / std0[0]
])
print(data1)
corr0 = np.corrcoef(yd, rowvar=False)
print('Output correlation')
data2 = corr0[0, 0:6]
print(data2)
print('Aggregate investment rate')
xmom = np.zeros(4)
xmom[0] = np.sum(Xvec[drop:simT + drop]) / simT
xmom[1] = np.sum(np.power(Xvec[drop:simT + drop] - xmom[0], 2)) / simT
xmom[2] = np.sum(np.power(Xvec[drop:simT + drop] - xmom[0], 3)) / simT
xmom[3] = np.sum(np.power(Xvec[drop:simT + drop] - xmom[0], 4)) / simT
# persistence
rho1 = np.dot(Xvec[drop:simT + drop - 1] - xmom[0],
Xvec[drop + 1:simT + drop] - xmom[0])
rho1 = rho1 / np.sum(np.power(Xvec[drop:simT + drop - 1] - xmom[0], 2))
# standard deviation
sig1 = np.power(xmom[1], 0.5)
# skewness
g1 = xmom[2] / np.power(xmom[1], 1.5)
# excess kurtosis
g2 = xmom[3] / np.power(xmom[1], 2) - 3.0
data3 = np.array([rho1, sig1, g1, g2, 0, 0])
print(data3)
data = np.vstack([data1, data2, data3])
np.savetxt(filename + ".csv", data, delimiter=",", fmt='%.4f')
return Yvec, Ivec, Nvec, Kvec, Cvec, Kpvec, Zvec
def initialize_approx_diffuse(self, obs_cov=0, initial_state_cov=1e6):
# sigma_epsilon = 2.0 # affects the measurement error
# sigma_xi = 1.0 # affects the local level
# sigma_omega = 1.0 # affects the seasonality
# self.state_cov[0,0] = sigma_xi ** 2
# self.state_cov[1,1] = sigma_omega ** 2
# self.obs_cov[0,0] = sigma_epsilon ** 2
from statsmodels.tsa.filters.hp_filter import hpfilter
for series_idx in range(self.k_series):
# Eliminate missing data to estimate starting parameters
endog = self.endog[series_idx, :]
exog = (
self.exog[series_idx, ...]
if self.exog is not None and self.exog.ndim == 3
else self.exog
)
if np.any(np.isnan(endog)):
mask = ~np.isnan(endog).squeeze()
endog = endog[mask]
if exog is not None:
# WARN: currently unused
exog = exog[mask]
# Level / trend variances
# (Use the HP filter to get initial estimates of variances)
_start_params = {}
resid, trend1 = hpfilter(endog)
if self.stochastic_trend:
cycle2, trend2 = hpfilter(trend1)
_start_params["trend_var"] = np.std(trend2) ** 2
if self.stochastic_level:
_start_params["level_var"] = np.std(cycle2) ** 2
elif self.stochastic_level:
_start_params["level_var"] = np.std(trend1) ** 2
# The variance of the residual term can be used for all variances,
# just to get something in the right order of magnitude.
var_resid = np.var(resid)
# Seasonal
if self.stochastic_seasonal:
_start_params["seasonal_var"] = var_resid
# Frequency domain seasonal
if self.stochastic_freq_seasonal:
_start_params["freq_seasonal_var"] = var_resid
offset = 0
# level
self.state_cov[series_idx, offset, offset] = _start_params["level_var"]
# trend
if self.trend:
offset += 1
self.state_cov[series_idx, offset, offset] = _start_params["trend_var"]
# seasonal
if self.seasonal:
offset += 1
self.state_cov[series_idx, offset, offset] = _start_params[
"seasonal_var"
]
# account for added seasonal components
offset += self._k_seasonal_states - 1
# freq_seasonal
for _ in range(self._k_freq_seas_states):
offset += 1
self.state_cov[series_idx, offset, offset] = _start_params[
"freq_seasonal_var"
]
self.obs_cov[0, 0] = obs_cov
self.initial_value = np.zeros(self.k_states)
self.initial_covariance = np.eye(self.k_states) * initial_state_cov
def _estimate_phase(self, imAnalyze):
# Step-1: compute inter-frame similarity matrix
simMat = self._compute_frame_similarity(imAnalyze)
# find the optimal key frame and use it to decompose
spectralEntropy = np.zeros((simMat.shape[0], 1))
simMat_Trend = np.zeros_like(simMat)
simMat_Seasonal = np.zeros_like(simMat)
for fid in range(simMat.shape[0]):
ts = simMat[fid, ]
# decompose into trend and seasonal parts
if self.detrend_method == 'lowess':
# lowess regression
ts_seasonal, ts_trend = detrend_lowess(ts,
frac=self.lowess_frac,
mode=self.lowess_mode)
else:
# hoedrick-prescott filter
ts_seasonal, ts_trend = hpfilter(ts, lamb=self.hp_lamda)
# compute periodogram entropy of the seasonal part
freq, power = scipy.signal.periodogram(ts_seasonal)
# store result
simMat_Trend[fid, ] = ts_trend
simMat_Seasonal[fid, ] = ts_seasonal
spectralEntropy[fid] = scipy.stats.entropy(power)
fid_best = np.argmin(spectralEntropy)
ts = simMat[fid_best, :]
ts_trend = simMat_Trend[fid_best, :]
ts_seasonal = simMat_Seasonal[fid_best, :]
print "Chose frame %d as key frame" % fid_best
# estimate period from the periodogram
freq, power = scipy.signal.periodogram(ts_seasonal)
maxPowerLoc = np.argmax(power)
period = 1.0/freq[maxPowerLoc]
print "Estimated period = %.2f frames" % period
print "Estimated number of periods = %.2f" % (ts_seasonal.size / period)
#beatsPerMinute = period * 60.0 / framesPerSecDownsmp
#print "beats per minute at %f fps = %f" % (framesPerSecDownsmp, beatsPerMinute)
# compute analytic signal, instantaneous phase and amplitude
ts_analytic = scipy.signal.hilbert(ts_seasonal - ts_seasonal.mean())
ts_instaamp = np.abs(ts_analytic)
ts_instaphase = np.arctan2(np.imag(ts_analytic), np.real(ts_analytic))
ts_instaphase_nmzd = (ts_instaphase + np.pi) / (2 * np.pi)
# estimate instantaneous phase of trend component - breathing
ts_trend_analytic = scipy.signal.hilbert(ts_trend - ts_trend.mean())
ts_trend_instaamp = np.abs(ts_trend_analytic)
ts_trend_instaphase = np.arctan2(np.imag(ts_trend_analytic), np.real(ts_trend_analytic))
ts_trend_instaphase_nmzd = (ts_trend_instaphase + np.pi) / (2 * np.pi)
# learn mapping from phase to similarity
resp_ind = []
if self.respiration_present: # is set to True when breathing is present
# identify frames with bad influence by respiration
w = self.resp_phase_cutoff
resp_ind = np.argwhere(
np.logical_or(ts_trend_instaphase_nmzd < w,
ts_trend_instaphase_nmzd > 1.0 - w)
).ravel()
print 'Frames with bad respiration influence = %.2f%%' % (
100.0 * len(resp_ind) / len(ts))
# find similarity bounds at each phase using lowess
phaseord_est = np.argsort(ts_instaphase_nmzd)
phaseord_est_wout_resp = [fid for fid in phaseord_est
if fid not in resp_ind]
fid_lowess = fid_best
# fid_lowess = phaseord_est_wout_resp[0]
assert(fid_lowess not in resp_ind)
ph_wout_resp = ts_instaphase_nmzd[phaseord_est_wout_resp]
sim_wout_resp = simMat[fid_lowess, phaseord_est_wout_resp]
sim_lowess_reg = LowessRegression()
sim_lowess_reg.fit(ph_wout_resp, sim_wout_resp, is_sorted=True)
ts_lowess = sim_lowess_reg.predict(ts_instaphase_nmzd)
# store results
self.simMat_Trend_ = simMat_Trend
self.simMat_Seasonal_ = simMat_Seasonal
self.spectralEntropy_ = spectralEntropy
self.fid_best_ = fid_best
self.ts_ = ts
self.ts_trend_ = ts_trend
self.ts_seasonal_ = ts_seasonal
self.period_ = period
self.ts_analytic_ = ts_analytic
self.ts_instaamp_ = ts_instaamp
self.ts_instaphase_ = ts_instaphase
self.ts_instaphase_nmzd_ = ts_instaphase_nmzd
self.ts_trend_analytic_ = ts_trend_analytic
self.ts_trenda_instaamp_ = ts_trend_instaamp
self.ts_trend_instaphase_ = ts_trend_instaphase
self.ts_trend_instaphase_nmzd_ = ts_trend_instaphase_nmzd
self.resp_ind_ = resp_ind
self.sim_lowess_reg_ = sim_lowess_reg
self.ts_lowess_ = ts_lowess
self.fid_lowess_ = fid_lowess
A[i, j] = -2
k = 0
for lamda in range(lamda1, lamda2 + 1, step):
for i in range(T):
As = np.delete(A, i, axis=1)
la = np.dot(inv(np.eye(T - 1) - lamda * (np.dot(As.T, As))),
np.delete(ffr, i))
Gtk[i] = np.sum(la)
GCV[k] = (1 / T + 2 / lamda) * np.sum((ffr - Gtk) * (ffr - Gtk))
k += 1
# Value of lamda corresponding to min GCV value
lamdamin = lamda1 + np.argmin(GCV) * step
return lamdamin
# Reading in the Input File
input_name = 'INPUT.csv'
df = pd.read_csv(input_name, index_col=0, parse_dates=True)
#Determine optimum
t1 = clock()
opt_lambda = optimum_lambda(df, 10000, 20000)
t2 = clock()
print('Time: {0} s'.format(t2 - t1))
print(opt_lambda)
#Trend component for optimum lamda
_, trend = hpfilter(df[1:], opt_lambda)
