def data2AB(data, x0=None):
n = data.shape[0]
T = data.shape[1]
YY = np.dot(data[:, 1:], data[:, 1:].T)
XX = np.dot(data[:, :-1], data[:, :-1].T)
YX = np.dot(data[:, 1:], data[:, :-1].T)
model = VAR(data.T)
r = model.fit(1)
A = r.coefs[0,:,:]
# A = np.ones((n,n))
B = np.ones((n, n))
np.fill_diagonal(B, 0)
B[np.triu_indices(n)] = 0
K = np.int(scipy.sum(abs(B)))#abs(A)+abs(B)))
a_idx = np.where(A != 0)
b_idx = np.where(B != 0)
np.fill_diagonal(B, 1)
try:
s = x0.shape
x = x0
except AttributeError:
x = np.r_[A.flatten(), 0.1*scipy.randn(K)]
o = optimize.fmin_bfgs(nllf2, x,
args=(np.double(A), np.double(B),
YY, XX, YX, T, a_idx, b_idx),
gtol=1e-12, maxiter=500,
disp=False, full_output=True)
A, B = x2M(o[0], np.double(A), np.double(B), a_idx, b_idx)
B = B+B.T
return A, B
def VARprocess(df,log=False):
# Log transformation, relative difference and drop NULL values
if (log):
df = np.log(df+0.1).diff().dropna()
# Vector Autoregression Process generation
maxAttr = len(df.columns)
# Find the right lag order
orderFound = False
while orderFound!=True:
try:
model = VAR(df.ix[:,0:maxAttr])
order = model.select_order()
orderFound = True
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
if str(exc_obj)=="data already contains a constant.":
maxAttr = maxAttr - 1
else:
maxAttr = int(str(exc_obj).split("-th")[0])-1
print "Exception, reducing to n_attributes ",maxAttr
orderFound = False
n_lags = max(order.iteritems(), key=operator.itemgetter(1))[1]
method = max(order.iteritems(), key=operator.itemgetter(1))[0]
print "n_lags ",n_lags
print "method ",method
results = model.fit(maxlags=n_lags, ic=method)
return results
def data2VARgraph_model(data, pval=0.05):
model = VAR(data.T)
r = model.fit(1)
A = r.coefs[0,:,:]
n = A.shape[0]
g = {str(i):{} for i in range(1,n+1)}
for i in range(n):
for j in range(n):
if np.abs(A[j,i]) > pval: g[str(i+1)][str(j+1)] = set([(0,1)])
return g, r
def get_fedea_on_gdp():
qbuilder = inquisitor.Inquisitor(token)
df = qbuilder.series(ticker = ['ESE.940000D259D.Q.ES','FEEA.PURE064A.M.ES'])
df.dropna(inplace = True)
df['fedea'] = df['FEEA.PURE064A.M.ES'].diff()
df['gdp'] = df['ESE.940000D259D.Q.ES'] / 100
data1 = df[['fedea','gdp']]
data1.dropna(inplace = True)
model1 = VAR(data1)
results1 = model1.fit(4)
irf1 = results1.irf(8)
fedea_on_gdp = irf1.orth_lr_effects[1,0] / data1['fedea'].std()
return fedea_on_gdp
def get_irf(nd, subset):
'''
http://statsmodels.sourceforge.net/0.6.0/vector_ar.html
'''
data = nd.reindex(columns=subset)
data = data.dropna()
data.describe()
model = VAR(data)
results = model.fit(6)
irf = results.irf(12)
cum_effects = irf.orth_cum_effects
return cum_effects[12,2,0]
def determineOrderOfP():
X_train = readVectorAutoRegressiveMethodXTrain()
for i in [1, 2, 3, 4, 5, 6, 7]:
vectorAutoRegressiveMethodModel = VAR(X_train)
vectorAutoRegressiveMethodModelResult = vectorAutoRegressiveMethodModel.fit(
i)
print('Order =', i)
print('AIC: ', vectorAutoRegressiveMethodModelResult.aic)
print('BIC: ', vectorAutoRegressiveMethodModelResult.bic)
print()
def vector_auto_reg(self, y, dates, p, clean_data="greedy"):
s = self.map_column_to_sheet(y[0])
v = np.copy(y)
v = np.append(v, dates)
# prepare data
dfClean = s.cleanData(v, clean_data)
time_series = dfClean[y]
dates = dfClean[dates]
time_series = time_series.set_index(dates)
# run pth-order VAR
model = VAR(time_series)
results = model.fit(p)
return results
def VAR_Model(modeldata):
model = VAR(modeldata)
res = {}
AIC = []
for i in range(100):
result = model.fit(i)
aic = result.aic
AIC.append(aic)
if (aic <= pr.AICvalue_limit) and (aic >= -pr.AICvalue_limit):
break
lag_order = i - 1
varmodel = model.fit(lag_order)
residuals = DataFrame(varmodel.resid)
rmean = abs(residuals.mean())
#print("Residual Error = {}".format(rmean[0]))
res.update({'Residual Mean': rmean, 'Lag Order': lag_order})
return varmodel, res
def var_forecast(train_df, test_df, params):
_order = params['order']
_input = list(params['input'])
_output = params['output']
_step = params.get('step', 1)
model = VAR(train_df[_input].values)
results = model.fit(_order)
lag_order = results.k_ar
params['order'] = lag_order
forecast = []
for i in np.arange(0, len(test_df) - lag_order - _step + 1):
fcst = results.forecast(test_df[_input].values[i:i + lag_order], _step)
forecast.append(fcst[-1])
forecast_df = pd.DataFrame(columns=test_df[_input].columns, data=forecast)
return forecast_df[_output].values
def var_model(data, look_ahead=5):
'''Fits a vector autoregression model to the data and forecasts closing
prices a number of days ahead equal to the look_ahead value.'''
data = data.set_index('date')
data = data.drop('index', axis=1)
model = VAR(data)
results = model.fit(maxlags=15)
forecast = pd.DataFrame(
results.forecast(data.values[0:], look_ahead),
columns=['close', 'high', 'low', 'open', 'volume', 'sentiment'])
#future = pd.date_range(start='1-1-2019', periods=5)
future = pd.date_range(start=data.iloc[-1].name + dt.timedelta(days=1),
periods=look_ahead)
forecast = forecast.set_index(future)
# dill.dump(forecast, open())
data_w_forecast = data.append(forecast)
return data_w_forecast
def causality_test(var1, var2):
data1 = pd.Series(var1, name='Var1')
data2 = pd.Series(var2, name='Var2')
mdata = pd.concat([data1, data2], axis=1)
mdata.index = pd.date_range('1950-01-01', periods=600, freq='M')
model = VAR(mdata)
results = model.fit(7)
foo = crit = results.test_causality('Var2', ['Var1'], kind='f')
crit = foo['crit_value']
stat = foo['statistic']
if (stat > crit):
cause = 1
else:
cause = 0
return cause
def trainVectorAutoRegressiveMethodModelOnFullDataset():
vectorAutoRegressiveMethodDataset = importVectorAutoRegressiveMethodDataset(
"M2SLMoneyStock.csv", "PCEPersonalSpending.csv")
#training model on the whole dataset
vectorAutoRegressiveMethodModel = VAR(vectorAutoRegressiveMethodDataset)
#we are taking p = 5 as we have created different models based on the different p values.
#Model gives minimum aic and bic for p =5
vectorAutoRegressiveMethodModelResult = vectorAutoRegressiveMethodModel.fit(
5)
#saving the model in pickle files
saveVectorAutoRegressiveMethodModelForFullDataset(
vectorAutoRegressiveMethodModelResult)
print(vectorAutoRegressiveMethodModelResult.summary())
def VAR_forecast(df):
"""Initiates Forecasting using Forecast on the passed dataset
Parameters
----------
df
The dateset containing historical-observations on day-ahead prices
Returns
-------
forecasted
A list of forecasted electricity prices for the next 24 hours
"""
column_name = "Day-ahead Price [EUR/MWh]"
# Open CSV File and set timestamp column as index
df.rename(columns={df.columns[0]: "cet_timestamp"}, inplace=True)
df["cet_timestamp"] = pd.to_datetime(df["cet_timestamp"], format="%Y-%m-%d %H:%M")
df.set_index("cet_timestamp", inplace=True)
# Impute and get only one column
df_diff = df.diff().dropna()
# Generate new dates
dates = list()
last_date = df.index[-1:][0]
for i in range(1, 25):
last_date += timedelta(hours=1)
dates.append(last_date.strftime("%Y-%m-%d %H:%M:%S"))
var_model = VAR(df_diff).fit(26)
var_forecast = var_model.forecast(y = var_model.y, steps=24)
var_forecast_df = pd.DataFrame(var_forecast, columns=df.columns, index= dates)
var_forecast_df = invert_transformation(df, var_forecast_df)
'''
# For mean absolute error
last_24hours = last_date - timedelta(hours=24)
# History - 24hours
history = df_diff[df_diff.index <= last_24hours]
var_mae_model = VAR(history).fit(26)
var_mae_forecast = var_mae_model.forecast(y = var_mae_model.y, steps=24)
mae = mean_absolute_error(history['Day-ahead Price [EUR/MWh]'].values, np.array(var_mae_forecast))
print(mae)
'''
return list(var_forecast_df[column_name].values), datetime.now().strftime("%Y-%m-%d")
def fit(self, p, transformation="dct"):
if p < 1:
raise ValueError(f"{p} is an invalid lag")
self.p = p
self.transformation = transformation
l_train_tensor = self.__apply_trans(
self.train, transformation,
2) # Applies the transformation across the rows
train_model_sets = self.__split_cols_into_model_sets(l_train_tensor)
# Fits all of the var models
fits = []
for i in range(self.matrix_shape[1]):
train_df = pd.DataFrame(train_model_sets[i])
model = VAR(train_df)
fit = model.fit(p)
fits.append(fit)
self.var_fits = fits
# Groups all of the coef matrix to coef tensors
coefs = np.empty((p, self.matrix_shape[1], self.matrix_shape[0],
self.matrix_shape[0]))
c = np.empty(self.matrix_shape)
for i in range(self.matrix_shape[1]):
curr_coefs = fits[i].coefs
for j in range(p):
coefs[j][i] = curr_coefs[j]
# Adds onto c
c[:, i] = fits[i].params[fits[i].params.index == "const"].iloc[0]
# Performs an inverse tranform to all of them
for i in range(p):
coefs[i] = self.__apply_inverse_trans(coefs[i], transformation, 0)
# Performs the inverse transformation to the const matrix
c = self.__apply_inverse_trans(c, transformation, 1)
self.coefs = coefs
self.c = c
def test_gc2(data, gc_format, maxlag=None, signif=0.05, verbose=False):
from statsmodels.tsa.api import VAR
model = VAR(data)
if maxlag:
res = model.fit(maxlag, verbose=verbose)
else:
res = model.fit(verbose=verbose)
gc_res = res.test_causality(gc_format[0],
gc_format[1],
signif=signif,
verbose=verbose)
# results = pd.Seires({k: v for k, v in gc_res.iteritems() if k in ['conclusion','pvalue']})
results = pd.Series(gc_res)
results['H0'] = "'{}' do not Granger-cause '{}'".format(
gc_format[1], gc_format[0])
results['VAR'] = res
results['best_order'] = (len(model.exog_names) - 1) / data.shape[1]
return results
def calc_granger_caulity(data):
"""Summary
Args:
data (array): array without nan values
Returns:
TYPE: Description
"""
from statsmodels.tsa.api import VAR
# data_dropna = data[~np.isnan(data).any(1)]
try:
model = VAR(data)
res = model.fit(verbose=False)
out = res.test_causality(0, 1, verbose=False)['statistic']
except ValueError as e:
# print 'calc_granger_caulity: ', 'most factors are zeros.'
out = np.nan
return out
def granger_causality(self, data):
columns = []
for i in range(data.shape[0]):
for j in range(data.shape[2]):
columns.append(str(i) + str(j))
# print(columns)
topic_oriented_data = data[0]
for i in range(1, len(data)):
topic_oriented_data = np.concatenate((topic_oriented_data, data[i]), 1)
topic_oriented_data = pandas.DataFrame(topic_oriented_data, columns=columns)
print(topic_oriented_data)
# print(type(topic_oriented_data))
var_model = VAR(topic_oriented_data)
results = var_model.fit(2)
gc_result = results.test_causality(columns, columns, kind='f')
print(gc_result.summary())
def granger(cause, effect, lag):
data = pd.DataFrame({'cause': cause, 'effect': effect})
return_vaule = 1
model = VAR(data)
try:
if lag == -1:
results = model.fit(maxlags=15, trend='nc', ic='aic')
else:
results = model.fit(lag)
except Exception:
# can not find a lag in interval [1, maxlags], that means they have no causality
return 1
try:
x = results.test_causality('effect', 'cause',
kind='wald').summary().data
except Exception:
return 0
return_vaule = x[1][2]
return return_vaule
def varmodel(self):
self.mvdfg.index = pd.to_datetime(self.mvdfg.index)
self.var_predicted = pd.DataFrame()
self.var_forecast = pd.DataFrame()
self.var_data_train = pd.DataFrame()
self.var_data_test = pd.DataFrame()
maxlag = 3
if splitdf.upper() == 'Y':
#Validation Model
self.var_data_train = self.mvdfg[(pd.to_datetime(self.mvdfg.index)) <= testdate]
self.var_data_test = self.mvdfg[(pd.to_datetime(self.mvdfg.index)) > testdate]
var_model = VAR(self.var_data_train)
results = var_model.fit(maxlags = maxlag, ic = 'aic')
print(results.summary())
lag_order = results.k_ar
var_steps = len(self.var_data_test)
pred_values = results.forecast(self.var_data_train.values[-lag_order:], var_steps)
self.predicted = pd.DataFrame(pred_values, index = self.mvdfg.index[-var_steps:], columns = self.mvdfg.columns)
self.var_predicted = self.predicted
#Forecast
startdate = self.mvdfg.index.max()+ pd.offsets.DateOffset(months = 1)
maxdate = self.mvdfg.index.max() + pd.offsets.DateOffset(months = forecaststeps + 1)
var_fc_index = np.asarray((pd.date_range(startdate, maxdate, freq = 'm').strftime('%Y-%m-01')))
var_fc_index = pd.to_datetime(var_fc_index)
var_forecast_model = VAR(self.mvdfg)
fc_results = var_forecast_model.fit(maxlags = maxlag, ic = 'aic')
print(fc_results.summary())
fc_lag_order = fc_results.k_ar
fc_values = fc_results.forecast(self.mvdfg.values[-fc_lag_order:], forecaststeps)
self.forecast = pd.DataFrame(fc_values, index = var_fc_index, columns = self.mvdfg.columns)
self.var_forecast = self.forecast
print(self.var_forecast)
return self.var_predicted, self.var_forecast
def gaussian_var_copula_entropy_rate(sample, p=None, robust=False, p_ic='hqic'):
"""
Estimates the entropy rate of the copula-uniform dual representation of a stationary Gaussian VAR(p) (or AR(p)) process from a sample path.
We recall that the copula-uniform representation of a :math:`\\mathbb{R}^d`-valued process :math:`\\{x_t\\} := \\{(x_{1t}, \\dots, x_{dt}) \\}`
is, by definition, the process :math:`\\{ u_t \\} := \\{ \\left( F_{1t}\\left(x_{1t}\\right), \\dots, F_{dt}\\left(x_{dt}\\right) \\right) \\}`
where :math:`F_{it}` is the cummulative density function of :math:`x_{it}`.
It can be shown that
.. math::
h\\left( \\{ x_t \\}\\right) = h\\left( \\{ u_t \\}\\right) + \\sum_{i=1}^d h\\left( x_{i*}\\right)
where :math:`h\\left(x_{i*}\\right)` is the entropy of the i-th coordinate process at any time.
Parameters
----------
sample: (T, d) np.array
Array of T sample observations of a :math:`d`-dimensional process.
p : int or None
Number of lags to compute for the autocovariance function. If :code:`p=None` (the default), it is inferred by fitting a VAR model on the sample, using as information criterion :code:`p_ic`.
robust: bool
If True, the Pearson autocovariance function is estimated by first estimating a Spearman rank correlation, and then inferring the equivalent Pearson autocovariance function, under the Gaussian assumption.
p_ic : str
The criterion used to learn the optimal value of :code:`p` (by fitting a VAR(p) model) when :code:`p=None`.
Should be one of 'hqic' (Hannan-Quinn Information Criterion), 'aic' (Akaike Information Criterion), 'bic' (Bayes Information Criterion) and 't-stat' (based on last lag).
Same as the 'ic' parameter of :code:`statsmodels.tsa.api.VAR`.
Returns
-------
h : float
The entropy rate of the copula-uniform dual representation of the input process.
p : int
Order of the VAR(p).
"""
_sample = sample[~np.isnan(sample).any(axis=1)] if len(sample.shape) > 1 else sample[~np.isnan(sample)]
if p == None:
# Fit an AR and use the fitted p.
max_lag = int(round(12*(_sample.shape[0]/100.)**(1/4.)))
if len(_sample.shape) == 1 or _sample.shape[1] == 1:
m = AR(_sample)
p = m.fit(ic=p_ic).k_ar
else:
m = VAR(_sample)
p = m.fit(ic=p_ic).k_ar
x = _sample if len(_sample.shape) > 1 else _sample[:, None]
res = -np.sum(0.5*np.log(2.*np.pi*np.e*np.var(x, axis=0)))
res += gaussian_var_entropy_rate(x, p, robust=robust)
return res, p
def fit_forecast(dataset):
cols = dataset.columns
# creating the train and validation set
train = dataset[:int(0.8 * (len(dataset)))]
valid = dataset[int(0.8 * (len(dataset))):]
train_differenced, round_no = remove_stationary(train)
model = VAR(train_differenced)
model_fit = model.fit()
# make prediction on validation
prediction = model_fit.forecast(model_fit.endog, steps=len(valid))
# converting predictions to dataframe
forecast = pd.DataFrame(prediction,
index=dataset.index[-len(valid):],
columns=cols)
if round_no != 0:
forecast = invert_transformation(train, forecast, (round_no == 2))
# check rmse
rmses = {}
for i in cols:
rmses[i + '_RMSE'] = sqrt(mean_squared_error(forecast[i], valid[i]))
return forecast, valid, rmses
def var_param_min_search(data, limit_p):
# Parameters of each minimum
param_aic = []
param_bic = []
# Minimums of each criterion
mins_aic = []
mins_bic = []
# Current minimums of each criterion
current_min_aic = None
current_min_bic = None
for i in range(limit_p):
model = VAR(data)
model_fit = model.fit(i)
current_aic = model_fit.aic
current_bic = model_fit.bic
# Check for new AIC minimum
if current_min_aic is None or current_min_aic > current_aic:
current_min_aic = current_aic
param_aic.append(str(i))
mins_aic.append(current_aic)
# Check for new BIC minimum
if current_min_bic is None or current_min_bic > current_bic:
current_min_bic = current_bic
param_bic.append(str(i))
mins_bic.append(current_bic)
res = {
'aic': {
'parameters': param_aic,
'mins': mins_aic
},
'bic': {
'parameters': param_bic,
'mins': mins_bic
}
}
return res
def fit(data, maxlag):
#with open('varModel.json') as f:
# data = json.load(f)
mdata = prepareData(data)
equation = dict()
equation["aic"] = []
equation["BIC"] = []
equation["hqic"] = []
equation["min"] = []
model = VAR(mdata)
for x in range(0, maxlag):
fitedModel = model.fit(x + 1)
equation["aic"].append(fitedModel.aic)
equation["BIC"].append(fitedModel.bic)
equation["hqic"].append(fitedModel.hqic)
minLag = model.fit(maxlags=maxlag, ic='bic')
equation["min"].append(minLag.aic)
equation["min"].append(minLag.bic)
equation["min"].append(minLag.hqic)
return equation
def evaluate_svar_model(X, p, s, feat_nm, agg_level):
feat_list = X.columns.values.tolist()
#add seasonal variables
decomposition = seasonal_decompose(X[feat_nm], model='additive', freq=s)
X['seasonal'] = decomposition.seasonal
trend = decomposition.trend
trend = trend.fillna(
method='ffill') # fill missing values with previous values
trend = trend.fillna(
method='bfill') # fill first missing value with the one before it
X['trend'] = trend
# prepare training dataset
train_size = len(X)
train = X
model = VAR(train)
model_fit = model.fit(p)
test_size = test_size_level[agg_level]
yhat = model_fit.forecast(train.values[:p],
train_size + max(test_size) - p)
index = feat_list.index(feat_nm) # index of feature we want to analyse
yhat_feat = [item[index]
for item in yhat] # model output relevant to that features
predictions = yhat_feat
pred0 = predictions[train_size - p + test_size[0] - 1]
pred1 = predictions[train_size - p + test_size[1] - 1]
pred2 = predictions[train_size - p + test_size[2] - 1]
avg0 = sum(predictions[train_size - p:train_size - p +
test_size[0]]) / test_size[0]
avg1 = sum(predictions[train_size - p:train_size - p +
test_size[1]]) / test_size[1]
avg2 = sum(predictions[train_size - p:train_size - p +
test_size[2]]) / test_size[2]
return [pred0, pred1, pred2, avg0, avg1, avg2]
def lag_selection(ytw, corp, tb):
import cs_data_analysis as da
from statsmodels.tsa.api import VAR
import numpy as np
'''
CS-Aaa-3MO CS-Aa-3MO CS-A-3MO CS-Baa-3MO CS-Aaa-1YR CS-Aa-1YR CS-A-1YR CS-Baa-1YR CS-Aaa-5YR CS-Aa-5YR CS-A-5YR CS-Baa-5YR
TB-3MO-TY TB-1YR-TY TB-5YR-TY
'''
debug(ytw.shape)
endog = ytw[[corp, tb]]
lag_count = 10
ic_aic = np.zeros((lag_count, 1)) # AIC, BIC, HQIC
ic_bic = np.zeros((lag_count, 1))
for i in range(lag_count):
debug(f"{'-'*4} period: {lag_count} {'-'*4}")
# https://www.statsmodels.org/stable/generated/statsmodels.tsa.vector_ar.var_model.VAR.fit.html
model = VAR(endog=endog)
model_fit = model.fit(maxlags=i + 1, trend='ct', verbose=True)
debug(f"aic: {model_fit.aic:.6f}")
debug(f"bic: {model_fit.bic:.6f}")
debug(f"hqic: {model_fit.hqic:.6f}")
ic_aic[i] = model_fit.aic
ic_bic[i] = model_fit.bic
results = model_fit.summary()
debug(results)
ic_aic_min, aic_model_min = np.min(ic_aic), np.argmin(ic_aic)
ic_bic_min, bic_model_min = np.min(ic_bic), np.argmin(ic_bic)
debug('Relative Likelihoods')
debug(np.exp((ic_aic_min - ic_aic) / 2))
debug(f'number of parameters in minimum AIC model {(aic_model_min + 1)}')
debug(np.exp((ic_bic_min - ic_bic) / 2))
debug(f'number of parameters in minimum BIC model {(bic_model_min + 1)}')
return aic_model_min + 1, bic_model_min + 1
def full_model(data, caused, L):
"""
:param dataX:
:type dataX: np.ndarray
:param dataY:
:type dataY: np.ndarray
:return:
"""
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
model = VAR(data)
model_fit = model.fit(L)
SSE = np.sum((model_fit.resid)**2)[caused]
#ONLY FOR DEBUGGING
#plt.plot(data['X'])
#plt.plot(model_fit.fittedvalues['X'])
#plt.show()
#DEBUGGING END
return SSE
def VARprocess(df, log=False):
"""
Description: This function applies Vector Auto Regression
Input: dataframe
Output: VARresults object
"""
# Log transformation, relative difference and drop NULL values
if (log):
df = np.log(df + 0.1).diff().dropna()
# Vector Autoregression Process generation
maxAttr = len(df.columns)
# Find the right lag order
orderFound = False
print "7.1.0 ----- Finding an order for the VAR"
maxIter = 0
while orderFound != True and maxIter < 15:
maxIter = maxIter + 1
try:
model = VAR(df)
order = model.select_order()
orderFound = True
print " !!! loop stuck"
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
#if str(exc_obj)=="data already contains a constant.":
maxAttr = maxAttr - 1
#else:
#maxAttr = int(str(exc_obj).split("-th")[0])-1
#print "Exception, reducing to n_attributes ",maxAttr
orderFound = False
print "7.1.1 ----- Model fitting"
if orderFound:
n_lags = max(order.iteritems(), key=operator.itemgetter(1))[1]
method = max(order.iteritems(), key=operator.itemgetter(1))[0]
results = model.fit(maxlags=n_lags, ic=method)
else:
results = model.fit()
return results
def data2AB(data, x0=None):
n = data.shape[0]
T = data.shape[1]
YY = np.dot(data[:, 1:], data[:, 1:].T)
XX = np.dot(data[:, :-1], data[:, :-1].T)
YX = np.dot(data[:, 1:], data[:, :-1].T)
model = VAR(data.T)
r = model.fit(1)
A = r.coefs[0, :, :]
#A = np.ones((n,n))
B = np.ones((n, n))
np.fill_diagonal(B, 0)
B[np.triu_indices(n)] = 0
K = np.int(scipy.sum(abs(B))) #abs(A)+abs(B)))
a_idx = np.where(A != 0)
b_idx = np.where(B != 0)
np.fill_diagonal(B, 1)
try:
s = x0.shape
x = x0
except AttributeError:
x = np.r_[A.flatten(), 0.1 * scipy.randn(K)]
o = optimize.fmin_bfgs(nllf2,
x,
args=(np.double(A), np.double(B), YY, XX, YX, T,
a_idx, b_idx),
gtol=1e-12,
maxiter=500,
disp=False,
full_output=True)
ipdb.set_trace()
A, B = x2M(o[0], np.double(A), np.double(B), a_idx, b_idx)
B = B + B.T
return A, B
def var_prediction(df, train_perc, incidence_file, window=18, diff=True):
# Limpiamos el df
df_aux = df.drop('Unnamed: 0', axis=1)
df_aux = df_aux.drop('tref_start', axis=1)
X = df_aux.values[:, :]
if diff:
X = np.diff(X, axis=0)
# Obtenemos estandarizador de valores
v = int(len(X) * train_perc)
scaler = StandardScaler()
X_train = scaler.fit_transform(X[:v])
# Entrenamos el modelo
model = VAR(X_train)
results = model.fit(window)
# df de validación con incidencias
df = df.iloc[v:]
incidencias = None
if incidence_file is not None:
inc = get_working_incidence(incidence_file)
df = generar_incidencias(df, inc).sort_values(by=['tref_start'])
# Array de incidencias
incidencias = df['incidencia'].values[window:]
df = df.drop('incidencia', axis=1)
# Obtenemos valores de la red reales
df = df.drop('Unnamed: 0', axis=1)
df = df.drop('tref_start', axis=1)
X = df.values[:, :]
if diff:
X = np.diff(X, axis=0)
if incidence_file is not None:
incidencias = incidencias[1:]
X = scaler.transform(X)
# Obtengamos predicciones
ys = X[window:]
yhats = []
for i in range(window, len(X)):
yhats.append(results.forecast(X[i - window:i], 1)[0])
return ys, np.array(yhats), incidencias
def infl_forecast_values(year='2001', month='02', n_steps=6):
# n_steps is how far into future you look
# crop the data depending on n_steps and date
orig_df = load_data()
date = form_date(year, month)
train, test = crop_data(orig_df, date, n_steps)
#take first difference
first_row, train_1 = take_diff(train)
first_YOY = first_row['YOY']
# create VAR model
model = VAR(train_1, freq='MS')
#for now fit to 4
results = model.fit(4)
lag_order = results.k_ar
prediction_input = train_1.values[-lag_order:]
# I want last column
infl_results = results.forecast(prediction_input, n_steps)[:, 1]
return infl_results
def process(data, cid):
""" Call make_stationary() to check for Stationarity and make the Time Series Stationary
Make a VAR model, call the fit method with the desired lag order.
Forecast VAR model and return the forcasted data.
"""
nobs = 1
df = data.copy()
df_differenced = make_stationary(
df) # check for Stationarity and make the Time Series Stationary
model = VAR(df_differenced) # Make a VAR model
model_fit = model.fit(10) # call fit method with lag order
model_fit.summary() # summary result of the model fitted
lag_order = model_fit.k_ar # Get the lag order
forecast_input = df_differenced.values[
-lag_order:] # Input data for forecasting
# Forecast and Invert the transformation to get the real forecast values
fc = model_fit.forecast(y=forecast_input, steps=nobs)
inp_file = os.getcwd() + cid
joblib.dump(model_fit, inp_file)
def run(self):
if not self._args:
return None
data = self._data_service.get_data(self._args)
split_data = {
self._args['dependent_variable']:
data[data['ticker'] == self._args['dependent_variable']]['close']
}
for i, ticker in enumerate(self._args['independent_variables']):
split_data[ticker] = data[data['ticker'] == ticker]['close']
data = pd.DataFrame(split_data).dropna()
model = VAR(data)
result = model.fit(2)
print(result.summary())
result.test_causality(self._args['dependent_variable'],
self._args['independent_variables'],
kind='f')
return result
def temporal_detect_individual(target_idx, dta, maxlag):
num_ts = len(dta[0])
len_ts = len(dta)
tmp_target = [ dta[j][target_idx] for j in range(len_ts) ]
res_lag = []
for i in range(num_ts):
if i != target_idx:
tmp_ts = [ dta[j][i] for j in range(len_ts) ]
tmp_x = zip(tmp_target, tmp_ts )
# print np.shape(tmp_x)
model = VAR(tmp_x)
best_lag = model.select_order(maxlag, verbose= False)
res_lag.append(best_lag)
return res_lag
def GMMGranger(k, t, n):
bet = 0
yes = 0
while bet <= n - 1:
xseries = GMM(k, t)
yseries = GMM(k + 3, t)
data = pd.DataFrame([xseries, yseries]).transpose()
model = VAR(np.asarray(data))
try:
results = model.fit(maxlags=15, ic='aic', trend='nc')
except:
continue
bet += 1
if results.test_causality(0, 1,
kind='wald').summary().data[1][2] > 0.05:
if results.test_causality(1, 0,
kind='wald').summary().data[1][2] > 0.05:
yes += 1
return float(yes) / n
#accuracy=GMMGranger(5,200,100)
#print accuracy
def test_gc(data, maxlag=None, signif=0.05, verbose=False):
"""Summary
Apply granger causaulity test into permutation of all columns
Args:
data (TYPE): Description
maxlag (None, optional): Description
signif (float, optional): Description
verbose (bool, optional): Description
Returns:
TYPE: dataframe
"""
from statsmodels.tsa.api import VAR
if isinstance(data, pd.core.frame.DataFrame):
colns = data.columns
arr = data.values
else:
arr = np.array(data)
model = VAR(arr)
if maxlag:
res = model.fit(maxlag, verbose=verbose)
else:
res = model.fit(verbose=verbose)
gc_test = []
obs_name = res.names
for c1, c2 in permutations(obs_name, 2):
gc_res = res.test_causality(c1, c2, signif=signif, verbose=verbose)
coln1, coln2 = colns[[obs_name.index(c1), obs_name.index(c2)]]
gc_res = pd.Series(gc_res, name=(coln1, coln2))
gc_res['H0'] = "'{}' do not Granger-cause '{}'".format(coln2, coln1)
gc_test.append(gc_res)
results = pd.DataFrame(gc_test)
results['VAR'] = model
results['best_order'] = (len(model.exog_names) - 1) / data.shape[1]
return results
import numpy as np
import statsmodels.api as sm
from statsmodels.tsa.api import VAR
# some example data
mdata = sm.datasets.macrodata.load().data
mdata = mdata[['realgdp','realcons','realinv']]
names = mdata.dtype.names
data = mdata.view((float,3))
use_growthrate = False #True #False
if use_growthrate:
data = 100 * 4 * np.diff(np.log(data), axis=0)
model = VAR(data, names=names)
res = model.fit(4)
nobs_all = data.shape[0]
#in-sample 1-step ahead forecasts
fc_in = np.array([np.squeeze(res.forecast(model.y[t-20:t], 1))
for t in range(nobs_all-6,nobs_all)])
print fc_in - res.fittedvalues[-6:]
#out-of-sample 1-step ahead forecasts
fc_out = np.array([np.squeeze(VAR(data[:t]).fit(2).forecast(data[t-20:t], 1))
for t in range(nobs_all-6,nobs_all)])
print fc_out - data[nobs_all-6:nobs_all]
#print df
df = df.fillna(0)
original_df[m] = df
stat_df = df.diff().dropna()
#get rid of columns that are zeros at the end , we just assume they will continue to be zeros
for col_name in stat_df.columns.values:
if stat_df[col_name][-1] == 0 and stat_df[col_name][-2] == 0:# and stat_df[col_name][-3] == 0:
print col_name
del stat_df[col_name]
no_forecast.setdefault(m,[]).append(col_name)
#print stat_df
forecast_cols[m] = stat_df.columns.values
#new_df = stat_df[['P17','P15','P16']]
model = VAR(stat_df)
maxlags = 3
try:
results = model.fit(maxlags, ic='aic', verbose=True)
except Exception,exc:
maxlags = 1
results = model.fit(maxlags, ic='aic', verbose=True)
#if m == 'M2':
# import pdb
# pdb.set_trace()
#import pdb
#pdb.set_trace()
# results = model.fit(4)
#print results.summary()
lag_order = results.k_ar
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.api as sm
from statsmodels.tsa.api import VAR
from scipy.signal import lfilter
mdata = sm.datasets.macrodata.load().data
mdata = mdata[["realgdp", "realcons", "realinv"]]
names = mdata.dtype.names
data = mdata.view((float, 3))
data = np.diff(np.log(data), axis=0)
model = VAR(data)
res = model.fit(2)
res.plot_sample_acorr()
irf = res.irf(10)
irf.plot()
plt.show()
plt.savefig("image.png")
res.plot_forecast(5)
res.fevd().plot()
plt.show()
plt.savefig("image2.png")
data = pd.read_csv("/home/dusty/Econ8310/DataSets/pollutionBeijing.csv")
format = '%Y-%m-%d %H:%M:%S'
data['datetime'] = pd.to_datetime(data['datetime'], format=format)
data.set_index(pd.DatetimeIndex(data['datetime']), inplace=True)
# Select variables for VAR model
varData = data[['pm2.5','TEMP','PRES', 'Iws']].dropna()[:-50]
test = data[['pm2.5','TEMP','PRES', 'Iws']].dropna()[-50:]
# endVal = varData.loc["2014-01-04 00:00:00"]
# varData = varData.diff(1)
model = VAR(varData) # define the model and data
# model.select_order() # uses information criteria to select
# model order
reg = model.fit(30) # order chosen based on BIC criterion
# Forecasting
fcast = reg.forecast(varData['2013-01-04':].values, steps = 50)
def dediff(todaysVal, forecast):
future = forecast
for i in range(np.shape(forecast)[0]):
if (i==0):
def estimate_VAR():
df = load_external()
d = load_es_uncertainty()
df1 = load_eu_uncertainty()
nd = d.join(df).join(df1)
plot_index_comparison(nd)
plot_eu_epu(nd)
plot_cinco_elpais(nd)
nd = transform_data(nd)
plot_epu_gdp(nd)
benchmark_subset = ['EPU','europe', 'fedea', 'inflation', 'differential']
nd['EPU'] = nd['policy'].diff(periods = 1)
data = nd.reindex(columns=benchmark_subset)
data = data.dropna()
data.describe()
model = VAR(data)
results = model.fit(6)
irf = results.irf(12)
irf.plot(orth=True, impulse='EPU', subplot_params = {'fontsize' : 12})
#irf.plot_cum_effects(orth=True, impulse='EPU', subplot_params = {'fontsize' : 12}) #
cum_effects = irf.orth_cum_effects
fedea_on_gdp = get_fedea_on_gdp()
elasticity = -100*fedea_on_gdp*cum_effects[12,2,0]
print 'Effects of a 1 sd uncertainty shock on gdp growth (negative): %0.3f%%' % elasticity
print 'Inflation increases by %0.2f' % (100* cum_effects[12,3,0], )
print 'Bond spreads increase by %0.1f basis points' % (100* cum_effects[12,4,0], )
full_sset = ['ibex','vol','resid','europe', 'fedea', 'inflation', 'differential' ]
def get_irf(nd, subset):
'''
http://statsmodels.sourceforge.net/0.6.0/vector_ar.html
'''
data = nd.reindex(columns=subset)
data = data.dropna()
data.describe()
model = VAR(data)
results = model.fit(6)
irf = results.irf(12)
cum_effects = irf.orth_cum_effects
return cum_effects[12,2,0]
for colname in colnames:
nd['uncert'] = nd[colname] / nd.articles
nd['uncert'] = nd['uncert'] / nd['uncert'].mean() * 100
nd['uncert'] = nd['uncert'].diff(periods = 1)
subset = ['uncert','europe', 'fedea', 'inflation', 'differential' ]
cum_effect= get_irf(nd, subset)
print '**%s** | %d | %.04f' % (colname, nd[colname].sum(), 100*fedea_on_gdp*cum_effect)
aa = d.mean()[colnames]
plt.figure(6)
h = plt.bar(range(len(aa)),aa,label = list(aa.index) )
plt.subplots_adjust(bottom=0.3)
xticks_pos = [0.65*patch.get_width() + patch.get_xy()[0] for patch in h]
plt.xticks(xticks_pos, list(aa.index), ha='right', rotation=45)
plt.savefig(os.path.join(rootdir, 'figures','frequency_types.%s' % fig_fmt), format=fig_fmt)
from statsmodels.tsa.base.datetools import dates_from_str
import pandas
mdata = ds.macrodata.load_pandas().data
# prepare the dates index
dates = mdata[['year', 'quarter']].astype(int).astype('S4')
quarterly = dates["year"] + "Q" + dates["quarter"]
quarterly = dates_from_str(quarterly)
mdata = mdata[['realgdp','realcons','realinv']]
mdata.index = pandas.DatetimeIndex(quarterly)
data = np.log(mdata).diff().dropna()
model = VAR(data)
est = model.fit(maxlags=2)
def plot_input():
est.plot()
def plot_acorr():
est.plot_acorr()
def plot_irf():
est.irf().plot()
def plot_irf_cum():
irf = est.irf()
irf.plot_cum_effects()