def test_dataframe(self):
df = pd.DataFrame(self.arr_2d)
appended = tsatools.add_trend(df)
expected = df.copy()
expected['const'] = self.c
tm.assert_frame_equal(expected, appended)
prepended = tsatools.add_trend(df, prepend=True)
expected = df.copy()
expected.insert(0, 'const', self.c)
tm.assert_frame_equal(expected, prepended)
df = pd.DataFrame(self.arr_2d)
appended = tsatools.add_trend(df, trend='t')
expected = df.copy()
expected['trend'] = self.t
tm.assert_frame_equal(expected, appended)
df = pd.DataFrame(self.arr_2d)
appended = tsatools.add_trend(df, trend='ctt')
expected = df.copy()
expected['const'] = self.c
expected['trend'] = self.t
expected['trend_squared'] = self.t**2
tm.assert_frame_equal(expected, appended)
def test_series(self):
s = pd.Series(self.arr_1d)
appended = tsatools.add_trend(s)
expected = pd.DataFrame(s)
expected['const'] = self.c
tm.assert_frame_equal(expected, appended)
prepended = tsatools.add_trend(s, prepend=True)
expected = pd.DataFrame(s)
expected.insert(0, 'const', self.c)
tm.assert_frame_equal(expected, prepended)
s = pd.Series(self.arr_1d)
appended = tsatools.add_trend(s, trend='ct')
expected = pd.DataFrame(s)
expected['const'] = self.c
expected['trend'] = self.t
tm.assert_frame_equal(expected, appended)
def test_mixed_recarray(self):
dt = np.dtype([('c0', np.float64), ('c1', np.int8), ('c2', 'S4')])
ra = np.array([(1.0, 1, 'aaaa'), (1.1, 2, 'bbbb')],
dtype=dt).view(np.recarray)
added = tsatools.add_trend(ra, trend='ct')
dt = np.dtype([('c0', np.float64), ('c1', np.int8), ('c2', 'S4'),
('const', np.float64), ('trend', np.float64)])
expected = np.array([(1.0, 1, 'aaaa', 1.0, 1.0),
(1.1, 2, 'bbbb', 1.0, 2.0)],
dtype=dt).view(np.recarray)
assert_equal(added, expected)
def test_duplicate_const(self):
# TODO: parametrize?
df = pd.DataFrame(self.c)
for trend in ['c', 'ct']:
for data in [self.c, df]:
with pytest.raises(ValueError):
tsatools.add_trend(x=data,
trend=trend,
has_constant='raise')
skipped = tsatools.add_trend(self.c, trend='c')
assert_equal(skipped, self.c[:, None])
skipped_const = tsatools.add_trend(self.c,
trend='ct',
has_constant='skip')
expected = np.vstack((self.c, self.t)).T
assert_equal(skipped_const, expected)
added = tsatools.add_trend(self.c, trend='c', has_constant='add')
expected = np.vstack((self.c, self.c)).T
assert_equal(added, expected)
added = tsatools.add_trend(self.c, trend='ct', has_constant='add')
expected = np.vstack((self.c, self.c, self.t)).T
assert_equal(added, expected)
def add_constant(data, prepend=True, has_constant='skip'):
"""
Adds a column of ones to an array
Parameters
----------
data : array-like
``data`` is the column-ordered design matrix
prepend : bool
If true, the constant is in the first column. Else the constant is
appended (last column).
has_constant : str {'raise', 'add', 'skip'}
Behavior if ``data`` already has a constant. The default will return
data without adding another constant. If 'raise', will raise an
error if a constant is present. Using 'add' will duplicate the
constant, if one is present.
Returns
-------
data : array, recarray or DataFrame
The original values with a constant (column of ones) as the first or
last column. Returned value depends on input type.
Notes
-----
When the input is recarray or a pandas Series or DataFrame, the added
column's name is 'const'.
"""
if _is_using_pandas(data, None) or _is_recarray(data):
from sm2.tsa.tsatools import add_trend
return add_trend(data, trend='c',
prepend=prepend,
has_constant=has_constant)
# Special case for NumPy
x = np.asanyarray(data)
if x.ndim == 1:
x = x[:, None]
elif x.ndim > 2: # pragma: no cover
raise ValueError('Only implementd 2-dimensional arrays')
is_nonzero_const = np.ptp(x, axis=0) == 0
is_nonzero_const &= np.all(x != 0.0, axis=0)
if is_nonzero_const.any():
if has_constant == 'skip':
return x
elif has_constant == 'raise':
raise ValueError("data already contains a constant")
x = [np.ones(x.shape[0]), x]
x = x if prepend else x[::-1]
return np.column_stack(x)
def _stackX(self, k_ar, trend):
"""
Private method to build the RHS matrix for estimation.
Columns are trend terms then lags.
"""
endog = self.endog
X = lagmat(endog, maxlag=k_ar, trim='both')
k_trend = util.get_trendorder(trend)
if k_trend:
X = add_trend(X, prepend=True, trend=trend)
self.k_trend = k_trend # TODO: Don't set this here
return X
def test_recarray(self):
df = pd.DataFrame(self.arr_2d)
recarray = df.to_records(index=False)
appended = tsatools.add_trend(recarray)
expected = pd.DataFrame(self.arr_2d)
expected['const'] = self.c
expected = expected.to_records(index=False)
assert_equal(expected, appended)
prepended = tsatools.add_trend(recarray, prepend=True)
expected = pd.DataFrame(self.arr_2d)
expected.insert(0, 'const', self.c)
expected = expected.to_records(index=False)
assert_equal(expected, prepended)
appended = tsatools.add_trend(recarray, trend='ctt')
expected = pd.DataFrame(self.arr_2d)
expected['const'] = self.c
expected['trend'] = self.t
expected['trend_squared'] = self.t**2
expected = expected.to_records(index=False)
assert_equal(expected, appended)
def test_array(self):
base = np.vstack((self.arr_1d, self.c, self.t, self.t**2)).T
assert_equal(tsatools.add_trend(self.arr_1d), base[:, :2])
assert_equal(tsatools.add_trend(self.arr_1d, trend='t'), base[:,
[0, 2]])
assert_equal(tsatools.add_trend(self.arr_1d, trend='ct'), base[:, :3])
assert_equal(tsatools.add_trend(self.arr_1d, trend='ctt'), base)
base = np.hstack(
(self.c[:, None], self.t[:, None], self.t[:,
None]**2, self.arr_2d))
assert_equal(tsatools.add_trend(self.arr_2d, prepend=True),
base[:, [0, 3, 4]])
assert_equal(tsatools.add_trend(self.arr_2d, trend='t', prepend=True),
base[:, [1, 3, 4]])
assert_equal(tsatools.add_trend(self.arr_2d, trend='ct', prepend=True),
base[:, [0, 1, 3, 4]])
assert_equal(
tsatools.add_trend(self.arr_2d, trend='ctt', prepend=True), base)
def get_var_endog(y, lags, trend='c', has_constant='skip'):
"""
Make predictor matrix for VAR(p) process
Z := (Z_0, ..., Z_T).T (T x Kp)
Z_t = [1 y_t y_{t-1} ... y_{t - p + 1}] (Kp x 1)
Ref: Lütkepohl p.70 (transposed)
has_constant can be 'raise', 'add', or 'skip'. See add_constant.
"""
nobs = len(y)
# Ravel C order, need to put in descending order
Z = np.array([y[t - lags: t][::-1].ravel() for t in range(lags, nobs)])
# Add constant, trend, etc.
if trend != 'nc':
Z = tsatools.add_trend(Z, prepend=True, trend=trend,
has_constant=has_constant)
return Z
def test_unknown_trend(self):
with pytest.raises(ValueError):
tsatools.add_trend(x=self.arr_1d, trend='unknown')
def test_dataframe_duplicate(self):
df = pd.DataFrame(self.arr_2d, columns=['const', 'trend'])
tsatools.add_trend(df, trend='ct')
tsatools.add_trend(df, trend='ct', prepend=True)
def coint(y0,
y1,
trend='c',
method='aeg',
maxlag=None,
autolag='aic',
return_results=None):
"""Test for no-cointegration of a univariate equation
The null hypothesis is no cointegration. Variables in y0 and y1 are
assumed to be integrated of order 1, I(1).
This uses the augmented Engle-Granger two-step cointegration test.
Constant or trend is included in 1st stage regression, i.e. in
cointegrating equation.
**Warning:** The autolag default has changed compared to statsmodels 0.8.
In 0.8 autolag was always None, no the keyword is used and defaults to
'aic'. Use `autolag=None` to avoid the lag search.
Parameters
----------
y1 : array_like, 1d
first element in cointegrating vector
y2 : array_like
remaining elements in cointegrating vector
trend : str {'c', 'ct'}
trend term included in regression for cointegrating equation
* 'c' : constant
* 'ct' : constant and linear trend
* also available quadratic trend 'ctt', and no constant 'nc'
method : string
currently only 'aeg' for augmented Engle-Granger test is available.
default might change.
maxlag : None or int
keyword for `adfuller`, largest or given number of lags
autolag : string
keyword for `adfuller`, lag selection criterion.
* if None, then maxlag lags are used without lag search
* if 'AIC' (default) or 'BIC', then the number of lags is chosen
to minimize the corresponding information criterion
* 't-stat' based choice of maxlag. Starts with maxlag and drops a
lag until the t-statistic on the last lag length is significant
using a 5%-sized test
return_results : bool
for future compatibility, currently only tuple available.
If True, then a results instance is returned. Otherwise, a tuple
with the test outcome is returned.
Set `return_results=False` to avoid future changes in return.
Returns
-------
coint_t : float
t-statistic of unit-root test on residuals
pvalue : float
MacKinnon's approximate, asymptotic p-value based on MacKinnon (1994)
crit_value : dict
Critical values for the test statistic at the 1 %, 5 %, and 10 %
levels based on regression curve. This depends on the number of
observations.
Notes
-----
The Null hypothesis is that there is no cointegration, the alternative
hypothesis is that there is cointegrating relationship. If the pvalue is
small, below a critical size, then we can reject the hypothesis that there
is no cointegrating relationship.
P-values and critical values are obtained through regression surface
approximation from MacKinnon 1994 and 2010.
TODO: We could handle gaps in data by dropping rows with nans in the
auxiliary regressions. Not implemented yet, currently assumes no nans
and no gaps in time series.
References
----------
MacKinnon, J.G. 1994 "Approximate Asymptotic Distribution Functions for
Unit-Root and Cointegration Tests." Journal of Business & Economics
Statistics, 12.2, 167-76.
MacKinnon, J.G. 2010. "Critical Values for Cointegration Tests."
Queen's University, Dept of Economics Working Papers 1227.
http://ideas.repec.org/p/qed/wpaper/1227.html
"""
trend = trend.lower()
if trend not in ['c', 'nc', 'ct', 'ctt']: # pragma: no cover
raise ValueError("trend option %s not understood" % trend)
y0 = np.asarray(y0)
y1 = np.asarray(y1)
if y1.ndim < 2:
y1 = y1[:, None]
nobs, k_vars = y1.shape
k_vars += 1 # add 1 for y0
if trend == 'nc':
xx = y1
else:
xx = add_trend(y1, trend=trend, prepend=False)
res_co = OLS(y0, xx).fit()
if res_co.rsquared < 1 - np.sqrt(np.finfo(np.double).eps):
res_adf = adfuller(res_co.resid,
maxlag=maxlag,
autolag=autolag,
regression='nc')
else:
warnings.warn(
"y0 and y1 are perfectly colinear. Cointegration test "
"is not reliable in this case.", CollinearityWarning)
# Edge case where series are too similar
res_adf = (-np.inf, )
# no constant or trend, see egranger in Stata and MacKinnon
if trend == 'nc':
crit = [np.nan] * 3 # 2010 critical values not available
else:
crit = mackinnoncrit(N=k_vars, regression=trend, nobs=nobs - 1)
# nobs - 1, the -1 is to match egranger in Stata, I don't know why.
# TODO: check nobs or df = nobs - k
pval_asy = mackinnonp(res_adf[0], regression=trend, N=k_vars)
return res_adf[0], pval_asy, crit
def adfuller(x,
maxlag=None,
regression="c",
autolag='AIC',
store=False,
regresults=False):
"""
Augmented Dickey-Fuller unit root test
The Augmented Dickey-Fuller test can be used to test for a unit root in a
univariate process in the presence of serial correlation.
Parameters
----------
x : array_like, 1d
data series
maxlag : int
Maximum lag which is included in test, default 12*(nobs/100)^{1/4}
regression : {'c','ct','ctt','nc'}
Constant and trend order to include in regression
* 'c' : constant only (default)
* 'ct' : constant and trend
* 'ctt' : constant, and linear and quadratic trend
* 'nc' : no constant, no trend
autolag : {'AIC', 'BIC', 't-stat', None}
* if None, then maxlag lags are used
* if 'AIC' (default) or 'BIC', then the number of lags is chosen
to minimize the corresponding information criterion
* 't-stat' based choice of maxlag. Starts with maxlag and drops a
lag until the t-statistic on the last lag length is significant
using a 5%-sized test
store : bool
If True, then a result instance is returned additionally to
the adf statistic. Default is False
regresults : bool, optional
If True, the full regression results are returned. Default is False
Returns
-------
adf : float
Test statistic
pvalue : float
MacKinnon's approximate p-value based on MacKinnon (1994, 2010)
usedlag : int
Number of lags used
nobs : int
Number of observations used for the ADF regression and calculation of
the critical values
critical values : dict
Critical values for the test statistic at the 1 %, 5 %, and 10 %
levels. Based on MacKinnon (2010)
icbest : float
The maximized information criterion if autolag is not None.
resstore : ResultStore, optional
A dummy class with results attached as attributes
Notes
-----
The null hypothesis of the Augmented Dickey-Fuller is that there is a unit
root, with the alternative that there is no unit root. If the pvalue is
above a critical size, then we cannot reject that there is a unit root.
The p-values are obtained through regression surface approximation from
MacKinnon 1994, but using the updated 2010 tables. If the p-value is close
to significant, then the critical values should be used to judge whether
to reject the null.
The autolag option and maxlag for it are described in Greene.
Examples
--------
See example notebook
References
----------
.. [*] W. Green. "Econometric Analysis," 5th ed., Pearson, 2003.
.. [*] Hamilton, J.D. "Time Series Analysis". Princeton, 1994.
.. [*] MacKinnon, J.G. 1994. "Approximate asymptotic distribution
functions for unit-root and cointegration tests. `Journal of Business
and Economic Statistics` 12, 167-76.
.. [*] MacKinnon, J.G. 2010. "Critical Values for Cointegration Tests."
Queen's University, Dept of Economics, Working Papers. Available at
http://ideas.repec.org/p/qed/wpaper/1227.html
"""
if regresults:
store = True
trenddict = {None: 'nc', 0: 'c', 1: 'ct', 2: 'ctt'}
if regression is None or isinstance(regression, integer_types):
regression = trenddict[regression]
regression = regression.lower()
if regression not in ['c', 'nc', 'ct', 'ctt']: # pragma: no cover
raise ValueError("regression option %s not understood" % regression)
x = np.asarray(x)
nobs = x.shape[0]
if maxlag is None:
# from Greene referencing Schwert 1989
maxlag = int(np.ceil(12. * np.power(nobs / 100., 1 / 4.)))
xdiff = np.diff(x)
xdall = lagmat(xdiff[:, None], maxlag, trim='both', original='in')
nobs = xdall.shape[0] # pylint: disable=E1103
xdall[:, 0] = x[-nobs - 1:-1] # replace 0 xdiff with level of x
xdshort = xdiff[-nobs:]
if store:
resstore = ResultsStore()
if autolag:
if regression != 'nc':
fullRHS = add_trend(xdall, regression, prepend=True)
else:
fullRHS = xdall
# add +1 for level
startlag = fullRHS.shape[1] - xdall.shape[1] + 1
# search for lag length with smallest information criteria
# Note: use the same number of observations to have comparable IC
# aic and bic: smaller is better
icbest, bestlag, alres = _autolag(OLS,
xdshort,
fullRHS,
startlag,
maxlag,
autolag,
regresults=True)
if regresults:
resstore.autolag_results = alres
bestlag -= startlag # convert to lag not column index
# rerun ols with best autolag
xdall = lagmat(xdiff[:, None], bestlag, trim='both', original='in')
nobs = xdall.shape[0] # pylint: disable=E1103
xdall[:, 0] = x[-nobs - 1:-1] # replace 0 xdiff with level of x
xdshort = xdiff[-nobs:]
usedlag = bestlag
else:
usedlag = maxlag
icbest = None
if regression != 'nc':
resols = OLS(xdshort, add_trend(xdall[:, :usedlag + 1],
regression)).fit()
else:
resols = OLS(xdshort, xdall[:, :usedlag + 1]).fit()
adfstat = resols.tvalues[0]
# adfstat = (resols.params[0]-1.0)/resols.bse[0]
# the "asymptotically correct" z statistic is obtained as
# nobs/(1-np.sum(resols.params[1:-(trendorder+1)])) (resols.params[0] - 1)
# I think this is the statistic that is used for series that are integrated
# for orders higher than I(1), ie., not ADF but cointegration tests.
# Get approx p-value and critical values
pvalue = mackinnonp(adfstat, regression=regression, N=1)
critvalues = mackinnoncrit(N=1, regression=regression, nobs=nobs)
critvalues = {
"1%": critvalues[0],
"5%": critvalues[1],
"10%": critvalues[2]
}
if store:
resstore.resols = resols
resstore.maxlag = maxlag
resstore.usedlag = usedlag
resstore.adfstat = adfstat
resstore.critvalues = critvalues
resstore.nobs = nobs
resstore.H0 = ("The coefficient on the lagged level equals 1 - "
"unit root")
resstore.HA = "The coefficient on the lagged level < 1 - stationary"
resstore.icbest = icbest
resstore._str = 'Augmented Dickey-Fuller Test Results'
return adfstat, pvalue, critvalues, resstore
else:
if not autolag:
return adfstat, pvalue, usedlag, nobs, critvalues
else:
return adfstat, pvalue, usedlag, nobs, critvalues, icbest