def test__reduce_dict():
data = OrderedDict(zip(list(product('mf', 'oy', 'wn')), [1] * 8))
eq(_reduce_dict(data, ('m', )), 4)
eq(_reduce_dict(data, ('m', 'o')), 2)
eq(_reduce_dict(data, ('m', 'o', 'w')), 1)
data = OrderedDict(zip(list(product('mf', 'oy', 'wn')), lrange(8)))
eq(_reduce_dict(data, ('m', )), 6)
eq(_reduce_dict(data, ('m', 'o')), 1)
eq(_reduce_dict(data, ('m', 'o', 'w')), 0)
def test_mosaic_very_complex():
# make a scattermatrix of mosaic plots to show the correlations between
# each pair of variable in a dataset. Could be easily converted into a
# new function that does this automatically based on the type of data
key_name = ['gender', 'age', 'health', 'work']
key_base = (['male', 'female'], ['old',
'young'], ['healty',
'ill'], ['work', 'unemployed'])
keys = list(product(*key_base))
data = OrderedDict(zip(keys, range(1, 1 + len(keys))))
props = {}
props[('male', 'old')] = {'color': 'r'}
props[('female', )] = {'color': 'pink'}
L = len(key_base)
fig, axes = pylab.subplots(L, L)
for i in range(L):
for j in range(L):
m = set(range(L)).difference(set((i, j)))
if i == j:
axes[i, i].text(0.5,
0.5,
key_name[i],
ha='center',
va='center')
axes[i, i].set_xticks([])
axes[i, i].set_xticklabels([])
axes[i, i].set_yticks([])
axes[i, i].set_yticklabels([])
else:
ji = max(i, j)
ij = min(i, j)
temp_data = OrderedDict([((k[ij], k[ji]) + tuple(k[r]
for r in m),
v) for k, v in iteritems(data)])
keys = list(iterkeys(temp_data))
for k in keys:
value = _reduce_dict(temp_data, k[:2])
temp_data[k[:2]] = value
del temp_data[k]
mosaic(temp_data,
ax=axes[i, j],
axes_label=False,
properties=props,
gap=0.05,
horizontal=i > j)
pylab.suptitle('old males should look bright red, (plot 4 of 4)')
#pylab.show()
pylab.close('all')
def test_mosaic_simple():
# display a simple plot of 4 categories of data, splitted in four
# levels with increasing size for each group
# creation of the levels
key_set = (['male',
'female'], ['old', 'adult',
'young'], ['worker',
'unemployed'], ['healty', 'ill'])
# the cartesian product of all the categories is
# the complete set of categories
keys = list(product(*key_set))
data = OrderedDict(zip(keys, range(1, 1 + len(keys))))
# which colours should I use for the various categories?
# put it into a dict
props = {}
#males and females in blue and red
props[('male', )] = {'color': 'b'}
props[('female', )] = {'color': 'r'}
# all the groups corresponding to ill groups have a different color
for key in keys:
if 'ill' in key:
if 'male' in key:
props[key] = {'color': 'BlueViolet', 'hatch': '+'}
else:
props[key] = {'color': 'Crimson', 'hatch': '+'}
# mosaic of the data, with given gaps and colors
mosaic(data, gap=0.05, properties=props, axes_label=False)
pylab.suptitle('syntetic data, 4 categories (plot 2 of 4)')
#pylab.show()
pylab.close('all')
def test_recursive_split():
keys = list(product('mf'))
data = OrderedDict(zip(keys, [1] * len(keys)))
res = _hierarchical_split(data, gap=0)
assert_(list(iterkeys(res)) == keys)
res[('m', )] = (0.0, 0.0, 0.5, 1.0)
res[('f', )] = (0.5, 0.0, 0.5, 1.0)
keys = list(product('mf', 'yao'))
data = OrderedDict(zip(keys, [1] * len(keys)))
res = _hierarchical_split(data, gap=0)
assert_(list(iterkeys(res)) == keys)
res[('m', 'y')] = (0.0, 0.0, 0.5, 1 / 3)
res[('m', 'a')] = (0.0, 1 / 3, 0.5, 1 / 3)
res[('m', 'o')] = (0.0, 2 / 3, 0.5, 1 / 3)
res[('f', 'y')] = (0.5, 0.0, 0.5, 1 / 3)
res[('f', 'a')] = (0.5, 1 / 3, 0.5, 1 / 3)
res[('f', 'o')] = (0.5, 2 / 3, 0.5, 1 / 3)
def _normalize_data(data, index):
"""normalize the data to a dict with tuples of strings as keys
right now it works with:
0 - dictionary (or equivalent mappable)
1 - pandas.Series with simple or hierarchical indexes
2 - numpy.ndarrays
3 - everything that can be converted to a numpy array
4 - pandas.DataFrame (via the _normalize_dataframe function)
"""
# if data is a dataframe we need to take a completely new road
# before coming back here. Use the hasattr to avoid importing
# pandas explicitly
if hasattr(data, 'pivot') and hasattr(data, 'groupby'):
data = _normalize_dataframe(data, index)
index = None
# can it be used as a dictionary?
try:
items = list(iteritems(data))
except AttributeError:
# ok, I cannot use the data as a dictionary
# Try to convert it to a numpy array, or die trying
data = np.asarray(data)
temp = OrderedDict()
for idx in np.ndindex(data.shape):
name = tuple(i for i in idx)
temp[name] = data[idx]
data = temp
items = list(iteritems(data))
# make all the keys a tuple, even if simple numbers
data = OrderedDict([_tuplify(k), v] for k, v in items)
categories_levels = _categories_level(list(iterkeys(data)))
# fill the void in the counting dictionary
indexes = product(*categories_levels)
contingency = OrderedDict([(k, data.get(k, 0)) for k in indexes])
data = contingency
# reorder the keys order according to the one specified by the user
# or if the index is None convert it into a simple list
# right now it doesn't do any check, but can be modified in the future
index = lrange(len(categories_levels)) if index is None else index
contingency = OrderedDict()
for key, value in iteritems(data):
new_key = tuple(key[i] for i in index)
contingency[new_key] = value
data = contingency
return data
def summary_model(results):
'''Create a dict with information about the model
'''
def time_now(*args, **kwds):
now = datetime.datetime.now()
return now.strftime('%Y-%m-%d %H:%M')
info = OrderedDict()
info['Model:'] = lambda x: x.model.__class__.__name__
info['Model Family:'] = lambda x: x.family.__class.__name__
info['Link Function:'] = lambda x: x.family.link.__class__.__name__
info['Dependent Variable:'] = lambda x: x.model.endog_names
info['Date:'] = time_now
info['No. Observations:'] = lambda x: "%#6d" % x.nobs
info['Df Model:'] = lambda x: "%#6d" % x.df_model
info['Df Residuals:'] = lambda x: "%#6d" % x.df_resid
info['Converged:'] = lambda x: x.mle_retvals['converged']
info['No. Iterations:'] = lambda x: x.mle_retvals['iterations']
info['Method:'] = lambda x: x.method
info['Norm:'] = lambda x: x.fit_options['norm']
info['Scale Est.:'] = lambda x: x.fit_options['scale_est']
info['Cov. Type:'] = lambda x: x.fit_options['cov']
info['R-squared:'] = lambda x: "%#8.3f" % x.rsquared
info['Adj. R-squared:'] = lambda x: "%#8.3f" % x.rsquared_adj
info['Pseudo R-squared:'] = lambda x: "%#8.3f" % x.prsquared
info['AIC:'] = lambda x: "%8.4f" % x.aic
info['BIC:'] = lambda x: "%8.4f" % x.bic
info['Log-Likelihood:'] = lambda x: "%#8.5g" % x.llf
info['LL-Null:'] = lambda x: "%#8.5g" % x.llnull
info['LLR p-value:'] = lambda x: "%#8.5g" % x.llr_pvalue
info['Deviance:'] = lambda x: "%#8.5g" % x.deviance
info['Pearson chi2:'] = lambda x: "%#6.3g" % x.pearson_chi2
info['F-statistic:'] = lambda x: "%#8.4g" % x.fvalue
info['Prob (F-statistic):'] = lambda x: "%#6.3g" % x.f_pvalue
info['Scale:'] = lambda x: "%#8.5g" % x.scale
out = OrderedDict()
for key, func in iteritems(info):
try:
out[key] = func(results)
# NOTE: some models don't have loglike defined (RLM), so that's NIE
except (AttributeError, KeyError, NotImplementedError):
pass
return out
def _categories_level(keys):
"""use the Ordered dict to implement a simple ordered set
return each level of each category
[[key_1_level_1,key_2_level_1],[key_1_level_2,key_2_level_2]]
"""
res = []
for i in zip(*(keys)):
tuplefied = _tuplify(i)
res.append(list(OrderedDict([(j, None) for j in tuplefied])))
return res
def test_axes_labeling():
from numpy.random import rand
key_set = (['male', 'female'], ['old', 'adult', 'young'],
['worker', 'unemployed'], ['yes', 'no'])
# the cartesian product of all the categories is
# the complete set of categories
keys = list(product(*key_set))
data = OrderedDict(zip(keys, rand(len(keys))))
lab = lambda k: ''.join(s[0] for s in k)
fig, (ax1, ax2) = pylab.subplots(1, 2, figsize=(16, 8))
mosaic(data, ax=ax1, labelizer=lab, horizontal=True, label_rotation=45)
mosaic(data, ax=ax2, labelizer=lab, horizontal=False,
label_rotation=[0, 45, 90, 0])
#fig.tight_layout()
fig.suptitle("correct alignment of the axes labels")
def termcolumns(self, query_term, dict=False):
"""
Return a list of the indices of all columns associated
to a given term.
"""
if self.hasterm(query_term):
names = query_term.names()
value = OrderedDict()
for name in names:
value[name] = self._names.index(name)
else:
raise ValueError('term not in formula')
if dict:
return value
else:
return list(itervalues(value))
def _key_splitting(rect_dict, keys, values, key_subset, horizontal, gap):
"""
Given a dictionary where each entry is a rectangle, a list of key and
value (count of elements in each category) it split each rect accordingly,
as long as the key start with the tuple key_subset. The other keys are
returned without modification.
"""
result = OrderedDict()
L = len(key_subset)
for name, (x, y, w, h) in iteritems(rect_dict):
if key_subset == name[:L]:
# split base on the values given
divisions = _split_rect(x, y, w, h, values, horizontal, gap)
for key, rect in zip(keys, divisions):
result[name + (key,)] = rect
else:
result[name] = (x, y, w, h)
return result
def _ecdf(x):
nobs = len(x)
return np.arange(1, nobs + 1) / float(nobs)
multitest_methods_names = {
'b': 'Bonferroni',
'fdr_bh': 'FDR Benjamini-Hochberg',
'fdr_by': 'FDR Benjamini-Yekutieli'
}
_alias_list = [['b', 'bonf', 'bonferroni'],
['fdr_bh', 'fdr_i', 'fdr_p', 'fdri', 'fdrp'],
['fdr_by', 'fdr_n', 'fdr_c', 'fdrn', 'fdrcorr']]
multitest_alias = OrderedDict()
for m in _alias_list:
multitest_alias[m[0]] = m[0]
for a in m[1:]:
multitest_alias[a] = m[0]
def summary(self, title=None, alpha=.05):
"""
Summarize the #1lab_results of running MICE.
Parameters
-----------
title : string, optional
Title for the top table. If not None, then this replaces
the default title
alpha : float
Significance level for the confidence intervals
Returns
-------
smry : Summary instance
This holds the summary tables and text, which can be
printed or converted to various output formats.
"""
from statsmodels.iolib import summary2
from statsmodels.compat.collections import OrderedDict
smry = summary2.Summary()
float_format = "%8.3f"
info = OrderedDict()
info["Method:"] = "MICE"
info["Model:"] = self.model_class.__name__
info["Dependent variable:"] = self.endog_names
info["Sample size:"] = "%d" % self.model.data.data.shape[0]
info["Scale"] = "%.2f" % self.scale
info["Num. imputations"] = "%d" % len(self.model.results_list)
smry.add_dict(info, align='l', float_format=float_format)
param = summary2.summary_params(self, alpha=alpha)
param["FMI"] = self.frac_miss_info
smry.add_df(param, float_format=float_format)
smry.add_title(title=title, results=self)
return smry
def summary_frame(self, what='all', alpha=0.05):
# TODO: finish and cleanup
import pandas as pd
from statsmodels.compat.collections import OrderedDict
#ci_obs = self.conf_int(alpha=alpha, obs=True) # need to split
ci_mean = self.conf_int(alpha=alpha)
to_include = OrderedDict()
to_include['mean'] = self.predicted_mean
to_include['mean_se'] = self.se_mean
to_include['mean_ci_lower'] = ci_mean[:, 0]
to_include['mean_ci_upper'] = ci_mean[:, 1]
self.table = to_include
#OrderedDict doesn't work to preserve sequence
# pandas dict doesn't handle 2d_array
#data = np.column_stack(list(to_include.values()))
#names = ....
res = pd.DataFrame(to_include,
index=self.row_labels,
columns=to_include.keys())
return res
def _col_info(result, more_info=None):
'''Stack model info in a column
'''
model_info = summary_model(result)
default_info_ = OrderedDict()
default_info_['Model:'] = lambda x: x.get('Model:')
default_info_['No. Observations:'] = lambda x: x.get('No. Observations:')
default_info_['R-squared:'] = lambda x: x.get('R-squared:')
default_info_['Adj. R-squared:'] = lambda x: x.get('Adj. R-squared:')
default_info_['Pseudo R-squared:'] = lambda x: x.get('Pseudo R-squared:')
default_info_['F-statistic:'] = lambda x: x.get('F-statistic:')
default_info_['Covariance Type:'] = lambda x: x.get('Covariance Type:')
default_info_['Eeffects:'] = lambda x: x.get('Effects:')
default_info_['Covariance Type:'] = lambda x: x.get('Covariance Type:')
default_info = default_info_.copy()
for k,v in default_info_.items():
if v(model_info):
default_info[k] = v(model_info)
else:
default_info.pop(k) # pop the item whose value is none.
if more_info is None:
more_info = default_info
else:
if not isinstance(more_info,list):
more_info = [more_info]
for i in more_info:
try:
default_info[i] = getattr(result,i)
except (AttributeError, KeyError, NotImplementedError) as e:
raise e
more_info = default_info
try:
out = pd.DataFrame(more_info, index=[result.model.endog_names]).T
except (AttributeError):
out = pd.DataFrame(more_info, index=result.model.dependent.vars).T
return out
def hdrboxplot(data,
ncomp=2,
alpha=None,
threshold=0.95,
bw=None,
xdata=None,
labels=None,
ax=None):
"""
High Density Region boxplot
Parameters
----------
data : sequence of ndarrays or 2-D ndarray
The vectors of functions to create a functional boxplot from. If a
sequence of 1-D arrays, these should all be the same size.
The first axis is the function index, the second axis the one along
which the function is defined. So ``data[0, :]`` is the first
functional curve.
ncomp : int, optional
Number of components to use. If None, returns the as many as the
smaller of the number of rows or columns in data.
alpha : list of floats between 0 and 1, optional
Extra quantile values to compute. Default is None
threshold : float between 0 and 1, optional
Percentile threshold value for outliers detection. High value means
a lower sensitivity to outliers. Default is `0.95`.
bw: array_like or str, optional
If an array, it is a fixed user-specified bandwidth. If `None`, set to
`normal_reference`. If a string, should be one of:
- normal_reference: normal reference rule of thumb (default)
- cv_ml: cross validation maximum likelihood
- cv_ls: cross validation least squares
xdata : ndarray, optional
The independent variable for the data. If not given, it is assumed to
be an array of integers 0..N-1 with N the length of the vectors in
`data`.
labels : sequence of scalar or str, optional
The labels or identifiers of the curves in `data`. If not given,
outliers are labeled in the plot with array indices.
ax : Matplotlib AxesSubplot instance, optional
If given, this subplot is used to plot in instead of a new figure being
created.
Returns
-------
fig : Matplotlib figure instance
If `ax` is None, the created figure. Otherwise the figure to which
`ax` is connected.
hdr_res : HdrResults instance
An `HdrResults` instance with the following attributes:
- 'median', array. Median curve.
- 'hdr_50', array. 50% quantile band. [sup, inf] curves
- 'hdr_90', list of array. 90% quantile band. [sup, inf]
curves.
- 'extra_quantiles', list of array. Extra quantile band.
[sup, inf] curves.
- 'outliers', ndarray. Outlier curves.
Notes
-----
The median curve is the curve with the highest probability on the reduced
space of a Principal Component Analysis (PCA).
Outliers are defined as curves that fall outside the band corresponding
to the quantile given by `threshold`.
The non-outlying region is defined as the band made up of all the
non-outlying curves.
Behind the scene, the dataset is represented as a matrix. Each line
corresponding to a 1D curve. This matrix is then decomposed using Principal
Components Analysis (PCA). This allows to represent the data using a finite
number of modes, or components. This compression process allows to turn the
functional representation into a scalar representation of the matrix. In
other words, you can visualize each curve from its components. Each curve
is thus a point in this reduced space. With 2 components, this is called a
bivariate plot (2D plot).
In this plot, if some points are adjacent (similar components), it means
that back in the original space, the curves are similar. Then, finding the
median curve means finding the higher density region (HDR) in the reduced
space. Moreover, the more you get away from this HDR, the more the curve is
unlikely to be similar to the other curves.
Using a kernel smoothing technique, the probability density function (PDF)
of the multivariate space can be recovered. From this PDF, it is possible to
compute the density probability linked to the cluster of points and plot
its contours.
Finally, using these contours, the different quantiles can be extracted
along with the median curve and the outliers.
Steps to produce the HDR boxplot include:
1. Compute a multivariate kernel density estimation
2. Compute contour lines for quantiles 90%, 50% and `alpha` %
3. Plot the bivariate plot
4. Compute median curve along with quantiles and outliers curves.
References
----------
[1] R.J. Hyndman and H.L. Shang, "Rainbow Plots, Bagplots, and Boxplots for
Functional Data", vol. 19, pp. 29-45, 2010.
Examples
--------
Load the El Nino dataset. Consists of 60 years worth of Pacific Ocean sea
surface temperature data.
>>> import matplotlib.pyplot as plt
>>> import statsmodels.api as sm
>>> data = sm.datasets.elnino.load()
Create a functional boxplot. We see that the years 1982-83 and 1997-98 are
outliers; these are the years where El Nino (a climate pattern
characterized by warming up of the sea surface and higher air pressures)
occurred with unusual intensity.
>>> fig = plt.figure()
>>> ax = fig.add_subplot(111)
>>> res = sm.graphics.hdrboxplot(data.raw_data[:, 1:],
... labels=data.raw_data[:, 0].astype(int),
... ax=ax)
>>> ax.set_xlabel("Month of the year")
>>> ax.set_ylabel("Sea surface temperature (C)")
>>> ax.set_xticks(np.arange(13, step=3) - 1)
>>> ax.set_xticklabels(["", "Mar", "Jun", "Sep", "Dec"])
>>> ax.set_xlim([-0.2, 11.2])
>>> plt.show()
.. plot:: plots/graphics_functional_hdrboxplot.py
See Also
--------
banddepth, rainbowplot, fboxplot
"""
fig, ax = utils.create_mpl_ax(ax)
if labels is None:
# For use with pandas, get the labels
if hasattr(data, 'index'):
labels = data.index
else:
labels = np.arange(len(data))
data = np.asarray(data)
if xdata is None:
xdata = np.arange(data.shape[1])
n_samples, dim = data.shape
# PCA and bivariate plot
pca = PCA(data, ncomp=ncomp)
data_r = pca.factors
# Create gaussian kernel
ks_gaussian = KDEMultivariate(data_r,
bw=bw,
var_type='c' * data_r.shape[1])
# Boundaries of the n-variate space
bounds = np.array([data_r.min(axis=0), data_r.max(axis=0)]).T
# Compute contour line of pvalue linked to a given probability level
if alpha is None:
alpha = [threshold, 0.9, 0.5]
else:
alpha.extend([threshold, 0.9, 0.5])
alpha = list(set(alpha))
alpha.sort(reverse=True)
n_quantiles = len(alpha)
pdf_r = ks_gaussian.pdf(data_r).flatten()
pvalues = [
np.percentile(pdf_r, (1 - alpha[i]) * 100, interpolation='linear')
for i in range(n_quantiles)
]
# Find mean, outliers curves
if have_de_optim:
median = differential_evolution(lambda x: -ks_gaussian.pdf(x),
bounds=bounds,
maxiter=5).x
else:
median = brute(lambda x: -ks_gaussian.pdf(x),
ranges=bounds,
finish=fmin)
outliers_idx = np.where(pdf_r < pvalues[alpha.index(threshold)])[0]
labels_outlier = [labels[i] for i in outliers_idx]
outliers = data[outliers_idx]
# Find HDR given some quantiles
def _band_quantiles(band):
"""Find extreme curves for a quantile band.
From the `band` of quantiles, the associated PDF extrema values
are computed. If `min_alpha` is not provided (single quantile value),
`max_pdf` is set to `1E6` in order not to constrain the problem on high
values.
An optimization is performed per component in order to find the min and
max curves. This is done by comparing the PDF value of a given curve
with the band PDF.
Parameters
----------
band : array_like
alpha values ``(max_alpha, min_alpha)`` ex: ``[0.9, 0.5]``
Returns
-------
band_quantiles : list of 1-D array
``(max_quantile, min_quantile)`` (2, n_features)
"""
min_pdf = pvalues[alpha.index(band[0])]
try:
max_pdf = pvalues[alpha.index(band[1])]
except IndexError:
max_pdf = 1E6
band = [min_pdf, max_pdf]
pool = Pool()
data = zip(range(dim),
itertools.repeat((band, pca, bounds, ks_gaussian)))
band_quantiles = pool.map(_min_max_band, data)
pool.terminate()
pool.close()
band_quantiles = list(zip(*band_quantiles))
return band_quantiles
extra_alpha = [
i for i in alpha if 0.5 != i and 0.9 != i and threshold != i
]
if extra_alpha != []:
extra_quantiles = [
y for x in extra_alpha for y in _band_quantiles([x])
]
else:
extra_quantiles = []
# Inverse transform from n-variate plot to dataset dataset's shape
median = _inverse_transform(pca, median)[0]
hdr_90 = _band_quantiles([0.9, 0.5])
hdr_50 = _band_quantiles([0.5])
hdr_res = HdrResults({
"median": median,
"hdr_50": hdr_50,
"hdr_90": hdr_90,
"extra_quantiles": extra_quantiles,
"outliers": outliers,
"outliers_idx": outliers_idx
})
# Plots
ax.plot(np.array([xdata] * n_samples).T,
data.T,
c='c',
alpha=.1,
label=None)
ax.plot(xdata, median, c='k', label='Median')
fill_betweens = []
fill_betweens.append(
ax.fill_between(xdata,
*hdr_50,
color='gray',
alpha=.4,
label='50% HDR'))
fill_betweens.append(
ax.fill_between(xdata,
*hdr_90,
color='gray',
alpha=.3,
label='90% HDR'))
if len(extra_quantiles) != 0:
ax.plot(np.array([xdata] * len(extra_quantiles)).T,
np.array(extra_quantiles).T,
c='y',
ls='-.',
alpha=.4,
label='Extra quantiles')
if len(outliers) != 0:
for ii, outlier in enumerate(outliers):
label = str(labels_outlier[ii]
) if labels_outlier is not None else 'Outliers'
ax.plot(xdata, outlier, ls='--', alpha=0.7, label=label)
handles, labels = ax.get_legend_handles_labels()
# Proxy artist for fill_between legend entry
# See http://matplotlib.org/1.3.1/users/legend_guide.html
plt = _import_mpl()
for label, fill_between in zip(['50% HDR', '90% HDR'], fill_betweens):
p = plt.Rectangle((0, 0), 1, 1, fc=fill_between.get_facecolor()[0])
handles.append(p)
labels.append(label)
by_label = OrderedDict(zip(labels, handles))
if len(outliers) != 0:
by_label.pop('Median')
by_label.pop('50% HDR')
by_label.pop('90% HDR')
ax.legend(by_label.values(), by_label.keys(), loc='best')
return fig, hdr_res
def _hierarchical_split(count_dict, horizontal=True, gap=0.05):
"""
Split a square in a hierarchical way given a contingency table.
Hierarchically split the unit square in alternate directions
in proportion to the subdivision contained in the contingency table
count_dict. This is the function that actually perform the tiling
for the creation of the mosaic plot. If the gap array has been specified
it will insert a corresponding amount of space (proportional to the
unit lenght), while retaining the proportionality of the tiles.
Parameters
----------
count_dict : dict
Dictionary containing the contingency table.
Each category should contain a non-negative number
with a tuple as index. It expects that all the combination
of keys to be representes; if that is not true, will
automatically consider the missing values as 0
horizontal : bool
The starting direction of the split (by default along
the horizontal axis)
gap : float or array of floats
The list of gaps to be applied on each subdivision.
If the lenght of the given array is less of the number
of subcategories (or if it's a single number) it will extend
it with exponentially decreasing gaps
Returns
----------
base_rect : dict
A dictionary containing the result of the split.
To each key is associated a 4-tuple of coordinates
that are required to create the corresponding rectangle:
0 - x position of the lower left corner
1 - y position of the lower left corner
2 - width of the rectangle
3 - height of the rectangle
"""
# this is the unit square that we are going to divide
base_rect = OrderedDict([(tuple(), (0, 0, 1, 1))])
# get the list of each possible value for each level
categories_levels = _categories_level(list(iterkeys(count_dict)))
L = len(categories_levels)
# recreate the gaps vector starting from an int
if not np.iterable(gap):
gap = [gap / 1.5 ** idx for idx in range(L)]
# extend if it's too short
if len(gap) < L:
last = gap[-1]
gap = list(*gap) + [last / 1.5 ** idx for idx in range(L)]
# trim if it's too long
gap = gap[:L]
# put the count dictionay in order for the keys
# this will allow some code simplification
count_ordered = OrderedDict([(k, count_dict[k])
for k in list(product(*categories_levels))])
for cat_idx, cat_enum in enumerate(categories_levels):
# get the partial key up to the actual level
base_keys = list(product(*categories_levels[:cat_idx]))
for key in base_keys:
# for each partial and each value calculate how many
# observation we have in the counting dictionary
part_count = [_reduce_dict(count_ordered, key + (partial,))
for partial in cat_enum]
# reduce the gap for subsequents levels
new_gap = gap[cat_idx]
# split the given subkeys in the rectangle dictionary
base_rect = _key_splitting(base_rect, cat_enum, part_count, key,
horizontal, new_gap)
horizontal = not horizontal
return base_rect
def summary_model(results):
'''Create a dict with information about the model
'''
def time_now(*args, **kwds):
now = datetime.datetime.now()
return now.strftime('%Y-%m-%d %H:%M')
info = OrderedDict()
# I added some informations of Panel regression from the package linearmodels.
# Panel regression has some different attribute names, but it doesn't matter here.
info['Model:'] = lambda x: x.model.__class__.__name__
info['Model Family:'] = lambda x: x.family.__class.__name__
info['Link Function:'] = lambda x: x.family.link.__class__.__name__
info['Dependent Variable:'] = lambda x: x.model.endog_names
# 1
info['Dependent Variable:'] = lambda x: x.model.dependent.vars[0]
info['Date:'] = time_now
info['No. Observations:'] = lambda x: "%#6d" % x.nobs
info['Df Model:'] = lambda x: "%#6d" % x.df_model
info['Df Residuals:'] = lambda x: "%#6d" % x.df_resid
info['Converged:'] = lambda x: x.mle_retvals['converged']
info['No. Iterations:'] = lambda x: x.mle_retvals['iterations']
info['Method:'] = lambda x: x.method
info['Norm:'] = lambda x: x.fit_options['norm']
info['Scale Est.:'] = lambda x: x.fit_options['scale_est']
info['Cov. Type:'] = lambda x: x.fit_options['cov']
# 2
#I add the x.cov_type because there is no attribute fit_options like OLS model
info['Covariance Type:'] = lambda x: x.cov_type
info['Covariance Type:'] = lambda x: x._cov_type
info['R-squared:'] = lambda x: "%#8.3f" % x.rsquared
info['Adj. R-squared:'] = lambda x: "%#8.3f" % x.rsquared_adj
info['Pseudo R-squared:'] = lambda x: "%#8.3f" % x.prsquared
info['AIC:'] = lambda x: "%8.4f" % x.aic
info['BIC:'] = lambda x: "%8.4f" % x.bic
info['Log-Likelihood:'] = lambda x: "%#8.5g" % x.llf
# 3
info['Log-Likelihood:'] = lambda x: "%#8.5g" % x.loglike
info['LL-Null:'] = lambda x: "%#8.5g" % x.llnull
info['LLR p-value:'] = lambda x: "%#8.5g" % x.llr_pvalue
info['Deviance:'] = lambda x: "%#8.5g" % x.deviance
info['Pearson chi2:'] = lambda x: "%#6.3g" % x.pearson_chi2
info['F-statistic:'] = lambda x: "%#8.4g" % x.fvalue
# 4
info['F-statistic:'] = lambda x: "%#8.4g" % x.f_statistic.stat
info['Prob (F-statistic):'] = lambda x: "%#6.3g" % x.f_pvalue
# 5
info['Prob (F-statistic):'] = lambda x: "%#6.3g" % x.f_statistic.pval
info['Scale:'] = lambda x: "%#8.5g" % x.scale
# 6
info['Effects:'] = lambda x: ','.join(['%#8s' % i for i in x.included_effects])
out = OrderedDict()
for key, func in iteritems(info):
try:
out[key] = func(results)
# NOTE: some models don't have loglike defined (RLM), so that's NIE
except (AttributeError, KeyError, NotImplementedError):
pass
return out
def __init__(self,
endog,
exog=None,
order=(1, 0),
trend='c',
error_cov_type='unstructured',
measurement_error=False,
enforce_stationarity=True,
enforce_invertibility=True,
**kwargs):
# Model parameters
self.error_cov_type = error_cov_type
self.measurement_error = measurement_error
self.enforce_stationarity = enforce_stationarity
self.enforce_invertibility = enforce_invertibility
# Save the given orders
self.order = order
self.trend = trend
# Model orders
self.k_ar = int(order[0])
self.k_ma = int(order[1])
self.k_trend = int(self.trend == 'c')
# Check for valid model
if trend not in ['c', 'nc']:
raise ValueError('Invalid trend specification.')
if error_cov_type not in ['diagonal', 'unstructured']:
raise ValueError('Invalid error covariance matrix type'
' specification.')
if self.k_ar == 0 and self.k_ma == 0:
raise ValueError('Invalid VARMAX(p,q) specification; at least one'
' p,q must be greater than zero.')
# Warn for VARMA model
if self.k_ar > 0 and self.k_ma > 0:
warn('Estimation of VARMA(p,q) models is not generically robust,'
' due especially to identification issues.')
# Exogenous data
self.k_exog = 0
if exog is not None:
exog_is_using_pandas = _is_using_pandas(exog, None)
if not exog_is_using_pandas:
exog = np.asarray(exog)
# Make sure we have 2-dimensional array
if exog.ndim == 1:
if not exog_is_using_pandas:
exog = exog[:, None]
else:
exog = pd.DataFrame(exog)
self.k_exog = exog.shape[1]
# Note: at some point in the future might add state regression, as in
# SARIMAX.
self.mle_regression = self.k_exog > 0
# We need to have an array or pandas at this point
if not _is_using_pandas(endog, None):
endog = np.asanyarray(endog)
# Model order
# Used internally in various places
_min_k_ar = max(self.k_ar, 1)
self._k_order = _min_k_ar + self.k_ma
# Number of states
k_endog = endog.shape[1]
k_posdef = k_endog
k_states = k_endog * self._k_order
# By default, initialize as stationary
kwargs.setdefault('initialization', 'stationary')
# By default, use LU decomposition
kwargs.setdefault('inversion_method', INVERT_UNIVARIATE | SOLVE_LU)
# Initialize the state space model
super(VARMAX, self).__init__(endog,
exog=exog,
k_states=k_states,
k_posdef=k_posdef,
**kwargs)
# Initialize the parameters
self.parameters = OrderedDict()
self.parameters['trend'] = self.k_endog * self.k_trend
self.parameters['ar'] = self.k_endog**2 * self.k_ar
self.parameters['ma'] = self.k_endog**2 * self.k_ma
self.parameters['regression'] = self.k_endog * self.k_exog
if self.error_cov_type == 'diagonal':
self.parameters['state_cov'] = self.k_endog
# These parameters fill in a lower-triangular matrix which is then
# dotted with itself to get a positive definite matrix.
elif self.error_cov_type == 'unstructured':
self.parameters['state_cov'] = (int(self.k_endog *
(self.k_endog + 1) / 2))
self.parameters['obs_cov'] = self.k_endog * self.measurement_error
self.k_params = sum(self.parameters.values())
# Initialize known elements of the state space matrices
# If we have exog effects, then the state intercept needs to be
# time-varying
if self.k_exog > 0:
self.ssm['state_intercept'] = np.zeros((self.k_states, self.nobs))
# The design matrix is just an identity for the first k_endog states
idx = np.diag_indices(self.k_endog)
self.ssm[('design', ) + idx] = 1
# The transition matrix is described in four blocks, where the upper
# left block is in companion form with the autoregressive coefficient
# matrices (so it is shaped k_endog * k_ar x k_endog * k_ar) ...
if self.k_ar > 0:
idx = np.diag_indices((self.k_ar - 1) * self.k_endog)
idx = idx[0] + self.k_endog, idx[1]
self.ssm[('transition', ) + idx] = 1
# ... and the lower right block is in companion form with zeros as the
# coefficient matrices (it is shaped k_endog * k_ma x k_endog * k_ma).
idx = np.diag_indices((self.k_ma - 1) * self.k_endog)
idx = (idx[0] + (_min_k_ar + 1) * self.k_endog,
idx[1] + _min_k_ar * self.k_endog)
self.ssm[('transition', ) + idx] = 1
# The selection matrix is described in two blocks, where the upper
# block selects the all k_posdef errors in the first k_endog rows
# (the upper block is shaped k_endog * k_ar x k) and the lower block
# also selects all k_posdef errors in the first k_endog rows (the lower
# block is shaped k_endog * k_ma x k).
idx = np.diag_indices(self.k_endog)
self.ssm[('selection', ) + idx] = 1
idx = idx[0] + _min_k_ar * self.k_endog, idx[1]
if self.k_ma > 0:
self.ssm[('selection', ) + idx] = 1
# Cache some indices
if self.trend == 'c' and self.k_exog == 0:
self._idx_state_intercept = np.s_['state_intercept', :k_endog]
elif self.k_exog > 0:
self._idx_state_intercept = np.s_['state_intercept', :k_endog, :]
if self.k_ar > 0:
self._idx_transition = np.s_['transition', :k_endog, :]
else:
self._idx_transition = np.s_['transition', :k_endog, k_endog:]
if self.error_cov_type == 'diagonal':
self._idx_state_cov = (('state_cov', ) +
np.diag_indices(self.k_endog))
elif self.error_cov_type == 'unstructured':
self._idx_lower_state_cov = np.tril_indices(self.k_endog)
if self.measurement_error:
self._idx_obs_cov = ('obs_cov', ) + np.diag_indices(self.k_endog)
# Cache some slices
def _slice(key, offset):
length = self.parameters[key]
param_slice = np.s_[offset:offset + length]
offset += length
return param_slice, offset
offset = 0
self._params_trend, offset = _slice('trend', offset)
self._params_ar, offset = _slice('ar', offset)
self._params_ma, offset = _slice('ma', offset)
self._params_regression, offset = _slice('regression', offset)
self._params_state_cov, offset = _slice('state_cov', offset)
self._params_obs_cov, offset = _slice('obs_cov', offset)
'ho': 'Hommel',
'fdr_bh': 'FDR Benjamini-Hochberg',
'fdr_by': 'FDR Benjamini-Yekutieli',
'fdr_tsbh': 'FDR 2-stage Benjamini-Hochberg',
'fdr_tsbky': 'FDR 2-stage Benjamini-Krieger-Yekutieli',
'fdr_gbs': 'FDR adaptive Gavrilov-Benjamini-Sarkar'
}
_alias_list = [['b', 'bonf', 'bonferroni'], ['s', 'sidak'], ['h', 'holm'],
['hs', 'holm-sidak'], ['sh', 'simes-hochberg'],
['ho', 'hommel'], ['fdr_bh', 'fdr_i', 'fdr_p', 'fdri', 'fdrp'],
['fdr_by', 'fdr_n', 'fdr_c', 'fdrn', 'fdrcorr'],
['fdr_tsbh', 'fdr_2sbh'],
['fdr_tsbky', 'fdr_2sbky', 'fdr_twostage'], ['fdr_gbs']]
multitest_alias = OrderedDict()
for m in _alias_list:
multitest_alias[m[0]] = m[0]
for a in m[1:]:
multitest_alias[a] = m[0]
def multipletests(pvals,
alpha=0.05,
method='hs',
is_sorted=False,
returnsorted=False):
'''test results and p-value correction for multiple tests
Parameters
