def stack_sparse_frame(frame):
"""
Only makes sense when fill_value is NaN
"""
lengths = [s.sp_index.npoints for _, s in frame.iteritems()]
nobs = sum(lengths)
# this is pretty fast
minor_labels = np.repeat(np.arange(len(frame.columns)), lengths)
inds_to_concat = []
vals_to_concat = []
for _, series in frame.iteritems():
if not np.isnan(series.fill_value):
raise Exception('This routine assumes NaN fill value')
int_index = series.sp_index.to_int_index()
inds_to_concat.append(int_index.indices)
vals_to_concat.append(series.sp_values)
major_labels = np.concatenate(inds_to_concat)
stacked_values = np.concatenate(vals_to_concat)
index = MultiIndex(levels=[frame.index, frame.columns],
labels=[major_labels, minor_labels])
lp = LongPanel(stacked_values.reshape((nobs, 1)),
index=index,
columns=['foo'])
return lp.sortlevel(level=0)
def _unstack_vector(self, vec, index=None):
if index is None:
index = self._y_trans.index
panel = LongPanel(vec.reshape((len(vec), 1)),
index=index,
columns=['dummy'])
return panel.to_wide()['dummy']
def to_long(self, filter_observations=True):
"""
Convert SparsePanel to (dense) LongPanel
Returns
-------
lp : LongPanel
"""
if not filter_observations:
raise Exception('filter_observations=False not supported for '
'SparsePanel.to_long')
I, N, K = self.shape
counts = np.zeros(N * K, dtype=int)
d_values = {}
d_indexer = {}
for item in self.items:
frame = self[item]
values, major, minor = _stack_sparse_info(frame)
# values are stacked column-major
indexer = minor * N + major
counts.put(indexer, counts.take(indexer) + 1) # cuteness
d_values[item] = values
d_indexer[item] = indexer
# have full set of observations for each item
mask = counts == I
# for each item, take mask values at index locations for those sparse
# values, and use that to select values
values = np.column_stack([d_values[item][mask.take(d_indexer[item])]
for item in self.items])
inds, = mask.nonzero()
# still column major
major_labels = inds % N
minor_labels = inds // N
index = MultiIndex(levels=[self.major_axis, self.minor_axis],
labels=[major_labels, minor_labels])
lp = LongPanel(values, index=index, columns=self.items)
return lp.sortlevel(level=0)
def _convertDummies(dummies, mapping):
# cleans up the names of the generated dummies
new_items = []
for item in dummies.items:
if not mapping:
var = str(item)
if isinstance(item, float):
var = '%g' % item
new_items.append(var)
else:
# renames the dummies if a conversion dict is provided
new_items.append(mapping[int(item)])
dummies = LongPanel(dummies.values, index=dummies.index, columns=new_items)
return dummies
def _var_beta_panel(y, x, beta, xx, rmse, cluster_axis, nw_lags, nobs, df,
nw_overlap):
from pandas.core.frame import group_agg
from pandas.core.panel import LongPanel
xx_inv = math.inv(xx)
yv = y.values
if cluster_axis is None:
if nw_lags is None:
return xx_inv * (rmse**2)
else:
resid = yv - np.dot(x.values, beta)
m = (x.values.T * resid).T
xeps = math.newey_west(m, nw_lags, nobs, df, nw_overlap)
return np.dot(xx_inv, np.dot(xeps, xx_inv))
else:
Xb = np.dot(x.values, beta).reshape((len(x.values), 1))
resid = LongPanel(yv[:, None] - Xb, index=y.index, columns=['resid'])
if cluster_axis == 1:
x = x.swapaxes()
resid = resid.swapaxes()
m = group_agg(x.values * resid.values, x.index._bounds,
lambda x: np.sum(x, axis=0))
if nw_lags is None:
nw_lags = 0
xox = 0
for i in range(len(x.major_axis)):
xox += math.newey_west(m[i:i + 1], nw_lags, nobs, df, nw_overlap)
return np.dot(xx_inv, np.dot(xox, xx_inv))
