def test_wls_panel(self):
y = tm.makeTimeDataFrame()
x = Panel({'x1': tm.makeTimeDataFrame(),
'x2': tm.makeTimeDataFrame()})
y.iloc[[1, 7], y.columns.get_loc('A')] = np.nan
y.iloc[[6, 15], y.columns.get_loc('B')] = np.nan
y.iloc[[3, 20], y.columns.get_loc('C')] = np.nan
y.iloc[[5, 11], y.columns.get_loc('D')] = np.nan
stack_y = y.stack()
stack_x = DataFrame(dict((k, v.stack())
for k, v in x.iteritems()))
weights = x.std('items')
stack_weights = weights.stack()
stack_y.index = stack_y.index._tuple_index
stack_x.index = stack_x.index._tuple_index
stack_weights.index = stack_weights.index._tuple_index
with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
result = ols(y=y, x=x, weights=1 / weights)
with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
expected = ols(y=stack_y, x=stack_x, weights=1 / stack_weights)
assert_almost_equal(result.beta, expected.beta)
for attr in ['resid', 'y_fitted']:
rvals = getattr(result, attr).stack().values
evals = getattr(expected, attr).values
assert_almost_equal(rvals, evals)
def test_wls_panel(self):
y = tm.makeTimeDataFrame()
x = Panel({'x1' : tm.makeTimeDataFrame(),
'x2' : tm.makeTimeDataFrame()})
y.ix[[1, 7], 'A'] = np.nan
y.ix[[6, 15], 'B'] = np.nan
y.ix[[3, 20], 'C'] = np.nan
y.ix[[5, 11], 'D'] = np.nan
stack_y = y.stack()
stack_x = DataFrame(dict((k, v.stack())
for k, v in x.iteritems()))
weights = x.std('items')
stack_weights = weights.stack()
stack_y.index = stack_y.index.get_tuple_index()
stack_x.index = stack_x.index.get_tuple_index()
stack_weights.index = stack_weights.index.get_tuple_index()
result = ols(y=y, x=x, weights=1/weights)
expected = ols(y=stack_y, x=stack_x, weights=1/stack_weights)
assert_almost_equal(result.beta, expected.beta)
for attr in ['resid', 'y_fitted']:
rvals = getattr(result, attr).stack().values
evals = getattr(expected, attr).values
assert_almost_equal(rvals, evals)
def test_ix_setitem_slice_dataframe(self):
a = Panel(items=[1, 2, 3], major_axis=[11, 22, 33], minor_axis=[111, 222, 333])
b = DataFrame(np.random.randn(2, 3), index=[111, 333], columns=[1, 2, 3])
a.ix[:, 22, [111, 333]] = b
assert_frame_equal(a.ix[:, 22, [111, 333]], b)
def test_wls_panel(self):
y = tm.makeTimeDataFrame()
x = Panel({"x1": tm.makeTimeDataFrame(), "x2": tm.makeTimeDataFrame()})
y.ix[[1, 7], "A"] = np.nan
y.ix[[6, 15], "B"] = np.nan
y.ix[[3, 20], "C"] = np.nan
y.ix[[5, 11], "D"] = np.nan
stack_y = y.stack()
stack_x = DataFrame(dict((k, v.stack()) for k, v in x.iteritems()))
weights = x.std("items")
stack_weights = weights.stack()
stack_y.index = stack_y.index.get_tuple_index()
stack_x.index = stack_x.index.get_tuple_index()
stack_weights.index = stack_weights.index.get_tuple_index()
result = ols(y=y, x=x, weights=1 / weights)
expected = ols(y=stack_y, x=stack_x, weights=1 / stack_weights)
assert_almost_equal(result.beta, expected.beta)
for attr in ["resid", "y_fitted"]:
rvals = getattr(result, attr).stack().values
evals = getattr(expected, attr).values
assert_almost_equal(rvals, evals)
def test_delitem_and_pop(self):
expected = self.panel['ItemA']
result = self.panel.pop('ItemA')
assert_frame_equal(expected, result)
self.assert_('ItemA' not in self.panel.items)
del self.panel['ItemB']
self.assert_('ItemB' not in self.panel.items)
self.assertRaises(Exception, self.panel.__delitem__, 'ItemB')
values = np.empty((3, 3, 3))
values[0] = 0
values[1] = 1
values[2] = 2
panel = Panel(values, range(3), range(3), range(3))
# did we delete the right row?
panelc = panel.copy()
del panelc[0]
assert_frame_equal(panelc[1], panel[1])
assert_frame_equal(panelc[2], panel[2])
panelc = panel.copy()
del panelc[1]
assert_frame_equal(panelc[0], panel[0])
assert_frame_equal(panelc[2], panel[2])
panelc = panel.copy()
del panelc[2]
assert_frame_equal(panelc[1], panel[1])
assert_frame_equal(panelc[0], panel[0])
def test_wls_panel(self):
y = tm.makeTimeDataFrame()
x = Panel({"x1": tm.makeTimeDataFrame(), "x2": tm.makeTimeDataFrame()})
y.ix[[1, 7], "A"] = np.nan
y.ix[[6, 15], "B"] = np.nan
y.ix[[3, 20], "C"] = np.nan
y.ix[[5, 11], "D"] = np.nan
stack_y = y.stack()
stack_x = DataFrame(dict((k, v.stack()) for k, v in compat.iteritems(x)))
weights = x.std("items")
stack_weights = weights.stack()
stack_y.index = stack_y.index._tuple_index
stack_x.index = stack_x.index._tuple_index
stack_weights.index = stack_weights.index._tuple_index
with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
result = ols(y=y, x=x, weights=1 / weights)
with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
expected = ols(y=stack_y, x=stack_x, weights=1 / stack_weights)
assert_almost_equal(result.beta, expected.beta)
for attr in ["resid", "y_fitted"]:
rvals = getattr(result, attr).stack().values
evals = getattr(expected, attr).values
assert_almost_equal(rvals, evals)
def __call__(self, forecasts):
items = list(range(len(forecasts)))
means = [forecast.mean for forecast in forecasts]
means = zip(items, means)
stds = [forecast.sd for forecast in forecasts]
stds = zip(items, stds)
means = Panel({item:frame for item, frame in means})
stds = Panel({item:frame for item, frame in stds})
return MeanForecast(means.mean(axis = "items"), stds.mean(axis = "items"))
def test_multiindex_blocks(self):
ind = MultiIndex.from_tuples([("a", 1), ("a", 2), ("b", 1)], names=["first", "second"])
wp = Panel(self.panel._data)
wp.items = ind
f1 = wp["a"]
self.assert_((f1.items == [1, 2]).all())
f1 = wp[("b", 1)]
self.assert_((f1.columns == ["A", "B", "C", "D"]).all())
def test_multiindex_blocks(self):
ind = MultiIndex.from_tuples([('a', 1), ('a', 2), ('b', 1)],
names=['first', 'second'])
wp = Panel(self.panel._data)
wp.items = ind
f1 = wp['a']
self.assert_((f1.items == [1, 2]).all())
f1 = wp[('b',1)]
self.assert_((f1.columns == ['A', 'B', 'C', 'D']).all())
def test_ctor_dict(self):
itema = self.panel["ItemA"]
itemb = self.panel["ItemB"]
d = {"A": itema, "B": itemb[5:]}
d2 = {"A": itema._series, "B": itemb[5:]._series}
d3 = {"A": None, "B": DataFrame(itemb[5:]._series), "C": DataFrame(itema._series)}
wp = Panel.from_dict(d)
wp2 = Panel.from_dict(d2) # nested Dict
wp3 = Panel.from_dict(d3)
self.assert_(wp.major_axis.equals(self.panel.major_axis))
assert_panel_equal(wp, wp2)
# intersect
wp = Panel.from_dict(d, intersect=True)
self.assert_(wp.major_axis.equals(itemb.index[5:]))
# use constructor
assert_panel_equal(Panel(d), Panel.from_dict(d))
assert_panel_equal(Panel(d2), Panel.from_dict(d2))
assert_panel_equal(Panel(d3), Panel.from_dict(d3))
# a pathological case
d4 = {"A": None, "B": None}
wp4 = Panel.from_dict(d4)
assert_panel_equal(Panel(d4), Panel(items=["A", "B"]))
# cast
dcasted = dict((k, v.reindex(wp.major_axis).fillna(0)) for k, v in d.iteritems())
result = Panel(dcasted, dtype=int)
expected = Panel(dict((k, v.astype(int)) for k, v in dcasted.iteritems()))
assert_panel_equal(result, expected)
def test_ix_align(self):
from pandas import Series
b = Series(np.random.randn(10))
b.sort()
df_orig = Panel(np.random.randn(3, 10, 2))
df = df_orig.copy()
df.ix[0, :, 0] = b
assert_series_equal(df.ix[0, :, 0].reindex(b.index), b)
df = df_orig.swapaxes(0, 1)
df.ix[:, 0, 0] = b
assert_series_equal(df.ix[:, 0, 0].reindex(b.index), b)
df = df_orig.swapaxes(1, 2)
df.ix[0, 0, :] = b
assert_series_equal(df.ix[0, 0, :].reindex(b.index), b)
def test_ctor_dict(self):
itema = self.panel['ItemA']
itemb = self.panel['ItemB']
d = {'A' : itema, 'B' : itemb[5:]}
d2 = {'A' : itema._series, 'B' : itemb[5:]._series}
d3 = {'A' : DataFrame(itema._series),
'B' : DataFrame(itemb[5:]._series)}
wp = Panel.from_dict(d)
wp2 = Panel.from_dict(d2) # nested Dict
wp3 = Panel.from_dict(d3)
self.assert_(wp.major_axis.equals(self.panel.major_axis))
assert_panel_equal(wp, wp2)
# intersect
wp = Panel.from_dict(d, intersect=True)
self.assert_(wp.major_axis.equals(itemb.index[5:]))
# use constructor
assert_panel_equal(Panel(d), Panel.from_dict(d))
assert_panel_equal(Panel(d2), Panel.from_dict(d2))
assert_panel_equal(Panel(d3), Panel.from_dict(d3))
# cast
result = Panel(d, dtype=int)
expected = Panel(dict((k, v.astype(int)) for k, v in d.iteritems()))
def var_beta(self):
"""Returns the covariance of beta."""
result = {}
result_index = self._result_index
for i in range(len(self._var_beta_raw)):
dm = DataFrame(self._var_beta_raw[i], columns=self.beta.columns, index=self.beta.columns)
result[result_index[i]] = dm
return Panel.from_dict(result, intersect=False)
def test_from_dict_mixed_orient(self):
df = tm.makeDataFrame()
df["foo"] = "bar"
data = {"k1": df, "k2": df}
panel = Panel.from_dict(data, orient="minor")
self.assert_(panel["foo"].values.dtype == np.object_)
self.assert_(panel["A"].values.dtype == np.float64)
def test_update_from_dict(self):
pan = Panel(
{
"one": DataFrame([[1.5, np.nan, 3], [1.5, np.nan, 3], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]]),
"two": DataFrame([[1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]]),
}
)
other = {"two": DataFrame([[3.6, 2.0, np.nan], [np.nan, np.nan, 7]])}
pan.update(other)
expected = Panel(
{
"two": DataFrame([[3.6, 2.0, 3], [1.5, np.nan, 7], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]]),
"one": DataFrame([[1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]]),
}
)
assert_panel_equal(pan, expected)
def test_update_nooverwrite(self):
pan = Panel(
[
[[1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
[[1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
]
)
other = Panel([[[3.6, 2.0, np.nan], [np.nan, np.nan, 7]]], items=[1])
pan.update(other, overwrite=False)
expected = Panel(
[
[[1.5, np.nan, 3], [1.5, np.nan, 3], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
[[1.5, 2.0, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
]
)
assert_panel_equal(pan, expected)
def test_update_filtered(self):
pan = Panel(
[
[[1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
[[1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
]
)
other = Panel([[[3.6, 2.0, np.nan], [np.nan, np.nan, 7]]], items=[1])
pan.update(other, filter_func=lambda x: x > 2)
expected = Panel(
[
[[1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
[[1.5, np.nan, 3], [1.5, np.nan, 7], [1.5, np.nan, 3.0], [1.5, np.nan, 3.0]],
]
)
assert_panel_equal(pan, expected)
def test_ix_frame_align(self):
from pandas import DataFrame
df = DataFrame(np.random.randn(2, 10))
df.sort_index(inplace=True)
p_orig = Panel(np.random.randn(3, 10, 2))
p = p_orig.copy()
p.ix[0, :, :] = df
out = p.ix[0, :, :].T.reindex(df.index, columns=df.columns)
assert_frame_equal(out, df)
p = p_orig.copy()
p.ix[0] = df
out = p.ix[0].T.reindex(df.index, columns=df.columns)
assert_frame_equal(out, df)
p = p_orig.copy()
p.ix[0, [0, 1, 3, 5], -2:] = df
out = p.ix[0, [0, 1, 3, 5], -2:]
assert_frame_equal(out, df.T.reindex([0, 1, 3, 5], p.minor_axis[-2:]))
def test_from_dict_mixed_orient(self):
df = tm.makeDataFrame()
df['foo'] = 'bar'
data = {'k1' : df,
'k2' : df}
panel = Panel.from_dict(data, orient='minor')
self.assert_(panel['foo'].values.dtype == np.object_)
self.assert_(panel['A'].values.dtype == np.float64)
def _filter_data(self):
"""
"""
data = self._x_orig
cat_mapping = {}
if isinstance(data, LongPanel):
data = data.to_wide()
else:
if isinstance(data, Panel):
data = data.copy()
if not isinstance(data, SparsePanel):
data, cat_mapping = self._convert_x(data)
if not isinstance(data, Panel):
data = Panel.from_dict(data, intersect=True)
x_names = data.items
if self._weights is not None:
data['__weights__'] = self._weights
# Filter x's without y (so we can make a prediction)
filtered = data.to_long()
# Filter all data together using to_long
# convert to DataFrame
y = self._y_orig
if isinstance(y, Series):
y = y.unstack()
data['__y__'] = y
data_long = data.to_long()
x_filt = filtered.filter(x_names)
if self._weights:
weights_filt = filtered['__weights__']
else:
weights_filt = None
x = data_long.filter(x_names)
y = data_long['__y__']
if self._weights:
weights = data_long['__weights__']
else:
weights = None
return x, x_filt, y, weights, weights_filt, cat_mapping
def resid(self):
"""
Returns the DataFrame containing the residuals of the VAR regressions.
Each column x1 contains the residuals generated by regressing the x1
column of the input against the lagged input.
Returns
-------
DataFrame
"""
d = dict([(key, value.resid)
for (key, value) in self.ols_results.iteritems()])
return Panel.fromDict(d)
def _filter_data(self):
"""
"""
data = self._x_orig
cat_mapping = {}
if isinstance(data, LongPanel):
data = data.to_wide()
else:
if isinstance(data, Panel):
data = data.copy()
if not isinstance(data, SparsePanel):
data, cat_mapping = self._convert_x(data)
if not isinstance(data, Panel):
data = Panel.from_dict(data, intersect=True)
x_names = data.items
if self._weights is not None:
data['__weights__'] = self._weights
# Filter x's without y (so we can make a prediction)
filtered = data.to_long()
# Filter all data together using to_long
# convert to DataFrame
y = self._y_orig
if isinstance(y, Series):
y = y.unstack()
data['__y__'] = y
data_long = data.to_long()
x_filt = filtered.filter(x_names)
x = data_long.filter(x_names)
y = data_long['__y__']
if self._weights:
weights = data_long['__weights__']
else:
weights = None
return x, x_filt, y, weights, cat_mapping
def test_logical_with_nas(self):
d = Panel({
'ItemA': {
'a': [np.nan, False]
},
'ItemB': {
'a': [True, True]
}
})
result = d['ItemA'] | d['ItemB']
expected = DataFrame({'a': [np.nan, True]})
assert_frame_equal(result, expected)
# this is autodowncasted here
result = d['ItemA'].fillna(False) | d['ItemB']
expected = DataFrame({'a': [True, True]})
assert_frame_equal(result, expected)
def test_to_frame_multi_major(self):
idx = MultiIndex.from_tuples([(1, 'one'), (1, 'two'), (2, 'one'),
(2, 'two')])
df = DataFrame([[1, 'a', 1], [2, 'b', 1], [3, 'c', 1], [4, 'd', 1]],
columns=['A', 'B', 'C'],
index=idx)
wp = Panel({'i1': df, 'i2': df})
expected_idx = MultiIndex.from_tuples([
(1, 'one', 'A'), (1, 'one', 'B'), (1, 'one', 'C'), (1, 'two', 'A'),
(1, 'two', 'B'), (1, 'two', 'C'), (2, 'one', 'A'), (2, 'one', 'B'),
(2, 'one', 'C'), (2, 'two', 'A'), (2, 'two', 'B'), (2, 'two', 'C')
],
names=[None, None, 'minor'])
expected = DataFrame(
{
'i1': [1, 'a', 1, 2, 'b', 1, 3, 'c', 1, 4, 'd', 1],
'i2': [1, 'a', 1, 2, 'b', 1, 3, 'c', 1, 4, 'd', 1]
},
index=expected_idx)
result = wp.to_frame()
assert_frame_equal(result, expected)
wp.iloc[0, 0].iloc[0] = np.nan # BUG on setting. GH #5773
result = wp.to_frame()
assert_frame_equal(result, expected[1:])
idx = MultiIndex.from_tuples([(1, 'two'), (1, 'one'), (2, 'one'),
(np.nan, 'two')])
df = DataFrame([[1, 'a', 1], [2, 'b', 1], [3, 'c', 1], [4, 'd', 1]],
columns=['A', 'B', 'C'],
index=idx)
wp = Panel({'i1': df, 'i2': df})
ex_idx = MultiIndex.from_tuples([(1, 'two', 'A'), (1, 'two', 'B'),
(1, 'two', 'C'), (1, 'one', 'A'),
(1, 'one', 'B'), (1, 'one', 'C'),
(2, 'one', 'A'), (2, 'one', 'B'),
(2, 'one', 'C'), (np.nan, 'two', 'A'),
(np.nan, 'two', 'B'),
(np.nan, 'two', 'C')],
names=[None, None, 'minor'])
expected.index = ex_idx
result = wp.to_frame()
assert_frame_equal(result, expected)
def _aggregate_generic(self, func, *args, **kwargs):
result = {}
axis = self.axis
obj = self._obj_with_exclusions
for name in self.primary:
data = self.get_group(name, obj=obj)
try:
result[name] = func(data, *args, **kwargs)
except Exception:
wrapper = lambda x: func(x, *args, **kwargs)
result[name] = data.apply(wrapper, axis=axis)
result = Panel.fromDict(result, intersect=False)
if axis > 0:
result = result.swapaxes(0, axis)
return result
def test_panel_dups(self):
# GH 4960
# duplicates in an index
# items
data = np.random.randn(5, 100, 5)
no_dup_panel = Panel(data, items=list("ABCDE"))
panel = Panel(data, items=list("AACDE"))
expected = no_dup_panel['A']
result = panel.iloc[0]
assert_frame_equal(result, expected)
expected = no_dup_panel['E']
result = panel.loc['E']
assert_frame_equal(result, expected)
# major
data = np.random.randn(5, 5, 5)
no_dup_panel = Panel(data, major_axis=list("ABCDE"))
panel = Panel(data, major_axis=list("AACDE"))
expected = no_dup_panel.loc[:, 'A']
result = panel.iloc[:, 0]
assert_frame_equal(result, expected)
expected = no_dup_panel.loc[:, 'E']
result = panel.loc[:, 'E']
assert_frame_equal(result, expected)
# minor
data = np.random.randn(5, 100, 5)
no_dup_panel = Panel(data, minor_axis=list("ABCDE"))
panel = Panel(data, minor_axis=list("AACDE"))
expected = no_dup_panel.loc[:, :, 'A']
result = panel.iloc[:, :, 0]
assert_frame_equal(result, expected)
expected = no_dup_panel.loc[:, :, 'E']
result = panel.loc[:, :, 'E']
assert_frame_equal(result, expected)
def test_dropna(self):
p = Panel(np.random.randn(4, 5, 6), major_axis=list("abcde"))
p.ix[:, ["b", "d"], 0] = np.nan
result = p.dropna(axis=1)
exp = p.ix[:, ["a", "c", "e"], :]
assert_panel_equal(result, exp)
result = p.dropna(axis=1, how="all")
assert_panel_equal(result, p)
p.ix[:, ["b", "d"], :] = np.nan
result = p.dropna(axis=1, how="all")
exp = p.ix[:, ["a", "c", "e"], :]
assert_panel_equal(result, exp)
p = Panel(np.random.randn(4, 5, 6), items=list("abcd"))
p.ix[["b"], :, 0] = np.nan
result = p.dropna()
exp = p.ix[["a", "c", "d"]]
assert_panel_equal(result, exp)
result = p.dropna(how="all")
assert_panel_equal(result, p)
p.ix["b"] = np.nan
result = p.dropna(how="all")
exp = p.ix[["a", "c", "d"]]
assert_panel_equal(result, exp)
def test_dropna(self):
p = Panel(np.random.randn(4, 5, 6), major_axis=list('abcde'))
p.ix[:, ['b', 'd'], 0] = np.nan
result = p.dropna(axis=1)
exp = p.ix[:, ['a', 'c', 'e'], :]
assert_panel_equal(result, exp)
result = p.dropna(axis=1, how='all')
assert_panel_equal(result, p)
p.ix[:, ['b', 'd'], :] = np.nan
result = p.dropna(axis=1, how='all')
exp = p.ix[:, ['a', 'c', 'e'], :]
assert_panel_equal(result, exp)
p = Panel(np.random.randn(4, 5, 6), items=list('abcd'))
p.ix[['b'], :, 0] = np.nan
result = p.dropna()
exp = p.ix[['a', 'c', 'd']]
assert_panel_equal(result, exp)
result = p.dropna(how='all')
assert_panel_equal(result, p)
p.ix['b'] = np.nan
result = p.dropna(how='all')
exp = p.ix[['a', 'c', 'd']]
assert_panel_equal(result, exp)
class TestPanel(PanelTests, CheckIndexing, SafeForSparse):
def test_constructor_cast(self):
# can't cast
data = [[['foo', 'bar', 'baz']]]
pytest.raises(ValueError, Panel, data, dtype=float)
def test_constructor_empty_panel(self):
empty = Panel()
assert len(empty.items) == 0
assert len(empty.major_axis) == 0
assert len(empty.minor_axis) == 0
def test_constructor_observe_dtype(self):
# GH #411
panel = Panel(items=lrange(3),
major_axis=lrange(3),
minor_axis=lrange(3),
dtype='O')
assert panel.values.dtype == np.object_
def test_constructor_dtypes(self):
# GH #797
def _check_dtype(panel, dtype):
for i in panel.items:
assert panel[i].values.dtype.name == dtype
# only nan holding types allowed here
for dtype in ['float64', 'float32', 'object']:
panel = Panel(items=lrange(2),
major_axis=lrange(10),
minor_axis=lrange(5),
dtype=dtype)
_check_dtype(panel, dtype)
for dtype in ['float64', 'float32', 'int64', 'int32', 'object']:
panel = Panel(np.array(np.random.randn(2, 10, 5), dtype=dtype),
items=lrange(2),
major_axis=lrange(10),
minor_axis=lrange(5),
dtype=dtype)
_check_dtype(panel, dtype)
for dtype in ['float64', 'float32', 'int64', 'int32', 'object']:
panel = Panel(np.array(np.random.randn(2, 10, 5), dtype='O'),
items=lrange(2),
major_axis=lrange(10),
minor_axis=lrange(5),
dtype=dtype)
_check_dtype(panel, dtype)
for dtype in ['float64', 'float32', 'int64', 'int32', 'object']:
panel = Panel(np.random.randn(2, 10, 5),
items=lrange(2),
major_axis=lrange(10),
minor_axis=lrange(5),
dtype=dtype)
_check_dtype(panel, dtype)
for dtype in ['float64', 'float32', 'int64', 'int32', 'object']:
df1 = DataFrame(np.random.randn(2, 5),
index=lrange(2),
columns=lrange(5))
df2 = DataFrame(np.random.randn(2, 5),
index=lrange(2),
columns=lrange(5))
panel = Panel.from_dict({'a': df1, 'b': df2}, dtype=dtype)
_check_dtype(panel, dtype)
def test_constructor_fails_with_not_3d_input(self):
msg = "The number of dimensions required is 3"
with pytest.raises(ValueError, match=msg):
Panel(np.random.randn(10, 2))
def test_ctor_orderedDict(self):
keys = list(set(np.random.randint(
0, 5000, 100)))[:50] # unique random int keys
d = OrderedDict([(k, mkdf(10, 5)) for k in keys])
p = Panel(d)
assert list(p.items) == keys
p = Panel.from_dict(d)
assert list(p.items) == keys
def test_from_dict_mixed_orient(self):
df = tm.makeDataFrame()
df['foo'] = 'bar'
data = {'k1': df, 'k2': df}
panel = Panel.from_dict(data, orient='minor')
assert panel['foo'].values.dtype == np.object_
assert panel['A'].values.dtype == np.float64
def test_constructor_error_msgs(self):
msg = (r"Shape of passed values is \(3, 4, 5\), "
r"indices imply \(4, 5, 5\)")
with pytest.raises(ValueError, match=msg):
Panel(np.random.randn(3, 4, 5), lrange(4), lrange(5), lrange(5))
msg = (r"Shape of passed values is \(3, 4, 5\), "
r"indices imply \(5, 4, 5\)")
with pytest.raises(ValueError, match=msg):
Panel(np.random.randn(3, 4, 5), lrange(5), lrange(4), lrange(5))
msg = (r"Shape of passed values is \(3, 4, 5\), "
r"indices imply \(5, 5, 4\)")
with pytest.raises(ValueError, match=msg):
Panel(np.random.randn(3, 4, 5), lrange(5), lrange(5), lrange(4))
def test_apply_slabs(self):
# with multi-indexes
# GH7469
index = MultiIndex.from_tuples([('one', 'a'), ('one', 'b'),
('two', 'a'), ('two', 'b')])
dfa = DataFrame(np.array(np.arange(12, dtype='int64')).reshape(4, 3),
columns=list("ABC"),
index=index)
dfb = DataFrame(np.array(np.arange(10, 22,
dtype='int64')).reshape(4, 3),
columns=list("ABC"),
index=index)
p = Panel({'f': dfa, 'g': dfb})
result = p.apply(lambda x: x.sum(), axis=0)
# on windows this will be in32
result = result.astype('int64')
expected = p.sum(0)
assert_frame_equal(result, expected)
def test_apply_no_or_zero_ndim(self):
# GH10332
self.panel = Panel(np.random.rand(5, 5, 5))
result_int = self.panel.apply(lambda df: 0, axis=[1, 2])
result_float = self.panel.apply(lambda df: 0.0, axis=[1, 2])
result_int64 = self.panel.apply(lambda df: np.int64(0), axis=[1, 2])
result_float64 = self.panel.apply(lambda df: np.float64(0.0),
axis=[1, 2])
expected_int = expected_int64 = Series([0] * 5)
expected_float = expected_float64 = Series([0.0] * 5)
assert_series_equal(result_int, expected_int)
assert_series_equal(result_int64, expected_int64)
assert_series_equal(result_float, expected_float)
assert_series_equal(result_float64, expected_float64)
def test_fillna(self):
# limit not implemented when only value is specified
p = Panel(np.random.randn(3, 4, 5))
p.iloc[0:2, 0:2, 0:2] = np.nan
pytest.raises(NotImplementedError, lambda: p.fillna(999, limit=1))
def test_to_frame_multi_major(self):
idx = MultiIndex.from_tuples([(1, 'one'), (1, 'two'), (2, 'one'),
(2, 'two')])
df = DataFrame([[1, 'a', 1], [2, 'b', 1], [3, 'c', 1], [4, 'd', 1]],
columns=['A', 'B', 'C'],
index=idx)
wp = Panel({'i1': df, 'i2': df})
expected_idx = MultiIndex.from_tuples([
(1, 'one', 'A'), (1, 'one', 'B'), (1, 'one', 'C'), (1, 'two', 'A'),
(1, 'two', 'B'), (1, 'two', 'C'), (2, 'one', 'A'), (2, 'one', 'B'),
(2, 'one', 'C'), (2, 'two', 'A'), (2, 'two', 'B'), (2, 'two', 'C')
],
names=[None, None, 'minor'])
expected = DataFrame(
{
'i1': [1, 'a', 1, 2, 'b', 1, 3, 'c', 1, 4, 'd', 1],
'i2': [1, 'a', 1, 2, 'b', 1, 3, 'c', 1, 4, 'd', 1]
},
index=expected_idx)
result = wp.to_frame()
assert_frame_equal(result, expected)
wp.iloc[0, 0].iloc[0] = np.nan # BUG on setting. GH #5773
result = wp.to_frame()
assert_frame_equal(result, expected[1:])
idx = MultiIndex.from_tuples([(1, 'two'), (1, 'one'), (2, 'one'),
(np.nan, 'two')])
df = DataFrame([[1, 'a', 1], [2, 'b', 1], [3, 'c', 1], [4, 'd', 1]],
columns=['A', 'B', 'C'],
index=idx)
wp = Panel({'i1': df, 'i2': df})
ex_idx = MultiIndex.from_tuples([(1, 'two', 'A'), (1, 'two', 'B'),
(1, 'two', 'C'), (1, 'one', 'A'),
(1, 'one', 'B'), (1, 'one', 'C'),
(2, 'one', 'A'), (2, 'one', 'B'),
(2, 'one', 'C'), (np.nan, 'two', 'A'),
(np.nan, 'two', 'B'),
(np.nan, 'two', 'C')],
names=[None, None, 'minor'])
expected.index = ex_idx
result = wp.to_frame()
assert_frame_equal(result, expected)
def test_to_frame_multi_major_minor(self):
cols = MultiIndex(levels=[['C_A', 'C_B'], ['C_1', 'C_2']],
codes=[[0, 0, 1, 1], [0, 1, 0, 1]])
idx = MultiIndex.from_tuples([(1, 'one'), (1, 'two'), (2, 'one'),
(2, 'two'), (3, 'three'), (4, 'four')])
df = DataFrame(
[[1, 2, 11, 12], [3, 4, 13, 14], ['a', 'b', 'w', 'x'],
['c', 'd', 'y', 'z'], [-1, -2, -3, -4], [-5, -6, -7, -8]],
columns=cols,
index=idx)
wp = Panel({'i1': df, 'i2': df})
exp_idx = MultiIndex.from_tuples([(1, 'one', 'C_A', 'C_1'),
(1, 'one', 'C_A', 'C_2'),
(1, 'one', 'C_B', 'C_1'),
(1, 'one', 'C_B', 'C_2'),
(1, 'two', 'C_A', 'C_1'),
(1, 'two', 'C_A', 'C_2'),
(1, 'two', 'C_B', 'C_1'),
(1, 'two', 'C_B', 'C_2'),
(2, 'one', 'C_A', 'C_1'),
(2, 'one', 'C_A', 'C_2'),
(2, 'one', 'C_B', 'C_1'),
(2, 'one', 'C_B', 'C_2'),
(2, 'two', 'C_A', 'C_1'),
(2, 'two', 'C_A', 'C_2'),
(2, 'two', 'C_B', 'C_1'),
(2, 'two', 'C_B', 'C_2'),
(3, 'three', 'C_A', 'C_1'),
(3, 'three', 'C_A', 'C_2'),
(3, 'three', 'C_B', 'C_1'),
(3, 'three', 'C_B', 'C_2'),
(4, 'four', 'C_A', 'C_1'),
(4, 'four', 'C_A', 'C_2'),
(4, 'four', 'C_B', 'C_1'),
(4, 'four', 'C_B', 'C_2')],
names=[None, None, None, None])
exp_val = [[1, 1], [2, 2], [11, 11], [12, 12], [3, 3], [4, 4],
[13, 13], [14, 14], ['a', 'a'], ['b', 'b'], ['w', 'w'],
['x', 'x'], ['c', 'c'], ['d', 'd'], ['y', 'y'], ['z', 'z'],
[-1, -1], [-2, -2], [-3, -3], [-4, -4], [-5, -5], [-6, -6],
[-7, -7], [-8, -8]]
result = wp.to_frame()
expected = DataFrame(exp_val, columns=['i1', 'i2'], index=exp_idx)
assert_frame_equal(result, expected)
def test_to_frame_multi_drop_level(self):
idx = MultiIndex.from_tuples([(1, 'one'), (2, 'one'), (2, 'two')])
df = DataFrame({'A': [np.nan, 1, 2]}, index=idx)
wp = Panel({'i1': df, 'i2': df})
result = wp.to_frame()
exp_idx = MultiIndex.from_tuples([(2, 'one', 'A'), (2, 'two', 'A')],
names=[None, None, 'minor'])
expected = DataFrame({'i1': [1., 2], 'i2': [1., 2]}, index=exp_idx)
assert_frame_equal(result, expected)
def test_panel_dups(self):
# GH 4960
# duplicates in an index
# items
data = np.random.randn(5, 100, 5)
no_dup_panel = Panel(data, items=list("ABCDE"))
panel = Panel(data, items=list("AACDE"))
expected = no_dup_panel['A']
result = panel.iloc[0]
assert_frame_equal(result, expected)
expected = no_dup_panel['E']
result = panel.loc['E']
assert_frame_equal(result, expected)
# major
data = np.random.randn(5, 5, 5)
no_dup_panel = Panel(data, major_axis=list("ABCDE"))
panel = Panel(data, major_axis=list("AACDE"))
expected = no_dup_panel.loc[:, 'A']
result = panel.iloc[:, 0]
assert_frame_equal(result, expected)
expected = no_dup_panel.loc[:, 'E']
result = panel.loc[:, 'E']
assert_frame_equal(result, expected)
# minor
data = np.random.randn(5, 100, 5)
no_dup_panel = Panel(data, minor_axis=list("ABCDE"))
panel = Panel(data, minor_axis=list("AACDE"))
expected = no_dup_panel.loc[:, :, 'A']
result = panel.iloc[:, :, 0]
assert_frame_equal(result, expected)
expected = no_dup_panel.loc[:, :, 'E']
result = panel.loc[:, :, 'E']
assert_frame_equal(result, expected)
def test_filter(self):
pass
def test_shift(self):
# mixed dtypes #6959
data = [('item ' + ch, makeMixedDataFrame()) for ch in list('abcde')]
data = dict(data)
mixed_panel = Panel.from_dict(data, orient='minor')
shifted = mixed_panel.shift(1)
assert_series_equal(mixed_panel.dtypes, shifted.dtypes)
def test_numpy_round(self):
values = [[[-3.2, 2.2], [0, -4.8213], [3.123, 123.12],
[-1566.213, 88.88], [-12, 94.5]],
[[-5.82, 3.5], [6.21, -73.272], [-9.087, 23.12],
[272.212, -99.99], [23, -76.5]]]
p = Panel(values,
items=['Item1', 'Item2'],
major_axis=date_range('1/1/2000', periods=5),
minor_axis=['A', 'B'])
msg = "the 'out' parameter is not supported"
with pytest.raises(ValueError, match=msg):
np.round(p, out=p)
# removing Panel before NumPy enforces, so just ignore
@pytest.mark.filterwarnings("ignore:Using a non-tuple:FutureWarning")
def test_multiindex_get(self):
ind = MultiIndex.from_tuples([('a', 1), ('a', 2), ('b', 1), ('b', 2)],
names=['first', 'second'])
wp = Panel(np.random.random((4, 5, 5)),
items=ind,
major_axis=np.arange(5),
minor_axis=np.arange(5))
f1 = wp['a']
f2 = wp.loc['a']
assert (f1.items == [1, 2]).all()
assert (f2.items == [1, 2]).all()
MultiIndex.from_tuples([('a', 1), ('a', 2), ('b', 1)],
names=['first', 'second'])
def test_repr_empty(self):
empty = Panel()
repr(empty)
@pytest.mark.parametrize(
'bad_kwarg, exception, msg',
[
# errors must be 'ignore' or 'raise'
({
'errors': 'something'
}, ValueError, 'The parameter errors must.*'),
({
'join': 'inner'
}, NotImplementedError, 'Only left join is supported')
])
def test_update_raise_bad_parameter(self, bad_kwarg, exception, msg):
pan = Panel([[[1.5, np.nan, 3.]]])
with pytest.raises(exception, match=msg):
pan.update(pan, **bad_kwarg)
def test_update_raise_on_overlap(self):
pan = Panel([[[1.5, np.nan, 3.], [1.5, np.nan, 3.], [1.5, np.nan, 3.],
[1.5, np.nan, 3.]],
[[1.5, np.nan, 3.], [1.5, np.nan, 3.], [1.5, np.nan, 3.],
[1.5, np.nan, 3.]]])
with pytest.raises(ValueError, match='Data overlaps'):
pan.update(pan, errors='raise')
@pytest.mark.parametrize('raise_conflict', [True, False])
def test_update_deprecation(self, raise_conflict):
pan = Panel([[[1.5, np.nan, 3.]]])
other = Panel([[[]]])
with tm.assert_produces_warning(FutureWarning):
pan.update(other, raise_conflict=raise_conflict)
def test_repr_empty(self):
empty = Panel()
repr(empty)
def test_dropna(self):
p = Panel(np.random.randn(4, 5, 6), major_axis=list('abcde'))
p.ix[:, ['b', 'd'], 0] = np.nan
result = p.dropna(axis=1)
exp = p.ix[:, ['a', 'c', 'e'], :]
assert_panel_equal(result, exp)
result = p.dropna(axis=1, how='all')
assert_panel_equal(result, p)
p.ix[:, ['b', 'd'], :] = np.nan
result = p.dropna(axis=1, how='all')
exp = p.ix[:, ['a', 'c', 'e'], :]
assert_panel_equal(result, exp)
p = Panel(np.random.randn(4, 5, 6), items=list('abcd'))
p.ix[['b'], :, 0] = np.nan
result = p.dropna()
exp = p.ix[['a', 'c', 'd']]
assert_panel_equal(result, exp)
result = p.dropna(how='all')
assert_panel_equal(result, p)
p.ix['b'] = np.nan
result = p.dropna(how='all')
exp = p.ix[['a', 'c', 'd']]
assert_panel_equal(result, exp)
def test_constructor_empty_panel(self):
empty = Panel()
assert len(empty.items) == 0
assert len(empty.major_axis) == 0
assert len(empty.minor_axis) == 0
def _prep_panel_data(data):
"""Converts the given data into a Panel."""
if isinstance(data, Panel):
return data
return Panel.fromDict(data)
def test_panel_np_all():
wp = Panel({"A": DataFrame({'b': [1, 2]})})
result = np.all(wp)
assert result == np.bool_(True)
def test_update_deprecation(self, raise_conflict):
pan = Panel([[[1.5, np.nan, 3.]]])
other = Panel([[[]]])
with tm.assert_produces_warning(FutureWarning):
pan.update(other, raise_conflict=raise_conflict)
def test_update_raise_bad_parameter(self, bad_kwarg, exception, msg):
pan = Panel([[[1.5, np.nan, 3.]]])
with pytest.raises(exception, match=msg):
pan.update(pan, **bad_kwarg)
def test_ctor_dict(self):
itema = self.panel['ItemA']
itemb = self.panel['ItemB']
d = {'A': itema, 'B': itemb[5:]}
d2 = {'A': itema._series, 'B': itemb[5:]._series}
d3 = {'A': DataFrame(itema._series), 'B': DataFrame(itemb[5:]._series)}
wp = Panel.from_dict(d)
wp2 = Panel.from_dict(d2) # nested Dict
wp3 = Panel.from_dict(d3)
self.assert_(wp.major_axis.equals(self.panel.major_axis))
assert_panel_equal(wp, wp2)
# intersect
wp = Panel.from_dict(d, intersect=True)
self.assert_(wp.major_axis.equals(itemb.index[5:]))
# use constructor
assert_panel_equal(Panel(d), Panel.from_dict(d))
assert_panel_equal(Panel(d2), Panel.from_dict(d2))
assert_panel_equal(Panel(d3), Panel.from_dict(d3))
# cast
result = Panel(d, dtype=int)
expected = Panel(dict((k, v.astype(int)) for k, v in d.iteritems()))
def not_hashable(self):
c_empty = Panel()
c = Panel(Panel([[[1]]]))
pytest.raises(TypeError, hash, c_empty)
pytest.raises(TypeError, hash, c)
def test_constructor_empty_panel(self):
empty = Panel()
self.assertEqual(len(empty.items), 0)
self.assertEqual(len(empty.major_axis), 0)
self.assertEqual(len(empty.minor_axis), 0)
def test_constructor_fails_with_not_3d_input(self):
msg = "The number of dimensions required is 3"
with pytest.raises(ValueError, match=msg):
Panel(np.random.randn(10, 2))
def test_fillna(self):
# limit not implemented when only value is specified
p = Panel(np.random.randn(3, 4, 5))
p.iloc[0:2, 0:2, 0:2] = np.nan
pytest.raises(NotImplementedError, lambda: p.fillna(999, limit=1))
def test_constructor_observe_dtype(self):
# GH #411
panel = Panel(items=lrange(3), major_axis=lrange(3),
minor_axis=lrange(3), dtype='O')
self.assertEqual(panel.values.dtype, np.object_)
def test_constructor_empty_panel(self):
empty = Panel()
self.assertTrue(len(empty.items) == 0)
self.assertTrue(len(empty.major_axis) == 0)
self.assertTrue(len(empty.minor_axis) == 0)
