def test_select(self):
wp = tm.makePanel()
# put/select ok
self.store.remove('wp')
self.store.put('wp', wp, table=True)
self.store.select('wp')
# non-table ok (where = None)
self.store.remove('wp')
self.store.put('wp2', wp, table=False)
self.store.select('wp2')
# selection on the non-indexable with a large number of columns
wp = Panel(np.random.randn(100, 100, 100),
items=['Item%03d' % i for i in xrange(100)],
major_axis=date_range('1/1/2000', periods=100),
minor_axis=['E%03d' % i for i in xrange(100)])
self.store.remove('wp')
self.store.append('wp', wp)
items = ['Item%03d' % i for i in xrange(80)]
result = self.store.select('wp', Term('items', items))
expected = wp.reindex(items=items)
tm.assert_panel_equal(expected, result)
def first_difference(self):
"""
Compute first differences of variables
Returns
-------
diffs : PanelData
Differenced values
"""
diffs = self.panel.values
diffs = diffs[:, 1:] - diffs[:, :-1]
diffs = Panel(diffs, items=self.panel.items,
major_axis=self.panel.major_axis[1:],
minor_axis=self.panel.minor_axis)
diffs = diffs.swapaxes(1, 2).to_frame(filter_observations=False)
diffs = diffs.reindex(self._frame.index).dropna(how='any')
return PanelData(diffs)
def test_select(self):
wp = tm.makePanel()
# put/select ok
self.store.remove('wp')
self.store.put('wp', wp, table=True)
self.store.select('wp')
# non-table ok (where = None)
self.store.remove('wp')
self.store.put('wp2', wp, table=False)
self.store.select('wp2')
# selection on the non-indexable with a large number of columns
wp = Panel(np.random.randn(100, 100, 100), items = [ 'Item%03d' % i for i in xrange(100) ],
major_axis=date_range('1/1/2000', periods=100), minor_axis = [ 'E%03d' % i for i in xrange(100) ])
self.store.remove('wp')
self.store.append('wp', wp)
items = [ 'Item%03d' % i for i in xrange(80) ]
result = self.store.select('wp', Term('items', items))
expected = wp.reindex(items = items)
tm.assert_panel_equal(expected, result)
def _read_panel_table(self, group, where=None):
table = getattr(group, 'table')
fields = table._v_attrs.fields
# create the selection
sel = Selection(table, where, table._v_attrs.index_kind)
sel.select()
fields = table._v_attrs.fields
columns = _maybe_convert(sel.values['column'],
table._v_attrs.columns_kind)
index = _maybe_convert(sel.values['index'], table._v_attrs.index_kind)
values = sel.values['values']
major = Factor.from_array(index)
minor = Factor.from_array(columns)
J, K = len(major.levels), len(minor.levels)
key = major.labels * K + minor.labels
if len(unique(key)) == len(key):
sorter, _ = lib.groupsort_indexer(com._ensure_int64(key), J * K)
sorter = com._ensure_platform_int(sorter)
# the data need to be sorted
sorted_values = values.take(sorter, axis=0)
major_labels = major.labels.take(sorter)
minor_labels = minor.labels.take(sorter)
block = block2d_to_block3d(sorted_values, fields, (J, K),
major_labels, minor_labels)
mgr = BlockManager([block],
[block.ref_items, major.levels, minor.levels])
wp = Panel(mgr)
else:
if not self._quiet: # pragma: no cover
print(
'Duplicate entries in table, taking most recently '
'appended')
# reconstruct
long_index = MultiIndex.from_arrays([index, columns])
lp = DataFrame(values, index=long_index, columns=fields)
# need a better algorithm
tuple_index = long_index._tuple_index
unique_tuples = lib.fast_unique(tuple_index)
unique_tuples = _asarray_tuplesafe(unique_tuples)
indexer = match(unique_tuples, tuple_index)
indexer = com._ensure_platform_int(indexer)
new_index = long_index.take(indexer)
new_values = lp.values.take(indexer, axis=0)
lp = DataFrame(new_values, index=new_index, columns=lp.columns)
wp = lp.to_panel()
if sel.column_filter:
new_minor = sorted(set(wp.minor_axis) & sel.column_filter)
wp = wp.reindex(minor=new_minor)
return wp
def _read_panel_table(self, group, where=None):
table = getattr(group, 'table')
fields = table._v_attrs.fields
# create the selection
sel = Selection(table, where, table._v_attrs.index_kind)
sel.select()
fields = table._v_attrs.fields
columns = _maybe_convert(sel.values['column'],
table._v_attrs.columns_kind)
index = _maybe_convert(sel.values['index'],
table._v_attrs.index_kind)
values = sel.values['values']
major = Factor(index)
minor = Factor(columns)
J, K = len(major.levels), len(minor.levels)
key = major.labels * K + minor.labels
if len(unique(key)) == len(key):
sorter, _ = lib.groupsort_indexer(key, J * K)
# the data need to be sorted
sorted_values = values.take(sorter, axis=0)
major_labels = major.labels.take(sorter)
minor_labels = minor.labels.take(sorter)
block = block2d_to_block3d(sorted_values, fields, (J, K),
major_labels, minor_labels)
mgr = BlockManager([block], [block.items,
major.levels, minor.levels])
wp = Panel(mgr)
else:
if not self._quiet: # pragma: no cover
print ('Duplicate entries in table, taking most recently '
'appended')
# reconstruct
long_index = MultiIndex.from_arrays([index, columns])
lp = DataFrame(values, index=long_index, columns=fields)
# need a better algorithm
tuple_index = long_index.get_tuple_index()
unique_tuples = lib.fast_unique(tuple_index)
unique_tuples = _asarray_tuplesafe(unique_tuples)
indexer = match(unique_tuples, tuple_index)
new_index = long_index.take(indexer)
new_values = lp.values.take(indexer, axis=0)
lp = DataFrame(new_values, index=new_index, columns=lp.columns)
wp = lp.to_panel()
if sel.column_filter:
new_minor = sorted(set(wp.minor_axis) & sel.column_filter)
wp = wp.reindex(minor=new_minor)
return wp
def _read_panel_table(self, group, where=None):
from pandas.core.index import unique_int64, Factor
from pandas.core.common import _asarray_tuplesafe
from pandas.core.internals import BlockManager
from pandas.core.reshape import block2d_to_block3d
table = getattr(group, "table")
# create the selection
sel = Selection(table, where)
sel.select()
fields = table._v_attrs.fields
columns = _maybe_convert(sel.values["column"], table._v_attrs.columns_kind)
index = _maybe_convert(sel.values["index"], table._v_attrs.index_kind)
values = sel.values["values"]
major = Factor(index)
minor = Factor(columns)
J, K = len(major.levels), len(minor.levels)
key = major.labels * K + minor.labels
if len(unique_int64(key)) == len(key):
sorter, _ = lib.groupsort_indexer(key, J * K)
# the data need to be sorted
sorted_values = values.take(sorter, axis=0)
major_labels = major.labels.take(sorter)
minor_labels = minor.labels.take(sorter)
block = block2d_to_block3d(sorted_values, fields, (J, K), major_labels, minor_labels)
mgr = BlockManager([block], [block.items, major.levels, minor.levels])
wp = Panel(mgr)
else:
if not self._quiet: # pragma: no cover
print ("Duplicate entries in table, taking most recently " "appended")
# reconstruct
long_index = MultiIndex.from_arrays([index, columns])
lp = DataFrame(values, index=long_index, columns=fields)
# need a better algorithm
tuple_index = long_index.get_tuple_index()
index_map = lib.map_indices_object(tuple_index)
unique_tuples = lib.fast_unique(tuple_index)
unique_tuples = _asarray_tuplesafe(unique_tuples)
indexer = lib.merge_indexer_object(unique_tuples, index_map)
new_index = long_index.take(indexer)
new_values = lp.values.take(indexer, axis=0)
lp = DataFrame(new_values, index=new_index, columns=lp.columns)
wp = lp.to_panel()
if sel.column_filter:
new_minor = sorted(set(wp.minor_axis) & sel.column_filter)
wp = wp.reindex(minor=new_minor)
return wp
