def test_groupsort_indexer():
a = np.random.randint(0, 1000, 100).astype(np.int64)
b = np.random.randint(0, 1000, 100).astype(np.int64)
result = lib.groupsort_indexer(a, 1000)[0]
# need to use a stable sort
expected = np.argsort(a, kind='mergesort')
assert(np.array_equal(result, expected))
# compare with lexsort
key = a * 1000 + b
result = lib.groupsort_indexer(key, 1000000)[0]
expected = np.lexsort((b, a))
assert(np.array_equal(result, expected))
def test_groupsort_indexer():
a = np.random.randint(0, 1000, 100).astype(np.int64)
b = np.random.randint(0, 1000, 100).astype(np.int64)
result = lib.groupsort_indexer(a, 1000)[0]
# need to use a stable sort
expected = np.argsort(a, kind='mergesort')
assert (np.array_equal(result, expected))
# compare with lexsort
key = a * 1000 + b
result = lib.groupsort_indexer(key, 1000000)[0]
expected = np.lexsort((b, a))
assert (np.array_equal(result, expected))
def _make_sorted_values_labels(self):
v = self.level
labs = self.index.labels
levs = self.index.levels
to_sort = labs[:v] + labs[v + 1:] + [labs[v]]
sizes = [len(x) for x in levs[:v] + levs[v + 1:] + [levs[v]]]
group_index = get_group_index(to_sort, sizes)
comp_index, obs_ids = _compress_group_index(group_index)
ngroups = len(obs_ids)
indexer = lib.groupsort_indexer(comp_index, ngroups)[0]
indexer = _ensure_platform_int(indexer)
self.sorted_values = com.take_2d(self.values, indexer, axis=0)
self.sorted_labels = [l.take(indexer) for l in to_sort]
def _make_sorted_values_labels(self):
v = self.level
labs = self.index.labels
levs = self.index.levels
to_sort = labs[:v] + labs[v + 1:] + [labs[v]]
sizes = [len(x) for x in levs[:v] + levs[v + 1:] + [levs[v]]]
group_index = get_group_index(to_sort, sizes)
comp_index, obs_ids = _compress_group_index(group_index)
ngroups = len(obs_ids)
indexer = lib.groupsort_indexer(comp_index, ngroups)[0]
indexer = _ensure_platform_int(indexer)
self.sorted_values = com.take_2d(self.values, indexer, axis=0)
self.sorted_labels = [l.take(indexer) for l in to_sort]
def _read_panel_table(self, group, where=None):
table = getattr(group, 'table')
fields = table._v_attrs.fields
# create the selection
sel = Selection(table, where, table._v_attrs.index_kind)
sel.select()
fields = table._v_attrs.fields
columns = _maybe_convert(sel.values['column'],
table._v_attrs.columns_kind)
index = _maybe_convert(sel.values['index'], table._v_attrs.index_kind)
values = sel.values['values']
major = Factor.from_array(index)
minor = Factor.from_array(columns)
J, K = len(major.levels), len(minor.levels)
key = major.labels * K + minor.labels
if len(unique(key)) == len(key):
sorter, _ = lib.groupsort_indexer(com._ensure_int64(key), J * K)
sorter = com._ensure_platform_int(sorter)
# the data need to be sorted
sorted_values = values.take(sorter, axis=0)
major_labels = major.labels.take(sorter)
minor_labels = minor.labels.take(sorter)
block = block2d_to_block3d(sorted_values, fields, (J, K),
major_labels, minor_labels)
mgr = BlockManager([block],
[block.ref_items, major.levels, minor.levels])
wp = Panel(mgr)
else:
if not self._quiet: # pragma: no cover
print(
'Duplicate entries in table, taking most recently '
'appended')
# reconstruct
long_index = MultiIndex.from_arrays([index, columns])
lp = DataFrame(values, index=long_index, columns=fields)
# need a better algorithm
tuple_index = long_index._tuple_index
unique_tuples = lib.fast_unique(tuple_index)
unique_tuples = _asarray_tuplesafe(unique_tuples)
indexer = match(unique_tuples, tuple_index)
indexer = com._ensure_platform_int(indexer)
new_index = long_index.take(indexer)
new_values = lp.values.take(indexer, axis=0)
lp = DataFrame(new_values, index=new_index, columns=lp.columns)
wp = lp.to_panel()
if sel.column_filter:
new_minor = sorted(set(wp.minor_axis) & sel.column_filter)
wp = wp.reindex(minor=new_minor)
return wp
def _read_panel_table(self, group, where=None):
table = getattr(group, 'table')
fields = table._v_attrs.fields
# create the selection
sel = Selection(table, where, table._v_attrs.index_kind)
sel.select()
fields = table._v_attrs.fields
columns = _maybe_convert(sel.values['column'],
table._v_attrs.columns_kind)
index = _maybe_convert(sel.values['index'], table._v_attrs.index_kind)
values = sel.values['values']
major = Factor(index)
minor = Factor(columns)
J, K = len(major.levels), len(minor.levels)
key = major.labels * K + minor.labels
if len(unique(key)) == len(key):
sorter, _ = lib.groupsort_indexer(com._ensure_int64(key), J * K)
sorter = com._ensure_platform_int(sorter)
# the data need to be sorted
sorted_values = values.take(sorter, axis=0)
major_labels = major.labels.take(sorter)
minor_labels = minor.labels.take(sorter)
block = block2d_to_block3d(sorted_values, fields, (J, K),
major_labels, minor_labels)
mgr = BlockManager([block], [block.ref_items,
major.levels, minor.levels])
wp = Panel(mgr)
else:
if not self._quiet: # pragma: no cover
print ('Duplicate entries in table, taking most recently '
'appended')
# reconstruct
long_index = MultiIndex.from_arrays([index, columns])
lp = DataFrame(values, index=long_index, columns=fields)
# need a better algorithm
tuple_index = long_index.get_tuple_index()
unique_tuples = lib.fast_unique(tuple_index)
unique_tuples = _asarray_tuplesafe(unique_tuples)
indexer = match(unique_tuples, tuple_index)
indexer = com._ensure_platform_int(indexer)
new_index = long_index.take(indexer)
new_values = lp.values.take(indexer, axis=0)
lp = DataFrame(new_values, index=new_index, columns=lp.columns)
wp = lp.to_panel()
if sel.column_filter:
new_minor = sorted(set(wp.minor_axis) & sel.column_filter)
wp = wp.reindex(minor=new_minor)
return wp
