def test_index_many_concat_c(self) -> None:
from datetime import date
i1 = IndexHierarchy.from_labels(
[[1, date(2019, 1, 1)], [2, date(2019, 1, 2)]],
index_constructors=[Index, IndexDate])
i2 = IndexHierarchy.from_labels(
[[2, date(2019, 1, 3)], [3, date(2019, 1, 4)]],
index_constructors=[Index, IndexDate])
i3 = IndexHierarchy.from_labels(
[[4, date(2019, 1, 5)], [5, date(2019, 1, 6)]],
index_constructors=[Index, IndexDate])
i4 = IndexHierarchy.from_labels([[4, date(2019, 1, 5)],
[5, date(2019, 1, 6)]])
i5 = index_many_concat((i1, i2, i3), cls_default=Index)
assert isinstance(i5, IndexHierarchy)
self.assertEqual(i5.index_types.to_pairs(),
((0, Index), (1, IndexDate)))
self.assertEqual(i5.values.tolist(),
[[1, date(2019, 1, 1)], [2, date(2019, 1, 2)],
[2, date(2019, 1, 3)], [3, date(2019, 1, 4)],
[4, date(2019, 1, 5)], [5, date(2019, 1, 6)]])
# with unaligned index types we fall back in Index
i6 = index_many_concat((i1, i2, i4), cls_default=Index)
assert isinstance(i6, IndexHierarchy)
self.assertEqual(i6.index_types.to_pairs(), ((0, Index), (1, Index)))
def test_index_many_concat_b(self) -> None:
idx0 = Index(('1997-01-01', '1997-01-02'), name='foo')
idx1 = IndexDate(('2020-01-01', '2020-01-02'), name='foo')
idx2 = IndexDate(('2020-02-01', '2020-02-02'))
post1 = index_many_concat((idx0, idx1), IndexGO)
self.assertEqual(post1.__class__, IndexGO)
post2 = index_many_concat((idx1, idx2), IndexGO)
self.assertEqual(post2.__class__, IndexDateGO)
def test_index_many_concat_a1(self) -> None:
idx0 = Index(('1997-01-01', '1997-01-02'), name='foo')
idx1 = IndexDate(('2020-01-01', '2020-01-02'), name='foo')
idx2 = IndexDate(('2020-02-01', '2020-02-02'))
post1 = index_many_concat((idx0, idx1), Index)
assert isinstance(post1, Index)
post1b = index_many_concat((idx0, idx1), Index, IndexDate)
assert isinstance(post1b, IndexDate)
post1c = index_many_concat((idx0, idx1, idx2), Index, IndexDate)
assert isinstance(post1c, IndexDate)
self.assertEqual(post1.values.tolist(), [
'1997-01-01', '1997-01-02',
datetime.date(2020, 1, 1),
datetime.date(2020, 1, 2)
])
self.assertEqual(post1.name, 'foo')
self.assertEqual(post1.__class__, Index)
def test_index_many_concat_a2(self) -> None:
# idx0 = Index(('1997-01-01', '1997-01-02'), name='foo')
idx1 = IndexDate(('2020-01-01', '2020-01-02'), name='foo')
idx2 = IndexDate(('2020-02-01', '2020-02-02'))
post2 = index_many_concat((idx1, idx2), Index)
assert isinstance(post2, Index)
self.assertEqual(post2.__class__, IndexDate)
self.assertEqual(post2.values.tolist(), [
datetime.date(2020, 1, 1),
datetime.date(2020, 1, 2),
datetime.date(2020, 2, 1),
datetime.date(2020, 2, 2)
])
def test_index_many_concat_e(self) -> None:
idx1 = IndexDateGO(('2020-01-01', '2020-01-02'))
idx2 = IndexDateGO(('2020-02-01', '2020-02-02'))
post1 = index_many_concat((idx1, idx2), cls_default=Index)
self.assertEqual(post1.__class__, IndexDate)
self.assertEqual(
post1.values.tolist(), #type: ignore
[
datetime.date(2020, 1, 1),
datetime.date(2020, 1, 2),
datetime.date(2020, 2, 1),
datetime.date(2020, 2, 2)
])
def test_index_many_concat_d(self) -> None:
from datetime import date
i1 = IndexHierarchy.from_labels(
[[1, date(2019, 1, 1)], [2, date(2019, 1, 2)]],
index_constructors=[Index, IndexDate])
i2 = IndexHierarchy.from_labels(
[[2, date(2019, 1, 3)], [3, date(2019, 1, 4)]],
index_constructors=[Index, IndexDate])
post1 = index_many_concat((i1, i2), cls_default=IndexGO)
self.assertEqual(post1.__class__, IndexHierarchyGO)
assert isinstance(post1, IndexHierarchy)
self.assertEqual(post1.values.tolist(),
[[1, date(2019, 1, 1)], [2, date(2019, 1, 2)],
[2, date(2019, 1, 3)], [3, date(2019, 1, 4)]])
def from_frames(
self,
frames: tp.Iterable['Frame'],
*,
include_index: bool = True,
include_columns: bool = True,
axis: int = 0,
union: bool = True,
name: NameType = None,
fill_value: object = np.nan,
) -> None:
'''Given an iterable of Frames, write out an NPZ or NPY directly, without building up an intermediary Frame. If axis 0, the Frames must be block compatible; if axis 1, the Frames must have the same number of rows. For both axis, if included, concatenated indices must be unique or aligned.
Args:
frames:
*
include_index:
include_columns:
axis:
union:
name:
fill_value:
'''
if not self._writeable:
raise UnsupportedOperation('Open with mode "w" to write.')
from static_frame.core.type_blocks import TypeBlocks
from static_frame.core.frame import Frame
frames = [
f if isinstance(f, Frame) else f.to_frame(axis) for f in frames
] # type: ignore
# NOTE: based on Frame.from_concat
if axis == 1: # stacks columns (extends rows horizontally)
if include_columns:
try:
columns = index_many_concat(
(f._columns for f in frames),
Index,
)
except ErrorInitIndexNonUnique:
raise RuntimeError(
'Column names after horizontal concatenation are not unique; set include_columns to None to ignore.'
)
else:
columns = None
if include_index:
index = index_many_set(
(f._index for f in frames),
Index,
union=union,
)
else:
raise RuntimeError(
'Must include index for horizontal alignment.')
def blocks() -> tp.Iterator[np.ndarray]:
for f in frames:
if len(f.index) != len(index) or (f.index != index).any():
f = f.reindex(index=index, fill_value=fill_value)
for block in f._blocks._blocks:
yield block
elif axis == 0: # stacks rows (extends columns vertically)
if include_index:
try:
index = index_many_concat((f._index for f in frames),
Index)
except ErrorInitIndexNonUnique:
raise RuntimeError(
'Index names after vertical concatenation are not unique; set include_index to None to ignore'
)
else:
index = None
if include_columns:
columns = index_many_set(
(f._columns for f in frames),
Index,
union=union,
)
else:
raise RuntimeError(
'Must include columns for vertical alignment.')
def blocks() -> tp.Iterator[np.ndarray]:
type_blocks = []
previous_f: tp.Optional[Frame] = None
block_compatible = True
reblock_compatible = True
for f in frames:
if len(f.columns) != len(columns) or (f.columns !=
columns).any():
f = f.reindex(columns=columns, fill_value=fill_value)
type_blocks.append(f._blocks)
# column size is all the same by this point
if previous_f is not None: # after the first
if block_compatible:
block_compatible &= f._blocks.block_compatible(
previous_f._blocks,
axis=1) # only compare columns
if reblock_compatible:
reblock_compatible &= f._blocks.reblock_compatible(
previous_f._blocks)
previous_f = f
yield from TypeBlocks.vstack_blocks_to_blocks(
type_blocks=type_blocks,
block_compatible=block_compatible,
reblock_compatible=reblock_compatible,
)
else:
raise AxisInvalid(f'no support for {axis}')
self.from_arrays(
blocks=blocks(),
index=index,
columns=columns,
name=name,
axis=1, # blocks are normalized for horizontal concat
)