def test_index_many_concat_c(self) -> None: from datetime import date i1 = IndexHierarchy.from_labels( [[1, date(2019, 1, 1)], [2, date(2019, 1, 2)]], index_constructors=[Index, IndexDate]) i2 = IndexHierarchy.from_labels( [[2, date(2019, 1, 3)], [3, date(2019, 1, 4)]], index_constructors=[Index, IndexDate]) i3 = IndexHierarchy.from_labels( [[4, date(2019, 1, 5)], [5, date(2019, 1, 6)]], index_constructors=[Index, IndexDate]) i4 = IndexHierarchy.from_labels([[4, date(2019, 1, 5)], [5, date(2019, 1, 6)]]) i5 = index_many_concat((i1, i2, i3), cls_default=Index) assert isinstance(i5, IndexHierarchy) self.assertEqual(i5.index_types.to_pairs(), ((0, Index), (1, IndexDate))) self.assertEqual(i5.values.tolist(), [[1, date(2019, 1, 1)], [2, date(2019, 1, 2)], [2, date(2019, 1, 3)], [3, date(2019, 1, 4)], [4, date(2019, 1, 5)], [5, date(2019, 1, 6)]]) # with unaligned index types we fall back in Index i6 = index_many_concat((i1, i2, i4), cls_default=Index) assert isinstance(i6, IndexHierarchy) self.assertEqual(i6.index_types.to_pairs(), ((0, Index), (1, Index)))
def test_index_many_concat_b(self) -> None: idx0 = Index(('1997-01-01', '1997-01-02'), name='foo') idx1 = IndexDate(('2020-01-01', '2020-01-02'), name='foo') idx2 = IndexDate(('2020-02-01', '2020-02-02')) post1 = index_many_concat((idx0, idx1), IndexGO) self.assertEqual(post1.__class__, IndexGO) post2 = index_many_concat((idx1, idx2), IndexGO) self.assertEqual(post2.__class__, IndexDateGO)
def test_index_many_concat_a1(self) -> None: idx0 = Index(('1997-01-01', '1997-01-02'), name='foo') idx1 = IndexDate(('2020-01-01', '2020-01-02'), name='foo') idx2 = IndexDate(('2020-02-01', '2020-02-02')) post1 = index_many_concat((idx0, idx1), Index) assert isinstance(post1, Index) post1b = index_many_concat((idx0, idx1), Index, IndexDate) assert isinstance(post1b, IndexDate) post1c = index_many_concat((idx0, idx1, idx2), Index, IndexDate) assert isinstance(post1c, IndexDate) self.assertEqual(post1.values.tolist(), [ '1997-01-01', '1997-01-02', datetime.date(2020, 1, 1), datetime.date(2020, 1, 2) ]) self.assertEqual(post1.name, 'foo') self.assertEqual(post1.__class__, Index)
def test_index_many_concat_a2(self) -> None: # idx0 = Index(('1997-01-01', '1997-01-02'), name='foo') idx1 = IndexDate(('2020-01-01', '2020-01-02'), name='foo') idx2 = IndexDate(('2020-02-01', '2020-02-02')) post2 = index_many_concat((idx1, idx2), Index) assert isinstance(post2, Index) self.assertEqual(post2.__class__, IndexDate) self.assertEqual(post2.values.tolist(), [ datetime.date(2020, 1, 1), datetime.date(2020, 1, 2), datetime.date(2020, 2, 1), datetime.date(2020, 2, 2) ])
def test_index_many_concat_e(self) -> None: idx1 = IndexDateGO(('2020-01-01', '2020-01-02')) idx2 = IndexDateGO(('2020-02-01', '2020-02-02')) post1 = index_many_concat((idx1, idx2), cls_default=Index) self.assertEqual(post1.__class__, IndexDate) self.assertEqual( post1.values.tolist(), #type: ignore [ datetime.date(2020, 1, 1), datetime.date(2020, 1, 2), datetime.date(2020, 2, 1), datetime.date(2020, 2, 2) ])
def test_index_many_concat_d(self) -> None: from datetime import date i1 = IndexHierarchy.from_labels( [[1, date(2019, 1, 1)], [2, date(2019, 1, 2)]], index_constructors=[Index, IndexDate]) i2 = IndexHierarchy.from_labels( [[2, date(2019, 1, 3)], [3, date(2019, 1, 4)]], index_constructors=[Index, IndexDate]) post1 = index_many_concat((i1, i2), cls_default=IndexGO) self.assertEqual(post1.__class__, IndexHierarchyGO) assert isinstance(post1, IndexHierarchy) self.assertEqual(post1.values.tolist(), [[1, date(2019, 1, 1)], [2, date(2019, 1, 2)], [2, date(2019, 1, 3)], [3, date(2019, 1, 4)]])
def from_frames( self, frames: tp.Iterable['Frame'], *, include_index: bool = True, include_columns: bool = True, axis: int = 0, union: bool = True, name: NameType = None, fill_value: object = np.nan, ) -> None: '''Given an iterable of Frames, write out an NPZ or NPY directly, without building up an intermediary Frame. If axis 0, the Frames must be block compatible; if axis 1, the Frames must have the same number of rows. For both axis, if included, concatenated indices must be unique or aligned. Args: frames: * include_index: include_columns: axis: union: name: fill_value: ''' if not self._writeable: raise UnsupportedOperation('Open with mode "w" to write.') from static_frame.core.type_blocks import TypeBlocks from static_frame.core.frame import Frame frames = [ f if isinstance(f, Frame) else f.to_frame(axis) for f in frames ] # type: ignore # NOTE: based on Frame.from_concat if axis == 1: # stacks columns (extends rows horizontally) if include_columns: try: columns = index_many_concat( (f._columns for f in frames), Index, ) except ErrorInitIndexNonUnique: raise RuntimeError( 'Column names after horizontal concatenation are not unique; set include_columns to None to ignore.' ) else: columns = None if include_index: index = index_many_set( (f._index for f in frames), Index, union=union, ) else: raise RuntimeError( 'Must include index for horizontal alignment.') def blocks() -> tp.Iterator[np.ndarray]: for f in frames: if len(f.index) != len(index) or (f.index != index).any(): f = f.reindex(index=index, fill_value=fill_value) for block in f._blocks._blocks: yield block elif axis == 0: # stacks rows (extends columns vertically) if include_index: try: index = index_many_concat((f._index for f in frames), Index) except ErrorInitIndexNonUnique: raise RuntimeError( 'Index names after vertical concatenation are not unique; set include_index to None to ignore' ) else: index = None if include_columns: columns = index_many_set( (f._columns for f in frames), Index, union=union, ) else: raise RuntimeError( 'Must include columns for vertical alignment.') def blocks() -> tp.Iterator[np.ndarray]: type_blocks = [] previous_f: tp.Optional[Frame] = None block_compatible = True reblock_compatible = True for f in frames: if len(f.columns) != len(columns) or (f.columns != columns).any(): f = f.reindex(columns=columns, fill_value=fill_value) type_blocks.append(f._blocks) # column size is all the same by this point if previous_f is not None: # after the first if block_compatible: block_compatible &= f._blocks.block_compatible( previous_f._blocks, axis=1) # only compare columns if reblock_compatible: reblock_compatible &= f._blocks.reblock_compatible( previous_f._blocks) previous_f = f yield from TypeBlocks.vstack_blocks_to_blocks( type_blocks=type_blocks, block_compatible=block_compatible, reblock_compatible=reblock_compatible, ) else: raise AxisInvalid(f'no support for {axis}') self.from_arrays( blocks=blocks(), index=index, columns=columns, name=name, axis=1, # blocks are normalized for horizontal concat )