def test_yarn_drop_a(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3), name='b1') b2 = Bus.from_frames((f4, ), name='b2') b3 = Bus.from_frames((f5, f6), name='b3') y1 = Yarn.from_buses((b1, b2, b3), retain_labels=False) y2 = y1.drop['f3':'f5'] #type: ignore self.assertEqual(len(y2._series), 2) # 2 buses remain self.assertEqual([(f.name, f.shape) for f in y2.values], [('f1', (4, 2)), ('f2', (4, 5)), ('f6', (6, 4))]) y3 = y1.drop[y1.index.isin(('f1', 'f6'))] self.assertEqual([(f.name, f.shape) for f in y3.values], [('f2', (4, 5)), ('f3', (2, 2)), ('f4', (2, 8)), ('f5', (4, 4))]) y4 = y1.drop['f4'] self.assertEqual([(f.name, f.shape) for f in y4.values], [('f1', (4, 2)), ('f2', (4, 5)), ('f3', (2, 2)), ('f5', (4, 4)), ('f6', (6, 4))]) y5 = y1.drop[['f4', 'f5', 'f6']] self.assertEqual([(f.name, f.shape) for f in y5.values], [('f1', (4, 2)), ('f2', (4, 5)), ('f3', (2, 2))])
def test_yarn_drop_b(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3), name='b1') b2 = Bus.from_frames((f4, ), name='b2') b3 = Bus.from_frames((f5, f6), name='b3') y1 = Yarn.from_buses((b1, b2, b3), retain_labels=False) y2 = y1.drop.iloc[2:5] self.assertEqual(len(y2._series), 2) # 2 buses remain self.assertEqual([(f.name, f.shape) for f in y2.values], [('f1', (4, 2)), ('f2', (4, 5)), ('f6', (6, 4))]) y3 = y1.drop.iloc[np.array([True, False, False, False, False, True])] self.assertEqual([(f.name, f.shape) for f in y3.values], [('f2', (4, 5)), ('f3', (2, 2)), ('f4', (2, 8)), ('f5', (4, 4))]) y4 = y1.drop.iloc[3] self.assertEqual([(f.name, f.shape) for f in y4.values], [('f1', (4, 2)), ('f2', (4, 5)), ('f3', (2, 2)), ('f5', (4, 4)), ('f6', (6, 4))]) y5 = y1.drop.iloc[[3, 4, 5]] self.assertEqual([(f.name, f.shape) for f in y5.values], [('f1', (4, 2)), ('f2', (4, 5)), ('f3', (2, 2))])
def test_yarn_equals_e(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3), name='a') b2 = Bus.from_frames((f4, f5, f6), name='b') y1 = Yarn.from_buses((b1, b2), retain_labels=True) with temp_file('.zip') as fp1, temp_file('.zip') as fp2: b1.to_zip_pickle(fp1) b2.to_zip_pickle(fp2) bus_a = Bus.from_zip_pickle(fp1, max_persist=1).rename('a') bus_b = Bus.from_zip_pickle(fp2, max_persist=1).rename('b') y2 = Yarn.from_buses((bus_a, bus_b), retain_labels=True) self.assertEqual(y2.status['loaded'].sum(), 0) self.assertTrue(y1.equals(y2)) self.assertEqual(y2.status['loaded'].sum(), 2)
def test_yarn_from_buses_a(self) -> None: f1 = ff.parse('s(4,4)|v(int,float)').rename('f1') f2 = ff.parse('s(4,4)|v(str)').rename('f2') f3 = ff.parse('s(4,4)|v(bool)').rename('f3') b1 = Bus.from_frames((f1, f2, f3), name='a') f4 = ff.parse('s(4,4)|v(int,float)').rename('f4') f5 = ff.parse('s(4,4)|v(str)').rename('f5') b2 = Bus.from_frames((f4, f5), name='b') y1 = Yarn.from_buses((b1, b2), retain_labels=True) self.assertEqual(len(y1), 5) self.assertEqual(y1.index.shape, (5, 2)) self.assertEqual(y1.shape, (5, )) self.assertEqual(y1.size, 5) self.assertEqual(y1.dtype, object) self.assertEqual(y1.ndim, 1) y3 = y1[('a', 'f2'):] #type: ignore self.assertEqual(y3.shape, (4, )) y2 = Yarn.from_buses((b1, b2), retain_labels=False) self.assertEqual(len(y2), 5) self.assertEqual(y2.index.shape, (5, )) self.assertEqual(y1.shape, (5, )) self.assertEqual(y1.size, 5) self.assertEqual(y1.dtype, object) self.assertEqual(y1.ndim, 1)
def test_yarn_items_b(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3)) b2 = Bus.from_frames((f4, f5, f6)) with temp_file('.zip') as fp1, temp_file('.zip') as fp2: b1.to_zip_pickle(fp1) b2.to_zip_pickle(fp2) bus_a = Bus.from_zip_pickle(fp1, max_persist=1).rename('a') bus_b = Bus.from_zip_pickle(fp2, max_persist=1).rename('b') y1 = Yarn.from_buses((bus_a, bus_b), retain_labels=False) s1 = y1.to_series() self.assertEqual([(label, f.shape) for label, f in s1.items()], [('f1', (4, 2)), ('f2', (4, 5)), ('f3', (2, 2)), ('f4', (2, 8)), ('f5', (4, 4)), ('f6', (6, 4))])
def test_yarn_items_a(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3)) b2 = Bus.from_frames((f4, f5, f6)) with temp_file('.zip') as fp1, temp_file('.zip') as fp2: b1.to_zip_pickle(fp1) b2.to_zip_pickle(fp2) bus_a = Bus.from_zip_pickle(fp1, max_persist=1).rename('a') bus_b = Bus.from_zip_pickle(fp2, max_persist=1).rename('b') y1 = Yarn.from_buses((bus_a, bus_b), retain_labels=False) labels = [] for label, frame in y1.items(): self.assertTrue(frame.__class__ is Frame) labels.append(label) self.assertEqual(labels, list(y1.index)) self.assertEqual(y1.status['loaded'].sum(), 2) self.assertEqual( y1.status.loc[y1.status['loaded']].index.values.tolist(), ['f3', 'f6'])
def test_yarn_init_b(self) -> None: f1 = ff.parse('s(4,4)|v(int,float)').rename('f1') f2 = ff.parse('s(4,4)|v(str)').rename('f2') f3 = ff.parse('s(4,4)|v(bool)').rename('f3') b1 = Bus.from_frames((f1, f2, f3)) f4 = ff.parse('s(4,4)|v(int,float)').rename('f4') f5 = ff.parse('s(4,4)|v(str)').rename('f5') b2 = Bus.from_frames((f4, f5)) y1 = Yarn((b1, b2), index=tuple('abcde')) self.assertEqual(y1.index.values.tolist(), list('abcde')) self.assertEqual(y1[['a', 'c', 'e']].shape, (3, )) y2 = Yarn((b1, b2)) self.assertEqual(y2.index.values.tolist(), list(range(5))) self.assertEqual(y2[2:].shape, (3, )) y3 = Yarn((b2, ), index=('2021-01-01', '2021-02-15'), index_constructor=IndexDate) self.assertEqual(y3.index.__class__, IndexDate) self.assertEqual( y3.index.values.tolist(), [datetime.date(2021, 1, 1), datetime.date(2021, 2, 15)]) with self.assertRaises(ErrorInitYarn): y4 = Yarn((b2, ), index=range(5))
def test_yarn_loc_e(self) -> None: f1 = ff.parse('s(4,4)|v(int,float)').rename('f1') f2 = ff.parse('s(4,4)|v(str)').rename('f2') f3 = ff.parse('s(4,4)|v(bool)').rename('f3') b1 = Bus.from_frames((f1, f2, f3), name='a') f4 = ff.parse('s(4,4)|v(int,float)').rename('f4') f5 = ff.parse('s(4,4)|v(str)').rename('f5') b2 = Bus.from_frames((f4, f5), name='b') f6 = ff.parse('s(2,4)|v(int,float)').rename('f6') f7 = ff.parse('s(4,2)|v(str)').rename('f7') b3 = Bus.from_frames((f6, f7), name='c') y1 = Yarn.from_buses((b1, b2, b3), retain_labels=False, name='foo') y2 = y1['f2':'f6'] #type: ignore self.assertEqual(y2.shapes.to_pairs(), (('f2', (4, 4)), ('f3', (4, 4)), ('f4', (4, 4)), ('f5', (4, 4)), ('f6', (2, 4)))) self.assertEqual(y2['f5'].to_pairs(), ((0, ((0, 'zjZQ'), (1, 'zO5l'), (2, 'zEdH'), (3, 'zB7E'))), (1, ((0, 'zaji'), (1, 'zJnC'), (2, 'zDdR'), (3, 'zuVU'))), (2, ((0, 'ztsv'), (1, 'zUvW'), (2, 'zkuW'), (3, 'zmVj'))), (3, ((0, 'z2Oo'), (1, 'z5l6'), (2, 'zCE3'), (3, 'zr4u')))))
def test_yarn_from_concat_b(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3)) b2 = Bus.from_frames((f4, f5, f6)) with temp_file('.zip') as fp1, temp_file('.zip') as fp2: b1.to_zip_pickle(fp1) b2.to_zip_pickle(fp2) bus_a = Bus.from_zip_pickle(fp1, max_persist=1).rename('a') bus_b = Bus.from_zip_pickle(fp2, max_persist=1).rename('b') y1 = Yarn.from_concat((Yarn.from_buses((bus_a, ), retain_labels=True), Yarn.from_buses((bus_b, ), retain_labels=True))) y2 = Yarn.from_concat((y1, y1), index=IndexAutoFactory) self.assertEqual(y2[3].shape, (2, 8)) self.assertEqual(y2[0].shape, (4, 2)) self.assertEqual(y2[5].shape, (6, 4)) y3 = y2.iloc[4:] self.assertEqual(y3.shape, (8, ))
def test_yarn_max_persist(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3)) b2 = Bus.from_frames((f4, f5, f6)) with temp_file('.zip') as fp1, temp_file('.zip') as fp2: b1.to_zip_pickle(fp1) b2.to_zip_pickle(fp2) bus_a = Bus.from_zip_pickle(fp1, max_persist=1).rename('a') bus_b = Bus.from_zip_pickle(fp2, max_persist=1).rename('b') y1 = Yarn.from_buses((bus_a, bus_b), retain_labels=False) self.assertEqual(y1.nbytes, 0) self.assertEqual(y1.status['loaded'].sum(), 0) self.assertEqual(y1['f2'].shape, (4, 5)) self.assertEqual(y1['f6'].shape, (6, 4)) self.assertEqual(y1.nbytes, 352) self.assertEqual(y1.status['loaded'].sum(), 2) self.assertEqual(y1.shapes.to_pairs(), (('f1', None), ('f2', (4, 5)), ('f3', None), ('f4', None), ('f5', None), ('f6', (6, 4)))) self.assertEqual(y1.mloc.isna().sum(), 4) self.assertEqual((y1.dtypes == float).sum().sum(), 9)
def test_yarn_relabel_a(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3)) b2 = Bus.from_frames((f4, )) b3 = Bus.from_frames((f5, f6)) y1 = Yarn((b1, b2, b3)) self.assertEqual( y1.relabel(lambda x: f'--{x}--').loc['--4--'].shape, (4, 4)) # None is a no-op self.assertEqual(y1.relabel(None).loc[4].shape, (4, 4)) with self.assertRaises(RelabelInvalid): y1.relabel({3, 4, 5}) self.assertEqual( y1.relabel(tuple('abcdef'))['d':].status['shape'].to_pairs(), (('d', (2, 8)), ('e', (4, 4)), ('f', (6, 4)))) y2 = Yarn((b1, b2, b3), index=tuple('abcdef')) self.assertEqual(y2.index.values.tolist(), ['a', 'b', 'c', 'd', 'e', 'f']) self.assertEqual( y2.relabel(IndexAutoFactory).index.values.tolist(), [0, 1, 2, 3, 4, 5])
def test_yarn_unpersist_a(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') f5 = ff.parse('s(4,4)').rename('f5') f6 = ff.parse('s(6,4)').rename('f6') b1 = Bus.from_frames((f1, f2, f3)) b2 = Bus.from_frames((f4, f5, f6)) with temp_file('.zip') as fp1, temp_file('.zip') as fp2: b1.to_zip_pickle(fp1) b2.to_zip_pickle(fp2) bus_a = Bus.from_zip_pickle(fp1, max_persist=1).rename('a') bus_b = Bus.from_zip_pickle(fp2, max_persist=1).rename('b') y1 = Yarn.from_buses((bus_a, bus_b), retain_labels=False) self.assertEqual(len(tuple(y1.items())), 6) self.assertEqual(y1.status['loaded'].sum(), 2) y1.unpersist() self.assertEqual(y1.status['loaded'].sum(), 0) self.assertEqual(len(tuple(y1.items())), 6) self.assertEqual(y1.status['loaded'].sum(), 2)
def test_yarn_dtypes_a(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') b1 = Bus.from_frames((f1, f2), name='a') b2 = Bus.from_frames((f3, f4), name='b') y1 = Yarn.from_buses((b1, b2), retain_labels=False) self.assertEqual(y1.dtypes.shape, (4, 8))
def test_yarn_size_a(self) -> None: f1 = ff.parse('s(4,4)|v(int,float)').rename('f1') f2 = ff.parse('s(4,4)|v(str)').rename('f2') f3 = ff.parse('s(4,4)|v(bool)').rename('f3') b1 = Bus.from_frames((f1, f2, f3), name='a') f4 = ff.parse('s(4,4)|v(int,float)').rename('f4') f5 = ff.parse('s(4,4)|v(str)').rename('f5') b2 = Bus.from_frames((f4, f5), name='b') y1 = Yarn.from_buses((b1, b2), retain_labels=False, name='foo') self.assertEqual(y1.size, 5)
def test_yarn_equals_b(self) -> None: f1 = ff.parse('s(4,2)').rename('f1') f2 = ff.parse('s(4,5)').rename('f2') f3 = ff.parse('s(2,2)').rename('f3') f4 = ff.parse('s(2,8)').rename('f4') b1 = Bus.from_frames((f1, f2), name='a') b2 = Bus.from_frames((f3, f4), name='b') y1 = Yarn.from_buses((b1, b2), retain_labels=False) y2 = Yarn.from_buses((b1, ), retain_labels=False) # fails on length self.assertFalse(y1.equals(y2))
def test_buses_to_hierarchy_a(self) -> None: f1 = ff.parse('s(4,4)|v(int,float)').rename('f1') f2 = ff.parse('s(4,4)|v(str)').rename('f2') f3 = ff.parse('s(4,4)|v(bool)').rename('f3') b1 = Bus.from_frames((f1, f2, f3), name='a') f4 = ff.parse('s(4,4)|v(int,float)').rename('f4') f5 = ff.parse('s(4,4)|v(str)').rename('f5') b2 = Bus.from_frames((f4, f5), name='b') post = buses_to_hierarchy((b1, b2), (b1.name, b2.name), deepcopy_from_bus=False, init_exception_cls=ErrorInitYarn) self.assertEqual(post.values.tolist(), [['a', 'f1'], ['a', 'f2'], ['a', 'f3'], ['b', 'f4'], ['b', 'f5']])
def test_buses_to_hierarchy_b(self) -> None: f1 = ff.parse('s(4,4)|v(int,float)').rename('f1') f2 = ff.parse('s(4,4)|v(str)').rename('f2') f3 = ff.parse('s(4,4)|v(bool)').rename('f3') b1 = Bus.from_frames((f1, f2, f3), name='foo') f4 = ff.parse('s(4,4)|v(int,float)').rename('f4') f5 = ff.parse('s(4,4)|v(str)').rename('f5') b2 = Bus.from_frames((f4, f5), name='foo') with self.assertRaises(ErrorInitYarn): _ = buses_to_hierarchy((b1, b2), (b1.name, b2.name), deepcopy_from_bus=False, init_exception_cls=ErrorInitYarn)
def test_bus_to_hierarchy_b(self) -> None: class CustomError(Exception): pass tree1 = dict(a_I=Index((1,2,3)), a_II=Index((1,2,3))) tree2 = dict(b_I=Index((1,2,3)), b_II=Index((1,2,3))) tree3 = dict(c_I=Index((1,2,3)), c_II=Index((1,2,3))) index1 = IndexHierarchy.from_tree(tree1) index2 = IndexHierarchy.from_tree(tree2) index3 = IndexHierarchy.from_tree(tree3) values = np.arange(36).reshape(6,6) # Align all the frames on columns! f1 = Frame(values, index=index1, columns=index1, name='f1') f2 = Frame(values, index=index2, columns=index1, name='f2') f3 = Frame(values, index=index3, columns=index1, name='f3') b1 = Bus.from_frames((f1, f2, f3)) def test_assertions(hierarchy: IndexHierarchy, opposite: Index) -> None: expected_tree = dict(f1=tree1, f2=tree2, f3=tree3) self.compare_trees(hierarchy.to_tree(), expected_tree) self.assertTrue(index1.equals(opposite)) test_assertions(*bus_to_hierarchy(b1, axis=0, deepcopy_from_bus=False, init_exception_cls=CustomError)) test_assertions(*bus_to_hierarchy(b1, axis=0, deepcopy_from_bus=True, init_exception_cls=CustomError)) # Cannot do this since the frames do not share the same index with self.assertRaises(CustomError): bus_to_hierarchy(b1, axis=1, deepcopy_from_bus=False, init_exception_cls=CustomError) with self.assertRaises(CustomError): bus_to_hierarchy(b1, axis=1, deepcopy_from_bus=True, init_exception_cls=CustomError) # Align all the frames on index! f1 = Frame(values, index=index1, columns=index1, name='f1') f2 = Frame(values, index=index1, columns=index2, name='f2') f3 = Frame(values, index=index1, columns=index3, name='f3') b1 = Bus.from_frames((f1, f2, f3)) test_assertions(*bus_to_hierarchy(b1, axis=1, deepcopy_from_bus=False, init_exception_cls=CustomError)) test_assertions(*bus_to_hierarchy(b1, axis=1, deepcopy_from_bus=True, init_exception_cls=CustomError)) # Cannot do this since the frames do not share the same columns with self.assertRaises(CustomError): bus_to_hierarchy(b1, axis=0, deepcopy_from_bus=False, init_exception_cls=CustomError) with self.assertRaises(CustomError): bus_to_hierarchy(b1, axis=0, deepcopy_from_bus=True, init_exception_cls=CustomError)
def test_bus_nbytes_a(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(a=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(a=(10,20), b=(50,60)), index=('p', 'q'), name='f3') b1 = Bus.from_frames((f1, f2, f3)) with temp_file('.zip') as fp: b1.to_zip_pickle(fp) b2 = Bus.from_zip_pickle(fp) f2_loaded = b2['f2'] self.assertEqual(b2.nbytes, 48) f3_loaded = b2['f3'] self.assertEqual(b2.nbytes, 80) f1_loaded = b2['f1'] self.assertEqual(b2.nbytes, 112)
def test_bus_shapes_a(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(a=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(a=(10,20), b=(50,60)), index=('p', 'q'), name='f3') b1 = Bus.from_frames((f1, f2, f3)) with temp_file('.zip') as fp: b1.to_zip_pickle(fp) b2 = Bus.from_zip_pickle(fp) f2_loaded = b2['f2'] self.assertEqual(b2.shapes.to_pairs(), (('f1', None), ('f2', (3, 2)), ('f3', None))) f3_loaded = b2['f3'] self.assertEqual(b2.shapes.to_pairs(), (('f1', None), ('f2', (3, 2)), ('f3', (2, 2 ))) )
def test_bus_keys_a(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(c=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(d=(10,20), b=(50,60)), index=('p', 'q'), name='f3') f4 = Frame.from_dict( dict(q=(None,None), r=(np.nan,np.nan)), index=(1000, 1001), name='f4') b1 = Bus.from_frames((f1, f2, f3, f4)) self.assertEqual(b1.keys().values.tolist(), ['f1', 'f2', 'f3', 'f4']) self.assertEqual(b1.values[2].name, 'f3') with temp_file('.zip') as fp: b1.to_zip_pickle(fp) b2 = Bus.from_zip_pickle(fp) self.assertFalse(b2._loaded_all) self.assertEqual(b2.keys().values.tolist(), ['f1', 'f2', 'f3', 'f4']) self.assertFalse(b2._loaded.any()) # accessing values forces loading all self.assertEqual(b2.values[2].name, 'f3') self.assertTrue(b2._loaded_all)
def test_bus_init_a(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='foo') f2 = Frame.from_dict( dict(a=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='bar') b1 = Bus.from_frames((f1, f2)) self.assertEqual(b1.keys().values.tolist(), ['foo', 'bar']) with temp_file('.zip') as fp: b1.to_zip_tsv(fp) b2 = Bus.from_zip_tsv(fp) f3 = b2['bar'] f4 = b2['foo'] # import ipdb; ipdb.set_trace() zs = StoreZipTSV(fp) zs.write(b1.items()) f3 = zs.read('foo') self.assertEqual( f3.to_pairs(0), (('a', (('x', 1), ('y', 2))), ('b', (('x', 3), ('y', 4)))) )
def test_bus_status_a(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(c=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(d=(10,20), b=(50,60)), index=('p', 'q'), name='f3') b1 = Bus.from_frames((f1, f2, f3)) with temp_file('.zip') as fp: b1.to_zip_pickle(fp) b2 = Bus.from_zip_pickle(fp) status = b2.status self.assertEqual(status.shape, (3, 4)) # force load all tuple(b2.items()) self.assertEqual( b2.status.to_pairs(0), (('loaded', (('f1', True), ('f2', True), ('f3', True))), ('size', (('f1', 4.0), ('f2', 6.0), ('f3', 4.0))), ('nbytes', (('f1', 32.0), ('f2', 48.0), ('f3', 32.0))),('shape', (('f1', (2, 2)), ('f2', (3, 2)), ('f3', (2, 2))))) )
def test_bus_init_a(self) -> None: f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)), index=('x', 'y'), name='foo') f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)), index=('x', 'y', 'z'), name='bar') config = StoreConfigMap.from_config(StoreConfig(index_depth=1)) b1 = Bus.from_frames((f1, f2), config=config) self.assertEqual(b1.keys().values.tolist(), ['foo', 'bar']) with temp_file('.zip') as fp: b1.to_zip_tsv(fp) b2 = Bus.from_zip_tsv(fp) f3 = b2['bar'] f4 = b2['foo'] # import ipdb; ipdb.set_trace() zs = StoreZipTSV(fp) zs.write(b1.items()) # how to show that this derived getitem has derived type? f3 = zs.read('foo', config=config['foo']) self.assertEqual(f3.to_pairs(0), (('a', (('x', 1), ('y', 2))), ('b', (('x', 3), ('y', 4)))))
def test_bus_to_xlsx_a(self) -> None: f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict(dict(c=(1, 2, 3), b=(4, 5, 6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict(dict(d=(10, 20), b=(50, 60)), index=('p', 'q'), name='f3') config = StoreConfigMap.from_config( StoreConfig(index_depth=1, columns_depth=1, include_columns=True, include_index=True)) b1 = Bus.from_frames((f1, f2, f3), config=config) with temp_file('.xlsx') as fp: b1.to_xlsx(fp) b2 = Bus.from_xlsx(fp, config=config) tuple(b2.items()) # force loading all for frame in (f1, f2, f3): self.assertEqualFrames(frame, b2[frame.name])
def test_bus_init_c(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='foo') f2 = Frame.from_dict( dict(a=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='bar') config = StoreConfigMap.from_config(StoreConfig(index_depth=1)) b1 = Bus.from_frames((f1, f2), config=config) self.assertEqual(b1.keys().values.tolist(), ['foo', 'bar']) with temp_file('.zip') as fp: b1.to_zip_csv(fp) b2 = Bus.from_zip_csv(fp, config=config) f1_loaded = b2['foo'] f2_loaded = b2['bar'] self.assertEqualFrames(f1, f1_loaded) self.assertEqualFrames(f2, f2_loaded)
def test_bus_to_parquet_a(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(c=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(d=(10,20), b=(50,60)), index=('p', 'q'), name='f3') config = StoreConfig( index_depth=1, columns_depth=1, include_columns=True, include_index=True ) b1 = Bus.from_frames((f1, f2, f3), config=config) with temp_file('.zip') as fp: b1.to_zip_parquet(fp) b2 = Bus.from_zip_parquet(fp, config=config) tuple(b2.items()) # force loading all for frame in (f1, f2, f3): # parquet brings in characters as objects, thus forcing different dtypes self.assertEqualFrames(frame, b2[frame.name], compare_dtype=False)
def test_bus_mloc_c(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(c=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(d=(10,20), b=(50,60)), index=('p', 'q'), name='f3') b1 = Bus.from_frames((f1, f2, f3)) with temp_file('.zip') as fp: b1.to_zip_pickle(fp) b2 = Bus.from_zip_pickle(fp) f2_loaded = b2['f2'] mloc1 = b2.mloc f3_loaded = b2['f3'] f1_loaded = b2['f1'] self.assertEqual(mloc1['f2'], b2.mloc.loc['f2'])
def test_bus_equals_b(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(c=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(d=(10,20), b=(50,60)), index=('p', 'q'), name='f3') b1 = Bus.from_frames((f1, f2, f3)) self.assertTrue(b1.equals(b1)) class BusDerived(Bus): pass b2 = BusDerived.from_frames((f1, f2, f3)) self.assertFalse(b1.equals(b2, compare_class=True)) self.assertTrue(b1.equals(b2, compare_class=False)) self.assertFalse(b1.equals('foo', compare_class=False))
def test_bus_to_hdf5_a(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict( dict(c=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict( dict(d=(10,20), b=(50,60)), index=('p', 'q'), name='f3') frames = (f1, f2, f3) config = StoreConfigMap.from_frames(frames) b1 = Bus.from_frames(frames, config=config) with temp_file('.h5') as fp: b1.to_hdf5(fp) b2 = Bus.from_hdf5(fp, config=config) tuple(b2.items()) # force loading all for frame in frames: self.assertEqualFrames(frame, b2[frame.name])