def test_store_config_map_a(self) -> None: sc1 = StoreConfig(index_depth=3, columns_depth=3) sc1m = StoreConfigMap.from_config(sc1) self.assertEqual(sc1m['a'].index_depth, 3) self.assertEqual(sc1m['b'].index_depth, 3) sc2 = StoreConfig(include_index=False) sc2m = StoreConfigMap.from_config(sc2) self.assertEqual(sc2m['a'].include_index, False) self.assertEqual(sc2m['b'].include_index, False)
def test_bus_to_xlsx_a(self) -> None: f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)), index=('x', 'y'), name='f1') f2 = Frame.from_dict(dict(c=(1, 2, 3), b=(4, 5, 6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_dict(dict(d=(10, 20), b=(50, 60)), index=('p', 'q'), name='f3') config = StoreConfigMap.from_config( StoreConfig(index_depth=1, columns_depth=1, include_columns=True, include_index=True)) b1 = Bus.from_frames((f1, f2, f3), config=config) with temp_file('.xlsx') as fp: b1.to_xlsx(fp) b2 = Bus.from_xlsx(fp, config=config) tuple(b2.items()) # force loading all for frame in (f1, f2, f3): self.assertEqualFrames(frame, b2[frame.name])
def test_bus_init_a(self) -> None: f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)), index=('x', 'y'), name='foo') f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)), index=('x', 'y', 'z'), name='bar') config = StoreConfigMap.from_config(StoreConfig(index_depth=1)) b1 = Bus.from_frames((f1, f2), config=config) self.assertEqual(b1.keys().values.tolist(), ['foo', 'bar']) with temp_file('.zip') as fp: b1.to_zip_tsv(fp) b2 = Bus.from_zip_tsv(fp) f3 = b2['bar'] f4 = b2['foo'] # import ipdb; ipdb.set_trace() zs = StoreZipTSV(fp) zs.write(b1.items()) # how to show that this derived getitem has derived type? f3 = zs.read('foo', config=config['foo']) self.assertEqual(f3.to_pairs(0), (('a', (('x', 1), ('y', 2))), ('b', (('x', 3), ('y', 4)))))
def test_bus_init_c(self) -> None: f1 = Frame.from_dict( dict(a=(1,2), b=(3,4)), index=('x', 'y'), name='foo') f2 = Frame.from_dict( dict(a=(1,2,3), b=(4,5,6)), index=('x', 'y', 'z'), name='bar') config = StoreConfigMap.from_config(StoreConfig(index_depth=1)) b1 = Bus.from_frames((f1, f2), config=config) self.assertEqual(b1.keys().values.tolist(), ['foo', 'bar']) with temp_file('.zip') as fp: b1.to_zip_csv(fp) b2 = Bus.from_zip_csv(fp, config=config) f1_loaded = b2['foo'] f2_loaded = b2['bar'] self.assertEqualFrames(f1, f1_loaded) self.assertEqualFrames(f2, f2_loaded)
def test_store_xlsx_write_b(self) -> None: f1 = Frame.from_records( ((None, np.nan, 50, 'a'), (None, -np.inf, -50, 'b'), (None, 60.4, -50, 'c')), index=('p', 'q', 'r'), columns=IndexHierarchy.from_product(('I', 'II'), ('a', 'b')), ) config_map = StoreConfigMap.from_config( StoreConfig(include_index=True, include_columns=True)) with temp_file('.xlsx') as fp: st = StoreXLSX(fp) st.write(((STORE_LABEL_DEFAULT, f1),), config=config_map) c = StoreConfig( index_depth=f1.index.depth, columns_depth=f1.columns.depth ) f2 = st.read(STORE_LABEL_DEFAULT, config=c) # just a sample column for now self.assertEqual( f1[HLoc[('II', 'a')]].values.tolist(), f2[HLoc[('II', 'a')]].values.tolist() ) self.assertEqualFrames(f1, f2)
def test_store_sqlite_read_many_a(self) -> None: f1 = Frame.from_dict(dict(x=(1, 2, -5, 200), y=(3, 4, -5, -3000)), index=IndexHierarchy.from_product(('I', 'II'), ('a', 'b')), name='f1') f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_records(((10, 20, 50, 60), (50.0, 60.4, -50, -60)), index=('p', 'q'), columns=IndexHierarchy.from_product( ('I', 'II'), ('a', 'b')), name='f3') f4 = Frame.from_records( ( (10, 20, 50, False, 10, 20, 50, False), (50.0, 60.4, -50, True, 50.0, 60.4, -50, True), (234, 44452, 0, False, 234, 44452, 0, False), (4, -4, 2000, True, 4, -4, 2000, True), (10, 20, 50, False, 10, 20, 50, False), (50.0, 60.4, -50, True, 50.0, 60.4, -50, True), (234, 44452, 0, False, 234, 44452, 0, False), (4, -4, 2000, True, 4, -4, 2000, True), ), index=IndexHierarchy.from_product( ('top', 'bottom'), ('far', 'near'), ('left', 'right')), columns=IndexHierarchy.from_product(('I', 'II'), ('a', 'b'), (1, 2)), name='f4') frames = (f1, f2, f3, f4) config_map_write = StoreConfigMap.from_config( StoreConfig(include_index=True, include_columns=True)) with temp_file('.sqlite') as fp: st1 = StoreSQLite(fp) st1.write(((f.name, f) for f in frames), config=config_map_write) labels = tuple( st1.labels()) # this will read from file, not in memory self.assertEqual(tuple(f.name for f in frames), labels) config_map_read: tp.Dict[tp.Hashable, StoreConfig] = {} for i, name in enumerate(labels): f_src = frames[i] c = StoreConfig(index_depth=f_src.index.depth, columns_depth=f_src.columns.depth) config_map_read[name] = c for i, f_loaded in enumerate( st1.read_many(labels, config=config_map_read)): f_src = frames[i] self.assertEqualFrames(f_src, f_loaded, compare_dtype=False)
def test_store_xlsx_write_a(self) -> None: f1 = Frame.from_dict(dict(x=(1, 2, -5, 200), y=(3, 4, -5, -3000)), index=IndexHierarchy.from_product(('I', 'II'), ('a', 'b')), name='f1') f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)), index=('x', 'y', 'z'), name='f2') f3 = Frame.from_records(((10, 20, 50, 60), (50.0, 60.4, -50, -60)), index=('p', 'q'), columns=IndexHierarchy.from_product( ('I', 'II'), ('a', 'b')), name='f3') f4 = Frame.from_records( ( (10, 20, 50, False, 10, 20, 50, False), (50.0, 60.4, -50, True, 50.0, 60.4, -50, True), (234, 44452, 0, False, 234, 44452, 0, False), (4, -4, 2000, True, 4, -4, 2000, True), (10, 20, 50, False, 10, 20, 50, False), (50.0, 60.4, -50, True, 50.0, 60.4, -50, True), (234, 44452, 0, False, 234, 44452, 0, False), (4, -4, 2000, True, 4, -4, 2000, True), ), index=IndexHierarchy.from_product( ('top', 'bottom'), ('far', 'near'), ('left', 'right')), columns=IndexHierarchy.from_product(('I', 'II'), ('a', 'b'), (1, 2)), name='f4') frames = (f1, f2, f3, f4) config_map = StoreConfigMap.from_config( StoreConfig(include_index=True, include_columns=True)) with temp_file('.xlsx') as fp: st1 = StoreXLSX(fp) st1.write(((f.name, f) for f in frames), config=config_map) # import ipdb; ipdb.set_trace() sheet_names = tuple( st1.labels()) # this will read from file, not in memory self.assertEqual(tuple(f.name for f in frames), sheet_names) for i, name in enumerate(sheet_names): f_src = frames[i] c = StoreConfig(index_depth=f_src.index.depth, columns_depth=f_src.columns.depth) f_loaded = st1.read(name, config=c) self.assertEqualFrames(f_src, f_loaded, check_dtypes=False)
def test_bus_update_series_cache_iloc(self) -> None: f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)), index=('x', 'y'), name='foo') config = StoreConfigMap.from_config(StoreConfig(index_depth=1)) # simulating a Bus with a FrameDefferred but no Store, just for testing s1 = Series((f1, FrameDeferred), index=('p', 'q')) b1 = Bus(s1, config=config) self.assertFalse(b1._loaded_all) with self.assertRaises(RuntimeError): b1._update_series_cache_iloc(1)
def test_store_zip_pickle_c(self) -> None: f1 = FrameGO.from_dict(dict(a=(1, 2), b=(3, 4)), index=('x', 'y'), name='foo') config = StoreConfig(index_depth=1, include_index=True) config_map = StoreConfigMap.from_config(config) with temp_file('.zip') as fp: st = StoreZipPickle(fp) # raise if trying to store a FrameGO with self.assertRaises(NotImplementedError): st.write(((f1.name, f1), ))
def test_store_zip_pickle_b(self) -> None: f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)), index=('x', 'y'), name='foo') config = StoreConfig(index_depth=1, include_index=True) config_map = StoreConfigMap.from_config(config) with temp_file('.zip') as fp: st = StoreZipPickle(fp) st.write(((f1.name, f1), )) frame_stored = st.read(f1.name) self.assertEqual(frame_stored.shape, f1.shape)
def test_store_xlsx_read_a(self) -> None: f1 = Frame.from_elements([1, 2, 3], index=('a', 'b', 'c'), columns=('x', )) config_map = StoreConfigMap.from_config( StoreConfig(include_index=False, include_columns=True)) with temp_file('.xlsx') as fp: st = StoreXLSX(fp) st.write(((None, f1), ), config=config_map) c = StoreConfig(index_depth=0, columns_depth=f1.columns.depth) f2 = st.read(None, config=c) self.assertTrue((f1.values == f2.values).all()) self.assertEqual(f2.to_pairs(0), (('x', ((0, 1), (1, 2), (2, 3))), ))
def test_store_xlsx_read_b(self) -> None: index = IndexHierarchy.from_product(('left', 'right'), ('up', 'down')) columns = IndexHierarchy.from_labels(((100, -5, 20), )) f1 = Frame.from_elements([1, 2, 3, 4], index=index, columns=columns) config_map = StoreConfigMap.from_config( StoreConfig(include_index=False, include_columns=True)) with temp_file('.xlsx') as fp: st = StoreXLSX(fp) st.write(((None, f1), ), config=config_map) c = StoreConfig(index_depth=0, columns_depth=f1.columns.depth) f2 = st.read(None, config=c) self.assertTrue((f1.values == f2.values).all()) self.assertEqual(f2.to_pairs(0), (((100, -5, 20), ((0, 1), (1, 2), (2, 3), (3, 4))), ))
def test_bus_mloc_b(self) -> None: f1 = Frame.from_dict(dict(a=(1, 2, 3)), index=('x', 'y', 'z'), name='f1') f2 = Frame.from_dict(dict(x=(10, 20, 30)), index=('q', 'r', 's'), name='f2') config = StoreConfigMap.from_config(StoreConfig(index_depth=1)) b1 = Bus.from_frames((f1, f2), config=config) with temp_file('.xlsx') as fp: b1.to_xlsx(fp) b2 = Bus.from_xlsx(fp, config=config) # only show memory locations for loaded Frames self.assertTrue(b2.iloc[1].equals(f2)) self.assertEqual( (b2.mloc == None).to_pairs(), #pylint: disable=C0121 (('f1', True), ('f2', False)))