def test_bus_max_persist_3(self) -> None:
def items() -> tp.Iterator[tp.Tuple[str, Frame]]:
for i in range(4):
yield str(i), Frame(np.arange(i, i+10).reshape(2, 5))
s = Series.from_items(items(), dtype=object)
b1 = Bus(s)
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp, config=config, max_persist=4)
_ = b2.iloc[[0, 1]]
_ = b2.iloc[[2, 3]]
self.assertTrue(b2._loaded_all)
_ = b2.iloc[[1, 0]]
self.assertEqual(list(b2._last_accessed.keys()),
['2', '3', '1', '0'])
_ = b2.iloc[3]
self.assertEqual(list(b2._last_accessed.keys()),
['2', '1', '0', '3'])
_ = b2.iloc[:3]
self.assertEqual(list(b2._last_accessed.keys()),
['3', '0', '1', '2'])
def test_frame_to_npz_a(self) -> None:
# this forced a RuntimeError: File size unexpectedly exceeded ZIP64 limit error until the `force_zip64` parameter was added
a1 = np.arange(15_000 * 18_000).reshape(15_000, 18_000)
f1 = Frame(a1)
with temp_file('.npz') as fp:
f1.to_npz(fp)
def test_store_zip_parquet_a(self) -> None:
f1, f2, f3 = get_test_framesA()
with temp_file('.zip') as fp:
for read_max_workers in (1, 2):
config = StoreConfig(index_depth=1,
include_index=True,
columns_depth=1,
read_max_workers=read_max_workers)
st = StoreZipParquet(fp)
st.write((f.name, f) for f in (f1, f2, f3))
f1_post = st.read('foo', config=config)
self.assertTrue(
f1.equals(f1_post, compare_name=True, compare_class=True))
f2_post = st.read('bar', config=config)
self.assertTrue(
f2.equals(f2_post, compare_name=True, compare_class=True))
f3_post = st.read('baz', config=config)
self.assertTrue(
f3.equals(f3_post, compare_name=True, compare_class=True))
def test_store_xlsx_write_b(self) -> None:
f1 = Frame.from_records(
((None, np.nan, 50, 'a'), (None, -np.inf, -50, 'b'), (None, 60.4, -50, 'c')),
index=('p', 'q', 'r'),
columns=IndexHierarchy.from_product(('I', 'II'), ('a', 'b')),
)
config_map = StoreConfigMap.from_config(
StoreConfig(include_index=True, include_columns=True))
with temp_file('.xlsx') as fp:
st = StoreXLSX(fp)
st.write(((STORE_LABEL_DEFAULT, f1),), config=config_map)
c = StoreConfig(
index_depth=f1.index.depth,
columns_depth=f1.columns.depth
)
f2 = st.read(STORE_LABEL_DEFAULT, config=c)
# just a sample column for now
self.assertEqual(
f1[HLoc[('II', 'a')]].values.tolist(),
f2[HLoc[('II', 'a')]].values.tolist() )
self.assertEqualFrames(f1, f2)
def test_store_zip_tsv_a(self) -> None:
f1, f2, f3 = get_test_framesA()
with temp_file('.zip') as fp:
st = StoreZipTSV(fp)
st.write((f.name, f) for f in (f1, f2, f3))
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels, ('foo.txt', 'bar.txt', 'baz.txt'))
for label, frame in ((f.name, f) for f in (f1, f2, f3)):
for read_max_workers in (None, 1, 2):
config = StoreConfig(index_depth=1,
read_max_workers=read_max_workers)
frame_stored = st.read(label, config=config)
self.assertEqual(frame_stored.shape, frame.shape)
self.assertTrue((frame_stored == frame).all().all())
self.assertEqual(frame.to_pairs(0),
frame_stored.to_pairs(0))
frame_stored_2 = st.read(label,
config=config,
container_type=FrameGO)
self.assertEqual(frame_stored_2.__class__, FrameGO)
self.assertEqual(frame_stored_2.shape, frame.shape)
def test_store_sqlite_write_d(self) -> None:
f1 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)),
index=('x', 'y', 'z'),
name='f2')
frames = (f1, )
with temp_file('.sqlite') as fp:
config = StoreConfig(include_index=False)
st1 = StoreSQLite(fp)
st1.write(((f.name, f) for f in frames), config=config)
f2 = st1.read(f1.name, config=config)
self.assertEqual(f2.to_pairs(0), (('a', ((0, 1), (1, 2), (2, 3))),
('b', ((0, 4), (1, 5), (2, 6)))))
# getting the default config
f3 = st1.read(f1.name, config=None)
self.assertEqual(f3.to_pairs(0), (('a', ((0, 1), (1, 2), (2, 3))),
('b', ((0, 4), (1, 5), (2, 6)))))
def test_store_zip_pickle_a(self) -> None:
f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)),
index=('x', 'y'),
name='foo')
f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)),
index=('x', 'y', 'z'),
name='bar')
f3 = Frame.from_dict(dict(a=(10, 20), b=(50, 60)),
index=('p', 'q'),
name='baz')
with temp_file('.zip') as fp:
st = StoreZipPickle(fp)
st.write((f.name, f) for f in (f1, f2, f3))
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels,
('foo.pickle', 'bar.pickle', 'baz.pickle'))
for label, frame in ((f.name, f) for f in (f1, f2, f3)):
frame_stored = st.read(label)
self.assertEqual(frame_stored.shape, frame.shape)
self.assertTrue((frame_stored == frame).all().all())
self.assertEqual(frame.to_pairs(0), frame_stored.to_pairs(0))
def test_bus_nbytes_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(a=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(a=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
b1 = Bus.from_frames((f1, f2, f3))
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp)
f2_loaded = b2['f2']
self.assertEqual(b2.nbytes, 48)
f3_loaded = b2['f3']
self.assertEqual(b2.nbytes, 80)
f1_loaded = b2['f1']
self.assertEqual(b2.nbytes, 112)
def test_bus_dtypes_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(c=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(d=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
b1 = Bus.from_frames((f1, f2, f3))
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp)
self.assertEqual(b2.dtypes.to_pairs(0), ())
f2_loaded = b2['f2']
self.assertEqual(b2.dtypes.to_pairs(0),
(('c', (('f1', None), ('f2', np.dtype('int64')), ('f3', None))), ('b', (('f1', None), ('f2', np.dtype('int64')), ('f3', None))))
)
f3_loaded = b2['f3']
self.assertEqual(b2.dtypes.to_pairs(0),
(('b', (('f1', None), ('f2', np.dtype('int64')), ('f3', np.dtype('int64')))), ('c', (('f1', None), ('f2', np.dtype('int64')), ('f3', None))), ('d', (('f1', None), ('f2', None), ('f3', np.dtype('int64')))))
)
def test_store_xlsx_read_e(self) -> None:
f1 = Frame.from_records(((np.inf, np.inf), (-np.inf, -np.inf)),
index=('p', 'q'),
columns=('a', 'b'),
name='f1')
sc1 = StoreConfig(columns_depth=1, index_depth=1)
with temp_file('.xlsx') as fp:
st = StoreXLSX(fp)
st.write(((STORE_LABEL_DEFAULT, f1), ))
f1 = st.read(STORE_LABEL_DEFAULT, config=sc1, store_filter=None)
self.assertEqual(f1.to_pairs(0),
(('a', (('p', 'inf'), ('q', '-inf'))),
('b', (('p', 'inf'), ('q', '-inf')))))
f2 = st.read(STORE_LABEL_DEFAULT,
config=sc1,
store_filter=StoreFilter())
self.assertEqual(f2.to_pairs(0),
(('a', (('p', np.inf), ('q', -np.inf))),
('b', (('p', np.inf), ('q', -np.inf)))))
def test_store_xlsx_read_many_f(self) -> None:
records = (
(2, 2, 'a', False, None),
(30, 73, 'd', True, None),
(None, None, None, None, None),
(None, None, None, None, None),
)
f1 = Frame.from_records(records, columns=('p', 'q', 'r', 's', 't'))
with temp_file('.xlsx') as fp:
f1.to_xlsx(fp,
label='f1',
include_index=False,
include_columns=False)
st1 = StoreXLSX(fp)
c = StoreConfig(
index_depth=3, # force coverage
columns_depth=0,
trim_nadir=True,
)
f2 = next(st1.read_many(('f1', ), config=c))
self.assertEqual(f2.shape, (2, 1))
self.assertEqual(f2.to_pairs(), ((0, (((2, 2, 'a'), False),
((30, 73, 'd'), True))), ))
def test_store_xlsx_read_d(self) -> None:
f1 = Frame.from_records(((10, 20, 50, 60), (50.0, 60.4, -50, -60)),
index=('p', 'q'),
columns=('a', 'b', 'c', 'd'),
name='f1')
sc1 = StoreConfig(include_index=False, include_columns=True)
sc2 = StoreConfig(columns_depth=0, index_depth=0)
with temp_file('.xlsx') as fp:
st = StoreXLSX(fp)
st.write(((STORE_LABEL_DEFAULT, f1), ), config=sc1)
f2 = st.read(STORE_LABEL_DEFAULT) # get default config
self.assertEqual(f2.to_pairs(0),
(('a', ((0, 10), (1, 50))), ('b', ((0, 20.0),
(1, 60.4))),
('c', ((0, 50), (1, -50))), ('d', ((0, 60),
(1, -60)))))
f3 = st.read(STORE_LABEL_DEFAULT, config=sc2)
self.assertEqual(f3.to_pairs(0),
((0, ((0, 'a'), (1, 10), (2, 50))),
(1, ((0, 'b'), (1, 20), (2, 60.4))),
(2, ((0, 'c'), (1, 50), (2, -50))),
(3, ((0, 'd'), (1, 60), (2, -60)))))
def test_store_sqlite_write_b(self) -> None:
f1 = Frame.from_dict(
dict(
x=(Fraction(3,2), Fraction(1,2), Fraction(2,3), Fraction(3,7)),
y=(3,4,-5,-3000)),
index=IndexHierarchy.from_product(('I', 'II'), ('a', 'b')),
name='f1')
frames = (f1,)
with temp_file('.sqlite') as fp:
st1 = StoreSQLite(fp)
st1.write((f.name, f) for f in frames)
f_loaded = st1.read(f1.name,
index_depth=f1.index.depth,
columns_depth=f1.columns.depth
)
# for now, Fractions come back as strings
self.assertEqual(
f_loaded['x'].to_pairs(),
((('I', 'a'), '3/2'), (('I', 'b'), '1/2'), (('II', 'a'), '2/3'), (('II', 'b'), '3/7'))
)
def test_bus_max_persist_a(self) -> None:
def items() -> tp.Iterator[tp.Tuple[str, Frame]]:
for i in range(20):
yield str(i), Frame(np.arange(i, i+10).reshape(2, 5))
s = Series.from_items(items(), dtype=object)
b1 = Bus(s)
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp, config=config, max_persist=3)
for i in b2.index:
_ = b2[i]
self.assertTrue(b2._loaded.sum() <= 3)
# after iteration only the last three are loaded
self.assertEqual(b2._loaded.tolist(),
[False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True])
def test_bus_mloc_c(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(c=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(d=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
b1 = Bus.from_frames((f1, f2, f3))
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp)
f2_loaded = b2['f2']
mloc1 = b2.mloc
f3_loaded = b2['f3']
f1_loaded = b2['f1']
self.assertEqual(mloc1['f2'], b2.mloc.loc['f2'])
def test_bus_status_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(c=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(d=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
b1 = Bus.from_frames((f1, f2, f3))
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp)
status = b2.status
self.assertEqual(status.shape, (3, 4))
# force load all
tuple(b2.items())
self.assertEqual(
b2.status.to_pairs(0), (('loaded', (('f1', True), ('f2', True), ('f3', True))), ('size', (('f1', 4.0), ('f2', 6.0), ('f3', 4.0))), ('nbytes', (('f1', 32.0), ('f2', 48.0), ('f3', 32.0))),('shape', (('f1', (2, 2)), ('f2', (3, 2)), ('f3', (2, 2)))))
)
def test_bus_init_c(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='foo')
f2 = Frame.from_dict(
dict(a=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='bar')
config = StoreConfigMap.from_config(StoreConfig(index_depth=1))
b1 = Bus.from_frames((f1, f2), config=config)
self.assertEqual(b1.keys().values.tolist(),
['foo', 'bar'])
with temp_file('.zip') as fp:
b1.to_zip_csv(fp)
b2 = Bus.from_zip_csv(fp, config=config)
f1_loaded = b2['foo']
f2_loaded = b2['bar']
self.assertEqualFrames(f1, f1_loaded)
self.assertEqualFrames(f2, f2_loaded)
def test_bus_keys_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(c=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(d=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
f4 = Frame.from_dict(
dict(q=(None,None), r=(np.nan,np.nan)),
index=(1000, 1001),
name='f4')
b1 = Bus.from_frames((f1, f2, f3, f4))
self.assertEqual(b1.keys().values.tolist(), ['f1', 'f2', 'f3', 'f4'])
self.assertEqual(b1.values[2].name, 'f3')
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp)
self.assertFalse(b2._loaded_all)
self.assertEqual(b2.keys().values.tolist(), ['f1', 'f2', 'f3', 'f4'])
self.assertFalse(b2._loaded.any())
# accessing values forces loading all
self.assertEqual(b2.values[2].name, 'f3')
self.assertTrue(b2._loaded_all)
def test_batch_to_zip_pickle_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(a=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(a=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
b1 = Batch.from_frames((f1, f2, f3))
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp, config=config)
b2 = Batch.from_zip_pickle(fp, config=config)
frames = dict(b2.items())
for frame in (f1, f2, f3):
# parquet brings in characters as objects, thus forcing different dtypes
self.assertEqualFrames(frame, frames[frame.name], compare_dtype=False)
def test_bus_to_xlsx_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(c=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(d=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
config = StoreConfigMap.from_config(
StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
))
b1 = Bus.from_frames((f1, f2, f3), config=config)
with temp_file('.xlsx') as fp:
b1.to_xlsx(fp)
b2 = Bus.from_xlsx(fp, config=config)
tuple(b2.items()) # force loading all
for frame in (f1, f2, f3):
self.assertEqualFrames(frame, b2[frame.name])
def test_store_xlsx_read_many_d(self) -> None:
records = (
(2, 2, 'a', False, None),
(30, 73, 'd', True, None),
(None, None, None, None, None),
(None, None, None, None, None),
)
columns = IndexHierarchy.from_labels((
('a', 1), ('a', 2), ('b', 1), ('b', 2), (None, None)
))
f1 = Frame.from_records(records, columns=columns)
with temp_file('.xlsx') as fp:
f1.to_xlsx(fp, label='f1', include_index=False, include_columns=True)
st1 = StoreXLSX(fp)
c = StoreConfig(
index_depth=0,
columns_depth=2,
trim_nadir=True,
)
f2 = next(st1.read_many(('f1',), config=c))
self.assertEqual(f2.shape, (2, 4))
self.assertEqual(f2.to_pairs(),
((('a', 1), ((0, 2), (1, 30))), (('a', 2), ((0, 2), (1, 73))), (('b', 1), ((0, 'a'), (1, 'd'))), (('b', 2), ((0, False), (1, True)))))
def test_bus_init_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='foo')
f2 = Frame.from_dict(
dict(a=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='bar')
config = StoreConfigMap.from_config(StoreConfig(index_depth=1))
b1 = Bus.from_frames((f1, f2), config=config)
self.assertEqual(b1.keys().values.tolist(),
['foo', 'bar'])
with temp_file('.zip') as fp:
b1.to_zip_tsv(fp)
b2 = Bus.from_zip_tsv(fp)
f3 = b2['bar']
f4 = b2['foo']
# import ipdb; ipdb.set_trace()
zs = StoreZipTSV(fp)
zs.write(b1.items())
# how to show that this derived getitem has derived type?
f3 = zs.read('foo', config=config['foo'])
self.assertEqual(
f3.to_pairs(0),
(('a', (('x', 1), ('y', 2))), ('b', (('x', 3), ('y', 4))))
)
def test_store_read_many_single_thread_weak_cache(self) -> None:
f1, f2, f3 = get_test_framesA()
with temp_file('.zip') as fp:
st = StoreZipTSV(fp)
st.write((f.name, f) for f in (f1, f2, f3))
kwargs = dict(config_map=StoreConfigMap.from_initializer(
StoreConfig(index_depth=1)),
constructor=st._container_type_to_constructor(Frame),
container_type=Frame)
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels, ('foo.txt', 'bar.txt', 'baz.txt'))
self.assertEqual(0, len(list(st._weak_cache)))
# Result is not held onto!
next(st._read_many_single_thread(('foo', ), **kwargs))
self.assertEqual(0, len(list(st._weak_cache)))
# Result IS held onto!
frame = next(st._read_many_single_thread(('foo', ), **kwargs))
self.assertEqual(1, len(list(st._weak_cache)))
# Reference in our weak_cache _is_ `frame`
self.assertIs(frame, st._weak_cache['foo'])
del frame
# Reference is gone now!
self.assertEqual(0, len(list(st._weak_cache)))
def test_bus_to_xlsx_c(self) -> None:
'''
Test manipulating a file behind the Bus.
'''
f1 = Frame.from_dict(
dict(a=(1,2,3)),
index=('x', 'y', 'z'),
name='f1')
f2 = Frame.from_dict(
dict(x=(10,20,30)),
index=('q', 'r', 's'),
name='f2')
b1 = Bus.from_frames((f1,),)
with temp_file('.xlsx') as fp:
b1.to_xlsx(fp)
b2 = Bus.from_xlsx(fp)
f2.to_xlsx(fp)
with self.assertRaises(StoreFileMutation):
tuple(b2.items())
def test_display_active_a(self) -> None:
fp1 = DisplayActive._default_fp()
with temp_file('.json') as fp2:
DisplayActive.write(fp2) #type: ignore
DisplayActive.read(fp2) #type: ignore
def test_bus_to_xlsx_f(self) -> None:
f = Frame.from_records([
[np.datetime64('1983-02-20 05:34:18.763'), np.datetime64('2020-08-01')],
[np.datetime64('1975-03-20 05:20:18.001'), np.datetime64('2020-07-31')]
],
columns=(date(2020, 7, 31), date(2020, 8, 1)),
index=(datetime(2020, 7, 31, 14, 20, 8), datetime(2017, 4, 28, 2, 30, 2)),
name='frame')
b1 = Bus.from_frames([f])
with temp_file('.xlsx') as fp:
b1.to_xlsx(fp)
config = StoreConfig(include_index=True, index_depth=1)
b2 = Bus.from_xlsx(fp, config=config)
tuple(b2.items()) # force loading all
self.assertEqual(b2['frame'].index.values.tolist(),
[datetime(2020, 7, 31, 14, 20, 8),
datetime(2017, 4, 28, 2, 30, 2)])
self.assertEqual(b2['frame'].index.values.tolist(),
[datetime(2020, 7, 31, 14, 20, 8),
datetime(2017, 4, 28, 2, 30, 2)])
self.assertEqual(b2['frame'].values.tolist(),
[[datetime(1983, 2, 20, 5, 34, 18, 763000), datetime(2020, 8, 1, 0, 0)], [datetime(1975, 3, 20, 5, 20, 18, 1000), datetime(2020, 7, 31, 0, 0)]]
)
def test_store_zip_pickle_a(self) -> None:
f1, f2, f3 = get_test_framesA()
with temp_file('.zip') as fp:
st = StoreZipPickle(fp)
st.write((f.name, f) for f in (f1, f2, f3))
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels,
('foo.pickle', 'bar.pickle', 'baz.pickle'))
for label, frame in ((f.name, f) for f in (f1, f2, f3)):
frame_stored = st.read(label)
self.assertEqual(frame_stored.shape, frame.shape)
self.assertTrue((frame_stored == frame).all().all())
self.assertEqual(frame.to_pairs(0), frame_stored.to_pairs(0))
frame_stored_2 = st.read(label, container_type=FrameGO)
self.assertEqual(frame_stored_2.__class__, FrameGO)
self.assertEqual(frame_stored_2.shape, frame.shape)
frame_stored_3 = st.read(label, container_type=FrameHE)
self.assertEqual(frame_stored_3.__class__, FrameHE)
self.assertEqual(frame_stored_3.shape, frame.shape)
def test_bus_to_hdf5_a(self) -> None:
f1 = Frame.from_dict(
dict(a=(1,2), b=(3,4)),
index=('x', 'y'),
name='f1')
f2 = Frame.from_dict(
dict(c=(1,2,3), b=(4,5,6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_dict(
dict(d=(10,20), b=(50,60)),
index=('p', 'q'),
name='f3')
frames = (f1, f2, f3)
config = StoreConfigMap.from_frames(frames)
b1 = Bus.from_frames(frames, config=config)
with temp_file('.h5') as fp:
b1.to_hdf5(fp)
b2 = Bus.from_hdf5(fp, config=config)
tuple(b2.items()) # force loading all
for frame in frames:
self.assertEqualFrames(frame, b2[frame.name])
def test_store_zip_parquet_c(self) -> None:
f1, f2 = get_test_framesB()
config = StoreConfig(
index_depth=1,
include_index=True,
index_constructors=IndexDate,
columns_depth=1,
include_columns=True,
)
with temp_file('.zip') as fp:
st = StoreZipParquet(fp)
st.write(((f.name, f) for f in (f1, f2)), config=config)
post = tuple(
st.read_many(
('a', 'b'),
container_type=Frame,
config=config,
))
self.assertIs(post[0].index.__class__, IndexDate)
self.assertIs(post[1].index.__class__, IndexDate)
def test_bus_max_persist_b(self) -> None:
def items() -> tp.Iterator[tp.Tuple[str, Frame]]:
for i in range(20):
yield str(i), Frame(np.arange(i, i+10).reshape(2, 5))
s = Series.from_items(items(), dtype=object)
b1 = Bus(s)
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp, config=config, max_persist=1)
b3 = b2.iloc[10:]
self.assertEqual(b3._loaded.sum(), 1)
# only the last one is loasded
self.assertEqual(b3._loaded.tolist(),
[False, False, False, False, False, False, False, False, False, True]
)
self.assertEqual(b3.iloc[0].sum().sum(), 145)
self.assertEqual(b3._loaded.tolist(),
[True, False, False, False, False, False, False, False, False, False]
)
self.assertEqual(b3.iloc[4].sum().sum(), 185)
self.assertEqual(b3._loaded.tolist(),
[False, False, False, False, True, False, False, False, False, False]
)
