def test_bus_init_b(self) -> None:
with self.assertRaises(ErrorInitBus):
Bus(Series([1, 2, 3]))
with self.assertRaises(ErrorInitBus):
Bus(Series([3, 4], dtype=object))
def to_bus(self) -> 'Bus':
'''Realize the :obj:`Batch` as an :obj:`Bus`. Note that, as a :obj:`Bus` must have all labels (even if :obj:`Frame` are loaded lazily)
'''
return Bus(
Series.from_items(self.items(),
name=self._name,
dtype=DTYPE_OBJECT))
def test_bus_max_persist_a(self) -> None:
def items() -> tp.Iterator[tp.Tuple[str, Frame]]:
for i in range(20):
yield str(i), Frame(np.arange(i, i+10).reshape(2, 5))
s = Series.from_items(items(), dtype=object)
b1 = Bus(s)
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp, config=config, max_persist=3)
for i in b2.index:
_ = b2[i]
self.assertTrue(b2._loaded.sum() <= 3)
# after iteration only the last three are loaded
self.assertEqual(b2._loaded.tolist(),
[False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True])
def test_bus_max_persist_3(self) -> None:
def items() -> tp.Iterator[tp.Tuple[str, Frame]]:
for i in range(4):
yield str(i), Frame(np.arange(i, i+10).reshape(2, 5))
s = Series.from_items(items(), dtype=object)
b1 = Bus(s)
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp, config=config, max_persist=4)
_ = b2.iloc[[0, 1]]
_ = b2.iloc[[2, 3]]
self.assertTrue(b2._loaded_all)
_ = b2.iloc[[1, 0]]
self.assertEqual(list(b2._last_accessed.keys()),
['2', '3', '1', '0'])
_ = b2.iloc[3]
self.assertEqual(list(b2._last_accessed.keys()),
['2', '1', '0', '3'])
_ = b2.iloc[:3]
self.assertEqual(list(b2._last_accessed.keys()),
['3', '0', '1', '2'])
def test_bus_max_persist_b(self) -> None:
def items() -> tp.Iterator[tp.Tuple[str, Frame]]:
for i in range(20):
yield str(i), Frame(np.arange(i, i+10).reshape(2, 5))
s = Series.from_items(items(), dtype=object)
b1 = Bus(s)
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp, config=config, max_persist=1)
b3 = b2.iloc[10:]
self.assertEqual(b3._loaded.sum(), 1)
# only the last one is loasded
self.assertEqual(b3._loaded.tolist(),
[False, False, False, False, False, False, False, False, False, True]
)
self.assertEqual(b3.iloc[0].sum().sum(), 145)
self.assertEqual(b3._loaded.tolist(),
[True, False, False, False, False, False, False, False, False, False]
)
self.assertEqual(b3.iloc[4].sum().sum(), 185)
self.assertEqual(b3._loaded.tolist(),
[False, False, False, False, True, False, False, False, False, False]
)
def to_bus(self) -> 'Bus':
'''Realize the :obj:`Batch` as an :obj:`Bus`. Note that, as a :obj:`Bus` must have all labels (even if :obj:`Frame` are loaded lazily), this :obj:`Batch` will be exhausted.
'''
series = Series.from_items(self.items(),
name=self._name,
dtype=DTYPE_OBJECT)
return Bus(series, config=self._config)
def test_bus_max_persist_d(self) -> None:
def items() -> tp.Iterator[tp.Tuple[str, Frame]]:
for i in range(5):
yield str(i), Frame(np.arange(i, i+10).reshape(2, 5))
s = Series.from_items(items(), dtype=object)
b1 = Bus(s)
config = StoreConfig(
index_depth=1,
columns_depth=1,
include_columns=True,
include_index=True
)
with temp_file('.zip') as fp:
b1.to_zip_pickle(fp)
b2 = Bus.from_zip_pickle(fp, config=config, max_persist=3)
_ = b2.iloc[[0, 2, 4]]
self.assertEqual(b2._loaded.tolist(),
[True, False, True, False, True])
_ = b2.iloc[[1, 2, 3]]
self.assertEqual(b2._loaded.tolist(),
[False, True, True, True, False])
_ = b2.iloc[4]
self.assertEqual(b2._loaded.tolist(),
[False, False, True, True, True])
_ = b2.iloc[0]
self.assertEqual(b2._loaded.tolist(),
[True, False, False, True, True])
_ = b2.iloc[[2, 3, 4]]
self.assertEqual(b2._loaded.tolist(),
[False, False, True, True, True])
_ = b2.iloc[[0, 1]]
self.assertEqual(b2._loaded.tolist(),
[True, True, False, False, True])
_ = b2.iloc[0]
self.assertEqual(b2._loaded.tolist(),
[True, True, False, False, True])
_ = b2.iloc[[3, 4]]
self.assertEqual(b2._loaded.tolist(),
[True, False, False, True, True])
def test_bus_update_series_cache_iloc(self) -> None:
f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)),
index=('x', 'y'),
name='foo')
config = StoreConfigMap.from_config(StoreConfig(index_depth=1))
# simulating a Bus with a FrameDefferred but no Store, just for testing
s1 = Series((f1, FrameDeferred), index=('p', 'q'))
b1 = Bus(s1, config=config)
self.assertFalse(b1._loaded_all)
with self.assertRaises(RuntimeError):
b1._update_series_cache_iloc(1)
def test_bus_extract_loc_a(self) -> None:
f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)),
index=('x', 'y'),
name='foo')
f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)),
index=('x', 'y', 'z'),
name='bar')
f3 = Frame.from_dict(dict(d=(10, 20), b=(50, 60)),
index=('p', 'q'),
name='f3')
ih = IndexHierarchy.from_labels((('a', 1), ('b', 2), ('b', 1)))
s1 = Series((f1, f2, f3), index=ih, dtype=object)
# do not support IndexHierarchy, as lables are tuples, not strings
with self.assertRaises(ErrorInitBus):
b1 = Bus(s1)
