def test_bus_init_a(self) -> None:
f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)),
index=('x', 'y'),
name='foo')
f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)),
index=('x', 'y', 'z'),
name='bar')
config = StoreConfigMap.from_config(StoreConfig(index_depth=1))
b1 = Bus.from_frames((f1, f2), config=config)
self.assertEqual(b1.keys().values.tolist(), ['foo', 'bar'])
with temp_file('.zip') as fp:
b1.to_zip_tsv(fp)
b2 = Bus.from_zip_tsv(fp)
f3 = b2['bar']
f4 = b2['foo']
# import ipdb; ipdb.set_trace()
zs = StoreZipTSV(fp)
zs.write(b1.items())
# how to show that this derived getitem has derived type?
f3 = zs.read('foo', config=config['foo'])
self.assertEqual(f3.to_pairs(0), (('a', (('x', 1), ('y', 2))),
('b', (('x', 3), ('y', 4)))))
def from_zip_tsv(
cls,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None,
) -> 'Bus':
# take and store a StoreConfigMap
store = StoreZipTSV(fp)
return cls(cls._deferred_series(store.labels()),
store=store,
config=config)
def to_zip_tsv(self,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None) -> None:
'''
Write the complete :obj:`Bus` as a zipped archive of TSV files.
{args}
'''
store = StoreZipTSV(fp)
config = config if not config is None else self._config
store.write(self.items(), config=config)
def test_store_read_many_single_thread_weak_cache(self) -> None:
f1, f2, f3 = get_test_framesA()
with temp_file('.zip') as fp:
st = StoreZipTSV(fp)
st.write((f.name, f) for f in (f1, f2, f3))
kwargs = dict(config_map=StoreConfigMap.from_initializer(
StoreConfig(index_depth=1)),
constructor=st._container_type_to_constructor(Frame),
container_type=Frame)
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels, ('foo.txt', 'bar.txt', 'baz.txt'))
self.assertEqual(0, len(list(st._weak_cache)))
# Result is not held onto!
next(st._read_many_single_thread(('foo', ), **kwargs))
self.assertEqual(0, len(list(st._weak_cache)))
# Result IS held onto!
frame = next(st._read_many_single_thread(('foo', ), **kwargs))
self.assertEqual(1, len(list(st._weak_cache)))
# Reference in our weak_cache _is_ `frame`
self.assertIs(frame, st._weak_cache['foo'])
del frame
# Reference is gone now!
self.assertEqual(0, len(list(st._weak_cache)))
def to_zip_tsv(
self,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None,
) -> None:
'''
Write the complete :obj:`Bus` as a zipped archive of TSV files.
{args}
'''
store = StoreZipTSV(fp)
config = self._filter_config(config)
store.write(self._items_store(), config=config)
def test_store_zip_tsv_a(self) -> None:
f1, f2, f3 = get_test_framesA()
with temp_file('.zip') as fp:
st = StoreZipTSV(fp)
st.write((f.name, f) for f in (f1, f2, f3))
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels, ('foo.txt', 'bar.txt', 'baz.txt'))
for label, frame in ((f.name, f) for f in (f1, f2, f3)):
for read_max_workers in (None, 1, 2):
config = StoreConfig(index_depth=1,
read_max_workers=read_max_workers)
frame_stored = st.read(label, config=config)
self.assertEqual(frame_stored.shape, frame.shape)
self.assertTrue((frame_stored == frame).all().all())
self.assertEqual(frame.to_pairs(0),
frame_stored.to_pairs(0))
frame_stored_2 = st.read(label,
config=config,
container_type=FrameGO)
self.assertEqual(frame_stored_2.__class__, FrameGO)
self.assertEqual(frame_stored_2.shape, frame.shape)
def test_store_zip_tsv_a(self) -> None:
f1 = Frame.from_dict(dict(a=(1, 2), b=(3, 4)),
index=('x', 'y'),
name='foo')
f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)),
index=('x', 'y', 'z'),
name='bar')
f3 = Frame.from_dict(dict(a=(10, 20), b=(50, 60)),
index=('p', 'q'),
name='baz')
with temp_file('.zip') as fp:
st = StoreZipTSV(fp)
st.write((f.name, f) for f in (f1, f2, f3))
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels, ('foo.txt', 'bar.txt', 'baz.txt'))
config = StoreConfig(index_depth=1)
for label, frame in ((f.name, f) for f in (f1, f2, f3)):
frame_stored = st.read(label, config=config)
self.assertEqual(frame_stored.shape, frame.shape)
self.assertTrue((frame_stored == frame).all().all())
self.assertEqual(frame.to_pairs(0), frame_stored.to_pairs(0))
frame_stored_2 = st.read(label,
config=config,
container_type=FrameGO)
self.assertEqual(frame_stored_2.__class__, FrameGO)
self.assertEqual(frame_stored_2.shape, frame.shape)
def from_zip_tsv(
cls,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None,
) -> 'StoreClientMixin':
'''
Given a file path to zipped TSV :obj:`Bus` store, return a :obj:`Bus` instance.
{args}
'''
store = StoreZipTSV(fp)
return cls._from_store(store, config) #type: ignore
def from_zip_tsv(
cls,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
) -> 'StoreClientMixin':
'''
Given a file path to zipped TSV :obj:`Bus` store, return a :obj:`Bus` instance.
{args}
'''
store = StoreZipTSV(fp)
return cls._from_store(
store, #type: ignore
config=config,
max_persist=max_persist,
)
def from_zip_tsv(cls,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
index_constructor: IndexConstructor = None,
) -> 'Bus':
'''
Given a file path to zipped TSV :obj:`Bus` store, return a :obj:`Bus` instance.
{args}
'''
store = StoreZipTSV(fp)
return cls._from_store(store,
config=config,
max_persist=max_persist,
index_constructor=index_constructor,
)
def from_zip_tsv(cls,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None,
axis: int = 0,
retain_labels: bool,
deepcopy_from_bus: bool = False,
max_persist: tp.Optional[int] = None,
) -> 'Quilt':
'''
Given a file path to zipped TSV :obj:`Quilt` store, return a :obj:`Quilt` instance.
{args}
'''
store = StoreZipTSV(fp)
return cls._from_store(store,
config=config,
axis=axis,
retain_labels=retain_labels,
deepcopy_from_bus=deepcopy_from_bus,
max_persist=max_persist,
)
def from_zip_tsv(
cls,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None,
max_workers: tp.Optional[int] = None,
chunksize: int = 1,
use_threads: bool = False,
) -> 'Batch':
'''
Given a file path to zipped TSV :obj:`Batch` store, return a :obj:`Batch` instance.
{args}
'''
store = StoreZipTSV(fp)
return cls._from_store(
store,
config=config,
max_workers=max_workers,
chunksize=chunksize,
use_threads=use_threads,
)
def test_store_init_a(self) -> None:
with self.assertRaises(ErrorInitStore):
StoreZipTSV('test.txt') # must be a zip
def test_store_read_many_weak_cache_a(self) -> None:
def gen_test_frames() -> tp.Iterator[tp.Tuple[str, Frame]]:
f1, f2, f3 = get_test_framesA()
yield from ((f.name, f) for f in (f1, f2, f3))
for read_max_workers in (None, 1):
with temp_file('.zip') as fp:
st = StoreZipTSV(fp)
st.write(gen_test_frames())
kwargs = dict(
config=StoreConfig(index_depth=1,
read_max_workers=read_max_workers),
container_type=Frame,
)
labels = tuple(st.labels(strip_ext=True))
self.assertEqual(labels, ('foo', 'bar', 'baz'))
self.assertEqual(0, len(list(st._weak_cache)))
# Go through the pass where there are no cache hits!
# Don't hold onto the result!
list(st.read_many(labels, **kwargs))
self.assertEqual(0, len(list(st._weak_cache)))
# Hold onto all results
result = list(st.read_many(labels, **kwargs))
self.assertEqual(3, len(result))
self.assertEqual(3, len(list(st._weak_cache)))
del result
self.assertEqual(0, len(list(st._weak_cache)))
[frame] = list(st.read_many(('foo', ), **kwargs))
self.assertIs(frame, st._weak_cache['foo'])
# Go through pass where there are some cache hits!
# Don't hold onto the result!
list(st.read_many(labels, **kwargs))
self.assertEqual(1, len(list(st._weak_cache)))
# Hold onto all results
result = list(st.read_many(labels, **kwargs))
self.assertEqual(3, len(result))
self.assertEqual(3, len(list(st._weak_cache)))
# Go through pass where all labels are in the cache
result2 = list(st.read_many(labels, **kwargs))
self.assertEqual(len(result), len(result2))
for f1, f2 in zip(result, result2):
self.assertIs(f1, f2)
def to_zip_tsv(self,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None) -> None:
store = StoreZipTSV(fp)
config = config if not None else self._config
store.write(self.items(), config=config)
