def from_hdf5(cls,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None) -> 'Bus':
store = StoreHDF5(fp)
return cls(cls._deferred_series(store.labels()),
store=store,
config=config)
def test_store_hdf5_write_e(self) -> None:
# failure when including objects
f1 = Frame.from_dict(
{'0ax': (1,2,-5, 3), '3iv': (3,4,-5,-3000)},
name='foo',
)
frames = (f1,)
with temp_file('.hdf5') as fp:
st1 = StoreHDF5(fp)
st1.write(((f.name, f) for f in frames))
st2 = StoreHDF5(fp)
c1 = StoreConfig(index_depth=1)
f2 = st2.read('foo', config=c1)
self.assertEqual(f2.to_pairs(0),
(('0ax', ((0, 1), (1, 2), (2, -5), (3, 3))),
('3iv', ((0, 3), (1, 4), (2, -5), (3, -3000)))))
def test_store_hdf5_read_many_a(self) -> None:
f1 = Frame.from_dict(dict(x=(1, 2, -5, 200), y=(3, 4, -5, -3000)),
index=IndexHierarchy.from_product(('I', 'II'),
('a', 'b')),
name='f1')
f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_records(((10, 20, 50, 60), (50.0, 60.4, -50, -60)),
index=('p', 'q'),
columns=IndexHierarchy.from_product(
('I', 'II'), ('a', 'b')),
name='f3')
f4 = Frame.from_records(
(
(10, 20, 50, False, 10, 20, 50, False),
(50.0, 60.4, -50, True, 50.0, 60.4, -50, True),
(234, 44452, 0, False, 234, 44452, 0, False),
(4, -4, 2000, True, 4, -4, 2000, True),
(10, 20, 50, False, 10, 20, 50, False),
(50.0, 60.4, -50, True, 50.0, 60.4, -50, True),
(234, 44452, 0, False, 234, 44452, 0, False),
(4, -4, 2000, True, 4, -4, 2000, True),
),
index=IndexHierarchy.from_product(
('top', 'bottom'), ('far', 'near'), ('left', 'right')),
columns=IndexHierarchy.from_product(('I', 'II'), ('a', 'b'),
(1, 2)),
name='f4')
frames = (f1, f2, f3, f4)
config_map_write = StoreConfigMap.from_config(
StoreConfig(include_index=True, include_columns=True))
with temp_file('.hdf5') as fp:
st1 = StoreHDF5(fp)
st1.write(((f.name, f) for f in frames), config=config_map_write)
labels = tuple(
st1.labels()) # this will read from file, not in memory
self.assertEqual(tuple(f.name for f in frames), labels)
config_map_read: tp.Dict[tp.Hashable, StoreConfig] = {}
for i, name in enumerate(labels):
f_src = frames[i]
c = StoreConfig(index_depth=f_src.index.depth,
columns_depth=f_src.columns.depth)
config_map_read[name] = c
for i, f_loaded in enumerate(
st1.read_many(labels, config=config_map_read)):
f_src = frames[i]
self.assertEqualFrames(f_src, f_loaded, compare_dtype=False)
def to_hdf5(self,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None) -> None:
'''
Write the complete :obj:`Bus` as an HDF5 table.
{args}
'''
store = StoreHDF5(fp)
config = config if not config is None else self._config
store.write(self.items(), config=config)
def from_hdf5(
cls,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None) -> 'StoreClientMixin':
'''
Given a file path to a HDF5 :obj:`Bus` store, return a :obj:`Bus` instance.
{args}
'''
store = StoreHDF5(fp)
return cls._from_store(store, config) #type: ignore
def test_store_hdf5_write_b(self) -> None:
# failure when including objects
f1 = Frame.from_dict(dict(x=(1, 2, -5, object()),
y=(3, 4, -5, -3000)), )
frames = (f1, )
with temp_file('.hdf5') as fp:
st1 = StoreHDF5(fp)
with self.assertRaises(RuntimeError):
st1.write(((f.name, f) for f in frames))
def to_hdf5(self,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None) -> None:
'''
Write the complete :obj:`Bus` as an HDF5 table.
{args}
'''
store = StoreHDF5(fp)
config = self._filter_config(config)
store.write(self._items_store(), config=config)
def test_store_hdf5_write_a(self) -> None:
f1 = Frame.from_dict(dict(x=(1, 2, -5, 200), y=(3, 4, -5, -3000)),
index=IndexHierarchy.from_product(('I', 'II'),
('a', 'b')),
name='f1')
f2 = Frame.from_dict(dict(a=(1, 2, 3), b=(4, 5, 6)),
index=('x', 'y', 'z'),
name='f2')
f3 = Frame.from_records(((10, 20, 50, 60), (50.0, 60.4, -50, -60)),
index=('p', 'q'),
columns=IndexHierarchy.from_product(
('I', 'II'), ('a', 'b')),
name='f3')
f4 = Frame.from_records(
(
(10, 20, 50, False, 10, 20, 50, False),
(50.0, 60.4, -50, True, 50.0, 60.4, -50, True),
(234, 44452, 0, False, 234, 44452, 0, False),
(4, -4, 2000, True, 4, -4, 2000, True),
(10, 20, 50, False, 10, 20, 50, False),
(50.0, 60.4, -50, True, 50.0, 60.4, -50, True),
(234, 44452, 0, False, 234, 44452, 0, False),
(4, -4, 2000, True, 4, -4, 2000, True),
),
index=IndexHierarchy.from_product(
('top', 'bottom'), ('far', 'near'), ('left', 'right')),
columns=IndexHierarchy.from_product(('I', 'II'), ('a', 'b'),
(1, 2)),
name='f4')
frames = (f1, f2, f3, f4)
with temp_file('.hdf5') as fp:
st1 = StoreHDF5(fp)
st1.write((f.name, f) for f in frames)
sheet_names = tuple(
st1.labels()) # this will read from file, not in memory
self.assertEqual(tuple(f.name for f in frames), sheet_names)
for i, name in enumerate(sheet_names):
f_src = frames[i]
f_loaded = st1.read(name,
index_depth=f_src.index.depth,
columns_depth=f_src.columns.depth)
# print(f_loaded)
# self.assertEqualFrames(f_src, f_loaded)
# import ipdb; ipdb.set_trace()
pass
def from_hdf5(
cls,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
) -> 'StoreClientMixin':
'''
Given a file path to a HDF5 :obj:`Bus` store, return a :obj:`Bus` instance.
{args}
'''
store = StoreHDF5(fp)
return cls._from_store(
store, #type: ignore
config=config,
max_persist=max_persist,
)
def from_hdf5(cls,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
index_constructor: IndexConstructor = None,
) -> 'Bus':
'''
Given a file path to a HDF5 :obj:`Bus` store, return a :obj:`Bus` instance.
{args}
'''
store = StoreHDF5(fp)
return cls._from_store(store,
config=config,
max_persist=max_persist,
index_constructor=index_constructor,
)
def test_store_hdf5_write_c(self) -> None:
f1 = Frame.from_dict(dict(x=(True, False), y=('foo', 'bar')),
name='baz')
frames = (f1, )
with temp_file('.hdf5') as fp:
st1 = StoreHDF5(fp)
st1.write(((f.name, f) for f in frames))
f2 = st1.read('baz')
self.assertEqual(f2.columns.values.tolist(),
['__index0__', 'x', 'y'])
# cannot use dtypes on hdf5
config = StoreConfig(dtypes=(int, ))
with self.assertRaises(NotImplementedError):
f2 = st1.read('baz', config=config)
def test_store_hdf5_write_d(self) -> None:
# failure when including objects
f1 = Frame.from_dict(
dict(x=(1, 2, -5, 3), y=(3, 4, -5, -3000)),
name='foo',
)
frames = (f1, )
with temp_file('.hdf5') as fp:
st1 = StoreHDF5(fp)
st1.write(((f.name, f) for f in frames))
c1 = StoreConfig(index_depth=1, consolidate_blocks=False)
f2 = st1.read('foo', config=c1)
self.assertEqual(f2._blocks.shapes.tolist(), [(4, ), (4, )])
c2 = StoreConfig(index_depth=1, consolidate_blocks=True)
f3 = st1.read('foo', config=c2)
self.assertEqual(f3._blocks.shapes.tolist(), [(4, 2)])
def from_hdf5(
cls,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None,
max_workers: tp.Optional[int] = None,
chunksize: int = 1,
use_threads: bool = False,
) -> 'Batch':
'''
Given a file path to a HDF5 :obj:`Batch` store, return a :obj:`Batch` instance.
{args}
'''
store = StoreHDF5(fp)
return cls._from_store(
store,
config=config,
max_workers=max_workers,
chunksize=chunksize,
use_threads=use_threads,
)
def from_hdf5(cls,
fp: PathSpecifier,
*,
config: StoreConfigMapInitializer = None,
axis: int = 0,
retain_labels: bool,
deepcopy_from_bus: bool = False,
max_persist: tp.Optional[int] = None,
) -> 'Quilt':
'''
Given a file path to a HDF5 :obj:`Quilt` store, return a :obj:`Quilt` instance.
{args}
'''
store = StoreHDF5(fp)
return cls._from_store(store,
config=config,
axis=axis,
retain_labels=retain_labels,
deepcopy_from_bus=deepcopy_from_bus,
max_persist=max_persist,
)
def to_hdf5(self,
fp: PathSpecifier,
config: StoreConfigMapInitializer = None) -> None:
store = StoreHDF5(fp)
config = config if not None else self._config
store.write(self.items())
def from_hdf5(cls, fp: PathSpecifier) -> 'Bus':
store = StoreHDF5(fp)
return cls(cls._deferred_series(store.labels()), store=store)
def to_hdf5(self, fp: PathSpecifier) -> None:
store = StoreHDF5(fp)
store.write(self.items())
