def save(dataset, destination, name=None, compression=None):
"""Save Dataset into HDF5 file
Parameters
----------
dataset : `Dataset`
destination : `h5py.highlevel.File` or str
name : str, optional
compression : None or int or {'gzip', 'szip', 'lzf'}, optional
Level of compression for gzip, or another compression strategy.
"""
if not externals.exists('h5py'):
raise RuntimeError("Missing 'h5py' package -- saving is not possible.")
import h5py
from mvpa.base.hdf5 import obj2hdf
# look if we got an hdf file instance already
if isinstance(destination, h5py.highlevel.File):
own_file = False
hdf = destination
else:
own_file = True
hdf = h5py.File(destination, 'w')
obj2hdf(hdf, dataset, name, compression=compression)
# if we opened the file ourselves we close it now
if own_file:
hdf.close()
return
def saveload_warehouse():
"""Store all warehouse datasets into HDF5 and reload them.
"""
import h5py
from mvpa.base.hdf5 import obj2hdf, hdf2obj
tempdir = tempfile.mkdtemp()
# store the whole datasets warehouse in one hdf5 file
hdf = h5py.File(os.path.join(tempdir, 'myhdf5.hdf5'), 'w')
for d in datasets:
obj2hdf(hdf, datasets[d], d)
hdf.close()
hdf = h5py.File(os.path.join(tempdir, 'myhdf5.hdf5'), 'r')
rc_ds = {}
for d in hdf:
rc_ds[d] = hdf2obj(hdf[d])
hdf.close()
#cleanup temp dir
shutil.rmtree(tempdir, ignore_errors=True)
# return the reconstructed datasets (for use in datasets warehouse)
return rc_ds
def save(dataset, destination, name=None, compression=None):
"""Save Dataset into HDF5 file
Parameters
----------
dataset : `Dataset`
destination : `h5py.highlevel.File` or str
name : str, optional
compression : None or int or {'gzip', 'szip', 'lzf'}, optional
Level of compression for gzip, or another compression strategy.
"""
if not externals.exists('h5py'):
raise RuntimeError("Missing 'h5py' package -- saving is not possible.")
import h5py
from mvpa.base.hdf5 import obj2hdf
# look if we got an hdf file instance already
if isinstance(destination, h5py.highlevel.File):
own_file = False
hdf = destination
else:
own_file = True
hdf = h5py.File(destination, 'w')
obj2hdf(hdf, dataset, name, compression=compression)
# if we opened the file ourselves we close it now
if own_file:
hdf.close()
return
def saveload_warehouse():
"""Store all warehouse datasets into HDF5 and reload them.
"""
import h5py
from mvpa.base.hdf5 import obj2hdf, hdf2obj
tempdir = tempfile.mkdtemp()
# store the whole datasets warehouse in one hdf5 file
hdf = h5py.File(os.path.join(tempdir, 'myhdf5.hdf5'), 'w')
for d in datasets:
obj2hdf(hdf, datasets[d], d)
hdf.close()
hdf = h5py.File(os.path.join(tempdir, 'myhdf5.hdf5'), 'r')
rc_ds = {}
for d in hdf:
rc_ds[d] = hdf2obj(hdf[d])
hdf.close()
#cleanup temp dir
shutil.rmtree(tempdir, ignore_errors=True)
# return the reconstructed datasets (for use in datasets warehouse)
return rc_ds
def test_h5py_dataset_typecheck():
ds = datasets['uni2small']
_, fpath = tempfile.mkstemp('mvpa', 'test')
h5save(fpath, [[1, 2, 3]])
assert_raises(ValueError, AttrDataset.from_hdf5, fpath)
# this one just catches if there is such a group
assert_raises(ValueError, AttrDataset.from_hdf5, fpath, name='bogus')
hdf = h5py.File(fpath, 'w')
ds = AttrDataset([1, 2, 3])
obj2hdf(hdf, ds, name='non-bogus')
obj2hdf(hdf, [1, 2, 3], name='bogus')
hdf.close()
assert_raises(ValueError, AttrDataset.from_hdf5, fpath, name='bogus')
ds_loaded = AttrDataset.from_hdf5(fpath, name='non-bogus')
assert_array_equal(ds, ds_loaded) # just to do smth useful with ds ;)
# cleanup and ignore stupidity
os.remove(fpath)
def test_h5py_dataset_typecheck():
ds = datasets['uni2small']
_, fpath = tempfile.mkstemp('mvpa', 'test')
h5save(fpath, [[1, 2, 3]])
assert_raises(ValueError, AttrDataset.from_hdf5, fpath)
# this one just catches if there is such a group
assert_raises(ValueError, AttrDataset.from_hdf5, fpath, name='bogus')
hdf = h5py.File(fpath, 'w')
ds = AttrDataset([1, 2, 3])
obj2hdf(hdf, ds, name='non-bogus')
obj2hdf(hdf, [1, 2, 3], name='bogus')
hdf.close()
assert_raises(ValueError, AttrDataset.from_hdf5, fpath, name='bogus')
ds_loaded = AttrDataset.from_hdf5(fpath, name='non-bogus')
assert_array_equal(ds, ds_loaded) # just to do smth useful with ds ;)
# cleanup and ignore stupidity
os.remove(fpath)
