def test_concat(self):
"""Concatenate new data to an existing item in an existing file."""
# write a single item named 'item_0'
data1 = generate.full(1, self.dim, 10, items_root='item')
h5f.write(self.filename, 'group1', data1[0], data1[1], data1[2])
h5f.write(self.filename, 'group3', data1[0], data1[1], data1[2])
# concatenate new data to 'item_0'
data2 = generate.full(1, self.dim, 20, items_root='item')
h5f.write(self.filename, 'group2', data2[0], data2[1], data2[2])
with pytest.raises(IOError):
h5f.write(self.filename, 'group3', data2[0], data2[1], data2[2])
def test_npz(self):
# generate 10 items
data = generate.full(self.nfiles, 10, 30)
# write them in 10 mat files
for i in range(self.nfiles):
np.savez(self.matfiles[i],
labels=data[1][i], features=data[2][i])
# check data conservation in mat file
mat = np.load(self.matfiles[i])
assert (mat['features'] == data[2][i]).all()
assert (mat['labels'] == data[1][i]).all()
# convert it to h5features
conv = h5f.Converter(self.h5file, 'group')
for name in self.matfiles:
conv.convert(name)
conv.close()
# check convertion is correct
rdata = h5f.Reader(self.h5file, 'group').read()
assert len(rdata.items()) == self.nfiles
for i in range(self.nfiles):
assert (rdata.labels()[i] == data[1][i]).all()
assert (rdata.features()[i] == data[2][i]).all()
def setup(self):
self.filename = 'test.h5'
self.groupname = 'group'
self.nitems = 10
d = generate.full(self.nitems)
self.data = h5f.Data(d[0], d[1], d[2])
h5f.Writer(self.filename).write(self.data, self.groupname)
def setup(self):
self.file_v1 = 'v1.0.h5'
self.file_v2 = 'v1.1.h5'
self.teardown() # in case files already exist, remove it
items, times, features = generate.full(20, 10)
h5f_1_0.write(self.file_v1, 'features', items, times, features)
h5f_1_1.write(self.file_v2, 'features', items, times, features)
def setup(self):
self.filename = 'test.h5'
self.groupname = 'group'
self.nitems = 10
d = generate.full(self.nitems)
self.data = h5f.Data(d[0], d[1], d[2])
h5f.Writer(self.filename).write(self.data, self.groupname)
def setup(self):
self.file_v1 = 'v1.0.h5'
self.file_v2 = 'v1.1.h5'
self.teardown() # in case files already exist, remove it
items, times, features = generate.full(20,10)
h5f_1_0.write(self.file_v1, 'features', items, times, features)
h5f_1_1.write(self.file_v2, 'features', items, times, features)
def test_bad_file2(self):
with open(self.filename, 'w') as f:
f.write('This is not a HDF5 file')
data = generate.full(1)
with pytest.raises(IOError) as ioerror:
h5f.write(self.filename, 'group', data[0], data[1], data[2])
msg = str(ioerror.value)
assert self.filename in msg
assert 'not a HDF5 file' in msg
def test_bad_file2(self):
with open(self.filename, 'w') as f:
f.write('This is not a HDF5 file')
data = generate.full(1)
with pytest.raises(IOError) as ioerror:
h5f.write(self.filename, 'group', data[0], data[1], data[2])
msg = str(ioerror.value)
assert self.filename in msg
assert 'not a HDF5 file' in msg
def test_from_exemple(tmpdir):
filename = os.path.join(str(tmpdir), 'exemple.h5')
a1, a2, a3 = generate.full(100)
data = h5f.Data(a1, a2, a3)
h5f.Writer(filename).write(data, 'group')
with h5f.Reader(filename, 'group') as r:
rdata = r.read()
assert len(rdata.items()) == 100
assert data == rdata
def test_write(self):
files, times, features = generate.full(10, 2, 5)
h5f.write(self.filename, 'f', files, times, features)
with h5py.File(self.filename, 'r') as f:
assert ['f'] == list(f.keys ())
g = f.get('f')
assert list(g.keys()) == ['features', 'index', 'items', 'labels']
assert g.get('features').shape[1] == 2
assert g.get('index').shape == (10,)
assert g.get('items').shape == (10,)
assert g.get('features').shape[0] == g.get('labels').shape[0]
def test_write(self):
files, times, features = generate.full(10, 2, 5)
h5f.write(self.filename, 'f', files, times, features)
with h5py.File(self.filename, 'r') as f:
assert ['f'] == list(f.keys())
g = f.get('f')
assert list(g.keys()) == ['features', 'index', 'items', 'labels']
assert g.get('features').shape[1] == 2
assert g.get('index').shape == (10, )
assert g.get('items').shape == (10, )
assert g.get('features').shape[0] == g.get('labels').shape[0]
def test_2D_labels_one_frame(self):
data = generate.full(1, 5, 1, 2)
sio.savemat(self.matfile, {'labels':data[1][0], 'features':data[2][0]})
h5f.Converter(self.h5file, 'group').convert(self.matfile)
rdata = h5f.Reader(self.h5file, 'group').read()
assert rdata.items() == [os.path.splitext(self.matfile)[0]]
assert (rdata.features()[0] == data[2][0]).all()
assert (rdata.labels()[0] == data[1][0]).all()
assert rdata == h5f.Data([os.path.splitext(self.matfile)[0]],
data[1], data[2])
def _test_wr(self, labeldim):
"""Test retrieving labels and files after a write/read operation."""
items, t_gold, feat = generate.full(self.nbitems, tformat=labeldim)
write(self.filename, self.group, items, t_gold, feat)
t, _ = read(self.filename, self.group)
assert len(t) == self.nbitems
if not labeldim == 1:
assert all([tt.shape[1] == labeldim for tt in t.values()])
# build a dict from gold to compare with t
d = dict(zip(items, t_gold))
for dd, tt in zip(d, t):
assert tt == dd
def test_from_exemple():
filename = '/tmp/exemple.h5'
remove(filename)
a1, a2, a3 = generate.full(100)
data = h5f.Data(a1, a2, a3)
with h5f.Writer(filename) as w:
w.write(data, 'group')
with h5f.Reader(filename, 'group') as r:
rdata = r.read()
assert len(rdata.items()) == 100
assert data == rdata
remove(filename)
def _test_wr(self, labeldim):
"""Test retrieving labels and files after a write/read operation."""
items, t_gold, feat = generate.full(self.nbitems, tformat=labeldim)
write(self.filename, self.group, items, t_gold, feat)
t, _ = read(self.filename, self.group)
assert len(t) == self.nbitems
if not labeldim == 1:
assert all([tt.shape[1] == labeldim for tt in t.values()])
# build a dict from gold to compare with t
d = dict(zip(items, t_gold))
# d = {}
# for k, v in zip(items, t_gold): d[k] = v
# compare the two dicts
for dd, tt in zip(d, t): assert tt == dd
def test_2D_labels(self):
data = generate.full(self.nfiles, 5, 10, 2)
# write them in 10 mat files
for i in range(self.nfiles):
np.savez(self.matfiles[i],
labels=data[1][i], features=data[2][i])
conv = h5f.Converter(self.h5file, 'group')
for name in self.matfiles:
conv.convert(name)
conv.close()
rdata = h5f.Reader(self.h5file, 'group').read()
assert len(rdata.items()) == self.nfiles
for i in range(self.nfiles):
assert (rdata.labels()[i] == data[1][i]).all()
assert (rdata.features()[i] == data[2][i]).all()
def setup(self):
# init default parameters
self.filename = 'test.h5'
self.group = 'features'
remove(self.filename)
# create a simple feature file
items, times, feat = generate.full(10)
items2 = [i+'2' for i in items]
self.data = Data(items, times, feat)
self.data2 = Data(items2, times, feat)
self.writer = Writer(self.filename)
self.writer.write(self.data, self.group)
# read it with h5py
self.f = h5py.File(self.filename, 'r')
self.g = self.f.get(self.group)
def setup(self):
# init default parameters
self.filename = 'test.h5'
self.group = 'features'
remove(self.filename)
# create a simple feature file
items, times, feat = generate.full(10)
items2 = [i + '2' for i in items]
self.data = Data(items, times, feat)
self.data2 = Data(items2, times, feat)
self.writer = Writer(self.filename)
self.writer.write(self.data, self.group)
# read it with h5py
self.f = h5py.File(self.filename, 'r')
self.g = self.f.get(self.group)
def test_append(self):
"""Append a new item to an existing dataset."""
i, t, f = generate.full(30, self.dim, 40, items_root='File')
h5f.write(self.filename, 'group', i, t, f)
# append new item to existing dataset
features_added = np.zeros(shape=(1, self.dim))
times_added = np.linspace(0, 2, 1)
h5f.write(self.filename, 'group', ['File_31'], [times_added],
[features_added])
with pytest.raises(IOError) as err:
h5f.write(self.filename, 'group', ['File_3'], [times_added],
[features_added])
assert 'data is not appendable to the group' in str(err.value)
# read it
times_r, features_r = h5f.read(self.filename, 'group')
assert set(times_r.keys()) == set(i + ['File_31'])
assert set(features_r.keys()) == set(i + ['File_31'])
assert all(times_r['File_31'] == times_added)
assert (features_r['File_31'] == features_added).all()
def test_append(self):
"""Append a new item to an existing dataset."""
i, t, f = generate.full(30, self.dim, 40, items_root='File')
h5f.write(self.filename, 'group', i, t, f)
# append new item to existing dataset
features_added = np.zeros(shape=(1, self.dim))
times_added = np.linspace(0, 2, 1)
h5f.write(self.filename, 'group', ['File_31'],
[times_added], [features_added])
with pytest.raises(IOError) as err:
h5f.write(self.filename, 'group', ['File_3'],
[times_added], [features_added])
assert 'data can be added only at the end' in str(err.value)
# read it
times_r, features_r = h5f.read(self.filename, 'group')
assert set(times_r.keys()) == set(i+['File_31'])
assert set(features_r.keys()) == set(i+['File_31'])
assert all(times_r['File_31'] == times_added)
assert (features_r['File_31'] == features_added).all()
def setup(self):
items, self.data, feats = generate.full(10,tformat=1)
self.filename = 'test.h5'
self.teardown()
write(self.filename, 'group', items, self.data, feats)
self.group = h5py.File(self.filename, 'a')['group']
def test_write(self):
index.create_index(self.group, 0.1)
items, times, features = generate.full(100, dim=5, max_frames=10)
data = Data(items, times, features)
index.write_index(data, self.group, append=False)
def test_write(self):
index.create_index(self.group, 0.1)
items, times, features = generate.full(100, dim=5, max_frames=10)
data = Data(items, times, features)
index.write_index(data, self.group, append=False)
def test_bad_file(self):
a, b, c = generate.full(2)
with pytest.raises(IOError) as ioerror:
h5f.write('/silly/path/to/no/where.h5', 'group', a, b, c)
assert all([s in str(ioerror.value)]
for s in ['/silly/path', 'No such file'])
def test_raise_on_write_sparse():
a, b, c = generate.full(1)
with pytest.raises(NotImplementedError) as ioerror:
h5f.write('test.h5', 'group', a, b, c, dformat='sparse')
assert 'sparse' in str(ioerror.value)
def test_raise_on_write_sparse():
a, b, c = generate.full(1)
with pytest.raises(NotImplementedError) as ioerror:
h5f.write('test.h5', 'group', a, b, c, dformat='sparse')
assert 'sparse' in str(ioerror.value)
def test_bad_file(self):
a, b, c = generate.full(2)
with pytest.raises(IOError) as ioerror:
h5f.write('/silly/path/to/no/where.h5', 'group', a, b, c)
assert all([s in str(ioerror.value)]
for s in ['/silly/path', 'No such file'])
def setup(self):
items, self.data, feats = generate.full(10, tformat=1)
self.filename = 'test.h5'
self.teardown()
write(self.filename, 'group', items, self.data, feats)
self.group = h5py.File(self.filename, 'a')['group']
