def test_concatenate_array(self):
fc = conc.FileConcatenator(self.dummy_files)
with tempfile.TemporaryFile() as tf:
fc.concatenate(tf)
with h5py.File(tf, "r") as f:
target = np.ones((25, 7, 3))
target[10:, :, :] = 2.
np.testing.assert_array_equal(target, f["numpy_array"][()])
def test_concatenate_used_files(self):
fc = conc.FileConcatenator(self.dummy_files)
with tempfile.TemporaryFile() as tf:
fc.concatenate(tf)
with h5py.File(tf, "r") as f:
self.assertSequenceEqual(
f["used_files"][()].tolist(),
[n.encode("ascii", "ignore") for n in self.dummy_files],
)
def test_concatenate_recarray_with_groupid(self):
fc = conc.FileConcatenator(self.dummy_files)
with tempfile.TemporaryFile() as tf:
fc.concatenate(tf)
with h5py.File(tf, "r") as f:
target = np.array([(1, 3, 1)] * 25,
dtype=[('x', '<f8'), ('y', '<i8'),
("group_id", "<i8")])
target["x"][10:] = 4.
target["y"][10:] = 5.
target["group_id"] = np.arange(25)
np.testing.assert_array_equal(target, f["rec_array"][()])
def test_concatenate_attrs(self):
fc = conc.FileConcatenator(self.dummy_files)
with tempfile.TemporaryFile() as tf:
fc.concatenate(tf)
with h5py.File(tf, "r") as f:
target_attrs = dict(f.attrs)
target_dset_attrs = dict(f["numpy_array"].attrs)
with h5py.File(self.dummy_files[0], "r") as f:
source_attrs = dict(f.attrs)
source_dset_attrs = dict(f["numpy_array"].attrs)
self.assertDictEqual(source_attrs, target_attrs)
self.assertDictEqual(source_dset_attrs, target_dset_attrs)
def test_chunksize_larger_than_data(self):
fc = conc.FileConcatenator(self.dummy_files,
comptopts_update={
"chunksize": {
"numpy_array": (100, 7, 3),
"rec_array": 200
}
})
with tempfile.TemporaryFile() as tf:
fc.concatenate(tf)
with h5py.File(tf, "r") as f:
self.assertTupleEqual(f["rec_array"].chunks,
f["rec_array"].shape)
def test_get_cumu_rows(self):
fc = conc.FileConcatenator(self.dummy_files)
np.testing.assert_array_equal(fc.cumu_rows, [0, 10, 25])
def test_fc_get_comptopts_updates(self):
fc = conc.FileConcatenator(self.dummy_files,
comptopts_update={'chunksize': 1})
target_compt_opts = dict(self.compt_opts)
target_compt_opts["chunksize"] = 1
self.assertDictEqual(fc.comptopts, target_compt_opts)
def test_fc_get_comptopts(self):
fc = conc.FileConcatenator(self.dummy_files)
self.assertDictEqual(fc.comptopts, self.compt_opts)
def test_get_cumu_rows(self):
fc = conc.FileConcatenator(self.dummy_files)
self.assertDictEqual(fc.cumu_rows, {
"numpy_array": [0, 10, 25],
"rec_array": [0, 10, 25]
})