def test_merging_multidimensional_results(tmpdir):
filename = str(tmpdir.join('r.h5'))
data1 = {'a': np.zeros((2, 2, 2))}
data2 = {'a': np.ones((3, 2, 2))}
append_dict_h5(filename, data1)
append_dict_h5(filename, data2)
saved = load_dict_h5(filename)
assert np.all(np.concatenate([data1['a'], data2['a']]) == saved['a'])
def test_merging_results_with_varying_dimensionality(tmpdir):
filename = str(tmpdir.join('r.h5'))
data1 = {'a': np.zeros((1, 1, 2))}
data2 = {'a': np.ones((2, 2, 1))}
expected = np.array([
[[0., 0.], [np.nan, np.nan]],
[[1., np.nan], [1., np.nan]],
[[1., np.nan], [1., np.nan]]])
append_dict_h5(filename, data1)
append_dict_h5(filename, data2)
saved = load_dict_h5(filename)
assert expected.shape == saved['a'].shape
for a, b in zip(expected.flat, saved['a'].flat):
assert a == b or (np.isnan(a) and np.isnan(b))
def merge(cls, outdir, merged_filename, append=True):
"""Merge processed files together.
Parameters
----------
outdir : str
Directory with the output files.
merged_filename : str
Filename of file to save with the merged results.
append : bool
If ``True`` the merged data will be appended, otherwise the file
will be overwritten with the merged data.
"""
if not append:
save_dict_h5(merged_filename, {})
for filename in os.listdir(outdir):
if os.path.splitext(filename)[1] != '.h5':
continue
infile = os.path.join(outdir, filename)
append_dict_h5(merged_filename, load_dict_h5(infile))
