def test_merging_multidimensional_results(tmpdir): filename = str(tmpdir.join('r.h5')) data1 = {'a': np.zeros((2, 2, 2))} data2 = {'a': np.ones((3, 2, 2))} append_dict_h5(filename, data1) append_dict_h5(filename, data2) saved = load_dict_h5(filename) assert np.all(np.concatenate([data1['a'], data2['a']]) == saved['a'])
def test_merging_results_with_varying_dimensionality(tmpdir): filename = str(tmpdir.join('r.h5')) data1 = {'a': np.zeros((1, 1, 2))} data2 = {'a': np.ones((2, 2, 1))} expected = np.array([ [[0., 0.], [np.nan, np.nan]], [[1., np.nan], [1., np.nan]], [[1., np.nan], [1., np.nan]]]) append_dict_h5(filename, data1) append_dict_h5(filename, data2) saved = load_dict_h5(filename) assert expected.shape == saved['a'].shape for a, b in zip(expected.flat, saved['a'].flat): assert a == b or (np.isnan(a) and np.isnan(b))
def merge(cls, outdir, merged_filename, append=True): """Merge processed files together. Parameters ---------- outdir : str Directory with the output files. merged_filename : str Filename of file to save with the merged results. append : bool If ``True`` the merged data will be appended, otherwise the file will be overwritten with the merged data. """ if not append: save_dict_h5(merged_filename, {}) for filename in os.listdir(outdir): if os.path.splitext(filename)[1] != '.h5': continue infile = os.path.join(outdir, filename) append_dict_h5(merged_filename, load_dict_h5(infile))