def test_infile_with_none(tmpdir): filename = str(tmpdir.join('infile.h5')) pspace = (Param(a=[1.], b=[1.]) + Param(a=[2.], c=[2.])).build() save_dict_h5(filename, pspace) saved_pspace = load_dict_h5(filename) assert sorted(pspace.keys()) == sorted(saved_pspace.keys()) for k in pspace: for a, b in zip(pspace[k], saved_pspace[k]): assert a == b or (np.isnan(a)and np.isnan(b))
def test_worker(mapper, tmpdir): infile = os.path.join(str(tmpdir), 'in.h5') outfile = os.path.join(str(tmpdir), 'out.h5') save_dict_h5(infile, Param(a=range(7)).build()) worker = Worker(mapper) worker.start(square, infile, outfile) result = load_dict_h5(outfile) assert sorted(result['a']) == sorted(range(7)) assert sorted(result['x']) == [i ** 2 for i in range(7)]
def split(self): """Perform splitting of parameters space and save input files for processing.""" items_remaining = len(self.pspace) param_iter = self.pspace.iterate() for i, filename in enumerate(self._iter_filenames()): split_size = max( self.min_items, items_remaining // (self.max_splits - i)) items_remaining -= split_size block = dict_concat( [row for row in self._iter_n(param_iter, split_size)]) save_dict_h5(os.path.join(self.indir, filename), block)
def start(self, fn, infile, outfile): """Start processing a parameter space. Parameters ---------- fn : function Function to evaluate on the parameter space. infile : str Parameter space input filename. outfile : str Output filename for the results. """ pspace = Param(**load_dict_h5(infile)) data = self.mapper(fn, pspace, **self.mapper_kwargs) save_dict_h5(outfile, data)
def test_split_merge_roundtrip( self, tmpdir, pspace_size, max_splits, min_items, n_splits): splitter = Splitter( str(tmpdir), Param(x=range(pspace_size)), max_splits, min_items) splitter.split() for filename in os.listdir(splitter.indir): infile = os.path.join(splitter.indir, filename) outfile = os.path.join(splitter.outdir, filename) save_dict_h5(outfile, load_dict_h5(infile)) result_file = os.path.join(str(tmpdir), 'result.h5') Splitter.merge(splitter.outdir, result_file) result = load_dict_h5(result_file) assert sorted(result['x']) == sorted(range(pspace_size))
def merge(cls, outdir, merged_filename, append=True): """Merge processed files together. Parameters ---------- outdir : str Directory with the output files. merged_filename : str Filename of file to save with the merged results. append : bool If ``True`` the merged data will be appended, otherwise the file will be overwritten with the merged data. """ if not append: save_dict_h5(merged_filename, {}) for filename in os.listdir(outdir): if os.path.splitext(filename)[1] != '.h5': continue infile = os.path.join(outdir, filename) append_dict_h5(merged_filename, load_dict_h5(infile))