def test_infile_with_none(tmpdir):
filename = str(tmpdir.join('infile.h5'))
pspace = (Param(a=[1.], b=[1.]) + Param(a=[2.], c=[2.])).build()
save_dict_h5(filename, pspace)
saved_pspace = load_dict_h5(filename)
assert sorted(pspace.keys()) == sorted(saved_pspace.keys())
for k in pspace:
for a, b in zip(pspace[k], saved_pspace[k]):
assert a == b or (np.isnan(a)and np.isnan(b))
def test_worker(mapper, tmpdir):
infile = os.path.join(str(tmpdir), 'in.h5')
outfile = os.path.join(str(tmpdir), 'out.h5')
save_dict_h5(infile, Param(a=range(7)).build())
worker = Worker(mapper)
worker.start(square, infile, outfile)
result = load_dict_h5(outfile)
assert sorted(result['a']) == sorted(range(7))
assert sorted(result['x']) == [i ** 2 for i in range(7)]
def split(self):
"""Perform splitting of parameters space and save input files for
processing."""
items_remaining = len(self.pspace)
param_iter = self.pspace.iterate()
for i, filename in enumerate(self._iter_filenames()):
split_size = max(
self.min_items, items_remaining // (self.max_splits - i))
items_remaining -= split_size
block = dict_concat(
[row for row in self._iter_n(param_iter, split_size)])
save_dict_h5(os.path.join(self.indir, filename), block)
def start(self, fn, infile, outfile):
"""Start processing a parameter space.
Parameters
----------
fn : function
Function to evaluate on the parameter space.
infile : str
Parameter space input filename.
outfile : str
Output filename for the results.
"""
pspace = Param(**load_dict_h5(infile))
data = self.mapper(fn, pspace, **self.mapper_kwargs)
save_dict_h5(outfile, data)
def test_split_merge_roundtrip(
self, tmpdir, pspace_size, max_splits, min_items, n_splits):
splitter = Splitter(
str(tmpdir), Param(x=range(pspace_size)), max_splits, min_items)
splitter.split()
for filename in os.listdir(splitter.indir):
infile = os.path.join(splitter.indir, filename)
outfile = os.path.join(splitter.outdir, filename)
save_dict_h5(outfile, load_dict_h5(infile))
result_file = os.path.join(str(tmpdir), 'result.h5')
Splitter.merge(splitter.outdir, result_file)
result = load_dict_h5(result_file)
assert sorted(result['x']) == sorted(range(pspace_size))
def merge(cls, outdir, merged_filename, append=True):
"""Merge processed files together.
Parameters
----------
outdir : str
Directory with the output files.
merged_filename : str
Filename of file to save with the merged results.
append : bool
If ``True`` the merged data will be appended, otherwise the file
will be overwritten with the merged data.
"""
if not append:
save_dict_h5(merged_filename, {})
for filename in os.listdir(outdir):
if os.path.splitext(filename)[1] != '.h5':
continue
infile = os.path.join(outdir, filename)
append_dict_h5(merged_filename, load_dict_h5(infile))
