def test_simulate():
tmpdir = tempfile.mkdtemp(prefix='psweep_test_simulate_')
params = [{'a': 1}, {'a': 2}, {'a': 3}, {'a': 4}]
params_sim = [{'a': 88}, {'a': 99}]
calc_dir = "{}/calc".format(tmpdir)
calc_dir_sim = calc_dir + '.simulate'
df = ps.run(func, params, calc_dir=calc_dir)
df_sim = ps.run(func, params_sim, calc_dir=calc_dir, simulate=True)
dbfn = "{}/results.pk".format(calc_dir)
dbfn_sim = "{}/results.pk".format(calc_dir_sim)
assert len(df_sim) == 6
assert len(df) == 4
assert os.path.exists(dbfn)
assert os.path.exists(dbfn_sim)
assert df.equals(ps.df_read(dbfn))
assert df_sim.equals(ps.df_read(dbfn_sim))
assert df.iloc[:4].equals(df_sim.iloc[:4])
assert np.isnan(df_sim.result.values[-2:]).all()
df2 = ps.run(func, params_sim, calc_dir=calc_dir)
assert len(df2) == 6
assert df.iloc[:4].equals(df2.iloc[:4])
assert (df2.result.values[-2:] == np.array([880.0, 990.0])).all()
shutil.rmtree(tmpdir)
def test_run():
tmpdir = tempfile.mkdtemp(prefix='psweep_test_run_')
params = [{'a': 1}, {'a': 2}, {'a': 3}, {'a': 4}]
calc_dir = "{}/calc".format(tmpdir)
# run two times, updating the database, the second time,
# also write tmp results
df = ps.run(func, params, calc_dir=calc_dir)
assert len(df) == 4
assert len(df._run_id.unique()) == 1
assert len(df._pset_id.unique()) == 4
df = ps.run(func, params, calc_dir=calc_dir, poolsize=2, tmpsave=True)
assert len(df) == 8
assert len(df._run_id.unique()) == 2
assert len(df._pset_id.unique()) == 8
assert set(df.columns) == \
set(['_calc_dir', '_pset_id', '_run_id', '_time_utc', 'a', 'result'])
dbfn = "{}/results.pk".format(calc_dir)
assert os.path.exists(dbfn)
assert df.equals(ps.df_read(dbfn))
# tmp results of second run
run_id = df._run_id.unique()[-1]
for pset_id in df[df._run_id == run_id]._pset_id:
tmpsave_fn = "{calc_dir}/tmpsave/{run_id}/{pset_id}.pk".format(
calc_dir=calc_dir, run_id=run_id, pset_id=pset_id)
assert os.path.exists(tmpsave_fn)
shutil.rmtree(tmpdir)
def test_save():
tmpdir = tempfile.mkdtemp(prefix='psweep_test_run_')
params = [{'a': 1}, {'a': 2}, {'a': 3}, {'a': 4}]
calc_dir = "{}/calc".format(tmpdir)
dbfn = "{}/results.pk".format(calc_dir)
df = ps.run(func, params, calc_dir=calc_dir, save=False)
assert not os.path.exists(dbfn)
assert os.listdir(tmpdir) == []
df = ps.run(func, params, calc_dir=calc_dir, save=True)
assert os.path.exists(dbfn)
assert os.listdir(tmpdir) != []
shutil.rmtree(tmpdir)
def test_backup():
def func(pset):
# write stuff to calc_dir
dr = pj(pset['_calc_dir'], pset['_pset_id'])
ps.makedirs(dr)
fn = pj(dr, 'foo')
with open(fn, 'w') as fd:
fd.write("bar")
return ps.merge_dicts(pset, {'result': pset['a'] * 10})
tmpdir = tempfile.mkdtemp(prefix='psweep_test_run_')
params = [{'a': 1}, {'a': 2}, {'a': 3}, {'a': 4}]
calc_dir = "{}/calc".format(tmpdir)
# First run. backup_calc_dir does nothing yet. Test backup_script.
df = ps.run(func,
params,
calc_dir=calc_dir,
backup_script=__file__,
backup_calc_dir=True)
run_id = df._run_id.unique()[-1]
script_fn = "{}/backup_script/{}.py".format(calc_dir, run_id)
assert os.path.exists(script_fn)
with open(script_fn) as fd1, open(__file__) as fd2:
assert fd1.read() == fd2.read()
df1 = df.copy()
# Second run. This time, test backup_calc_dir.
df = ps.run(func, params, calc_dir=calc_dir, backup_calc_dir=True)
rex = re.compile(r"calc_[0-9-]+T[0-9:\.]+Z")
found = False
for name in os.listdir(tmpdir):
if rex.search(name) is not None:
backup_dir = pj(tmpdir, name)
found = True
break
assert found
print(os.listdir(backup_dir))
for pset_id in df1[df1._run_id == run_id]._pset_id:
tgt = pj(backup_dir, pset_id, 'foo')
assert os.path.exists(tgt)
assert os.path.exists(pj(backup_dir, 'results.pk'))
assert os.path.exists("{}/backup_script/{}.py".format(backup_dir, run_id))
def test_scripts():
tmpdir = tempfile.mkdtemp(prefix='psweep_test_bin_')
params = [{'a': 1}, {'a': 2}, {'a': 3}, {'a': 4}]
calc_dir = "{}/calc".format(tmpdir)
df = ps.run(func, params, calc_dir=calc_dir)
bindir = ps.fullpath(pj(os.path.dirname(__file__), '../../bin'))
db = pj(calc_dir, 'results.pk')
print(system("{}/psweepdb2json.py -o columns {}".format(bindir, db)))
print(system("{}/psweepdb2table.py -i -a -f simple {}".format(bindir, db)))
shutil.rmtree(tmpdir)
#!/usr/bin/env python3
import random
import multiprocessing as mp
from psweep import psweep as ps
def func(pset):
print(mp.current_process().name)
return {'result': random.random() * pset['a']}
if __name__ == '__main__':
params = ps.seq2dicts('a', [1, 2, 3, 4, 5, 6, 7, 8])
df = ps.run(func, params, poolsize=2)
print(df)
#!/usr/bin/env python3
import random
from psweep import psweep as ps
def func(pset):
return {'result': random.random() * pset['a']}
if __name__ == '__main__':
params = ps.seq2dicts('a', [1, 2, 3, 4])
df = ps.run(func, params)
print(df)
return {'result': random.random() * pset['a'] * pset['b']}
if __name__ == '__main__':
print("first run")
const = {'a': 11111, 'b': 55555}
params = []
disp_cols = []
values = dict(a=[1, 2, 3], b=[66, 77])
for study, seq in values.items():
# [{'a': 1}, {'a': 2}, {'a': 3}]
# [{'b': 66}, {'b': 77}]
params_1d = ps.seq2dicts(study, seq)
this_params = ps.loops2params(product(params_1d, [{'study': study}]))
this_params = [ps.merge_dicts(const, dct) for dct in this_params]
params += this_params
disp_cols.append(study)
disp_cols += ['_run_id']
df = ps.run(func,
params,
backup_script=__file__,
backup_calc_dir=True,
verbose=disp_cols)
print(df[disp_cols + ['result']])
#!/usr/bin/env python3
import random
from itertools import product
from psweep import psweep as ps
def func(pset):
return {'result': random.random() * pset['a'] * pset['b']}
if __name__ == '__main__':
sel = ['_calc_dir', 'a', 'b', 'result']
# 1st real run: produce some data, vary a, b constant
params = ps.loops2params(
product(ps.seq2dicts('a', [1, 2, 3, 4]), ps.seq2dicts('b', [100])))
df = ps.run(func, params)
print(df[sel])
# simulate run: check if new parameter grid is OK
params = ps.loops2params(
product(ps.seq2dicts('a', [5, 6]), ps.seq2dicts('b', [88, 99])))
df = ps.run(func, params, simulate=True)
print(df[sel])
# looks good, 2nd real run with new params
df = ps.run(func, params)
print(df[sel])
style_scales=np.concatenate(
(np.linspace(0.1, 1, 19), np.linspace(1.1, 3, 30))),
content_weight_blend=np.linspace(0, 1, 30),
style_weight=np.linspace(0, 200, 50),
tv_weight=np.linspace(0, 600, 50),
style_layer_weight_exp=np.concatenate(
(np.linspace(0, 1.3, 15), np.linspace(1.33, 20, 20))),
)
params = []
disp_cols = []
for study, seq_1d in vary.items():
params_1d = ps.seq2dicts(study, seq_1d)
this_params = ps.loops2params(product(params_1d, [{'study': study}]))
this_params = [ps.merge_dicts(const, dct) for dct in this_params]
params += this_params
disp_cols.append(study)
pd.options.display.max_rows = None
pd.options.display.max_columns = None
df = ps.run(gpu_worker,
params,
poolsize=4,
simulate=False,
verbose=disp_cols + ['study'],
tmpsave=False,
backup_script=__file__,
backup_calc_dir=True)