def test_waterfall():
# create the necessary results
result_1 = create_optimization_result()
result_2 = create_optimization_result()
# test a standard call
visualize.waterfall(result_1)
# test plotting of lists
visualize.waterfall([result_1, result_2])
def test_waterfall_with_nan_inf():
# create the necessary results, one with nan and inf, one without
result_1 = create_optimization_result_nan_inf()
result_2 = create_optimization_result()
# test a standard call
visualize.waterfall(result_1)
# test plotting of lists
visualize.waterfall([result_1, result_2])
def store_and_plot_pretraining(result: Result, pretraindir: str, prefix: str):
"""
Store optimziation results in HDF5 as well as csv for later reuse. Also
saves some visualization for debugging purposes.
:param result:
result from pretraining
:param pretraindir:
directory in which results and plots will be stored
:param prefix:
prefix for file names that can be used to differentiate between
different pretraining stages as well as models/datasets.
"""
# store full results as hdf5
rfile = os.path.join(pretraindir, prefix + '.hdf5')
if os.path.exists(rfile):
# temp bugfix for https://github.com/ICB-DCM/pyPESTO/issues/529
os.remove(rfile)
writer = OptimizationResultHDF5Writer(rfile)
writer.write(result, overwrite=True)
# store parameter values, this will be used in subsequent steps
parameter_df = pd.DataFrame(
[r for r in result.optimize_result.get_for_key('x') if r is not None],
columns=result.problem.x_names)
parameter_df.to_csv(os.path.join(pretraindir, prefix + '.csv'))
# do plotting
waterfall(result, scale_y='log10', offset_y=0.0)
plt.tight_layout()
plt.savefig(os.path.join(pretraindir, prefix + '_waterfall.pdf'))
if result.problem.dim_full < 2e3:
parameters(result)
plt.tight_layout()
plt.savefig(os.path.join(pretraindir, prefix + '_parameters.pdf'))
def test_waterfall_with_options():
# create the necessary results
result_1 = create_optimization_result()
result_2 = create_optimization_result()
# alternative figure size and plotting options
(_, _, ref3, _, ref_point) = create_plotting_options()
alt_fig_size = (9.0, 8.0)
with pytest.warns(UserWarning, match="Invalid lower bound"):
# Test with y-limits as vector and invalid lower bound
visualize.waterfall(
result_1,
reference=ref_point,
y_limits=[-0.5, 2.5],
start_indices=[0, 1, 4, 11],
size=alt_fig_size,
colors=[1.0, 0.3, 0.3, 0.5],
)
# Test with fully invalid bounds
with pytest.warns(UserWarning, match="Invalid bounds"):
visualize.waterfall(result_1, y_limits=[-1.5, 0.0])
# Test with y-limits as float
with pytest.warns(UserWarning, match="Offset specified by user"):
visualize.waterfall(
[result_1, result_2],
reference=ref3,
offset_y=-2.5,
start_indices=3,
y_limits=5.0,
)
# Test with linear scale
visualize.waterfall(
result_1, reference=ref3, scale_y='lin', offset_y=0.2, y_limits=5.0
)
for opt_result, names in zip(optimizer_result, par_names):
sorted_par_idx = [
names.index(name)
for name in problem.x_names
]
x_sorted = [opt_result['x'][sorted_par_idx[ix]] for ix in
range(len(problem.x_names))]
opt_result['x'] = x_sorted
result.optimize_result.append(opt_result)
result.optimize_result.sort()
prefix = '__'.join([MODEL, DATA, str(N_HIDDEN), OPTIMIZER])
waterfall(result, scale_y='log10', offset_y=0.0)
plot_and_save_fig(prefix + '__waterfall.pdf')
optimizer_history(result, scale_y='log10')
plot_and_save_fig(prefix + '__optimizer_trace.pdf')
parameters(result)
plot_and_save_fig(prefix + '__parameters.pdf')
optimizer_convergence(result)
plot_and_save_fig(prefix + '__optimizer_convergence.pdf')
fig_embedding, axes_embedding = plt.subplots(1, N_STARTS,
figsize=(18.5, 10.5))
embedding_fun = theano.function(
except (FileNotFoundError, IOError):
pass
all_results = sorted(
all_results, key=lambda r: r['result'].optimize_result.list[0]['fval'])
waterfall_results = [
r for r in all_results if r['optimizer'] in ALGO_COLORS
]
fmin = np.nanmin(
[r['result'].optimize_result.list[0]['fval'] for r in all_results])
waterfall(
[r['result'] for r in waterfall_results],
legends=[r['optimizer'] for r in waterfall_results],
colors=[ALGO_COLORS[r['optimizer']] for r in waterfall_results],
size=(4.25, 3.5),
)
plt.tight_layout()
plt.savefig(os.path.join('evaluation', f'{MODEL_NAME}_all_starts.pdf'))
waterfall(
[r['result'] for r in waterfall_results],
legends=[r['optimizer'] for r in waterfall_results],
colors=[ALGO_COLORS[r['optimizer']] for r in waterfall_results],
start_indices=range(int(int(n_starts) / 10)),
size=(4.25, 3.5),
)
plt.tight_layout()
plt.savefig(
os.path.join('evaluation',
optimizer=optimizer_tnc,
n_starts=n_starts,
history_options=history_options)
result1_dogleg = optimize.minimize(problem=problem1,
optimizer=optimizer_dogleg,
n_starts=n_starts,
history_options=history_options)
# Optimize second type of objective
result2 = optimize.minimize(problem=problem2,
optimizer=optimizer_tnc,
n_starts=n_starts)
###### Visualite and compare optimization results
# plot separated waterfalls
visualize.waterfall(result1_bfgs, size=(15, 6))
visualize.waterfall(result1_tnc, size=(15, 6))
visualize.waterfall(result1_dogleg, size=(15, 6))
# plot one list of waterfalls
visualize.waterfall([result1_bfgs, result1_tnc],
legends=['L-BFGS-B', 'TNC'],
start_indices=10,
scale_y='lin')
# retrieve second optimum
all_x = result1_bfgs.optimize_result.get_for_key('x')
all_fval = result1_bfgs.optimize_result.get_for_key('fval')
x = all_x[19]
fval = all_fval[19]
print('Second optimum at: ' + str(fval))
report_hess=False)
# do the optimization
ref = visualize.create_references(
x=np.asarray(petab_problem.x_nominal_scaled)[np.asarray(
petab_problem.x_free_indices)],
fval=problem.objective(
np.asarray(petab_problem.x_nominal_scaled)[np.asarray(
petab_problem.x_free_indices)]))
print(f'Reference fval: {ref[0]["fval"]}')
hdf_results_file = os.path.join('results', prefix + '.hdf5')
result = optimize.minimize(
problem=problem,
optimizer=optimizer,
n_starts=N_STARTS,
engine=engine,
options=options,
progress_bar=False,
filename=None,
)
visualize.waterfall(result, reference=ref, scale_y='log10')
plt.tight_layout()
plt.savefig(os.path.join('results', prefix + '_waterfall.pdf'))
writer = OptimizationResultHDF5Writer(hdf_results_file)
writer.write(result, overwrite=True)
