def test_plots_work_without_cat():
"""Basic smoke tests to make sure plotting doesn't crash."""
SPACE = [
Integer(1, 20, name='max_depth'),
Integer(2, 100, name='min_samples_split'),
Integer(5, 30, name='min_samples_leaf'),
Integer(1, 30, name='max_features'),
]
def objective(params):
clf = DecisionTreeClassifier(random_state=3,
**{
dim.name: val
for dim, val in zip(SPACE, params)
if dim.name != 'dummy'
})
return -np.mean(cross_val_score(clf, *load_breast_cancer(True)))
res = gp_minimize(objective, SPACE, n_calls=10, random_state=3)
plots.plot_convergence(res)
plots.plot_evaluations(res)
plots.plot_objective(res)
plots.plot_objective(res, minimum='expected_minimum')
plots.plot_objective(res,
sample_source='expected_minimum',
n_minimum_search=10)
plots.plot_objective(res, sample_source='result')
plots.plot_regret(res)
def test_plots_work():
"""Basic smoke tests to make sure plotting doesn't crash."""
SPACE = [
Integer(1, 20, name='max_depth'),
Integer(2, 100, name='min_samples_split'),
Integer(5, 30, name='min_samples_leaf'),
Integer(1, 30, name='max_features'),
Categorical(['gini', 'entropy'], name='criterion'),
Categorical(list('abcdefghij'), name='dummy'),
]
def objective(params):
clf = DecisionTreeClassifier(random_state=3,
**{
dim.name: val
for dim, val in zip(SPACE, params)
if dim.name != 'dummy'
})
return -np.mean(cross_val_score(clf, *load_breast_cancer(True)))
res = gp_minimize(objective, SPACE, n_calls=10, random_state=3)
plots.plot_convergence(res)
plots.plot_evaluations(res)
plots.plot_objective(res)
plots.plot_regret(res)
def plot_skopt_regret(opt_res, path):
"""Plots the regret plot from the skopt package.
Args:
opt_res (scipy.optimize.OptimizeResult): Optimization result object.
path (str): Directory at which to save plot.
"""
if not os.path.exists(path):
os.makedirs(path)
fig = plt.figure()
skplot.plot_regret(opt_res, ax=None, true_minumum=None, yscale=None)
fig = plt.gcf()
fig.tight_layout()
fig.savefig(path + "regret")
fig.show()
def make_plots(results, dir):
import matplotlib
import matplotlib.pyplot as plt
# https://matplotlib.org/faq/usage_faq.html#non-interactive-example
# https://matplotlib.org/api/_as_gen/matplotlib.pyplot.savefig.html
if not dir.exists():
dir.mkdir(parents=True, exist_ok=True)
# WIP: there is currently no support for plotting categorical variables...
# You have to manually checkout this pull request:
# git pr 675 # install https://github.com/tj/git-extras
# git pull origin master
# https://github.com/scikit-optimize/scikit-optimize/pull/675
from skopt.plots import plot_convergence
_ = plot_convergence(results)
plt.savefig(dir / 'plot_convergence.png')
from skopt.plots import plot_objective
_ = plot_objective(results)
plt.savefig(dir / 'plot_objective.png')
from skopt.plots import plot_regret
_ = plot_regret(results)
plt.savefig(dir / 'plot_regret.png')
from skopt.plots import plot_evaluations
_ = plot_evaluations(results)
plt.savefig(dir / 'plot_evaluations.png')
# compare convergence...
# https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/strategy-comparison.ipynb
# for all runs...
# from skopt.plots import plot_convergence
# plot = plot_convergence(("dummy_minimize", dummy_res),
# ("gp_minimize", gp_res),
# ("forest_minimize('rf')", rf_res),
# ("forest_minimize('et)", et_res),
# true_minimum=0.397887, yscale="log")
# plot.legend(loc="best", prop={'size': 6}, numpoints=1);
# parallel optimization
# in the yaml:
# parallel: X (if you have a small dataset.self.)
# https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/parallel-optimization.ipynb
# from sklearn.externals.joblib import Parallel, delayed
# x = optimizer.ask(n_points=4) # x is a list of n_points points
# y = Parallel()(delayed(branin)(v) for v in x) # evaluate points in parallel
# optimizer.tell(x, y)
# checkpoints?
# https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/interruptible-optimization.ipynb
# https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/store-and-load-results.ipynb
# poor man's solution:
# import pickle
# with open('my-optimizer.pkl', 'wb') as f:
# pickle.dump(opt, f)
# with open('my-optimizer.pkl', 'rb') as f:
# opt_restored = pickle.load(f)
def plot_regret(self, *args, **kwargs):
"""Calls skopt.plots.plot_regret."""
def _coconut_mock_13(self, *args, **kwargs):
return self, args, kwargs
while True:
from skopt.plots import plot_regret
try:
_coconut_tre_check_3 = plot_regret is _coconut_recursive_func_32
except _coconut.NameError:
_coconut_tre_check_3 = False
if _coconut_tre_check_3:
self, args, kwargs = _coconut_mock_13(self.get_skopt_result(),
*args, **kwargs)
continue
else:
return plot_regret(self.get_skopt_result(), *args, **kwargs)
return None
def _log_plot_regret(results, experiment, name='diagnostics'):
expect_not_a_run(experiment)
fig, ax = plt.subplots()
sk_plots.plot_regret(results, ax=ax)
experiment.log_image(name, fig)
def test_plots_work():
"""Basic smoke tests to make sure plotting doesn't crash."""
SPACE = [
Integer(1, 20, name='max_depth'),
Integer(2, 100, name='min_samples_split'),
Integer(5, 30, name='min_samples_leaf'),
Integer(1, 30, name='max_features'),
Categorical(['gini', 'entropy'], name='criterion'),
Categorical(list('abcdefghij'), name='dummy'),
]
def objective(params):
clf = DecisionTreeClassifier(random_state=3,
**{
dim.name: val
for dim, val in zip(SPACE, params)
if dim.name != 'dummy'
})
return -np.mean(cross_val_score(clf, *load_breast_cancer(True)))
res = gp_minimize(objective, SPACE, n_calls=10, random_state=3)
x = [[11, 52, 8, 14, 'entropy', 'f'], [14, 90, 10, 2, 'gini', 'a'],
[7, 90, 6, 14, 'entropy', 'f']]
samples = res.space.transform(x)
xi_ = [1., 10.5, 20.]
yi_ = [-0.9240883492576596, -0.9240745890422687, -0.9240586402439884]
xi, yi = partial_dependence_1D(res.space,
res.models[-1],
0,
samples,
n_points=3)
assert_array_almost_equal(xi, xi_)
assert_array_almost_equal(yi, yi_, 1e-3)
xi_ = [0, 1]
yi_ = [-0.9241087603770617, -0.9240188905968352]
xi, yi = partial_dependence_1D(res.space,
res.models[-1],
4,
samples,
n_points=3)
assert_array_almost_equal(xi, xi_)
assert_array_almost_equal(yi, yi_, 1e-3)
xi_ = [0, 1]
yi_ = [1., 10.5, 20.]
zi_ = [[-0.92412562, -0.92403575], [-0.92411186, -0.92402199],
[-0.92409591, -0.92400604]]
xi, yi, zi = partial_dependence_2D(res.space,
res.models[-1],
0,
4,
samples,
n_points=3)
assert_array_almost_equal(xi, xi_)
assert_array_almost_equal(yi, yi_)
assert_array_almost_equal(zi, zi_, 1e-3)
x_min, f_min = expected_minimum_random_sampling(res, random_state=1)
x_min2, f_min2 = expected_minimum(res, random_state=1)
x_min, f_min = expected_minimum_random_sampling(res, random_state=1)
x_min2, f_min2 = expected_minimum(res, random_state=1)
assert x_min == x_min2
assert f_min == f_min2
plots.plot_convergence(res)
plots.plot_evaluations(res)
plots.plot_objective(res)
plots.plot_objective(res, dimensions=["a", "b", "c", "d", "e", "f"])
plots.plot_objective(res, minimum='expected_minimum_random')
plots.plot_objective(res,
sample_source='expected_minimum_random',
n_minimum_search=10000)
plots.plot_objective(res, sample_source='result')
plots.plot_regret(res)
plots.plot_objective_2D(res, 0, 4)
plots.plot_histogram(res, 0, 4)
def save_hpo_plot_regret(hpo_result: 'Skopt HPO Object', path: str) -> None:
"""Plots and saves regret from HPO process"""
plt.figure(figsize=(18,10))
plot_regret(hpo_result)
plt.savefig(path, bbox_inches='tight', pad_inches=1)