args.append(rate) if algo in 'RMSProp ADADELTA ESGD'.split(): slots.append('rmsh={}') args.append(half) slots.append('rmsr={:.2e}') args.append(reg) return ' '.join(slots).format(*args) # Here we run a number of rosenbrock optimization algorithms and measure their # performance. Below we plot the results. algos = 'SGD NAG RMSProp RProp Adam ADADELTA ESGD'.split() results = ((make_label(loss, key), xs, ys) for key, (xs, ys, loss) in sorted(rosenbrock.test(algos), key=by_loss)) _, ax = plt.subplots(1, 1) for color, (label, xs, ys) in zip(rosenbrock.COLORS, results): ax.plot(xs, ys, 'o-', color=color, label=label, alpha=0.8, lw=2, markersize=5, mew=1, mec=color, mfc='none') # make a contour plot of the rosenbrock function surface. X, Y = np.meshgrid(np.linspace(-1.3, 1.3, 31), np.linspace(-0.9, 1.7, 31)) Z = 100 * (Y - X ** 2) ** 2 + (1 - X) ** 2 ax.plot([1], [1], 'x', mew=3, markersize=10, color='#111111') ax.contourf(X, Y, Z, np.logspace(-1, 3, 31), cmap='gray_r') ax.set_xlim(-1.3, 1.3)
This example is meant to show how optimization hyperparameters affect performance across different optimization algorithms. Due to the large number of optimizers that are evaluated in this example, it can take a good while to run. ''' import itertools import matplotlib.pyplot as plt import numpy as np import rosenbrock algos = 'NAG RMSProp Adam ADADELTA ESGD'.split() results = rosenbrock.test(algos, n=10, init=[-1] * 100, limit=1000) # Here we make plots of the marginal performance of each of the four # hyperparameters. These are intended to get a sense of how random # hyperparameter selection gives a decent idea of how different algorithms # perform. _, ((rate_ax, mu_ax), (half_ax, reg_ax)) = plt.subplots(2, 2) by_algo = itertools.groupby(sorted(results), lambda item: item[0][0]) for color, (algo, items) in zip(rosenbrock.COLORS, by_algo): items = list(items) values = np.zeros((len(items), 5), 'f') for i, ((_, rate, mu, half, reg), (_, _, loss)) in enumerate(items): values[i] = [rate, mu, half, reg, loss] rates, mus, halfs, regs, losses = values.T
This example is meant to show how optimization hyperparameters affect performance across different optimization algorithms. Due to the large number of optimizers that are evaluated in this example, it can take a good while to run. ''' import itertools import matplotlib.pyplot as plt import numpy as np import rosenbrock algos = 'NAG RMSProp Adam ADADELTA ESGD'.split() results = rosenbrock.test(algos, n=10, init=[-1] * 100, limit=1000) # Here we make plots of the marginal performance of each of the four # hyperparameters. These are intended to get a sense of how random # hyperparameter selection gives a decent idea of how different algorithms # perform. _, ((rate_ax, mu_ax), (half_ax, reg_ax)) = plt.subplots(2, 2) by_algo = itertools.groupby(sorted(results), lambda item: item[0][0]) for color, (algo, items) in zip(rosenbrock.COLORS, by_algo): items = list(items) values = np.zeros((len(items), 5), 'f') for i, ((_, rate, mu, half, reg), (_, _, loss)) in enumerate(items): values[i] = [rate, mu, half, reg, loss]