def test_rmsprop_optimizer_univar(self, x_start, tol):
"""Tests that rmsprop optimizer takes one and two steps correctly
for univariate functions."""
stepsize, gamma = 0.1, 0.5
rms_opt = RMSPropOptimizer(stepsize, decay=gamma)
univariate_funcs = [np.sin, lambda x: np.exp(x / 10.0), lambda x: x**2]
grad_uni_fns = [
lambda x: (np.cos(x), ),
lambda x: (np.exp(x / 10.0) / 10.0, ),
lambda x: (2 * x, ),
]
for gradf, f in zip(grad_uni_fns, univariate_funcs):
rms_opt.reset()
x_onestep = rms_opt.step(f, x_start)
past_grads = (1 - gamma) * gradf(x_start)[0] * gradf(x_start)[0]
adapt_stepsize = stepsize / np.sqrt(past_grads + 1e-8)
x_onestep_target = x_start - gradf(x_start)[0] * adapt_stepsize
assert np.allclose(x_onestep, x_onestep_target, atol=tol)
x_twosteps = rms_opt.step(f, x_onestep)
past_grads = (
1 - gamma) * gamma * gradf(x_start)[0] * gradf(x_start)[0] + (
1 - gamma) * gradf(x_onestep)[0] * gradf(x_onestep)[0]
adapt_stepsize = stepsize / np.sqrt(past_grads + 1e-8)
x_twosteps_target = x_onestep - gradf(
x_onestep)[0] * adapt_stepsize
assert np.allclose(x_twosteps, x_twosteps_target, atol=tol)
class A:
sgd_opt = GradientDescentOptimizer(stepsize)
mom_opt = MomentumOptimizer(stepsize, momentum=gamma)
nesmom_opt = NesterovMomentumOptimizer(stepsize, momentum=gamma)
adag_opt = AdagradOptimizer(stepsize)
rms_opt = RMSPropOptimizer(stepsize, decay=gamma)
adam_opt = AdamOptimizer(stepsize, beta1=gamma, beta2=delta)
def setUp(self):
self.sgd_opt = GradientDescentOptimizer(stepsize)
self.mom_opt = MomentumOptimizer(stepsize, momentum=gamma)
self.nesmom_opt = NesterovMomentumOptimizer(stepsize, momentum=gamma)
self.adag_opt = AdagradOptimizer(stepsize)
self.rms_opt = RMSPropOptimizer(stepsize, decay=gamma)
self.adam_opt = AdamOptimizer(stepsize, beta1=gamma, beta2=delta)
self.fnames = ['test_function_1', 'test_function_2', 'test_function_3']
self.univariate_funcs = [
np.sin, lambda x: np.exp(x / 10.), lambda x: x**2
]
self.grad_uni_fns = [
np.cos, lambda x: np.exp(x / 10.) / 10., lambda x: 2 * x
]
self.multivariate_funcs = [
lambda x: np.sin(x[0]) + np.cos(x[1]),
lambda x: np.exp(x[0] / 3) * np.tanh(x[1]),
lambda x: np.sum([x_**2 for x_ in x])
]
self.grad_multi_funcs = [
lambda x: np.array([np.cos(x[0]), -np.sin(x[1])]),
lambda x: np.array([
np.exp(x[0] / 3) / 3 * np.tanh(x[1]),
np.exp(x[0] / 3) * (1 - np.tanh(x[1])**2)
]), lambda x: np.array([2 * x_ for x_ in x])
]
self.mvar_mdim_funcs = [
lambda x: np.sin(x[0, 0]) + np.cos(x[1, 0]) - np.sin(x[0, 1]) + x[
1, 1], lambda x: np.exp(x[0, 0] / 3) * np.tanh(x[0, 1]),
lambda x: np.sum([x_[0]**2 for x_ in x])
]
self.grad_mvar_mdim_funcs = [
lambda x: np.array([[np.cos(x[0, 0]), -np.cos(x[0, 1])],
[-np.sin(x[1, 0]), 1.]]),
lambda x: np.array([[
np.exp(x[0, 0] / 3) / 3 * np.tanh(x[0, 1]),
np.exp(x[0, 0] / 3) * (1 - np.tanh(x[0, 1])**2)
], [0., 0.]]), lambda x: np.array([[2 * x_[0], 0.] for x_ in x])
]
self.class_fun = class_fun
self.quant_fun = quant_fun
self.hybrid_fun = hybrid_fun
self.hybrid_fun_nested = hybrid_fun_nested
self.hybrid_fun_flat = hybrid_fun_flat
self.hybrid_fun_mdarr = hybrid_fun_mdarr
self.hybrid_fun_mdlist = hybrid_fun_mdlist
self.mixed_list = [(0.2, 0.3), np.array([0.4, 0.2, 0.4]), 0.1]
self.mixed_tuple = (np.array([0.2, 0.3]), [0.4, 0.2, 0.4], 0.1)
self.nested_list = [[[0.2], 0.3], [0.1, [0.4]], -0.1]
self.flat_list = [0.2, 0.3, 0.1, 0.4, -0.1]
self.multid_array = np.array([[0.1, 0.2], [-0.1, -0.4]])
self.multid_list = [[0.1, 0.2], [-0.1, -0.4]]
def opt(opt_name):
stepsize, gamma, delta = 0.1, 0.5, 0.8
if opt_name == "gd":
return GradientDescentOptimizer(stepsize)
if opt_name == "nest":
return NesterovMomentumOptimizer(stepsize, momentum=gamma)
if opt_name == "moment":
return MomentumOptimizer(stepsize, momentum=gamma)
if opt_name == "ada":
return AdagradOptimizer(stepsize)
if opt_name == "rms":
return RMSPropOptimizer(stepsize, decay=gamma)
if opt_name == "adam":
return AdamOptimizer(stepsize, beta1=gamma, beta2=delta)
def test_rmsprop_optimizer_multivar(self, tol):
"""Tests that rmsprop optimizer takes one and two steps correctly
for multivariate functions."""
stepsize, gamma = 0.1, 0.5
rms_opt = RMSPropOptimizer(stepsize, decay=gamma)
multivariate_funcs = [
lambda x: np.sin(x[0]) + np.cos(x[1]),
lambda x: np.exp(x[0] / 3) * np.tanh(x[1]),
lambda x: np.sum([x_**2 for x_ in x]),
]
grad_multi_funcs = [
lambda x: (np.array([np.cos(x[0]), -np.sin(x[1])]), ),
lambda x: (np.array([
np.exp(x[0] / 3) / 3 * np.tanh(x[1]),
np.exp(x[0] / 3) * (1 - np.tanh(x[1])**2),
]), ),
lambda x: (np.array([2 * x_ for x_ in x]), ),
]
x_vals = np.linspace(-10, 10, 16, endpoint=False)
for gradf, f in zip(grad_multi_funcs, multivariate_funcs):
for jdx in range(len(x_vals[:-1])):
rms_opt.reset()
x_vec = x_vals[jdx:jdx + 2]
x_onestep = rms_opt.step(f, x_vec)
past_grads = (1 - gamma) * gradf(x_vec)[0] * gradf(x_vec)[0]
adapt_stepsize = stepsize / np.sqrt(past_grads + 1e-8)
x_onestep_target = x_vec - gradf(x_vec)[0] * adapt_stepsize
assert np.allclose(x_onestep, x_onestep_target, atol=tol)
x_twosteps = rms_opt.step(f, x_onestep)
past_grads = (
1 - gamma) * gamma * gradf(x_vec)[0] * gradf(x_vec)[0] + (
1 - gamma) * gradf(x_onestep)[0] * gradf(x_onestep)[0]
adapt_stepsize = stepsize / np.sqrt(past_grads + 1e-8)
x_twosteps_target = x_onestep - gradf(
x_onestep)[0] * adapt_stepsize
assert np.allclose(x_twosteps, x_twosteps_target, atol=tol)
def test_apply_grad(self, grad, args, tol):
"""
Test that the gradient can be applied correctly to a set of parameters
and that accumulation works correctly.
"""
stepsize, gamma, eps = 0.1, 0.5, 1e-8
sgd_opt = RMSPropOptimizer(stepsize, decay=gamma, eps=eps)
grad, args = np.array(grad), np.array(args, requires_grad=True)
a1 = (1 - gamma) * grad**2
expected = args - stepsize / np.sqrt(a1 + eps) * grad
res = sgd_opt.apply_grad(grad, args)
assert np.allclose(res, expected, atol=tol)
# Simulate a new step
grad = grad + args
args = expected
a2 = gamma * a1 + (1 - gamma) * grad**2
expected = args - stepsize / np.sqrt(a2 + eps) * grad
res = sgd_opt.apply_grad(grad, args)
assert np.allclose(res, expected, atol=tol)
assert opt._stepsize == eta2
def reset(opt):
if getattr(opt, "reset", None):
opt.reset()
@pytest.mark.parametrize(
"opt, opt_name",
[
(GradientDescentOptimizer(stepsize), "gd"),
(MomentumOptimizer(stepsize, momentum=gamma), "moment"),
(NesterovMomentumOptimizer(stepsize, momentum=gamma), "nest"),
(AdagradOptimizer(stepsize), "ada"),
(RMSPropOptimizer(stepsize, decay=gamma), "rms"),
(AdamOptimizer(stepsize, beta1=gamma, beta2=delta), "adam"),
(RotosolveOptimizer(), "roto"),
],
)
class TestOverOpts:
"""Tests keywords, multiple arguements, and non-training arguments in relevent optimizers"""
def test_kwargs(self, mocker, opt, opt_name, tol):
"""Test that the keywords get passed and alter the function"""
class func_wrapper:
@staticmethod
def func(x, c=1.0):
return (x - c)**2
x = 1.0