def sgd_fun_nesterov(obj_fun, grad_fun, x0, max_iters, callback, lr_fun):
return opt.sgd(obj_fun,
grad_fun,
x0,
max_iters,
callback,
lr_fun,
mass=0.9,
update='nesterov',
avgdecay=0)
def sgd_fun_mom(obj_fun, grad_fun, x0, max_iters, callback, lr_fun):
return opt.sgd(obj_fun,
grad_fun,
x0,
max_iters,
callback,
lr_fun,
mass=0.9,
update='regular',
avgdecay=0)
def sgd_fun_vanilla_nobatch(obj_fun, grad_fun, x0, max_iters, callback,
lr_fun):
return opt.sgd(obj_fun,
grad_fun,
x0,
max_iters,
callback,
lr_fun,
mass=0,
update='regular',
avgdecay=0)
jac=rosen_der,
hess=rosen_hess,
callback=reporter)
print('\nresults of Newton-CG')
print(res)
plot_trace(ps, 'Newton, obj {:0.4f}'.format(res.fun))
ps = [x0]
res = opt.minimize(rosen, x0, method='BFGS', jac=rosen_der, callback=reporter)
print('\nresults of BFGS')
print(res)
plot_trace(ps, 'BFGS, obj {:0.4f}'.format(res.fun))
ps = [x0]
lr = 0.0001
params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter,
optim.const_lr(lr), 0)
print('\nGD final params {}'.format(params))
plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj))
ps = [x0]
lr = 0.0002
params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter,
optim.const_lr(lr), 0)
print('\nGD final params {}'.format(params))
plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj))
ps = [x0]
lr = 0.0004
params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter,
optim.const_lr(lr), 0)
print('\nGD final params {}'.format(params))
def sgd_fun_nesterov(obj_fun, grad_fun, x0, max_iters, callback, lr_fun):
return opt.sgd(obj_fun, grad_fun, x0, max_iters, callback,
lr_fun, mass=0.9, update='nesterov', avgdecay=0)
def sgd_fun_mom(obj_fun, grad_fun, x0, max_iters, callback, lr_fun):
return opt.sgd(obj_fun, grad_fun, x0, max_iters, callback,
lr_fun, mass=0.9, update='regular', avgdecay=0)
def sgd_fun_vanilla_nobatch(obj_fun, grad_fun, x0, max_iters, callback, lr_fun):
return opt.sgd(obj_fun, grad_fun, x0, max_iters, callback,
lr_fun, mass=0, update='regular', avgdecay=0)
ps = [x0]
res = opt.minimize(rosen, x0, method='Newton-CG', jac=rosen_der, hess=rosen_hess, callback=reporter)
print '\nresults of Newton-CG'
print res
plot_trace(ps, 'Newton, obj {:0.4f}'.format(res.fun))
ps = [x0]
res = opt.minimize(rosen, x0, method='BFGS', jac=rosen_der, callback=reporter)
print '\nresults of BFGS'
print res
plot_trace(ps, 'BFGS, obj {:0.4f}'.format(res.fun))
ps = [x0]
lr = 0.0001
params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0)
print '\nGD final params {}'.format(params)
plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj))
ps = [x0]
lr = 0.0002
params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0)
print '\nGD final params {}'.format(params)
plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj))
ps = [x0]
lr = 0.0004
params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0)
print '\nGD final params {}'.format(params)
plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj))
