def sgd_fun_nesterov(obj_fun, grad_fun, x0, max_iters, callback, lr_fun): return opt.sgd(obj_fun, grad_fun, x0, max_iters, callback, lr_fun, mass=0.9, update='nesterov', avgdecay=0)
def sgd_fun_mom(obj_fun, grad_fun, x0, max_iters, callback, lr_fun): return opt.sgd(obj_fun, grad_fun, x0, max_iters, callback, lr_fun, mass=0.9, update='regular', avgdecay=0)
def sgd_fun_vanilla_nobatch(obj_fun, grad_fun, x0, max_iters, callback, lr_fun): return opt.sgd(obj_fun, grad_fun, x0, max_iters, callback, lr_fun, mass=0, update='regular', avgdecay=0)
jac=rosen_der, hess=rosen_hess, callback=reporter) print('\nresults of Newton-CG') print(res) plot_trace(ps, 'Newton, obj {:0.4f}'.format(res.fun)) ps = [x0] res = opt.minimize(rosen, x0, method='BFGS', jac=rosen_der, callback=reporter) print('\nresults of BFGS') print(res) plot_trace(ps, 'BFGS, obj {:0.4f}'.format(res.fun)) ps = [x0] lr = 0.0001 params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0) print('\nGD final params {}'.format(params)) plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj)) ps = [x0] lr = 0.0002 params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0) print('\nGD final params {}'.format(params)) plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj)) ps = [x0] lr = 0.0004 params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0) print('\nGD final params {}'.format(params))
ps = [x0] res = opt.minimize(rosen, x0, method='Newton-CG', jac=rosen_der, hess=rosen_hess, callback=reporter) print '\nresults of Newton-CG' print res plot_trace(ps, 'Newton, obj {:0.4f}'.format(res.fun)) ps = [x0] res = opt.minimize(rosen, x0, method='BFGS', jac=rosen_der, callback=reporter) print '\nresults of BFGS' print res plot_trace(ps, 'BFGS, obj {:0.4f}'.format(res.fun)) ps = [x0] lr = 0.0001 params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0) print '\nGD final params {}'.format(params) plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj)) ps = [x0] lr = 0.0002 params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0) print '\nGD final params {}'.format(params) plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj)) ps = [x0] lr = 0.0004 params, obj = optim.sgd(rosen, rosen_der, x0, 100, reporter, optim.const_lr(lr), 0) print '\nGD final params {}'.format(params) plot_trace(ps, 'GD({:0.4f}), obj {:0.4f}'.format(lr, obj))