A, b = permute_data(A,b) # Randomize order
reg = 0.005
def f(x, s=0, e=ndata):
y = dot(A[s:e,:],x) - b[s:e]
fval = 0.5*dot(y,y) + 0.5*reg*(e-s)*dot(x,x)
grad = dot(A[s:e,:].T, y) + reg*(e-s)*x
return (fval/ndata, grad/ndata)
x0 = 0.01*ones(m)
##########################################
# Stores the intermediate values for later plotting
phf_cb = convergence.PlottingCallback("Conf. Hessian free", ndata)
x, optinfo = phessianfree.optimize(f, x0, ndata, maxiter=20, callback=phf_cb, props={})
##########################################
# Stores the intermediate values for later plotting
hf_cb = convergence.PlottingCallback("Hessian free", ndata)
x, optinfo = phessianfree.optimize(f, x0, ndata, maxiter=14, callback=hf_cb, props={
'subsetVariant': 'cg',
'subsetObjective': False,
})
##########################################
lbfgs_wrapper = convergence.PlottingWrapper(f, "lbfgs", ndata)
logger.info("Running scipy's lbfgs implementation")
scipy.optimize.fmin_l_bfgs_b(lbfgs_wrapper, x0, m=10, maxfun=30, disp=5)
##########################################
scipy.optimize.fmin_l_bfgs_b(lbfgs_wrapper, copy(x0), m=30, maxfun=iters, disp=5)
##############################
# Stores the intermediate values for later plotting
phf_cb = convergence.PlottingCallback("phessianfree", ndata)
props = {
'subsetVariant': 'lbfgs',
'parts': 100,
'innerSolveAverage': False, # Should be used when parts is large
'solveFraction': 0.2,
'gradRelErrorBound': 0.1
}
logger.info("Running phessianfree with inner lbfgs linear solver")
x, optinfo = phessianfree.optimize(f, x0, ndata, maxiter=20, callback=phf_cb, props=props)
##############################
# Run with inner cg method as well
props = {
'subsetVariant': 'cg',
'parts': 100,
'solveFraction': 0.2,
'subsetObjective': False
}
phf_cb_cg = convergence.PlottingCallback("phessianfree cg", ndata)
logger.info("Running phessianfree with conjugate gradient linear solver")
#x, optinfo = phessianfree.optimize(f, x0, ndata, maxiter=iters, callback=phf_cb_cg, props=props)