def lbfgs(X, y, regularizer=None, lamduh=1.0, max_iter=100, tol=1e-4, family=Logistic, verbose=False, **kwargs): """L-BFGS solver using scipy.optimize implementation Parameters ---------- X : array-like, shape (n_samples, n_features) y : array-like, shape (n_samples,) regularizer : str or Regularizer lamduh : float max_iter : int maximum number of iterations to attempt before declaring failure to converge tol : float Maximum allowed change from prior iteration required to declare convergence family : Family verbose : bool, default False whether to print diagnostic information during convergence Returns ------- beta : array-like, shape (n_features,) """ pointwise_loss = family.pointwise_loss pointwise_gradient = family.pointwise_gradient if regularizer is not None: regularizer = Regularizer.get(regularizer) pointwise_loss = regularizer.add_reg_f(pointwise_loss, lamduh) pointwise_gradient = regularizer.add_reg_grad(pointwise_gradient, lamduh) n, p = X.shape beta0 = np.zeros(p) def compute_loss_grad(beta, X, y): loss_fn = pointwise_loss(beta, X, y) gradient_fn = pointwise_gradient(beta, X, y) loss, gradient = compute(loss_fn, gradient_fn) return loss, gradient.copy() with dask.config.set(fuse_ave_width=0): # optimizations slows this down beta, loss, info = fmin_l_bfgs_b(compute_loss_grad, beta0, fprime=None, args=(X, y), iprint=(verbose > 0) - 1, pgtol=tol, maxiter=max_iter) return beta
def proximal_grad(X, y, regularizer='l1', lamduh=0.1, family=Logistic, max_iter=100, tol=1e-8, **kwargs): """ Parameters ---------- X : array-like, shape (n_samples, n_features) y : array-like, shape (n_samples,) max_iter : int maximum number of iterations to attempt before declaring failure to converge tol : float Maximum allowed change from prior iteration required to declare convergence family : Family verbose : bool, default False whether to print diagnostic information during convergence Returns ------- beta : array-like, shape (n_features,) """ n, p = X.shape firstBacktrackMult = 0.1 nextBacktrackMult = 0.5 armijoMult = 0.1 stepGrowth = 1.25 stepSize = 1.0 recalcRate = 10 backtrackMult = firstBacktrackMult beta = np.zeros(p) regularizer = Regularizer.get(regularizer) for k in range(max_iter): # Compute the gradient if k % recalcRate == 0: Xbeta = X.dot(beta) func = family.loglike(Xbeta, y) gradient = family.gradient(Xbeta, X, y) Xbeta, func, gradient = persist( Xbeta, func, gradient) obeta = beta # Compute the step size lf = func for ii in range(100): beta = regularizer.proximal_operator(obeta - stepSize * gradient, stepSize * lamduh) step = obeta - beta Xbeta = X.dot(beta) Xbeta, beta = persist(Xbeta, beta) func = family.loglike(Xbeta, y) func = persist(func)[0] func = compute(func)[0] df = lf - func if df > 0: break stepSize *= backtrackMult if stepSize == 0: break df /= max(func, lf) if df < tol: break stepSize *= stepGrowth backtrackMult = nextBacktrackMult # L2-regularization returned a dask-array try: return beta.compute() except AttributeError: return beta
def admm(X, y, regularizer='l1', lamduh=0.1, rho=1, over_relax=1, max_iter=250, abstol=1e-4, reltol=1e-2, family=Logistic, **kwargs): """ Alternating Direction Method of Multipliers Parameters ---------- X : array-like, shape (n_samples, n_features) y : array-like, shape (n_samples,) regularizer : str or Regularizer lambuh : float rho : float over_relax : FLOAT max_iter : int maximum number of iterations to attempt before declaring failure to converge abstol, reltol : float family : Family Returns ------- beta : array-like, shape (n_features,) """ pointwise_loss = family.pointwise_loss pointwise_gradient = family.pointwise_gradient regularizer = Regularizer.get(regularizer) def create_local_gradient(func): @functools.wraps(func) def wrapped(beta, X, y, z, u, rho): return func(beta, X, y) + rho * (beta - z + u) return wrapped def create_local_f(func): @functools.wraps(func) def wrapped(beta, X, y, z, u, rho): return func(beta, X, y) + (rho / 2) * np.dot(beta - z + u, beta - z + u) return wrapped f = create_local_f(pointwise_loss) fprime = create_local_gradient(pointwise_gradient) nchunks = getattr(X, 'npartitions', 1) # nchunks = X.npartitions (n, p) = X.shape # XD = X.to_delayed().flatten().tolist() # yD = y.to_delayed().flatten().tolist() if isinstance(X, da.Array): XD = X.rechunk((None, X.shape[-1])).to_delayed().flatten().tolist() else: XD = [X] if isinstance(y, da.Array): yD = y.rechunk((None, y.shape[-1])).to_delayed().flatten().tolist() else: yD = [y] z = np.zeros(p) u = np.array([np.zeros(p) for i in range(nchunks)]) betas = np.array([np.ones(p) for i in range(nchunks)]) for k in range(max_iter): # x-update step new_betas = [delayed(local_update)(xx, yy, bb, z, uu, rho, f=f, fprime=fprime) for xx, yy, bb, uu in zip(XD, yD, betas, u)] new_betas = np.array(da.compute(*new_betas)) beta_hat = over_relax * new_betas + (1 - over_relax) * z # z-update step zold = z.copy() ztilde = np.mean(beta_hat + np.array(u), axis=0) z = regularizer.proximal_operator(ztilde, lamduh / (rho * nchunks)) # u-update step u += beta_hat - z # check for convergence primal_res = np.linalg.norm(new_betas - z) dual_res = np.linalg.norm(rho * (z - zold)) eps_pri = np.sqrt(p * nchunks) * abstol + reltol * np.maximum( np.linalg.norm(new_betas), np.sqrt(nchunks) * np.linalg.norm(z)) eps_dual = np.sqrt(p * nchunks) * abstol + \ reltol * np.linalg.norm(rho * u) if primal_res < eps_pri and dual_res < eps_dual: break return z
def proximal_grad(X, y, regularizer='l1', lamduh=0.1, family=Logistic, max_iter=100, tol=1e-8, verbose=False): n, p = X.shape firstBacktrackMult = 0.1 nextBacktrackMult = 0.5 armijoMult = 0.1 stepGrowth = 1.25 stepSize = 1.0 recalcRate = 10 backtrackMult = firstBacktrackMult beta = np.zeros(p) regularizer = Regularizer.get(regularizer) if verbose: print('# -f |df/f| |dx/x| step') print('----------------------------------------------') for k in range(max_iter): # Compute the gradient if k % recalcRate == 0: Xbeta = X.dot(beta) func = family.loglike(Xbeta, y) gradient = family.gradient(Xbeta, X, y) Xbeta, func, gradient = persist(Xbeta, func, gradient) obeta = beta # Compute the step size lf = func for ii in range(100): beta = regularizer.proximal_operator(obeta - stepSize * gradient, stepSize * lamduh) step = obeta - beta Xbeta = X.dot(beta) overflow = (Xbeta < 700).all() overflow, Xbeta, beta = persist(overflow, Xbeta, beta) overflow = compute(overflow)[0] # This prevents overflow if overflow: func = family.loglike(Xbeta, y) func = persist(func)[0] func = compute(func)[0] df = lf - func if df > 0: break stepSize *= backtrackMult if stepSize == 0: print('No more progress') break df /= max(func, lf) db = 0 if verbose: print('%2d %.6e %9.2e %.2e %.1e' % (k + 1, func, df, db, stepSize)) if df < tol: print('Converged') break stepSize *= stepGrowth backtrackMult = nextBacktrackMult return beta
def admm(X, y, regularizer='l1', lamduh=0.1, rho=1, over_relax=1, max_iter=250, abstol=1e-4, reltol=1e-2, family=Logistic): pointwise_loss = family.pointwise_loss pointwise_gradient = family.pointwise_gradient regularizer = Regularizer.get(regularizer) def create_local_gradient(func): @functools.wraps(func) def wrapped(beta, X, y, z, u, rho): return func(beta, X, y) + rho * (beta - z + u) return wrapped def create_local_f(func): @functools.wraps(func) def wrapped(beta, X, y, z, u, rho): return func(beta, X, y) + (rho / 2) * np.dot(beta - z + u, beta - z + u) return wrapped f = create_local_f(pointwise_loss) fprime = create_local_gradient(pointwise_gradient) nchunks = getattr(X, 'npartitions', 1) # nchunks = X.npartitions (n, p) = X.shape # XD = X.to_delayed().flatten().tolist() # yD = y.to_delayed().flatten().tolist() if isinstance(X, da.Array): XD = X.rechunk((None, X.shape[-1])).to_delayed().flatten().tolist() else: XD = [X] if isinstance(y, da.Array): yD = y.rechunk((None, y.shape[-1])).to_delayed().flatten().tolist() else: yD = [y] z = np.zeros(p) u = np.array([np.zeros(p) for i in range(nchunks)]) betas = np.array([np.ones(p) for i in range(nchunks)]) for k in range(max_iter): # x-update step new_betas = [ delayed(local_update)(xx, yy, bb, z, uu, rho, f=f, fprime=fprime) for xx, yy, bb, uu in zip(XD, yD, betas, u) ] new_betas = np.array(da.compute(*new_betas)) beta_hat = over_relax * new_betas + (1 - over_relax) * z # z-update step zold = z.copy() ztilde = np.mean(beta_hat + np.array(u), axis=0) z = regularizer.proximal_operator(ztilde, lamduh / (rho * nchunks)) # u-update step u += beta_hat - z # check for convergence primal_res = np.linalg.norm(new_betas - z) dual_res = np.linalg.norm(rho * (z - zold)) eps_pri = np.sqrt(p * nchunks) * abstol + reltol * np.maximum( np.linalg.norm(new_betas), np.sqrt(nchunks) * np.linalg.norm(z)) eps_dual = np.sqrt(p * nchunks) * abstol + \ reltol * np.linalg.norm(rho * u) if primal_res < eps_pri and dual_res < eps_dual: print("Converged!", k) break return z