def path_calc(X, y, X_holdout, y_holdout, alphas, paramgrid, colname = 'CV', yname = '', method = 'Elastic Net'): #make a copy of the parameters before popping things off copy_params = copy.deepcopy(paramgrid) fit_intercept = copy_params.pop('fit_intercept') precompute = copy_params.pop('precompute') copy_X = copy_params.pop('copy_X') normalize = False # this code adapted from sklearn ElasticNet fit function, which unfortunately doesn't accept multiple alphas at once X, y = check_X_y(X, y, accept_sparse='csc', order='F', dtype=[np.float64, np.float32], copy=copy_X and fit_intercept, multi_output=True, y_numeric=True) y = check_array(y, order='F', copy=False, dtype=X.dtype.type, ensure_2d=False) #this is the step that gives the data to find intercept if fit_intercept is true. X, y, X_offset, y_offset, X_scale, precompute, Xy = _pre_fit(X, y, None, precompute, normalize, fit_intercept, copy=False) y = np.squeeze(y) #do the path calculation, and tell how long it took print('Calculating path...') start_t = time.time() if method == 'Elastic Net': path_alphas, path_coefs, path_gaps, path_iters = enet_path(X, y, alphas=alphas, return_n_iter = True, **copy_params) if method == 'LASSO': path_alphas, path_coefs, path_gaps, path_iters = lasso_path(X, y, alphas=alphas, return_n_iter=True, **copy_params) dt = time.time() - start_t print('Took ' + str(dt) + ' seconds') #create some empty arrays to store the result y_pred_holdouts = np.empty(shape=(len(alphas),len(y_holdout))) intercepts = np.empty(shape=(len(alphas))) rmses = np.empty(shape=(len(alphas))) cvcols = [] for j in list(range(len(path_alphas))): coef_temp = path_coefs[:, j] if fit_intercept: coef_temp = coef_temp / X_scale intercept = y_offset - np.dot(X_offset, coef_temp.T) else: intercept = 0. y_pred_holdouts[j,:] = np.dot(X_holdout, path_coefs[:, j]) + intercept intercepts[j] = intercept rmses[j] = RMSE(y_pred_holdouts[j,:], y_holdout) cvcols.append(('predict','"'+ method + ' - ' + yname + ' - ' + colname + ' - Alpha:' + str(path_alphas[j]) + ' - ' + str(paramgrid) + '"')) return path_alphas, path_coefs, intercepts, path_iters, y_pred_holdouts, rmses, cvcols
def fit(self, X, y, M=None): # Fit the model using X, y as training data. # Parameters #---------- # X : array-like, shape (n_samples, n_features) # Training data. # y : array-like, shape (n_samples,) or (n_samples, n_targets) # Target values. # Returns #------- # self : object # returns an instance of self. coeff = stpforward(y, X, M) X, y, X_mean, y_mean, X_std, Gram, Xy = _pre_fit( X, y, None, self.precompute, self.normalize, self.fit_intercept, copy=True) self.coef_ = coeff # MODIFY HERE !!! return self
def fit(self, X, y, *args, **kwargs): X = np.asanyarray(X) y = np.asanyarray(y) # Centering Data X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, None, self.precompute, self.normalize, self.fit_intercept, copy=False) # Calling the class-specific train method self._fit(X, y, *args, **kwargs) # Fitting the intercept if required self._set_intercept(X_offset, y_offset, X_scale) self._trained = True return self
def fit(self, X, y, *args, **kwargs): X = np.asanyarray(X) y = np.asanyarray(y) # Centering Data X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, None, self.precompute, self.normalize, self.fit_intercept, copy=False) # Calling the class-specific train method self._fit(X, y, *args, **kwargs) # Fitting the intercept if required self._set_intercept(X_offset, y_offset, X_scale) self._trained = True return self
def path_calc(X, y, X_holdout, y_holdout, alphas, paramgrid, colname='CV', yname='', method='Elastic Net'): #make a copy of the parameters before popping things off copy_params = copy.deepcopy(paramgrid) fit_intercept = copy_params.pop('fit_intercept') precompute = copy_params.pop('precompute') copy_X = copy_params.pop('copy_X') normalize = False # this code adapted from sklearn ElasticNet fit function, which unfortunately doesn't accept multiple alphas at once X, y = check_X_y(X, y, accept_sparse='csc', order='F', dtype=[np.float64, np.float32], copy=copy_X and fit_intercept, multi_output=True, y_numeric=True) y = check_array(y, order='F', copy=False, dtype=X.dtype.type, ensure_2d=False) #this is the step that gives the data to find intercept if fit_intercept is true. X, y, X_offset, y_offset, X_scale, precompute, Xy = _pre_fit(X, y, None, precompute, normalize, fit_intercept, copy=False) y = np.squeeze(y) #do the path calculation, and tell how long it took print('Calculating path...') start_t = time.time() if method == 'Elastic Net': path_alphas, path_coefs, path_gaps, path_iters = enet_path( X, y, alphas=alphas, return_n_iter=True, **copy_params) if method == 'LASSO': path_alphas, path_coefs, path_gaps, path_iters = lasso_path( X, y, alphas=alphas, return_n_iter=True, **copy_params) dt = time.time() - start_t print('Took ' + str(dt) + ' seconds') #create some empty arrays to store the result y_pred_holdouts = np.empty(shape=(len(alphas), len(y_holdout))) intercepts = np.empty(shape=(len(alphas))) rmses = np.empty(shape=(len(alphas))) cvcols = [] for j in list(range(len(path_alphas))): coef_temp = path_coefs[:, j] if fit_intercept: coef_temp = coef_temp / X_scale intercept = y_offset - np.dot(X_offset, coef_temp.T) else: intercept = 0. y_pred_holdouts[j, :] = np.dot(X_holdout, path_coefs[:, j]) + intercept intercepts[j] = intercept rmses[j] = RMSE(y_pred_holdouts[j, :], y_holdout) cvcols.append( ('predict', '"' + method + ' - ' + yname + ' - ' + colname + ' - Alpha:' + str(path_alphas[j]) + ' - ' + str(paramgrid) + '"')) return path_alphas, path_coefs, intercepts, path_iters, y_pred_holdouts, rmses, cvcols
def fit(self, X, y, check_input=True): """Fit model with coordinate descent. Parameters ----------- X : ndarray or scipy.sparse matrix, (n_samples, n_features) Data y : ndarray, shape (n_samples,) or (n_samples, n_targets) Target check_input : boolean, (default=True) Allow to bypass several input checking. Don't use this parameter unless you know what you do. Notes ----- Coordinate descent is an algorithm that considers each column of data at a time hence it will automatically convert the X input as a Fortran-contiguous numpy array if necessary. To avoid memory re-allocation it is advised to allocate the initial data in memory directly using that format. """ if self.alpha == 0: warnings.warn( "With alpha=0, this algorithm does not converge " "well. You are advised to use the LinearRegression " "estimator", stacklevel=2) if isinstance(self.precompute, six.string_types): raise ValueError('precompute should be one of True, False or' ' array-like. Got %r' % self.precompute) # We expect X and y to be float64 or float32 Fortran ordered arrays # when bypassing checks if check_input: X, y = check_X_y(X, y, accept_sparse='csc', order='F', dtype=[np.float64, np.float32], copy=self.copy_X and self.fit_intercept, multi_output=True, y_numeric=True) y = check_array(y, order='F', copy=False, dtype=X.dtype.type, ensure_2d=False) X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, None, self.precompute, self.normalize, self.fit_intercept, copy=False) if y.ndim == 1: y = y[:, None] if Xy is not None and Xy.ndim == 1: Xy = Xy[:, None] n_samples, n_features = X.shape n_targets = y.shape[1] if self.selection not in ['cyclic', 'random']: raise ValueError("selection should be either random or cyclic.") if not self.warm_start or self.coef_ is None: coef_ = np.zeros((n_targets, n_features), dtype=X.dtype, order='F') else: coef_ = self.coef_ if coef_.ndim == 1: coef_ = coef_[None, :] dual_gaps_ = np.zeros(n_targets, dtype=X.dtype) self.n_iter_ = [] history = [] for k in xrange(n_targets): if self.mode == 'admm': this_coef, hist, this_iter = \ group_lasso_overlap( X, y[:, k], lamda=self.alpha, groups=self.groups, rho=self.rho, max_iter=self.max_iter, tol=self.tol, verbose=self.verbose, rtol=self.rtol) else: # paspal wrapper this_coef, hist, this_iter = \ group_lasso_overlap_paspal( X, y[:, k], lamda=self.alpha, groups=self.groups, rho=self.rho, max_iter=self.max_iter, tol=self.tol, verbose=self.verbose, rtol=self.rtol, matlab_engine=self.matlab_engine) coef_[k] = this_coef.ravel() history.append(hist) self.n_iter_.append(this_iter) if n_targets == 1: self.n_iter_ = self.n_iter_[0] self.coef_, self.dual_gap_ = map(np.squeeze, [coef_, dual_gaps_]) self._set_intercept(X_offset, y_offset, X_scale) # workaround since _set_intercept will cast self.coef_ into float64 self.coef_ = np.asarray(self.coef_, dtype=X.dtype) self.history_ = history # return self for chaining fit and predict calls return self
def l1l2_regularization( X, y, max_iter=100000, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None, precompute='auto', Xy=None, copy_X=True, coef_init=None, verbose=False, return_n_iter=False, positive=False, tol=1e-5, check_input=True, **params): if check_input: X = check_array(X, 'csc', dtype=[np.float64, np.float32], order='F', copy=copy_X) y = check_array(y, 'csc', dtype=X.dtype.type, order='F', copy=False, ensure_2d=False) if Xy is not None: # Xy should be a 1d contiguous array or a 2D C ordered array Xy = check_array(Xy, dtype=X.dtype.type, order='C', copy=False, ensure_2d=False) _, n_features = X.shape multi_output = False if y.ndim != 1: multi_output = True _, n_outputs = y.shape # MultiTaskElasticNet does not support sparse matrices from scipy import sparse if not multi_output and sparse.isspmatrix(X): if 'X_offset' in params: # As sparse matrices are not actually centered we need this # to be passed to the CD solver. X_sparse_scaling = params['X_offset'] / params['X_scale'] X_sparse_scaling = np.asarray(X_sparse_scaling, dtype=X.dtype) else: X_sparse_scaling = np.zeros(n_features, dtype=X.dtype) # X should be normalized and fit already if function is called # from ElasticNet.fit if check_input: X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, Xy, precompute, normalize=False, fit_intercept=False, copy=False) if alphas is None: # No need to normalize of fit_intercept: it has been done above alphas = _alpha_grid(X, y, Xy=Xy, l1_ratio=l1_ratio, fit_intercept=False, eps=eps, n_alphas=n_alphas, normalize=False, copy_X=False) else: alphas = np.sort(alphas)[::-1] # make sure alphas are properly ordered n_alphas = len(alphas) tol = params.get('tol', 1e-4) max_iter = params.get('max_iter', 1000) dual_gaps = np.empty(n_alphas) n_iters = [] rng = check_random_state(params.get('random_state', None)) selection = params.get('selection', 'cyclic') if selection not in ['random', 'cyclic']: raise ValueError("selection should be either random or cyclic.") random = (selection == 'random') if not multi_output: coefs = np.empty((n_features, n_alphas), dtype=X.dtype) else: coefs = np.empty((n_outputs, n_features, n_alphas), dtype=X.dtype) if coef_init is None: coef_ = np.asfortranarray(np.zeros(coefs.shape[:-1], dtype=X.dtype)) else: coef_ = np.asfortranarray(coef_init, dtype=X.dtype) for i, alpha in enumerate(alphas): l1_reg = alpha * l1_ratio * 2 # * n_samples l2_reg = alpha * (1.0 - l1_ratio) # * n_samples if not multi_output and sparse.isspmatrix(X): # model = cd_fast.sparse_enet_coordinate_descent( # coef_, l1_reg, l2_reg, X.data, X.indices, # X.indptr, y, X_sparse_scaling, # max_iter, tol, rng, random, positive) raise NotImplementedError() elif multi_output: # model = cd_fast.enet_coordinate_descent_multi_task( # coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random) raise NotImplementedError('Multi output not implemented') elif isinstance(precompute, np.ndarray): # We expect precompute to be already Fortran ordered when bypassing # checks if check_input: precompute = check_array(precompute, dtype=np.float64, order='C') # model = cd_fast.enet_coordinate_descent_gram( # coef_, l1_reg, l2_reg, precompute, Xy, y, max_iter, # tol, rng, random, positive) raise NotImplementedError() elif precompute is False: # model = cd_fast.enet_coordinate_descent( # coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, # positive) model = fista_l1l2( coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, positive) else: raise ValueError("Precompute should be one of True, False, " "'auto' or array-like. Got %r" % precompute) coef_, dual_gap_, eps_, n_iter_ = model coefs[..., i] = coef_ dual_gaps[i] = dual_gap_ n_iters.append(n_iter_) #if dual_gap_ > eps_: # TODO evaluate the dual gap if n_iter_ >= max_iter: import warnings warnings.warn('Objective did not converge.' + ' You might want' + ' to increase the number of iterations.' + ' Fitting data with very small alpha' + ' may cause precision problems.', ConvergenceWarning) if verbose: if verbose > 2: print(model) elif verbose > 1: print('Path: %03i out of %03i' % (i, n_alphas)) else: import sys sys.stderr.write('.') if return_n_iter: return alphas, coefs, dual_gaps, n_iters return alphas, coefs, dual_gaps
def l1l2_regularization(X, y, max_iter=100000, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None, precompute='auto', Xy=None, copy_X=True, coef_init=None, verbose=False, return_n_iter=False, positive=False, tol=1e-5, check_input=True, **params): if check_input: X = check_array(X, 'csc', dtype=[np.float64, np.float32], order='F', copy=copy_X) y = check_array(y, 'csc', dtype=X.dtype.type, order='F', copy=False, ensure_2d=False) if Xy is not None: # Xy should be a 1d contiguous array or a 2D C ordered array Xy = check_array(Xy, dtype=X.dtype.type, order='C', copy=False, ensure_2d=False) _, n_features = X.shape multi_output = False if y.ndim != 1: multi_output = True _, n_outputs = y.shape # MultiTaskElasticNet does not support sparse matrices from scipy import sparse if not multi_output and sparse.isspmatrix(X): if 'X_offset' in params: # As sparse matrices are not actually centered we need this # to be passed to the CD solver. X_sparse_scaling = params['X_offset'] / params['X_scale'] X_sparse_scaling = np.asarray(X_sparse_scaling, dtype=X.dtype) else: X_sparse_scaling = np.zeros(n_features, dtype=X.dtype) # X should be normalized and fit already if function is called # from ElasticNet.fit if check_input: X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, Xy, precompute, normalize=False, fit_intercept=False, copy=False) if alphas is None: # No need to normalize of fit_intercept: it has been done above alphas = _alpha_grid(X, y, Xy=Xy, l1_ratio=l1_ratio, fit_intercept=False, eps=eps, n_alphas=n_alphas, normalize=False, copy_X=False) else: alphas = np.sort(alphas)[::-1] # make sure alphas are properly ordered n_alphas = len(alphas) tol = params.get('tol', 1e-4) max_iter = params.get('max_iter', 1000) dual_gaps = np.empty(n_alphas) n_iters = [] rng = check_random_state(params.get('random_state', None)) selection = params.get('selection', 'cyclic') if selection not in ['random', 'cyclic']: raise ValueError("selection should be either random or cyclic.") random = (selection == 'random') if not multi_output: coefs = np.empty((n_features, n_alphas), dtype=X.dtype) else: coefs = np.empty((n_outputs, n_features, n_alphas), dtype=X.dtype) if coef_init is None: coef_ = np.asfortranarray(np.zeros(coefs.shape[:-1], dtype=X.dtype)) else: coef_ = np.asfortranarray(coef_init, dtype=X.dtype) for i, alpha in enumerate(alphas): l1_reg = alpha * l1_ratio * 2 # * n_samples l2_reg = alpha * (1.0 - l1_ratio) # * n_samples if not multi_output and sparse.isspmatrix(X): # model = cd_fast.sparse_enet_coordinate_descent( # coef_, l1_reg, l2_reg, X.data, X.indices, # X.indptr, y, X_sparse_scaling, # max_iter, tol, rng, random, positive) raise NotImplementedError() elif multi_output: # model = cd_fast.enet_coordinate_descent_multi_task( # coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random) raise NotImplementedError('Multi output not implemented') elif isinstance(precompute, np.ndarray): # We expect precompute to be already Fortran ordered when bypassing # checks if check_input: precompute = check_array(precompute, dtype=np.float64, order='C') # model = cd_fast.enet_coordinate_descent_gram( # coef_, l1_reg, l2_reg, precompute, Xy, y, max_iter, # tol, rng, random, positive) raise NotImplementedError() elif precompute is False: # model = cd_fast.enet_coordinate_descent( # coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, # positive) model = fista_l1l2(coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, positive) else: raise ValueError("Precompute should be one of True, False, " "'auto' or array-like. Got %r" % precompute) coef_, dual_gap_, eps_, n_iter_ = model coefs[..., i] = coef_ dual_gaps[i] = dual_gap_ n_iters.append(n_iter_) #if dual_gap_ > eps_: # TODO evaluate the dual gap if n_iter_ >= max_iter: import warnings warnings.warn( 'Objective did not converge.' + ' You might want' + ' to increase the number of iterations.' + ' Fitting data with very small alpha' + ' may cause precision problems.', ConvergenceWarning) if verbose: if verbose > 2: print(model) elif verbose > 1: print('Path: %03i out of %03i' % (i, n_alphas)) else: import sys sys.stderr.write('.') if return_n_iter: return alphas, coefs, dual_gaps, n_iters return alphas, coefs, dual_gaps
def enet_path_adaptive(X, y, mask, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None, precompute='auto', Xy=None, copy_X=True, coef_init=None, verbose=False, return_n_iter=False, positive=False, check_input=True, **params): """Compute elastic net path with coordinate descent The elastic net optimization function varies for mono and multi-outputs. For mono-output tasks it is:: 1 / (2 * n_samples) * ||y - Xw||^2_2 + alpha * l1_ratio * ||w||_1 + 0.5 * alpha * (1 - l1_ratio) * ||w||^2_2 For multi-output tasks it is:: (1 / (2 * n_samples)) * ||Y - XW||^Fro_2 + alpha * l1_ratio * ||W||_21 + 0.5 * alpha * (1 - l1_ratio) * ||W||_Fro^2 Where:: ||W||_21 = \sum_i \sqrt{\sum_j w_{ij}^2} i.e. the sum of norm of each row. Read more in the :ref:`User Guide <elastic_net>`. Parameters ---------- X : {array-like}, shape (n_samples, n_features) Training data. Pass directly as Fortran-contiguous data to avoid unnecessary memory duplication. If ``y`` is mono-output then ``X`` can be sparse. y : ndarray, shape (n_samples,) or (n_samples, n_outputs) Target values l1_ratio : float, optional float between 0 and 1 passed to elastic net (scaling between l1 and l2 penalties). ``l1_ratio=1`` corresponds to the Lasso eps : float Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3`` n_alphas : int, optional Number of alphas along the regularization path alphas : ndarray, optional List of alphas where to compute the models. If None alphas are set automatically precompute : True | False | 'auto' | array-like Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. The Gram matrix can also be passed as argument. Xy : array-like, optional Xy = np.dot(X.T, y) that can be precomputed. It is useful only when the Gram matrix is precomputed. copy_X : boolean, optional, default True If ``True``, X will be copied; else, it may be overwritten. coef_init : array, shape (n_features, ) | None The initial values of the coefficients. verbose : bool or integer Amount of verbosity. params : kwargs keyword arguments passed to the coordinate descent solver. return_n_iter : bool whether to return the number of iterations or not. positive : bool, default False If set to True, forces coefficients to be positive. check_input : bool, default True Skip input validation checks, including the Gram matrix when provided assuming there are handled by the caller when check_input=False. Returns ------- alphas : array, shape (n_alphas,) The alphas along the path where models are computed. coefs : array, shape (n_features, n_alphas) or \ (n_outputs, n_features, n_alphas) Coefficients along the path. dual_gaps : array, shape (n_alphas,) The dual gaps at the end of the optimization for each alpha. n_iters : array-like, shape (n_alphas,) The number of iterations taken by the coordinate descent optimizer to reach the specified tolerance for each alpha. (Is returned when ``return_n_iter`` is set to True). Notes ----- See examples/linear_model/plot_lasso_coordinate_descent_path.py for an example. See also -------- MultiTaskElasticNet MultiTaskElasticNetCV ElasticNet ElasticNetCV """ # We expect X and y to be already Fortran ordered when bypassing # checks if check_input: X = check_array(X, 'csc', dtype=[np.float64, np.float32], order='F', copy=copy_X) y = check_array(y, 'csc', dtype=X.dtype.type, order='F', copy=False, ensure_2d=False) if Xy is not None: # Xy should be a 1d contiguous array or a 2D C ordered array Xy = check_array(Xy, dtype=X.dtype.type, order='C', copy=False, ensure_2d=False) n_samples, n_features = X.shape multi_output = False if y.ndim != 1: multi_output = True _, n_outputs = y.shape # MultiTaskElasticNet does not support sparse matrices if not multi_output and sparse.isspmatrix(X): if 'X_offset' in params: # As sparse matrices are not actually centered we need this # to be passed to the CD solver. X_sparse_scaling = params['X_offset'] / params['X_scale'] X_sparse_scaling = np.asarray(X_sparse_scaling, dtype=X.dtype) else: X_sparse_scaling = np.zeros(n_features, dtype=X.dtype) # X should be normalized and fit already if function is called # from ElasticNet.fit if check_input: X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, Xy, precompute, normalize=False, fit_intercept=False, copy=False) if len(mask) != n_samples: tmp_alpha = np.zeros(n_samples) tmp_alpha[np.arange(len(mask))] = mask mask = tmp_alpha if alphas is None: # No need to normalize of fit_intercept: it has been done # above alphas = _alpha_grid(X, y, Xy=Xy, l1_ratio=l1_ratio, fit_intercept=False, eps=eps, n_alphas=n_alphas, normalize=False, copy_X=False) else: alphas = np.sort(alphas)[::-1] # make sure alphas are properly ordered n_alphas = len(alphas) tol = params.get('tol', 1e-4) max_iter = params.get('max_iter', 1000) dual_gaps = np.empty(n_alphas) n_iters = [] rng = check_random_state(params.get('random_state', None)) selection = params.get('selection', 'cyclic') if selection not in ['random', 'cyclic']: raise ValueError("selection should be either random or cyclic.") random = (selection == 'random') if not multi_output: coefs = np.empty((n_features, n_alphas), dtype=X.dtype) else: coefs = np.empty((n_outputs, n_features, n_alphas), dtype=X.dtype) if coef_init is None: coef_ = np.asfortranarray(np.zeros(coefs.shape[:-1], dtype=X.dtype)) else: coef_ = np.asfortranarray(coef_init, dtype=X.dtype) for i, alpha in enumerate(alphas): # a vector now l1_reg = alpha * l1_ratio * n_samples * mask # l2_reg = alpha * (1.0 - l1_ratio) * n_samples if not multi_output and sparse.isspmatrix(X): model = cd_fast_adaptive.sparse_enet_coordinate_descent_adaptive( coef_, l1_reg, l2_reg, X.data, X.indices, X.indptr, y, X_sparse_scaling, max_iter, tol, rng, random, positive) elif multi_output: l1_reg_scalar = alpha* l1_ratio * n_samples model = cd_fast_adaptive.enet_coordinate_descent_multi_task( coef_, l1_reg_scalar, l2_reg, X, y, max_iter, tol, rng, random) elif isinstance(precompute, np.ndarray): # We expect precompute to be already Fortran ordered when bypassing # checks if check_input: precompute = check_array(precompute, dtype=np.float64, order='C') model = cd_fast_adaptive.enet_coordinate_descent_gram_adaptive( coef_, l1_reg, l2_reg, precompute, Xy, y, max_iter, tol, rng, random, positive) elif precompute is False: model = cd_fast_adaptive.enet_coordinate_descent_adaptive( coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, positive) else: raise ValueError("Precompute should be one of True, False, " "'auto' or array-like. Got %r" % precompute) coef_, dual_gap_, eps_, n_iter_ = model coefs[..., i] = coef_ dual_gaps[i] = dual_gap_ n_iters.append(n_iter_) if dual_gap_ > eps_: warnings.warn('Objective did not converge.' + ' You might want' + ' to increase the number of iterations.' + ' Fitting data with very small alpha' + ' may cause precision problems.', ConvergenceWarning) if verbose: if verbose > 2: print(model) elif verbose > 1: print('Path: %03i out of %03i' % (i, n_alphas)) else: sys.stderr.write('.') if return_n_iter: return alphas, coefs, dual_gaps, n_iters return alphas, coefs, dual_gaps
def fit(self, X, y, check_input=True): if self.alpha == 0: warnings.warn( "With alpha=0, this algorithm does not converge " "well. You are advised to use the LinearRegression " "estimator", stacklevel=2) if (isinstance(self.precompute, six.string_types) and self.precompute == 'auto'): warnings.warn( "Setting precompute to 'auto', was found to be " "slower even when n_samples > n_features. Hence " "it will be removed in 0.18.", DeprecationWarning, stacklevel=2) if not (self.eta > 0 and self.eta < 1): self.eta = 0.5 warnings.warn( "Value given for eta is invalid. It must satisfy the " "constraint 0 < eta < 1. Setting eta to the default " "value (0.5).", stacklevel=2) if not (self.init_step > 0): self.init_step = 10 warnings.warn( "Value given for init_step is invalid. It must be " "a positive number. Setting init_step to the default " "value (10).", stacklevel=2) if check_input: # Ensure that X and y are float64 Fortran ordered arrays. # Also check for consistency in the dimensions, and that y doesn't # contain np.nan or np.inf entries. y = np.asarray(y, dtype=np.float64) X, y = check_X_y(X, y, accept_sparse='csc', dtype=np.float64, order='F', copy=self.copy_X and self.fit_intercept, multi_output=True, y_numeric=True) y = check_array(y, dtype=np.float64, order='F', copy=False, ensure_2d=False) # Centre and normalise the data X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, None, self.precompute, self.normalize, self.fit_intercept, copy=False) if y.ndim == 1: y = y[:, np.newaxis] if Xy is not None and Xy.ndim == 1: Xy = Xy[:, np.newaxis] n_samples, n_features = X.shape n_targets = y.shape[1] if not self.warm_start or self.coef_ is None: # Initial guess for coef_ is zero coef_ = np.zeros((n_targets, n_features), dtype=np.float64, order='F') else: # Use previous value of coef_ as initial guess coef_ = self.coef_ if coef_.ndim == 1: coef_ = coef_[np.newaxis, :] dual_gaps_ = np.zeros(n_targets, dtype=np.float64) self.n_iter_ = [] # Perform the optimisation for k in xrange(n_targets): if Xy is not None: this_Xy = Xy[:, k] else: this_Xy = None _, this_coef, this_dual_gap, this_iter = \ self.path(X, y[:, k], l1_ratio=self.l1_ratio, eps=None, eta = self.eta, init_step = self.init_step, n_alphas=None, alphas=[self.alpha], precompute=precompute, Xy=this_Xy, fit_intercept=False, adaptive_step=self.adaptive_step, normalize=False, copy_X=True, verbose=False, tol=self.tol, X_offset=X_offset, X_scale=X_scale, return_n_iter=True, coef_init=coef_[k], max_iter=self.max_iter, check_input=False) coef_[k] = this_coef[:, 0] dual_gaps_[k] = this_dual_gap[0] self.n_iter_.append(this_iter[0]) if n_targets == 1: self.n_iter_ = self.n_iter_[0] self.coef_, self.dual_gap_ = map(np.squeeze, [coef_, dual_gaps_]) self._set_intercept(X_offset, y_offset, X_scale) # return self for chaining fit and predict calls return self
def enet_path(X, y, l1_ratio=0.5, eps=1e-3, eta=0.5, init_step=10, adaptive_step=True, n_alphas=100, alphas=None, precompute='auto', Xy=None, copy_X=True, coef_init=None, verbose=False, return_n_iter=False, check_input=True, **params): """Compute elastic net path with coordinate descent The optimization function is:: 1 / (2 * n_samples) * ||y - Xw||^2_2 + alpha * l1_ratio * ||w||_1 + 0.5 * alpha * (1 - l1_ratio) * ||w||^2_2 Read more in the :ref:`User Guide <elastic_net>`. Parameters ---------- X : {array-like}, shape (n_samples, n_features) Training data. Pass directly as Fortran-contiguous data to avoid unnecessary memory duplication. If ``y`` is mono-output then ``X`` can be sparse. y : ndarray, shape (n_samples,) or (n_samples, n_outputs) Target values l1_ratio : float, optional float between 0 and 1 passed to elastic net (scaling between l1 and l2 penalties). ``l1_ratio=1`` corresponds to the Lasso eps : float Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3`` n_alphas : int, optional Number of alphas along the regularization path alphas : ndarray, optional List of alphas where to compute the models. If None alphas are set automatically eta : float, optional Shrinkage parameter for backtracking line search. It must satisfy 0 < eta < 1. init_step : float, optional Initial step size used for the backtracking line search. It must be a positive number. adaptive_step : boolean, optional, default True Whether to calculate the optimal step size or use an adaptive step size chosen through a backtracking line search. precompute : True | False | 'auto' | array-like Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. The Gram matrix can also be passed as argument. Xy : array-like, optional Xy = np.dot(X.T, y) that can be precomputed. It is useful only when the Gram matrix is precomputed. copy_X : boolean, optional, default True If ``True``, X will be copied; else, it may be overwritten. coef_init : array, shape (n_features, ) | None The initial values of the coefficients. verbose : bool or integer Amount of verbosity. params : kwargs keyword arguments passed to the coordinate descent solver. return_n_iter : bool whether to return the number of iterations or not. check_input : bool, default True Skip input validation checks, including the Gram matrix when provided assuming there are handled by the caller when check_input=False. Returns ------- alphas : array, shape (n_alphas,) The alphas along the path where models are computed. coefs : array, shape (n_features, n_alphas) or \ (n_outputs, n_features, n_alphas) Coefficients along the path. dual_gaps : array, shape (n_alphas,) The dual gaps at the end of the optimization for each alpha. n_iters : array-like, shape (n_alphas,) The number of iterations taken by the coordinate descent optimizer to reach the specified tolerance for each alpha. (Is returned when ``return_n_iter`` is set to True). See also -------- ElasticNet ElasticNetCV """ # Direct prox_fast to use fixed optimal step size by passing eta = 0 and # init_step = 0 (which would otherwise be invalid) if not adaptive_step: eta_ = 0 init_step_ = 0 else: eta_ = eta init_step_ = init_step # We expect X and y to be already float64 Fortran ordered when bypassing # checks if check_input: X = check_array(X, 'csc', dtype=np.float64, order='F', copy=copy_X) y = check_array(y, 'csc', dtype=np.float64, order='F', copy=False, ensure_2d=False) if Xy is not None: # Xy should be a 1d contiguous array or a 2D C ordered array Xy = check_array(Xy, dtype=np.float64, order='C', copy=False, ensure_2d=False) n_samples, n_features = X.shape # MultiTaskElasticNet does not support sparse matrices if sparse.isspmatrix(X): if 'X_offset' in params: # As sparse matrices are not actually centered we need this # to be passed to the CD solver. X_sparse_scaling = params['X_offset'] / params['X_scale'] else: X_sparse_scaling = np.zeros(n_features) # X should be normalized and fit already if function is called # from ElasticNet.fit if check_input: X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, Xy, precompute, normalize=False, fit_intercept=False, copy=False) if alphas is None: # No need to normalize of fit_intercept: it has been done # above alphas = _alpha_grid(X, y, Xy=Xy, l1_ratio=l1_ratio, fit_intercept=False, eps=eps, n_alphas=n_alphas, normalize=False, copy_X=False) else: alphas = np.sort(alphas)[::-1] # make sure alphas are properly ordered n_alphas = len(alphas) tol = params.get('tol', 1e-4) max_iter = params.get('max_iter', 10000) dual_gaps = np.empty(n_alphas) n_iters = [] coefs = np.empty((n_features, n_alphas), dtype=np.float64) if coef_init is None: coef_ = np.asfortranarray(np.zeros(coefs.shape[:-1])) else: coef_ = np.asfortranarray(coef_init) for i, alpha in enumerate(alphas): l1_reg = alpha * l1_ratio * n_samples l2_reg = alpha * (1.0 - l1_ratio) * n_samples if sparse.isspmatrix(X): model = prox_fast.sparse_enet_prox_gradient( coef_, l1_reg, l2_reg, X.data, X.indices, X.indptr, y, X_sparse_scaling, max_iter, tol) elif isinstance(precompute, np.ndarray): # We expect precompute to be already Fortran ordered when bypassing # checks if check_input: precompute = check_array(precompute, dtype=np.float64, order='C') model = prox_fast.enet_prox_gradient_gram(coef_, l1_reg, l2_reg, precompute, Xy, y, max_iter, eta_, init_step_, tol) elif precompute is False: model = prox_fast.enet_prox_gradient(coef_, l1_reg, l2_reg, X, y, max_iter, eta_, init_step_, tol) else: raise ValueError("Precompute should be one of True, False, " "'auto' or array-like") coef_, dual_gap_, tol_, n_iter_ = model coefs[..., i] = coef_ dual_gaps[i] = dual_gap_ n_iters.append(n_iter_) if dual_gap_ > tol_: warnings.warn( 'Objective did not converge.' + ' You might want' + ' to increase the number of iterations', ConvergenceWarning) if verbose: if verbose > 2: print(model) elif verbose > 1: print('Path: %03i out of %03i' % (i, n_alphas)) else: sys.stderr.write('.') if return_n_iter: return alphas, coefs, dual_gaps, n_iters return alphas, coefs, dual_gaps
def _path_residuals(X, y, train, test, path, path_params, alphas=None, l1_ratio=1, X_order=None, dtype=None): """Returns the MSE for the models computed by 'path' Parameters ---------- X : {array-like, sparse matrix}, shape (n_samples, n_features) Training data. y : array-like, shape (n_samples,) or (n_samples, n_targets) Target values train : list of indices The indices of the train set test : list of indices The indices of the test set path : callable function returning a list of models on the path. See enet_path for an example of signature path_params : dictionary Parameters passed to the path function alphas : array-like, optional Array of float that is used for cross-validation. If not provided, computed using 'path' l1_ratio : float, optional float between 0 and 1 passed to ElasticNet (scaling between l1 and l2 penalties). For ``l1_ratio = 0`` the penalty is an L2 penalty. For ``l1_ratio = 1`` it is an L1 penalty. For ``0 < l1_ratio < 1``, the penalty is a combination of L1 and L2 X_order : {'F', 'C', or None}, optional The order of the arrays expected by the path function to avoid memory copies dtype : a numpy dtype or None The dtype of the arrays expected by the path function to avoid memory copies """ X_train = X[train] y_train = y[train] X_test = X[test] y_test = y[test] fit_intercept = path_params['fit_intercept'] normalize = path_params['normalize'] if y.ndim == 1: precompute = path_params['precompute'] else: # No Gram variant of multi-task exists right now. # Fall back to default enet_multitask precompute = False X_train, y_train, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X_train, y_train, None, precompute, normalize, fit_intercept, copy=False) path_params = path_params.copy() path_params['Xy'] = Xy path_params['X_offset'] = X_offset path_params['X_scale'] = X_scale path_params['precompute'] = precompute path_params['copy_X'] = False path_params['alphas'] = alphas if 'l1_ratio' in path_params: path_params['l1_ratio'] = l1_ratio # Do the ordering and type casting here, as if it is done in the path, # X is copied and a reference is kept here X_train = check_array(X_train, 'csc', dtype=dtype, order=X_order) alphas, coefs, _ = path(X_train, y_train, **path_params) del X_train, y_train if y.ndim == 1: # Doing this so that it becomes coherent with multioutput. coefs = coefs[np.newaxis, :, :] y_offset = np.atleast_1d(y_offset) y_test = y_test[:, np.newaxis] if normalize: nonzeros = np.flatnonzero(X_scale) coefs[:, nonzeros] /= X_scale[nonzeros][:, np.newaxis] intercepts = y_offset[:, np.newaxis] - np.dot(X_offset, coefs) if sparse.issparse(X_test): n_order, n_features, n_alphas = coefs.shape # Work around for sparse matices since coefs is a 3-D numpy array. coefs_feature_major = np.rollaxis(coefs, 1) feature_2d = np.reshape(coefs_feature_major, (n_features, -1)) X_test_coefs = safe_sparse_dot(X_test, feature_2d) X_test_coefs = X_test_coefs.reshape(X_test.shape[0], n_order, -1) else: X_test_coefs = safe_sparse_dot(X_test, coefs) residues = X_test_coefs - y_test[:, :, np.newaxis] residues += intercepts this_mses = ((residues**2).mean(axis=0)).mean(axis=0) return this_mses
def fit(self, X, y, check_input=True): if check_input: X, y = check_X_y(X, y, accept_sparse='csc', order='F', dtype=[np.float64, np.float32], copy=self.copy_X and self.fit_intercept, multi_output=True, y_numeric=True) y = check_array(y, order='F', copy=False, dtype=X.dtype.type, ensure_2d=False) X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, None, self.precompute, self.normalize, self.fit_intercept, copy=False) # # Centering Data # if self.fit_intercept: # X, Xmean = center(X, return_mean=True) # y, ymean = center(y, return_mean=True) if y.ndim == 1: y = y[:, np.newaxis] if Xy is not None and Xy.ndim == 1: Xy = Xy[:, np.newaxis] n_samples, n_features = X.shape n_targets = y.shape[1] if self.selection not in ['cyclic', 'random']: raise ValueError("selection should be either random or cyclic.") if not self.warm_start or self.coef_ is None: coef_ = np.zeros((n_targets, n_features), dtype=X.dtype, order='F') else: coef_ = self.coef_ if coef_.ndim == 1: coef_ = coef_[np.newaxis, :] dual_gaps_ = np.zeros(n_targets, dtype=X.dtype) self.n_iter_ = [] for k in xrange(n_targets): if Xy is not None: this_Xy = Xy[:, k] else: this_Xy = None _, this_coef, this_dual_gap, this_iter = enet_admm_path( X, y[:, k], rho=self.rho, alpha=self.alpha, max_iter=self.max_iter, return_n_iter=True, abs_tol=self.abs_tol, rel_tol=self.rel_tol, tau=self.tau, mu=self.mu, alphas=[self.mu]) coef_[k] = this_coef[:, 0] dual_gaps_[k] = this_dual_gap[0] self.n_iter_.append(this_iter[0]) # # Fitting the intercept if required # if self.fit_intercept: # self._intercept = ymean - np.dot(Xmean, self.coef_) # else: # self._intercept = 0.0 if n_targets == 1: self.n_iter_ = self.n_iter_[0] self.coef_, self.dual_gap_ = map(np.squeeze, [coef_, dual_gaps_]) self._set_intercept(X_offset, y_offset, X_scale) self.coef_ = np.asarray(self.coef_, dtype=X.dtype) return self