def reconstruct_l2(f,tree,minfrac,maxfrac,alpha=1.0,suppress_warnings=True): randomize_folder_size(tree,minfrac,maxfrac) tree_intervals = np.array([node.lbound for node in tree.leaves()]) indices = np.digitize(f[0,:],tree_intervals) - 1 fy = np.zeros([tree.size]) fcts = np.zeros(np.shape(fy)) for (i,idx) in enumerate(indices): fcts[idx] += 1.0 n = fcts[idx] fy[idx] = ((n-1)/n)*fy[idx]+(1/n)*f[1,i] active_indices = np.where(fcts>0)[0] #print active_indices cl = tree.char_library(alpha) if suppress_warnings: with warnings.catch_warnings(): warnings.simplefilter('ignore', UserWarning) alphas,active_vars,coef_path = sklm.lars_path(cl[active_indices,:], fy[active_indices,:], method='lasso',max_iter=2000) else: alphas,active_vars,coef_path = sklm.lars_path(cl[active_indices,:], fy[active_indices,:],method='lasso', max_iter=2000) return tree_intervals, fy, alphas, active_vars, coef_path, active_indices
def test_lasso_lars_vs_lasso_cd(verbose=False): # Test that LassoLars and Lasso using coordinate descent give the # same results. X = 3 * diabetes.data alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso') lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8) for c, a in zip(lasso_path.T, alphas): if a == 0: continue lasso_cd.alpha = a lasso_cd.fit(X, y) error = linalg.norm(c - lasso_cd.coef_) assert_less(error, 0.01) # similar test, with the classifiers for alpha in np.linspace(1e-2, 1 - 1e-2, 20): clf1 = linear_model.LassoLars(alpha=alpha, normalize=False).fit(X, y) clf2 = linear_model.Lasso(alpha=alpha, tol=1e-8, normalize=False).fit(X, y) err = linalg.norm(clf1.coef_ - clf2.coef_) assert_less(err, 1e-3) # same test, with normalized data X = diabetes.data alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso') lasso_cd = linear_model.Lasso(fit_intercept=False, normalize=True, tol=1e-8) for c, a in zip(lasso_path.T, alphas): if a == 0: continue lasso_cd.alpha = a lasso_cd.fit(X, y) error = linalg.norm(c - lasso_cd.coef_) assert_less(error, 0.01)
def test_lasso_lars_vs_lasso_cd_early_stopping(verbose=False): # Test that LassoLars and Lasso using coordinate descent give the # same results when early stopping is used. # (test : before, in the middle, and in the last part of the path) alphas_min = [10, 0.9, 1e-4] for alpha_min in alphas_min: alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso', alpha_min=alpha_min) lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8) lasso_cd.alpha = alphas[-1] lasso_cd.fit(X, y) error = linalg.norm(lasso_path[:, -1] - lasso_cd.coef_) assert_less(error, 0.01) # same test, with normalization for alpha_min in alphas_min: alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso', alpha_min=alpha_min) lasso_cd = linear_model.Lasso(fit_intercept=True, normalize=True, tol=1e-8) lasso_cd.alpha = alphas[-1] lasso_cd.fit(X, y) error = linalg.norm(lasso_path[:, -1] - lasso_cd.coef_) assert_less(error, 0.01)
def test_lars_path_positive_constraint(): # this is the main test for the positive parameter on the lars_path method # the estimator classes just make use of this function # we do the test on the diabetes dataset # ensure that we get negative coefficients when positive=False # and all positive when positive=True # for method 'lar' (default) and lasso # Once deprecation of LAR + positive option is done use these: # assert_raises(ValueError, linear_model.lars_path, diabetes['data'], # diabetes['target'], method='lar', positive=True) with pytest.warns(DeprecationWarning, match="broken"): linear_model.lars_path(diabetes['data'], diabetes['target'], return_path=True, method='lar', positive=True) method = 'lasso' alpha, active, coefs = \ linear_model.lars_path(diabetes['data'], diabetes['target'], return_path=True, method=method, positive=False) assert_true(coefs.min() < 0) alpha, active, coefs = \ linear_model.lars_path(diabetes['data'], diabetes['target'], return_path=True, method=method, positive=True) assert_true(coefs.min() >= 0)
def test_no_path(): # Test that the ``return_path=False`` option returns the correct output alphas_, active_, coef_path_ = linear_model.lars_path(diabetes.data, diabetes.target, method="lar") alpha_, active, coef = linear_model.lars_path(diabetes.data, diabetes.target, method="lar", return_path=False) assert_array_almost_equal(coef, coef_path_[:, -1]) assert_true(alpha_ == alphas_[-1])
def test_no_path_precomputed(): # Test that the ``return_path=False`` option with Gram remains correct alphas_, _, coef_path_ = linear_model.lars_path( X, y, method='lar', Gram=G) alpha_, _, coef = linear_model.lars_path( X, y, method='lar', Gram=G, return_path=False) assert_array_almost_equal(coef, coef_path_[:, -1]) assert alpha_ == alphas_[-1]
def test_no_path(): # Test that the ``return_path=False`` option returns the correct output alphas_, _, coef_path_ = linear_model.lars_path( X, y, method='lar') alpha_, _, coef = linear_model.lars_path( X, y, method='lar', return_path=False) assert_array_almost_equal(coef, coef_path_[:, -1]) assert alpha_ == alphas_[-1]
def test_all_precomputed(): # Test that lars_path with precomputed Gram and Xy gives the right answer G = np.dot(X.T, X) Xy = np.dot(X.T, y) for method in 'lar', 'lasso': output = linear_model.lars_path(X, y, method=method) output_pre = linear_model.lars_path(X, y, Gram=G, Xy=Xy, method=method) for expected, got in zip(output, output_pre): assert_array_almost_equal(expected, got)
def test_no_path_precomputed(): # Test that the ``return_path=False`` option with Gram remains correct G = np.dot(diabetes.data.T, diabetes.data) alphas_, active_, coef_path_ = linear_model.lars_path(diabetes.data, diabetes.target, method="lar", Gram=G) alpha_, active, coef = linear_model.lars_path( diabetes.data, diabetes.target, method="lar", Gram=G, return_path=False ) assert_array_almost_equal(coef, coef_path_[:, -1]) assert_true(alpha_ == alphas_[-1])
def test_all_precomputed(): """ Test that lars_path with precomputed Gram and Xy gives the right answer """ X, y = diabetes.data, diabetes.target G = np.dot(X.T, X) Xy = np.dot(X.T, y) for method in "lar", "lasso": output = linear_model.lars_path(X, y, method=method) output_pre = linear_model.lars_path(X, y, Gram=G, Xy=Xy, method=method) for expected, got in zip(output, output_pre): assert_array_almost_equal(expected, got)
def test_no_path_all_precomputed(): # Test that the ``return_path=False`` option with Gram and Xy remains # correct X, y = 3 * diabetes.data, diabetes.target G = np.dot(X.T, X) Xy = np.dot(X.T, y) alphas_, active_, coef_path_ = linear_model.lars_path(X, y, method="lasso", Gram=G, Xy=Xy, alpha_min=0.9) print("---") alpha_, active, coef = linear_model.lars_path(X, y, method="lasso", Gram=G, Xy=Xy, alpha_min=0.9, return_path=False) assert_array_almost_equal(coef, coef_path_[:, -1]) assert_true(alpha_ == alphas_[-1])
def test_lasso_lars_vs_lasso_cd_positive(verbose=False): # Test that LassoLars and Lasso using coordinate descent give the # same results when using the positive option # This test is basically a copy of the above with additional positive # option. However for the middle part, the comparison of coefficient values # for a range of alphas, we had to make an adaptations. See below. # not normalized data X = 3 * diabetes.data alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso', positive=True) lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8, positive=True) for c, a in zip(lasso_path.T, alphas): if a == 0: continue lasso_cd.alpha = a lasso_cd.fit(X, y) error = linalg.norm(c - lasso_cd.coef_) assert_less(error, 0.01) # The range of alphas chosen for coefficient comparison here is restricted # as compared with the above test without the positive option. This is due # to the circumstance that the Lars-Lasso algorithm does not converge to # the least-squares-solution for small alphas, see 'Least Angle Regression' # by Efron et al 2004. The coefficients are typically in congruence up to # the smallest alpha reached by the Lars-Lasso algorithm and start to # diverge thereafter. See # https://gist.github.com/michigraber/7e7d7c75eca694c7a6ff for alpha in np.linspace(6e-1, 1 - 1e-2, 20): clf1 = linear_model.LassoLars(fit_intercept=False, alpha=alpha, normalize=False, positive=True).fit(X, y) clf2 = linear_model.Lasso(fit_intercept=False, alpha=alpha, tol=1e-8, normalize=False, positive=True).fit(X, y) err = linalg.norm(clf1.coef_ - clf2.coef_) assert_less(err, 1e-3) # normalized data X = diabetes.data alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso', positive=True) lasso_cd = linear_model.Lasso(fit_intercept=False, normalize=True, tol=1e-8, positive=True) for c, a in zip(lasso_path.T[:-1], alphas[:-1]): # don't include alpha=0 lasso_cd.alpha = a lasso_cd.fit(X, y) error = linalg.norm(c - lasso_cd.coef_) assert_less(error, 0.01)
def test_Lasso_Path(self): diabetes = datasets.load_diabetes() X = diabetes.data y = diabetes.target X /= X.std(axis=0) df = pdml.ModelFrame(diabetes) df.data /= df.data.std(axis=0, ddof=False) self.assert_numpy_array_almost_equal(df.data.values, X) eps = 5e-3 expected = lm.lasso_path(X, y, eps, fit_intercept=False) result = df.lm.lasso_path(eps=eps, fit_intercept=False) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2]) expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False) result = df.lm.enet_path(eps=eps, l1_ratio=0.8, fit_intercept=False) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2]) expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) result = df.lm.enet_path(eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2]) expected = lm.lars_path(X, y, method='lasso', verbose=True) result = df.lm.lars_path(method='lasso', verbose=True) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2])
def estPath(values): """estimates path Args: values: dict of x and y Returns: alphas: regularization lambdas coefs: coef matrix for features and alphas """ X,y = values["x"], values["y"] alphas, _, coefs = linear_model.lars_path(X, y, method='lasso', verbose=True) return alphas,coefs print alphas print coefs print coefs[:,1] exit(1) xx = np.sum(np.abs(coefs.T), axis=1) xx /= xx[-1] plt.plot(xx, coefs.T) ymin, ymax = plt.ylim() plt.vlines(xx, ymin, ymax, linestyle='dashed') plt.xlabel('|coef| / max|coef|') plt.ylabel('Coefficients') plt.title('LASSO Path') plt.axis('tight') plt.show() plt.savefid("larspath.png")
def test_lasso_lars_vs_lasso_cd_ill_conditioned(): # Test lasso lars on a very ill-conditioned design, and check that # it does not blow up, and stays somewhat close to a solution given # by the coordinate descent solver # Also test that lasso_path (using lars_path output style) gives # the same result as lars_path and previous lasso output style # under these conditions. rng = np.random.RandomState(42) # Generate data n, m = 70, 100 k = 5 X = rng.randn(n, m) w = np.zeros((m, 1)) i = np.arange(0, m) rng.shuffle(i) supp = i[:k] w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1) y = np.dot(X, w) sigma = 0.2 y += sigma * rng.rand(*y.shape) y = y.squeeze() lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method='lasso') _, lasso_coef2, _ = linear_model.lasso_path(X, y, alphas=lars_alphas, tol=1e-6, fit_intercept=False) assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
def test_simple(): # Principle of Lars is to keep covariances tied and decreasing # also test verbose output from io import StringIO import sys old_stdout = sys.stdout try: sys.stdout = StringIO() _, _, coef_path_ = linear_model.lars_path( X, y, method='lar', verbose=10) sys.stdout = old_stdout for i, coef_ in enumerate(coef_path_.T): res = y - np.dot(X, coef_) cov = np.dot(X.T, res) C = np.max(abs(cov)) eps = 1e-3 ocur = len(cov[C - eps < abs(cov)]) if i < X.shape[1]: assert ocur == i + 1 else: # no more than max_pred variables can go into the active set assert ocur == X.shape[1] finally: sys.stdout = old_stdout
def test_lasso_lars_vs_lasso_cd_ill_conditioned(): # Test lasso lars on a very ill-conditioned design, and check that # it does not blow up, and stays somewhat close to a solution given # by the coordinate descent solver rng = np.random.RandomState(42) # Generate data n, m = 80, 100 k = 5 X = rng.randn(n, m) w = np.zeros((m, 1)) i = np.arange(0, m) rng.shuffle(i) supp = i[:k] w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1) y = np.dot(X, w) sigma = 0.2 y += sigma * rng.rand(*y.shape) y = y.squeeze() with warnings.catch_warnings(record=True) as warning_list: warnings.simplefilter("always", UserWarning) lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method='lasso') assert_true(len(warning_list) > 0) assert_true(('Dropping a regressor' in warning_list[0].message.args[0]) or ('Early stopping' in warning_list[0].message.args[0])) lasso_coef = np.zeros((w.shape[0], len(lars_alphas))) for i, model in enumerate(linear_model.lasso_path(X, y, alphas=lars_alphas, tol=1e-6)): lasso_coef[:, i] = model.coef_ np.testing.assert_array_almost_equal(lars_coef, lasso_coef, decimal=1)
def test_lasso_path_return_models_vs_new_return_gives_same_coefficients(): # Test that lasso_path with lars_path style output gives the # same result # Some toy data X = np.array([[1, 2, 3.1], [2.3, 5.4, 4.3]]).T y = np.array([1, 2, 3.1]) alphas = [5., 1., .5] # Compute the lasso_path f = ignore_warnings coef_path = [e.coef_ for e in f(lasso_path)(X, y, alphas=alphas, return_models=True, fit_intercept=False)] # Use lars_path and lasso_path(new output) with 1D linear interpolation # to compute the the same path alphas_lars, _, coef_path_lars = lars_path(X, y, method='lasso') coef_path_cont_lars = interpolate.interp1d(alphas_lars[::-1], coef_path_lars[:, ::-1]) alphas_lasso2, coef_path_lasso2, _ = lasso_path(X, y, alphas=alphas, fit_intercept=False, return_models=False) coef_path_cont_lasso = interpolate.interp1d(alphas_lasso2[::-1], coef_path_lasso2[:, ::-1]) np.testing.assert_array_almost_equal(coef_path_cont_lasso(alphas), np.asarray(coef_path).T, decimal=1) np.testing.assert_array_almost_equal(coef_path_cont_lasso(alphas), coef_path_cont_lars(alphas), decimal=1)
def test_simple(): # Principle of Lars is to keep covariances tied and decreasing # also test verbose output from sklearn.externals.six.moves import cStringIO as StringIO import sys old_stdout = sys.stdout try: sys.stdout = StringIO() alphas_, active, coef_path_ = linear_model.lars_path( diabetes.data, diabetes.target, method="lar", verbose=10) sys.stdout = old_stdout for (i, coef_) in enumerate(coef_path_.T): res = y - np.dot(X, coef_) cov = np.dot(X.T, res) C = np.max(abs(cov)) eps = 1e-3 ocur = len(cov[C - eps < abs(cov)]) if i < X.shape[1]: assert_true(ocur == i + 1) else: # no more than max_pred variables can go into the active set assert_true(ocur == X.shape[1]) finally: sys.stdout = old_stdout
def test_singular_matrix(): """ Test when input is a singular matrix """ X1 = np.array([[1, 1.], [1., 1.]]) y1 = np.array([1, 1]) alphas, active, coef_path = linear_model.lars_path(X1, y1) assert_array_almost_equal(coef_path.T, [[0, 0], [1, 0], [1, 0]])
def test_lars_path_gram_equivalent(method, return_path): _assert_same_lars_path_result( linear_model.lars_path_gram( Xy=Xy, Gram=G, n_samples=n_samples, method=method, return_path=return_path), linear_model.lars_path( X, y, Gram=G, method=method, return_path=return_path))
def test_collinearity(): """Check that lars_path is robust to collinearity in input""" X = np.array([[3.0, 3.0, 1.0], [2.0, 2.0, 0.0], [1.0, 1.0, 0]]) y = np.array([1.0, 0.0, 0]) _, _, coef_path_ = linear_model.lars_path(X, y, alpha_min=0.01) assert_true(not np.isnan(coef_path_).any()) residual = np.dot(X, coef_path_[:, -1]) - y assert_less((residual ** 2).sum(), 1.0) # just make sure it's bounded n_samples = 10 X = np.random.rand(n_samples, 5) y = np.zeros(n_samples) _, _, coef_path_ = linear_model.lars_path( X, y, Gram="auto", copy_X=False, copy_Gram=False, alpha_min=0.0, method="lasso", verbose=0, max_iter=500 ) assert_array_almost_equal(coef_path_, np.zeros_like(coef_path_))
def test_lasso_gives_lstsq_solution(): """ Test that Lars Lasso gives least square solution at the end of the path """ alphas_, active, coef_path_ = linear_model.lars_path(X, y, method="lasso") coef_lstsq = np.linalg.lstsq(X, y)[0] assert_array_almost_equal(coef_lstsq, coef_path_[:, -1])
def regularization_path(self): n = len(self.C1) Sigma = np.linalg.pinv(-self.C1 + self.beta * np.eye(n)) A = np.dot(self.Phi.view, np.eye(n) + np.dot(Sigma, -self.C2)) b = np.dot(self.Phi.view, np.dot(Sigma, self.b)) alphas, _, coefs = lm.lars_path(A, b, eps=1e-6) # lst = [(model.alpha, model.coef_) for model in models] return zip(alphas, coefs.T)
def test_collinearity(): """Check that lars_path is robust to collinearity in input""" X = np.array([[3., 3., 1.], [2., 2., 0.], [1., 1., 0]]) y = np.array([1., 0., 0]) _, _, coef_path_ = linear_model.lars_path(X, y) assert (not np.isnan(coef_path_).any()) assert_array_almost_equal(np.dot(X, coef_path_[:, -1]), y)
def sets(self, x, y, n_keep): alphas, active, coef_path = linear_model.lars_path(x.values, y.values) sets = [] seen = set() print coef_path for coefs in coef_path.T: cols = [x.columns[i] for i in range(len(coefs)) if coefs[i] > 1e-9] if len(cols) >= n_keep: return cols return cols
def test_lars_path_positive_constraint(): # this is the main test for the positive parameter on the lars_path method # the estimator classes just make use of this function # we do the test on the diabetes dataset # ensure that we get negative coefficients when positive=False # and all positive when positive=True # for method 'lar' (default) and lasso for method in ["lar", "lasso"]: alpha, active, coefs = linear_model.lars_path( diabetes["data"], diabetes["target"], return_path=True, method=method, positive=False ) assert_true(coefs.min() < 0) alpha, active, coefs = linear_model.lars_path( diabetes["data"], diabetes["target"], return_path=True, method=method, positive=True ) assert_true(coefs.min() >= 0)
def test_collinearity(): """Check that lars_path is robust to collinearity in input""" X = np.array([[3., 3., 1.], [2., 2., 0.], [1., 1., 0]]) y = np.array([1., 0., 0]) _, _, coef_path_ = linear_model.lars_path(X, y, alpha_min=0.01) assert_true(not np.isnan(coef_path_).any()) residual = np.dot(X, coef_path_[:, -1]) - y assert_less((residual ** 2).sum(), 1.) # just make sure it's bounded
def regularized_model_features(X, y): alphas, _, coefs = lars_path(X.values, y, method='lasso', verbose=True) xx = np.sum(np.abs(coefs.T), axis=1) xx /= xx[-1] plt.plot(xx, coefs.T) ymin, ymax = plt.ylim() plt.xlabel('|coef| / max|coef|') plt.ylabel('Coefficients') plt.title('LASSO Path') plt.axis('tight') plt.show() plt.legend()
def test_singular_matrix(): # Test when input is a singular matrix # In this test the "drop for good strategy" of lars_path is necessary # to give a good answer X1 = np.array([[1, 1.], [1., 1.]]) y1 = np.array([1, 1]) with warnings.catch_warnings(record=True) as warning_list: warnings.simplefilter("always", UserWarning) alphas, active, coef_path = linear_model.lars_path(X1, y1) assert_true(len(warning_list) > 0) assert_true('Dropping a regressor' in warning_list[0].message.args[0]) assert_array_almost_equal(coef_path.T, [[0, 0], [1, 0]])
def test_X_none_gram_not_none(): with pytest.raises(ValueError, match="X cannot be None if Gram is not None"): lars_path(X=None, y=[1], Gram='not None')
def test_x_none_gram_none_raises_value_error(): # Test that lars_path with no X and Gram raises exception Xy = np.dot(X.T, y) with pytest.raises(ValueError): linear_model.lars_path(None, y, Gram=None, Xy=Xy)
def test_lasso_gives_lstsq_solution(): # Test that Lars Lasso gives least square solution at the end # of the path _, _, coef_path_ = linear_model.lars_path(X, y, method="lasso") coef_lstsq = np.linalg.lstsq(X, y)[0] assert_array_almost_equal(coef_lstsq, coef_path_[:, -1])
def test_singular_matrix(): # Test when input is a singular matrix X1 = np.array([[1, 1.], [1., 1.]]) y1 = np.array([1, 1]) alphas, active, coef_path = linear_model.lars_path(X1, y1) assert_array_almost_equal(coef_path.T, [[0, 0], [1, 0]])
def solve(self, fraction_evaluated, dim): eyAdj = self.linkfv(self.ey[:, dim]) - self.link.f(self.fnull[dim]) s = np.sum(self.maskMatrix, 1) # do feature selection if we have not well enumerated the space nonzero_inds = np.arange(self.M) log.debug("fraction_evaluated = {0}".format(fraction_evaluated)) # if self.l1_reg == "auto": # warnings.warn( # "l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a " \ # "conditional use of AIC to simply \"num_features(10)\"!" # ) if (self.l1_reg not in [ "auto", False, 0 ]) or (fraction_evaluated < 0.2 and self.l1_reg == "auto"): w_aug = np.hstack( (self.kernelWeights * (self.M - s), self.kernelWeights * s)) log.info("np.sum(w_aug) = {0}".format(np.sum(w_aug))) log.info("np.sum(self.kernelWeights) = {0}".format( np.sum(self.kernelWeights))) w_sqrt_aug = np.sqrt(w_aug) eyAdj_aug = np.hstack( (eyAdj, eyAdj - (self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim])))) eyAdj_aug *= w_sqrt_aug mask_aug = np.transpose( w_sqrt_aug * np.transpose(np.vstack( (self.maskMatrix, self.maskMatrix - 1)))) #var_norms = np.array([np.linalg.norm(mask_aug[:, i]) for i in range(mask_aug.shape[1])]) # select a fixed number of top features if isinstance(self.l1_reg, str) and self.l1_reg.startswith("num_features("): r = int(self.l1_reg[len("num_features("):-1]) nonzero_inds = lars_path(mask_aug, eyAdj_aug, max_iter=r)[1] # use an adaptive regularization method elif self.l1_reg == "auto" or self.l1_reg == "bic" or self.l1_reg == "aic": c = "aic" if self.l1_reg == "auto" else self.l1_reg nonzero_inds = np.nonzero( LassoLarsIC(criterion=c).fit(mask_aug, eyAdj_aug).coef_)[0] # use a fixed regularization coeffcient else: nonzero_inds = np.nonzero( Lasso(alpha=self.l1_reg).fit(mask_aug, eyAdj_aug).coef_)[0] if len(nonzero_inds) == 0: return np.zeros(self.M), np.ones(self.M) # eliminate one variable with the constraint that all features sum to the output eyAdj2 = eyAdj - self.maskMatrix[:, nonzero_inds[-1]] * ( self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim])) etmp = np.transpose( np.transpose(self.maskMatrix[:, nonzero_inds[:-1]]) - self.maskMatrix[:, nonzero_inds[-1]]) log.debug("etmp[:4,:] {0}".format(etmp[:4, :])) # solve a weighted least squares equation to estimate phi tmp = np.transpose( np.transpose(etmp) * np.transpose(self.kernelWeights)) tmp2 = np.linalg.inv(np.dot(np.transpose(tmp), etmp)) w = np.dot(tmp2, np.dot(np.transpose(tmp), eyAdj2)) log.debug("np.sum(w) = {0}".format(np.sum(w))) log.debug("self.link(self.fx) - self.link(self.fnull) = {0}".format( self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim]))) log.debug("self.fx = {0}".format(self.fx[dim])) log.debug("self.link(self.fx) = {0}".format(self.link.f(self.fx[dim]))) log.debug("self.fnull = {0}".format(self.fnull[dim])) log.debug("self.link(self.fnull) = {0}".format( self.link.f(self.fnull[dim]))) phi = np.zeros(self.M) phi[nonzero_inds[:-1]] = w phi[nonzero_inds[-1]] = (self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim])) - sum(w) log.info("phi = {0}".format(phi)) # clean up any rounding errors for i in range(self.M): if np.abs(phi[i]) < 1e-10: phi[i] = 0 return phi, np.ones(len(phi))
def run_simple_model(train_x, train_y, dev_x, dev_y, test_x, test_y, model_type, out_dir=None, class_weight=None): from sklearn import datasets, neighbors, linear_model, svm totalTime = 0 startTrainTime = time() logger.info("Start training...") if model_type == 'ARDRegression': model = linear_model.ARDRegression().fit(train_x, train_y) elif model_type == 'BayesianRidge': model = linear_model.BayesianRidge().fit(train_x, train_y) elif model_type == 'ElasticNet': model = linear_model.ElasticNet().fit(train_x, train_y) elif model_type == 'ElasticNetCV': model = linear_model.ElasticNetCV().fit(train_x, train_y) elif model_type == 'HuberRegressor': model = linear_model.HuberRegressor().fit(train_x, train_y) elif model_type == 'Lars': model = linear_model.Lars().fit(train_x, train_y) elif model_type == 'LarsCV': model = linear_model.LarsCV().fit(train_x, train_y) elif model_type == 'Lasso': model = linear_model.Lasso().fit(train_x, train_y) elif model_type == 'LassoCV': model = linear_model.LassoCV().fit(train_x, train_y) elif model_type == 'LassoLars': model = linear_model.LassoLars().fit(train_x, train_y) elif model_type == 'LassoLarsCV': model = linear_model.LassoLarsCV().fit(train_x, train_y) elif model_type == 'LassoLarsIC': model = linear_model.LassoLarsIC().fit(train_x, train_y) elif model_type == 'LinearRegression': model = linear_model.LinearRegression().fit(train_x, train_y) elif model_type == 'LogisticRegression': model = linear_model.LogisticRegression(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'LogisticRegressionCV': model = linear_model.LogisticRegressionCV(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'MultiTaskLasso': model = linear_model.MultiTaskLasso().fit(train_x, train_y) elif model_type == 'MultiTaskElasticNet': model = linear_model.MultiTaskElasticNet().fit(train_x, train_y) elif model_type == 'MultiTaskLassoCV': model = linear_model.MultiTaskLassoCV().fit(train_x, train_y) elif model_type == 'MultiTaskElasticNetCV': model = linear_model.MultiTaskElasticNetCV().fit(train_x, train_y) elif model_type == 'OrthogonalMatchingPursuit': model = linear_model.OrthogonalMatchingPursuit().fit(train_x, train_y) elif model_type == 'OrthogonalMatchingPursuitCV': model = linear_model.OrthogonalMatchingPursuitCV().fit(train_x, train_y) elif model_type == 'PassiveAggressiveClassifier': model = linear_model.PassiveAggressiveClassifier(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'PassiveAggressiveRegressor': model = linear_model.PassiveAggressiveRegressor().fit(train_x, train_y) elif model_type == 'Perceptron': model = linear_model.Perceptron(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'RandomizedLasso': model = linear_model.RandomizedLasso().fit(train_x, train_y) elif model_type == 'RandomizedLogisticRegression': model = linear_model.RandomizedLogisticRegression().fit(train_x, train_y) elif model_type == 'RANSACRegressor': model = linear_model.RANSACRegressor().fit(train_x, train_y) elif model_type == 'Ridge': model = linear_model.Ridge().fit(train_x, train_y) elif model_type == 'RidgeClassifier': model = linear_model.RidgeClassifier(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'RidgeClassifierCV': model = linear_model.RidgeClassifierCV(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'RidgeCV': model = linear_model.RidgeCV().fit(train_x, train_y) elif model_type == 'SGDClassifier': model = linear_model.SGDClassifier(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'SGDRegressor': model = linear_model.SGDRegressor().fit(train_x, train_y) elif model_type == 'TheilSenRegressor': model = linear_model.TheilSenRegressor().fit(train_x, train_y) elif model_type == 'lars_path': model = linear_model.lars_path().fit(train_x, train_y) elif model_type == 'lasso_path': model = linear_model.lasso_path().fit(train_x, train_y) elif model_type == 'lasso_stability_path': model = linear_model.lasso_stability_path().fit(train_x, train_y) elif model_type == 'logistic_regression_path': model = linear_model.logistic_regression_path(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'orthogonal_mp': model = linear_model.orthogonal_mp().fit(train_x, train_y) elif model_type == 'orthogonal_mp_gram': model = linear_model.orthogonal_mp_gram().fit(train_x, train_y) elif model_type == 'LinearSVC': model = svm.LinearSVC(class_weight=class_weight).fit(train_x, train_y) elif model_type == 'SVC': model = svm.SVC(class_weight=class_weight, degree=3).fit(train_x, train_y) else: raise NotImplementedError('Model not implemented') logger.info("Finished training.") endTrainTime = time() trainTime = endTrainTime - startTrainTime logger.info("Training time : %d seconds" % trainTime) logger.info("Start predicting train set...") train_pred_y = model.predict(train_x) logger.info("Finished predicting train set.") logger.info("Start predicting test set...") test_pred_y = model.predict(test_x) logger.info("Finished predicting test set.") endTestTime = time() testTime = endTestTime - endTrainTime logger.info("Testing time : %d seconds" % testTime) totalTime += trainTime + testTime train_pred_y = np.round(train_pred_y) test_pred_y = np.round(test_pred_y) np.savetxt(out_dir + '/preds/best_test_pred' + '.txt', test_pred_y, fmt='%i') logger.info('[TRAIN] Acc: %.3f' % (accuracy_score(train_y, train_pred_y))) logger.info('[TEST] Acc: %.3f' % (accuracy_score(test_y, test_pred_y))) return accuracy_score(test_y, test_pred_y)
def compute_bench(samples_range, features_range): it = 0 results = dict() lars = np.empty((len(features_range), len(samples_range))) lars_gram = lars.copy() omp = lars.copy() omp_gram = lars.copy() max_it = len(samples_range) * len(features_range) for i_s, n_samples in enumerate(samples_range): for i_f, n_features in enumerate(features_range): it += 1 n_informative = n_features / 10 print('====================') print('Iteration %03d of %03d' % (it, max_it)) print('====================') # dataset_kwargs = { # 'n_train_samples': n_samples, # 'n_test_samples': 2, # 'n_features': n_features, # 'n_informative': n_informative, # 'effective_rank': min(n_samples, n_features) / 10, # #'effective_rank': None, # 'bias': 0.0, # } dataset_kwargs = { 'n_samples': 1, 'n_components': n_features, 'n_features': n_samples, 'n_nonzero_coefs': n_informative, 'random_state': 0 } print("n_samples: %d" % n_samples) print("n_features: %d" % n_features) y, X, _ = make_sparse_coded_signal(**dataset_kwargs) X = np.asfortranarray(X) gc.collect() print("benchmarking lars_path (with Gram):", end='') sys.stdout.flush() tstart = time() G = np.dot(X.T, X) # precomputed Gram matrix Xy = np.dot(X.T, y) lars_path(X, y, Xy=Xy, Gram=G, max_iter=n_informative) delta = time() - tstart print("%0.3fs" % delta) lars_gram[i_f, i_s] = delta gc.collect() print("benchmarking lars_path (without Gram):", end='') sys.stdout.flush() tstart = time() lars_path(X, y, Gram=None, max_iter=n_informative) delta = time() - tstart print("%0.3fs" % delta) lars[i_f, i_s] = delta gc.collect() print("benchmarking orthogonal_mp (with Gram):", end='') sys.stdout.flush() tstart = time() orthogonal_mp(X, y, precompute=True, n_nonzero_coefs=n_informative) delta = time() - tstart print("%0.3fs" % delta) omp_gram[i_f, i_s] = delta gc.collect() print("benchmarking orthogonal_mp (without Gram):", end='') sys.stdout.flush() tstart = time() orthogonal_mp(X, y, precompute=False, n_nonzero_coefs=n_informative) delta = time() - tstart print("%0.3fs" % delta) omp[i_f, i_s] = delta results['time(LARS) / time(OMP)\n (w/ Gram)'] = (lars_gram / omp_gram) results['time(LARS) / time(OMP)\n (w/o Gram)'] = (lars / omp) return results
#meresultc = memodelc.fit() db = pd.read_csv("2015_utilization_reduced.csv.gz") # Merge provider type du = db.groupby("npi")["provider_type"].agg("first") du = pd.DataFrame(du).reset_index() dr = pd.merge(dr, du, left_on="npi", right_on="npi") # Use lars to consider provider effects y, x = patsy.dmatrices("log_op_z ~ 0 + log_nonop_z + C(provider_type)", data=dr, return_type='dataframe') xa = np.asarray(x) ya = np.asarray(y)[:,0] xnames = x.columns.tolist() alphas, active, coefs = linear_model.lars_path(xa, ya, method='lars', verbose=True) # Display the first few variables selected by lars and the fitted # correlation. for k in range(1, 20): print(xnames[active[k-1]]) f = np.dot(xa, coefs[:, k]) print(np.corrcoef(ya, f)[0, 1]) # Fixed effects model for provider type pfemodelz = sm.OLS.from_formula("log_op_z ~ log_nonop_z + C(provider_type)", data=dr) pferesultz = pfemodelz.fit() # Basic mixed model (random intercepts by provider) pmemodelz = sm.MixedLM.from_formula("log_op_z ~ log_nonop_z", groups="provider_type", data=dr)
def compute_bench(samples_range, features_range): it = 0 results = defaultdict(lambda: []) max_it = len(samples_range) * len(features_range) for n_samples in samples_range: for n_features in features_range: it += 1 print('====================') print('Iteration %03d of %03d' % (it, max_it)) print('====================') dataset_kwargs = { 'n_samples': n_samples, 'n_features': n_features, 'n_informative': n_features / 10, 'effective_rank': min(n_samples, n_features) / 10, #'effective_rank': None, 'bias': 0.0, } print("n_samples: %d" % n_samples) print("n_features: %d" % n_features) X, y = make_regression(**dataset_kwargs) gc.collect() print("benchmarking lars_path (with Gram):", end='') sys.stdout.flush() tstart = time() G = np.dot(X.T, X) # precomputed Gram matrix Xy = np.dot(X.T, y) lars_path(X, y, Xy=Xy, Gram=G, method='lasso') delta = time() - tstart print("%0.3fs" % delta) results['lars_path (with Gram)'].append(delta) gc.collect() print("benchmarking lars_path (without Gram):", end='') sys.stdout.flush() tstart = time() lars_path(X, y, method='lasso') delta = time() - tstart print("%0.3fs" % delta) results['lars_path (without Gram)'].append(delta) gc.collect() print("benchmarking lasso_path (with Gram):", end='') sys.stdout.flush() tstart = time() lasso_path(X, y, precompute=True) delta = time() - tstart print("%0.3fs" % delta) results['lasso_path (with Gram)'].append(delta) gc.collect() print("benchmarking lasso_path (without Gram):", end='') sys.stdout.flush() tstart = time() lasso_path(X, y, precompute=False) delta = time() - tstart print("%0.3fs" % delta) results['lasso_path (without Gram)'].append(delta) return results
def graphical_lasso(TS, alpha=0.01, max_iter=100, convg_threshold=0.001): """ This function computes the graphical lasso algorithm as outlined in [1]. Params ------ TS (np.ndarray): Array consisting of $L$ observations from $N$ sensors. alpha (float, default=0.01): Coefficient of penalization, higher values means more sparseness convg_threshold (float, default=0.001): Stop the algorithm when the duality gap is below a certain threshold. Returns ------- cov (np.ndarray): Estimator of the inverse covariance matrix with sparsity. """ TS = TS.T if alpha < 1e-15: covariance_ = cov_estimator(TS) precision_ = np.linalg.pinv(TS) return covariance_, precision_ n_features = TS.shape[1] mle_estimate_ = cov_estimator(TS) covariance_ = mle_estimate_.copy() precision_ = np.linalg.pinv(mle_estimate_) indices = np.arange(n_features) for i in range(max_iter): for n in range(n_features): sub_estimate = covariance_[indices != n].T[indices != n] row = mle_estimate_[n, indices != n] #solve the lasso problem _, _, coefs_ = lars_path(sub_estimate, row, Xy=row, Gram=sub_estimate, alpha_min=alpha / (n_features - 1.), copy_Gram=True, method="lars") coefs_ = coefs_[:, -1] #just the last please. #update the precision matrix. precision_[n, n] = 1. / (covariance_[n, n] - np.dot(covariance_[indices != n, n], coefs_)) precision_[indices != n, n] = -precision_[n, n] * coefs_ precision_[n, indices != n] = -precision_[n, n] * coefs_ temp_coefs = np.dot(sub_estimate, coefs_) covariance_[n, indices != n] = temp_coefs covariance_[indices != n, n] = temp_coefs #if test_convergence( old_estimate_, new_estimate_, mle_estimate_, convg_threshold): if np.abs(_dual_gap(mle_estimate_, precision_, alpha)) < convg_threshold: break else: #this triggers if not break command occurs print( "The algorithm did not converge. Try increasing the max number of iterations." ) return covariance_, precision_
X = X1 Y = Y # In[98]: Y = dataset[:, 2] X = dataset[:, :13] X[np.isnan(X)] = 0 # In[99]: print("Regularization path using lars_path") # In[100]: alphas1, active1, coefs1 = lars_path(X, Y, method='lasso', verbose=True) # In[101]: print("Regularization path using lars_path") # In[102]: eps = 5e-6 # In[103]: alphas2, coefs2, _ = lasso_path(X, Y, eps) # In[104]:
# Linear Regression, Ridge or Poly # Split the data into 3 portions: 60% for training, 20% for validation (used to select the model), 20% for final testing evaluation. X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.2, random_state=42) X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=.25, random_state=42) #%% # Features selection with LASSO # Scale the variables std = StandardScaler() std.fit(X_train.values) X_tr = std.transform(X_train.values) # Finding the lars paths print("Computing regularization path using the LARS ...") alphas, _, coefs = lars_path(X_tr, y_train.values, method='lasso') # plotting the LARS path xx = np.sum(np.abs(coefs.T), axis=1) xx /= xx[-1] plt.figure(figsize=(8,8)) plt.plot(xx, coefs.T) ymin, ymax = plt.ylim() plt.vlines(xx, ymin, ymax, linestyle='dashed') plt.xlabel('|coef| / max|coef|') plt.ylabel('Coefficients') plt.title('LASSO Path') plt.axis('tight') plt.legend(X_train.columns) plt.show()
# License: BSD 3 clause import numpy as np import matplotlib.pyplot as plt from sklearn import linear_model from sklearn import datasets diabetes = datasets.load_diabetes() X = diabetes.data y = diabetes.target print("Computing regularization path using the LARS ...") alphas, active, coefs = linear_model.lars_path(X, y, method='lar', verbose=True, return_path=True) print(coefs) print(np.add(active, 1)) xx = np.sum(np.abs(coefs.T), axis=1) print(xx) xx /= xx[-1] plt.plot(xx, coefs.T) ymin, ymax = plt.ylim() plt.vlines(xx, ymin, ymax, linestyle='dashed') plt.xlabel('|coef| / max|coef|') plt.ylabel('Coefficients') plt.title('LASSO Path') plt.axis('tight')
print("Weights", clf.coef_) #plt.plot(test_range,[f(x) for x in test_range], label="True plot") #plt.plot(sample_range,data[1],linestyle=" ", marker="o") #plt.plot(test_range,[y(x) for x in test_range], label="True plot") #plt.show() print("intercept:", clf.intercept_) #Calculation of design matrix dm2 = np.array([[sample_range[i]**j for j in range(m + 1)] for i in range(n)], dtype='float') eps = 5e-3 alphas_lasso, _, coefs = linear_model.lars_path(dm2, np.array(data[1]), method='lasso', verbose=True) #print(coefs) xx = np.sum(np.abs(coefs.T), axis=1) xx /= xx[-1] neg_log_alphas_lasso = -np.log10(alphas_lasso) for i in range(m): plt.plot(xx, coefs[i], label="w" + str(i)) legend = plt.legend(loc='upper center', shadow=True, fontsize='x-large') #plt.plot(xx, coefs.T) #plt.plot(np.array([1,2,3]),np.array([[4,9],[5,9],[6,9]])) ymin, ymax = plt.ylim() plt.vlines(xx, ymin, ymax, linestyle='dashed') plt.xlabel('|coef| / |max_likelihood(coef)|') plt.ylabel('Coefficients')
# Author: Fabian Pedregosa <*****@*****.**> # Alexandre Gramfort <*****@*****.**> # License: BSD 3 clause get_ipython().run_line_magic('matplotlib', 'inline') import numpy as np import matplotlib.pyplot as plt from sklearn import linear_model from sklearn import datasets diabetes = datasets.load_diabetes() X = diabetes.data y = diabetes.target print("Computing regularization path using the LARS ...") alphas, _, coefs = linear_model.lars_path(X, y, method='lasso', verbose=True) xx = np.sum(np.abs(coefs.T), axis=1) xx /= xx[-1] plt.plot(xx, coefs.T) ymin, ymax = plt.ylim() plt.vlines(xx, ymin, ymax, linestyle='dashed') plt.xlabel('|coef| / max|coef| (Alpha)') plt.ylabel('Coefficients') plt.title('LASSO Path') plt.axis('tight') plt.show() # ### Ventajas LASSO: