def ode_integratedlasso_rank_vars(D, times, target, env=None, silent=True, interactions=True, rm_target=True): L = len(times) d = D.shape[1] // L n = D.shape[0] Xint = np.zeros([(L - 1) * n, d]) deltaY = np.zeros([(L - 1) * n]) for i in range(n): deltaY[(i) * (L - 1):(i + 1) * (L - 1)] = np.diff( D[i, target * L:(target + 1) * L]) for j in range(d): for i in range(n): tmp = D[i, j * L:(j + 1) * L] Xint[i * (L - 1):(i + 1) * (L - 1), j] = (tmp[:(L - 1)] + tmp[1:]) / 2 * np.diff(times) # remove NAs na_ind = np.logical_or(np.isnan(deltaY), (np.isnan(Xint) > 0).sum(axis=1)) deltaY = deltaY[~na_ind] Xint = Xint[~na_ind, ] # Perform lasso if interactions: dC2 = d * (d - 1) // 2 # combination var_names = np.zeros([d + dC2 + d], dtype=np.object) var_names[:d] = np.array([[]] + [[i] for i in range(d)], dtype=np.object)[1:] var_names[d:] = np.array([[]] + sum( ([[i, j] for j in range(i + 1)] for i in range(d)), []), dtype=np.object)[1:] Xint_interactions = np.zeros([n * (L - 1), len(var_names)]) Xint_interactions[:, :d] = Xint for i in range(d, len(var_names)): Xint_interactions[:, i] = Xint[:, var_names[i] [0]] * Xint[:, var_names[i][1]] fit = ElasticNet().fit(Xint_interactions, deltaY) sel_matrix = (np.abs(fit.coef_path_) > 1e-7) first_entrance = sel_matrix.max(axis=1) # find all rows without ones and set first entrance to Inf first_entrance[sel_matrix.sum(axis=1) == 0] = np.infty ranking = first_entrance.argsort() ranking = var_names[ranking] else: fit = ElasticNet().fit(Xint, deltaY) sel_matrix = fit.coef_path_ != 0 first_entrance = sel_matrix.max(axis=1) # find all rows without ones and set first entrance to Inf first_entrance[sel_matrix.sum(axis=1) == 0] = np.infty ranking = first_entrance.argsort() if rm_target: ranking = ranking[ranking != target] return ({'ranking': ranking, 'coef': fit.coef_})
def test_one_row_predict(self): # Verify that predicting on one row gives only one row of output m = ElasticNet(random_state=42) for X, y in self.inputs: m.fit(X, y) p = m.predict(X[0].reshape((1, -1))) assert p.shape == (1,)
def _notify(self, trainer): if not self.computed: X = trainer.X.cpu().detach().numpy() y = trainer.y.cpu().detach().numpy() glmnet = ElasticNet(n_splits=0, fit_intercept=False, lambda_path=np.exp( np.linspace(start=np.log(10**(-9)), stop=np.log(10), num=200))) glmnet.fit(X, y.squeeze()) self.lambdas, self.coefs = glmnet.lambda_path_, glmnet.coef_path_ # Swap axes for parameters learned by lasso path, so that lambda # corresponds to the same axis as time for gradient descent iterates. self.coefs = np.swapaxes(self.coefs, 0, 1) # So now self.coefs is of shape (n_lambda, n_params). for lambda_id in range(self.coefs.shape[0]): w_lambda = self.coefs[lambda_id, :] self.append_performance_metrics(w_lambda, trainer) if self.store_path is False: # Dummy array so that _aggregate_numeric_results works. self.coefs = [np.array([[-1], [-1]])] self.computed = True
def TL(self, x, y, pen_bic, pen_gic): m = ElasticNet() if len(x.shape) < 2: x.reshape((x.shape[0], 1)) m = m.fit(x, y) betas = m.coef_path_ intercepts = m.intercept_path_ BIC = np.inf for i in range(betas.shape[1]): RSS = np.sum((y - np.matmul(x, betas[:, i]) - intercepts[i])**2) k = np.sum(betas[:, i] != 0) BIC_new = RSS + pen_bic * k if BIC_new < BIC: BIC = BIC_new beta_bic = betas[:, i] intercept = intercepts[i] thresholds = beta_bic[beta_bic > 0] thresholds.sort() beta_gic = np.zeros_like(beta_bic) GIC = np.inf for delta in thresholds: beta_thres = deepcopy(beta_bic) beta_thres[beta_thres < delta] = 0 RSS = np.sum((y - np.matmul(x, beta_thres) - intercept)**2) k = np.sum(beta_thres != 0) GIC_new = RSS + pen_gic * k if GIC_new < GIC: GIC = GIC_new beta_gic = beta_thres return beta_gic, intercept
def test_one_row_predict_with_lambda(self): # One row to predict along with lambdas should give 2D output m = ElasticNet(random_state=42) for X, y in self.inputs: m.fit(X, y) p = m.predict(X[0].reshape((1, -1)), lamb=[20, 10]) assert p.shape == (1, 2)
def test_validate_rel_penalties(self): X = np.random.random(size=(50, 10)) w = np.random.random(size=(10, )) y = np.dot(X, w) enet = ElasticNet(alpha=.5) # Invalid use: # Passing in a rel_penalties vector that is too short. with self.assertRaises(ValueError): rel_pens = np.ones(shape=(9, )) enet._validate_rel_penalties(X, y, rel_penalties=rel_pens) # Invalid use: # Passing in a rel_penalties that matches the wrong dimenstion of X. with self.assertRaises(ValueError): rel_pens = np.ones(shape=(50, )) enet._validate_rel_penalties(X, y, rel_penalties=rel_pens) # Invalid use: # Passing in a rel_penalties containing a negative entry. with self.assertRaises(ValueError): rel_pens = np.ones(shape=(10, )) rel_pens[5] = -1 enet._validate_rel_penalties(X, y, rel_penalties=rel_pens) # Invalid use: # Passing in a rel_panalties with no positive entries. with self.assertRaises(ValueError): rel_pens = np.zeros(shape=(10, )) enet._validate_rel_penalties(X, y, rel_penalties=rel_pens) # Valid use: # Rel_penalties has the correct dimenstion with all non-negative # entries. rel_pens = np.ones(shape=(10, )) rel_pens[5] = 0 enet._validate_rel_penalties(X, y, rel_penalties=rel_pens)
def train_and_test(basename): d = np.load("train_test_" + basename + ".npz") train_x = d["train_x"] train_y = d["train_y"] test_x = d["test_x"] test_y = d["test_y"] countrylist = d["countrylist"] numcountries = train_x.shape[1] weights = np.zeros((numcountries,numcountries)) enets = [None]*numcountries enet_cvs = [None]*numcountries preds = np.zeros(test_y.shape) errors = np.zeros(test_y.shape) var_ratio = np.zeros((numcountries)) for i in range(numcountries): enets[i] = ElasticNet(alpha=.1) enet_cvs[i] = CVGlmNet(enets[i], n_folds=10, n_jobs=10) enet_cvs[i].fit(train_x, train_y[:,i]) bli = enet_cvs[i].best_lambda_idx weights[i,:] = enet_cvs[i].base_estimator.get_coefficients_from_lambda_idx(bli) preds[:,i] = enet_cvs[i].predict(test_x) errors[:,i] = test_y[:,i] - preds[:,i] var_truth = np.var(test_y[:,i]) var_err = np.var(errors[:,i]) var_ratio[i] = 1 - var_err/var_truth print("finished predicting country number %d" % i) np.savez("results_" + basename + ".npz", preds = preds, truth=test_y, errors = errors, var_ratio = var_ratio, countrylist = countrylist, weights=weights)
def test_ridge_models(self): '''Test that a pure ridge (alpha=0) model gives expected results for both dense and sparse matricies. We test that the ridge model, when fit on uncorrelated predictors, shrinks the parameter estiamtes uniformly. To see this, we generate linearly related data with a correlation free model matrix, then test that the array of ratios of fit parameters to true coefficients is a constant array. This test generates more samples than the others to guarentee that the data is sufficiently correlation free, otherwise the effect to be measured does not occur. ''' Xdn = np.random.random(size=(50000, 3)) Xsp = csc_matrix(Xdn) w = np.random.random(size=(3, )) for X in (Xdn, Xsp): for lam in np.linspace(0, 1, 10): y = np.dot(Xdn, w) enet = ElasticNet(alpha=0) enet.fit(X, y, lambdas=[lam]) ratios = enet._coefficients.ravel() / w norm_ratios = ratios / np.max(ratios) test = np.allclose(norm_ratios, 1, atol=.05) self.assertTrue(test)
def test_validate_weights(self): X = np.random.random(size=(50, 10)) w = np.random.random(size=(10, )) y = np.dot(X, w) enet = ElasticNet(alpha=.5) # Invalid use # Passing in a sample weight vector that is too short. with self.assertRaises(ValueError): sw = np.ones(shape=(49, )) enet._validate_weights(X, y, weights=sw) # Invalid use: # Passing in a weight vector that matches the wrong dimenstion of X. with self.assertRaises(ValueError): sw = np.ones(shape=(10, )) enet._validate_weights(X, y, weights=sw) # Invalid use: # Passing in a weight vector containing a negative entry. with self.assertRaises(ValueError): sw = np.ones(shape=(50, )) sw[25] = -1 enet._validate_weights(X, y, weights=sw) # Valid Use: # Weight vector of the correct dimension with all non-negative # entries. sw = np.ones(shape=(50, )) enet._validate_weights(X, y, weights=sw)
def test_max_features(self): x, y = self.inputs[3] max_features = 5 m = ElasticNet(n_splits=3, random_state=42, max_features=max_features) m = m.fit(x, y) num_features = np.count_nonzero(m.coef_) self.assertTrue(num_features <= max_features)
def test_random_state_cv(self): random_state = 133 m = ElasticNet(random_state=random_state) x, y = self.inputs[0] m.fit(x, y) print(dir(m._cv)) assert m._cv.random_state == random_state
def test_with_single_var(self): x = np.random.rand(500,1) y = (1.3 * x).ravel() m = ElasticNet(random_state=449065) m = m.fit(x, y) self.check_r2_score(y, m.predict(x), 0.90)
def test_predict_without_cv(self): x, y = self.inputs[0] m = ElasticNet(n_splits=0, random_state=340561) m = m.fit(x, y) # should not make prediction unless value is passed for lambda with self.assertRaises(ValueError): m.predict(x)
def test_coef_limits(self): x, y = self.inputs[0] lower_limits = np.repeat(-1, x.shape[1]) upper_limits = 0 m = ElasticNet(lower_limits=lower_limits, upper_limits=upper_limits, random_state=5934, alpha=0) m = m.fit(x, y) assert(np.all(m.coef_ >= -1)) assert(np.all(m.coef_ <= 0))
def test_linear_glmnet(benchmark, alpha, n_obs): rng = np.random.default_rng(SEED) error = rng.normal(loc=0, scale=1, size=n_obs) X = rng.normal(loc=5, scale=2, size=(n_obs, 4)) true_betas = np.array([1, -2, 0.5, 1]) y = X.dot(true_betas) + error m = ElasticNet(alpha=alpha) benchmark(m.fit, X, y)
def test_with_pandas_df(self): x, y = make_regression(random_state=561) df = pd.DataFrame(x) df['y'] = y m = ElasticNet(n_folds=3, random_state=123) m = m.fit(df.drop(['y'], axis=1), df.y) sanity_check_regression(m, x)
def test_with_no_predictor_variance(self): x = np.ones((500, 1)) y = np.random.rand(500) m = ElasticNet(random_state=561) msg = "All predictors have zero variance (glmnet error no. 7777)." with self.assertRaises(ValueError, msg=msg): m.fit(x, y)
def test_coef_limits(self): x, y = self.inputs[0] lower_limits = 0 upper_limits = np.repeat(1, x.shape[1]) m = ElasticNet(lower_limits=lower_limits, upper_limits=upper_limits, random_state=5934) m = m.fit(x, y) assert (np.all(m.coef_) >= 0) assert (np.all(m.coef_) <= 1)
def test_n_splits(self): x, y = self.inputs[0] for n in self.n_splits: m = ElasticNet(n_splits=n, random_state=6601) if n > 0 and n < 3: with self.assertRaisesRegexp(ValueError, "n_splits must be at least 3"): m = m.fit(x, y) else: m = m.fit(x, y) sanity_check_regression(m, x)
def test_fit_cv_glmnet_comparison(): rng = np.random.default_rng(SEED) error = rng.normal(loc=0, scale=1, size=100) X = rng.normal(loc=5, scale=2, size=(100, 4)) true_betas = np.array([1, -2, 0.5, 1]) y = X.dot(true_betas) + error m = Elnet(n_splits=3, random_state=182, scoring="r2") m.fit(X, y) m2 = ElasticNet(n_splits=3, random_state=182) m2.fit(X, y) np.testing.assert_almost_equal(m.lambda_max_, m2.lambda_max_) np.testing.assert_almost_equal(m.lambda_1se_, m2.lambda_best_[0])
def test_with_defaults(self): m = ElasticNet(random_state=2821) for x, y in self.inputs: m = m.fit(x, y) sanity_check_regression(m, x) # check selection of lambda_best self.assertTrue(m.lambda_best_inx_ <= m.lambda_max_inx_) # check full path predict p = m.predict(x, lamb=m.lambda_path_) self.assertEqual(p.shape[-1], m.lambda_path_.size)
def create_estimator(self): """ Create an estimator. Creates an estimator depending on the family of regression. :return: A scikit-learn estimator. """ if self.family == 'gaussian': estimator = ElasticNet(standardize=False, cut_point=0) elif self.family == 'binomial': estimator = LogitNet(standardize=False, cut_point=0) return estimator
def test_validate_matrix(self): '''Test the _validate_matrix method.''' Xdn = np.random.random(size=(50, 10)) enet = ElasticNet(alpha=.5) # Invalid use: # Passing in a sparse matrix in the incorrect format. with self.assertRaises(ValueError): Xsp = csr_matrix(Xdn) enet._validate_matrix(Xsp) # Valid use: # Passing in a matrix in compressed sparse column format. Xsp = csc_matrix(Xdn) enet._validate_matrix(Xsp)
def test_lambda_clip_warning(self): x, y = self.inputs[0] m = ElasticNet(n_splits=0, random_state=1729) m = m.fit(x, y) # we should get a warning when we ask for predictions at values of # lambda outside the range of lambda_path_ with self.assertWarns(RuntimeWarning): # note, lambda_path_ is in decreasing order m.predict(x, lamb=m.lambda_path_[0] + 1) with self.assertWarns(RuntimeWarning): m.predict(x, lamb=m.lambda_path_[-1] - 1)
def _parallel_permute_count_nonzero_penalised_coefs(xp, yp, lam_path, penalties, norm_num, is_regression): from glmnet import ElasticNet, LogitNet np.random.shuffle(yp) params = dict(alpha=norm_num, lambda_path=lam_path) pm = ElasticNet(**params) if is_regression else LogitNet(**params) pm.fit(xp, yp, relative_penalties=penalties) return np.sign( np.abs(np.squeeze(pm.coef_path_)) * vec_to_array(penalties)).sum(axis=0)
def test_relative_penalties(self): m1 = ElasticNet(random_state=4328) m2 = ElasticNet(random_state=4328) for x, y in self.inputs: p = x.shape[1] # m1 no relative penalties applied m1.fit(x, y) # find the nonzero indices from LASSO nonzero = np.nonzero(m1.coef_) # unpenalize those nonzero coefs penalty = np.repeat(1, p) penalty[nonzero] = 0 # refit the model with the unpenalized coefs m2.fit(x, y, relative_penalties=penalty) # verify that the unpenalized coef ests exceed the penalized ones # in absolute value assert(np.all(np.abs(m1.coef_) <= np.abs(m2.coef_)))
def test_edge_cases(self): '''Edge cases in model specification.''' X = np.random.random(size=(50, 10)) w = np.random.random(size=(10, )) y = np.dot(X, w) # Edge case # A single lambda is so big that it sets all estimated coefficients # to zero. This used to break the predict method. enet = ElasticNet(alpha=1) enet.fit(X, y, lambdas=[10**5]) _ = enet.predict(X) # Edge case # Multiple lambdas are so big as to set all estiamted coefficients # to zero. This used to break the predict method. enet = ElasticNet(alpha=1) enet.fit(X, y, lambdas=[10**5, 2 * 10**5]) _ = enet.predict(X) # Edge case: # Some predictors have zero varaince. This used to break lambda # max. X = np.random.random(size=(50, 10)) X[:, 2] = 0 X[:, 8] = 1 y = np.dot(X, w) enet = ElasticNet(alpha=.1) enet.fit(X, y) ol = enet.out_lambdas max_lambda_from_fortran = ol[1] * (ol[1] / ol[2]) max_lambda_from_python = enet._max_lambda(X, y) self.assertAlmostEqual(max_lambda_from_fortran, max_lambda_from_python, 4) # Edge case. # All predictors have zero variance. This is an error in # sepcification. with self.assertRaises(ValueError): X = np.ones(shape=(50, 10)) enet = ElasticNet(alpha=.1) enet.fit(X, y)
def run(self, epochs: int): print("Starting ElasticNet simulation.") # First, set up the data loaders, that will be used for computing # metrics via the self.executor object. self._reset_data_loaders() # The epochs parameter will be ignored but it is needed for # compatibility of the rprml.core.Executor class. X = self.train_dataset.X.cpu().detach().numpy() y = self.train_dataset.y.cpu().detach().numpy() # Lambda path has to be supplied in decreasing order. lambda_path = np.array(self.lambdas) lambda_path = -np.sort(-lambda_path) # Append infinity to the front of lambda_path. # We do this because the glmnet package modifies the first lambda. lambda_path = np.insert(lambda_path, 0, 1e10000, axis=0) glmnet = ElasticNet(alpha=self.alpha, n_splits=0, fit_intercept=False, standardize=False, lambda_path=lambda_path) glmnet.fit(X, y.squeeze()) lambdas, coefs = glmnet.lambda_path_, glmnet.coef_path_ # Swap axes for parameters learned by lasso path, so that lambda # corresponds to the first axis. coefs = np.swapaxes(coefs, 0, 1) # Remove the first value of lambda (because glmnet modifies its value) # and remove the associated fitted vector. lambdas = np.array(lambdas).flatten()[1:] coefs = np.array(coefs)[1:, :] # Save lambdas and alpha to the executor's history. self.executor.history['lambdas'] = lambdas self.executor.history['alpha'] = self.alpha # For each fitted model, compute the metrics registered to # self.executor. for lambda_id in range(coefs.shape[0]): w_lambda = coefs[lambda_id, :] w_lambda = torch.tensor(w_lambda, dtype=torch.float32, device=self.device) self.model.set_w(w_lambda) # Compute the metrics associated to w_lambda. self.trainer.fire_event(_iteration_level_event)
def FeatureSelection(df_x, xtrain, ytrain, exclude_cols=[]): # #### Gam # gam = LinearGAM(n_splines=4).gridsearch(xtrain, ytrain) # pvalues = np.array(gam.statistics_['p_values']) # important_x_idx_gam = [idx-1 for idx in np.where(pvalues < 0.1)[0]] # important_x_gam = df_x.iloc[:, important_x_idx_gam] #### Lasso lasso = ElasticNet(alpha=1, n_splits=10, random_state=123, n_jobs=4) lasso.fit(xtrain, ytrain) coeffs = lasso.coef_ important_x_dx_lasso = np.where(coeffs != 0.)[0] important_x_lasso = [col for col in df_x.columns[important_x_dx_lasso]] return (important_x_lasso, important_x_dx_lasso)
def glmnet_box(): m1 = ElasticNet(n_splits=20, scoring='r2', alpha=0) m1.fit(music_features, box_latitude_label) lat_r_squared = m1.score(music_features, box_latitude_label) print('GLMNET ridge lattitude r2 {}'.format(lat_r_squared)) plot_predictions( inverse_box_cox(m1.predict(music_features), lambda_lat, 90), latitude_label, 'ridge_latitude_residual.png', 'residual vs fitted latitude for Ridge') m1.fit(music_features, box_longitude_label) lon_r_squared = m1.score(music_features, box_longitude_label) print('GLMNET ridge longitude r2 {}'.format(lon_r_squared)) plot_predictions( inverse_box_cox(m1.predict(music_features), lambda_lon, 180), longitude_label, 'ridge_longitude_residual.png', 'residual vs fitted longitude for Ridge regression')