def test_Lasso_Path(self): diabetes = datasets.load_diabetes() X = diabetes.data y = diabetes.target X /= X.std(axis=0) df = pdml.ModelFrame(diabetes) df.data /= df.data.std(axis=0, ddof=False) self.assert_numpy_array_almost_equal(df.data.values, X) eps = 5e-3 expected = lm.lasso_path(X, y, eps, fit_intercept=False) result = df.lm.lasso_path(eps=eps, fit_intercept=False) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2]) expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False) result = df.lm.enet_path(eps=eps, l1_ratio=0.8, fit_intercept=False) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2]) expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) result = df.lm.enet_path(eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2]) expected = lm.lars_path(X, y, method='lasso', verbose=True) result = df.lm.lars_path(method='lasso', verbose=True) self.assert_numpy_array_almost_equal(expected[0], result[0]) self.assert_numpy_array_almost_equal(expected[1], result[1]) self.assert_numpy_array_almost_equal(expected[2], result[2])
def enetpath_vs_enet(self, drug_name, alphas=None, l1_ratio=0.5, nfeat=5, max_iter=1000, tol=1e-4, selection="cyclic", fit_intercept=False): """ #if X is not scaled, the enetpath and ElasticNet will give slightly # different results #if X is scale using:: from sklearn import preprocessing xscaled = preprocessing.scale(X) xscaled = pd.DataFrame(xscaled, columns=X.columns) """ # 1. use enet to loop over alphas and then plot the coefficients # along alpha for each feature # Get the data for the requested drug X, Y = self._get_one_drug_data(drug_name) # Run elasticnet for a bunch of alphas to get the coefficients alphas, coeffs, _ = enet_path(X, Y, l1_ratio=l1_ratio, alphas=alphas) # estimate the best alpha for later best_alpha = self.runCV(drug_name, verbose=False)['alpha'] # The final data, coeffs is sorted by coefficients on the smallest alpha coeffs = pd.DataFrame(coeffs, index=list(X.columns)) N = len(alphas) coeffs.sort_values(by=N-1, inplace=True) results = {"alphas": alphas, "coeffs": coeffs, "best_alpha": best_alpha} # the Viewer self._plot_enet(results) return results
def plot_enet_descent_path(X, y, l1_ratio, plot_file): # Compute paths eps = 5e-3 # the smaller it is the longer is the path # Reference the global image variable global image print("Computing regularization path using the elastic net.") alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=l1_ratio, fit_intercept=False) # Display results fig = plt.figure(1) ax = plt.gca() colors = cycle(['b', 'r', 'g', 'c', 'k']) neg_log_alphas_enet = -np.log10(alphas_enet) for coef_e, c in zip(coefs_enet, colors): l1 = plt.plot(neg_log_alphas_enet, coef_e, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') title = 'ElasticNet Path by alpha for l1_ratio = ' + str(l1_ratio) plt.title(title) plt.axis('tight') image = fig fig.savefig(plot_file) plt.close(fig) return image
def get_paths(self, num=10, n_alphas=10): l1_ratio_list = np.linspace(max(0.01, 1.0 / num), 1.0, endpoint=False) space_list = list() for l1 in l1_ratio_list: alphas, coefs, _ = enet_path(self.X_train, self.y_train, n_alphas=n_alphas, l1_ratio=l1) f = pd.DataFrame( coefs).transpose() # cols = features, one row per alpha g = f.applymap(lambda x: np.abs(x) > 0.0) nz = pd.DataFrame({ 'nz': g.sum(axis=1), 'alpha': alphas }) # number of non-zero coefs and min alpha needed nz.sort_values(by='nz', inplace=True) nz.reset_index(inplace=True, drop=True) alpha_max = nz['alpha'].max( ) # if alpha > alpha_max then all coefs are 0 space_list.append({ 'l1_ratio': l1, 'alpha_max': alpha_max, 'alpha_min': 0.0 }) return space_list
def worker( boot_inds, X, y, X_noise=0.01, y_noise=0.5, alpha=0.9, lambda_path=np.geomspace(1.0, 0.01, num=100), ): X_boot = X[boot_inds, :] y_boot = y[boot_inds] X_boot = scale( scale(X_boot + np.random.normal(scale=X_noise * 1e-6, size=X_boot.shape)) + np.random.normal(scale=X_noise, size=X_boot.shape)) y_boot = scale( scale(y_boot + np.random.normal(scale=y_noise * 1e-6, size=len(y_boot))) + np.random.normal(scale=y_noise, size=len(y_boot))) lambdas_enet, coefs_enet, _ = enet_path(X, y, l1_ratio=alpha, alphas=lambda_path, fit_intercept=False) return {"beta": coefs_enet != 0, "lambda_path": lambdas_enet}
def elastic_net_path(x_df,y_se,almax,almin,l1r,amin,a1se,intercept): X = x_df.values.copy() y = y_se.values.copy() labs = x_df.columns log10a = np.linspace(almax,almin,100) alps = 10**(log10a) # === Path calculation (by "enet_path") === alphas,coeffs,_ = enet_path(X,y,alphas=alps,l1_ratio=l1r,fit_intercept=intercept) # Plotting (path) plt.figure(figsize=(8,6)) axes = plt.gca() for coeff,lab in zip(coeffs,labs): plt.plot(log10a,coeff) axes.text(log10a[20],coeff[20],lab,fontsize=10) axes.text(log10a[40],coeff[40],lab,fontsize=10) axes.text(log10a[60],coeff[60],lab,fontsize=10) axes.text(log10a[80],coeff[80],lab,fontsize=10) axes.text(log10a[99],coeff[99],lab,fontsize=10) plt.axvline(np.log10(amin),linestyle="--",color="k",label="alpha min") plt.axvline(np.log10(a1se),linestyle=":",color="k",label="alpha 1se") plt.title("Regularization path\nenet_path (L1 ratio: "+str(l1r)+")") plt.xlabel('log10(alpha)') plt.ylabel('Coefficients') #plt.xlim(min(log10a)-0.1, max(log10a)+0.1) plt.axhline(0,linewidth=1,color="k") plt.grid() plt.show()
def plot_results(self): """Create the base regression plots as well as a regularization path plot.""" rc.REG.plot_results(self) path = linear_model.enet_path(self.independentVar, self.dependentVar, l1_ratio=self.l1_ratio,return_models=False, fit_intercept=False) alphas = path[0] #Vector of alphas coefs = (path[1]).T #Array of coefficients for each alpha viz.plot_regPath(alphas, coefs).plot()
def enetModel(Xfillna,yNor,l1_ratio,names): alphas, coefs, _ = linear_model.enet_path(Xfillna,yNor,l1_ratio = 1.0, \ fit_intercept=False, return_models=False) plot.plot(alphas,coefs.T) plot.xlabel('alpha') plot.ylabel('Coefficients') plot.title('Coefficient curves for debt classify data') plot.axis('tight') plot.semilogx() ax = plot.gca() ax.invert_xaxis() plot.show() nattr, nalpha = coefs.shape #find coefficient ordering nzList = [] for iAlpha in range(1,nalpha): coefList = list(coefs[:,iAlpha]) nzCoef = [index for index in range(nattr) if coefList[index] != 0.0] for q in nzCoef: if not(q in nzList): nzList.append(q) nameList = [names[nzList[i]] for i in range(len(nzList))] print('Attribute Ordered by How Early They Enter the Model') print(nameList)
def train(A, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None, positive=True, max_iter=1000): sample_num, feature_num = A.shape W = lil_matrix((feature_num, feature_num)) A = SparseMatrix(A) for j in range(feature_num): aj, Aj = A.lil_get_col_to_csc(j) if aj.nnz == 0: continue aj = aj.toarray().ravel() alphas, coefs, __ = linear_model.enet_path(Aj, aj, l1_ratio=l1_ratio, eps=eps, n_alphas=n_alphas, alphas=alphas, positive=positive, max_iter=max_iter) W[:j, j] = coefs[:j, -1].reshape(j, 1) W[j + 1:, j] = coefs[j:, -1].reshape(feature_num - j - 1, 1) return W
def train(self, trainset): # Here again: call base method before doing anything. env.AlgoBase.train(self, trainset) user_num = self.trainset.n_users item_num = self.trainset.n_items A = sparse.lil_matrix((user_num, item_num)) for u, i, r in trainset.all_ratings(): # A[u, i] = r if r > self.threshold: A[u, i] = 1 W = sparse.lil_matrix((item_num, item_num)) for j in range(item_num): aj = A.getcol(j).toarray().ravel() Aj = A.copy() Aj[:, j] = 0 Aj = Aj.tocsc() __, coefs, __ = linear_model.enet_path(Aj, aj, l1_ratio=self.l1_ratio, eps=self.eps, n_alphas=self.n_alphas, alphas=self.alphas, positive=self.positive, max_iter=self.max_iter) W[:, j] = coefs[:, -1].reshape(item_num, 1) self.estimator = A.tocsc() * W.tocsc()
def _fit_bootstrap_sample(X, y, func, L): """ Computes the regularization path for the regression y ~ X. Parameters ---------- X : array, shape (n_samples , n_features) y : array, shape (n_samples) func : string the function used for computing the regularization path (either 'lasso', 'elasticnet', or 'lars'). L : int length of the path. Returns ------- array, shape (n_features , L) 0 if the coefficient is null and 1 otherwise. """ if func == 'lasso': _, coef_path, _ = lasso_path(X, y, n_alphas=L) elif func == 'elasticnet': _, coef_path, _ = enet_path(X, y, nalphas=L) elif func == 'lars': _, _, coef_path = lars_path(X, y, max_iter=L - 1) return 1 * (coef_path != 0)
def _gen_cv_paths(self, alphas): """Helper function to generate cv paths.""" self.alphas, self.coefs_cv, _ = linear_model.enet_path( self.x_train, self.y_train, fit_intercept=self.intercept, alphas=alphas) self.coefs_cv = self.coefs_cv.T
def path_calc(X, y, X_holdout, y_holdout, alphas, paramgrid, colname = 'CV', yname = '', method = 'Elastic Net'): #make a copy of the parameters before popping things off copy_params = copy.deepcopy(paramgrid) fit_intercept = copy_params.pop('fit_intercept') precompute = copy_params.pop('precompute') copy_X = copy_params.pop('copy_X') normalize = False # this code adapted from sklearn ElasticNet fit function, which unfortunately doesn't accept multiple alphas at once X, y = check_X_y(X, y, accept_sparse='csc', order='F', dtype=[np.float64, np.float32], copy=copy_X and fit_intercept, multi_output=True, y_numeric=True) y = check_array(y, order='F', copy=False, dtype=X.dtype.type, ensure_2d=False) #this is the step that gives the data to find intercept if fit_intercept is true. X, y, X_offset, y_offset, X_scale, precompute, Xy = _pre_fit(X, y, None, precompute, normalize, fit_intercept, copy=False) y = np.squeeze(y) #do the path calculation, and tell how long it took print('Calculating path...') start_t = time.time() if method == 'Elastic Net': path_alphas, path_coefs, path_gaps, path_iters = enet_path(X, y, alphas=alphas, return_n_iter = True, **copy_params) if method == 'LASSO': path_alphas, path_coefs, path_gaps, path_iters = lasso_path(X, y, alphas=alphas, return_n_iter=True, **copy_params) dt = time.time() - start_t print('Took ' + str(dt) + ' seconds') #create some empty arrays to store the result y_pred_holdouts = np.empty(shape=(len(alphas),len(y_holdout))) intercepts = np.empty(shape=(len(alphas))) rmses = np.empty(shape=(len(alphas))) cvcols = [] for j in list(range(len(path_alphas))): coef_temp = path_coefs[:, j] if fit_intercept: coef_temp = coef_temp / X_scale intercept = y_offset - np.dot(X_offset, coef_temp.T) else: intercept = 0. y_pred_holdouts[j,:] = np.dot(X_holdout, path_coefs[:, j]) + intercept intercepts[j] = intercept rmses[j] = RMSE(y_pred_holdouts[j,:], y_holdout) cvcols.append(('predict','"'+ method + ' - ' + yname + ' - ' + colname + ' - Alpha:' + str(path_alphas[j]) + ' - ' + str(paramgrid) + '"')) return path_alphas, path_coefs, intercepts, path_iters, y_pred_holdouts, rmses, cvcols
def Plot_Enet_Path(self, path_lenght=5e-3, alphas=None): import matplotlib.pyplot as plt print("Computing regularization path using the elastic net...") alphas_enet, coefs_enet, _ = enet_path( self.X, self.Y, eps=path_lenght, l1_ratio=0.8, fit_intercept=False, alphas=alphas) print("Computing regularization path using the positve elastic net...") alphas_positive_enet, coefs_positive_enet, _ = enet_path( self.X, self.Y, eps=path_lenght, l1_ratio=0.8, positive=True, fit_intercept=False, alphas=alphas) plt.figure() ax = plt.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = plt.plot(-np.log10(alphas_enet), coefs_enet.T) l2 = plt.plot(-np.log10(alphas_positive_enet), coefs_positive_enet.T, linestyle='--') plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Elastic-Net and positive Elastic-Net') plt.legend((l1[-1], l2[-1]), ('Elastic-Net', 'positive Elastic-Net'), loc='lower left') plt.axis('tight') plt.show()
def coefficient_order(x_scaled, y_scaled): # 在整个数据集上训练,获取alphas, coefs alphas, coefs, _ = enet_path(x_scaled, y_scaled, l1_ratio=0.8, fit_intercept=False, return_models=False) plt.plot(alphas, coefs.T) plt.xlabel('alpha') plt.ylabel('coefficients') plt.axis('tight') plt.semilogx() ax = plt.gca() ax.invert_xaxis() plt.show() # find coefficient ordering 找到每一列中不为0的行 n_attr, n_alpha = coefs.shape nz_list = [] for i_alpha in range(1, n_alpha): coef_list = list(coefs[:, i_alpha]) nz_coef = [i for i in range(n_attr) if coef_list[i] != 0] for q in nz_coef: if q not in nz_list: nz_list.append(q) names = ['V' + str(i) for i in range(n_rows)] name_list = [names[nz_list[i]] for i in range(len(nz_list))] print('Attributes Ordered by How Early They Enter the Model') print(name_list) # find coefficients corresponding to best alpha value(来源于交叉验证) alphaStar = 0.020334883589342503 index_alphaStar = [ index for index in range(100) if alphas[index] > alphaStar ] index_star = max(index_alphaStar) coef_star = list(coefs[:, index_star]) print('Best coefficient values', coef_star) print("") # The coefficients on normalized attributes give another slightly different ordering abs_coef = [abs(a) for a in coef_star] coef_sort = sorted(abs_coef, reverse=True) idx_coef_size = [abs_coef.index(a) for a in coef_sort if a != 0] name_list_2 = [names[idx_coef_size[i]] for i in range(len(idx_coef_size))] print("Attributes Ordered by Coef Size at Optimum alpha") print(name_list_2)
def train(self, alpha, l1_ratio): with mlflow.start_run(source_name=self.current_file) as run: run_id = run.info.run_uuid print("run_id:", run_id) experiment_id = run.info.experiment_id print(" experiment_id:", experiment_id) clf = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42) clf.fit(self.train_x, self.train_y) predicted_qualities = clf.predict(self.test_x) (rmse, mae, r2) = self.eval_metrics(self.test_y, predicted_qualities) #print("Parameters:(alpha={}, l1_ratio={}):".format(alpha, l1_ratio)) print(" Parameters:") print(" alpha:", alpha) print(" l1_ratio:", l1_ratio) print(" Metrics:") print(" RMSE:", rmse) print(" MAE:", mae) print(" R2:", r2) mlflow.log_param("alpha", alpha) mlflow.log_param("l1_ratio", l1_ratio) mlflow.log_metric("rmse", rmse) mlflow.log_metric("r2", r2) mlflow.log_metric("mae", mae) mlflow.set_tag("data_path", self.data_path) mlflow.set_tag("exp_id", experiment_id) mlflow.set_tag("exp_name", self.experiment_name) mlflow.set_tag("run_origin", self.run_origin) mlflow.set_tag("platform", platform.system()) mlflow.sklearn.log_model(clf, "model") eps = 5e-3 # the smaller it is the longer is the path alphas_enet, coefs_enet, _ = enet_path(self.X, self.y, eps=eps, l1_ratio=l1_ratio, fit_intercept=False) plot_file = "wine_ElasticNet-paths.png" plot_utils.plot_enet_descent_path(self.X, self.y, l1_ratio, alphas_enet, coefs_enet, plot_file) mlflow.log_artifact(plot_file) return (experiment_id, run_id)
def sklearn_liner_model_regressions(xTrain, xTest, yTrain, yTest): modelForConsideration: DataFrame = pd.DataFrame() LinerModels = \ [ linear_model.ARDRegression(), linear_model.BayesianRidge(), linear_model.ElasticNet(), linear_model.ElasticNetCV(), linear_model.HuberRegressor(), linear_model.Lars(), linear_model.LarsCV(), linear_model.Lasso(), linear_model.LassoCV(), linear_model.LassoLars(), linear_model.LassoLarsCV(), linear_model.LassoLarsIC(), linear_model.LinearRegression(), linear_model.MultiTaskLasso(), linear_model.MultiTaskElasticNet(), linear_model.MultiTaskLassoCV(), linear_model.MultiTaskElasticNetCV(), linear_model.OrthogonalMatchingPursuit(), linear_model.OrthogonalMatchingPursuitCV(), linear_model.PassiveAggressiveClassifier(), linear_model.PassiveAggressiveRegressor(), linear_model.Perceptron(), linear_model.RANSACRegressor(), linear_model.Ridge(), linear_model.RidgeClassifier(), linear_model.RidgeClassifierCV(), linear_model.RidgeCV(), linear_model.SGDClassifier(), linear_model.SGDRegressor(), linear_model.TheilSenRegressor(), linear_model.enet_path(xTrain, yTrain), linear_model.lars_path(xTrain, yTrain), linear_model.lasso_path(xTrain, yTrain), # linear_model.LogisticRegression() # ,linear_model.LogisticRegressionCV(),linear_model.logistic_regression_path(xTrain, yTrain), linear_model.orthogonal_mp(xTrain, yTrain), linear_model.orthogonal_mp_gram(), linear_model.ridge_regression() ] for model in LinerModels: modelName: str = model.__class__.__name__ try: # print(f"Preparing Model {modelName}") if modelName == "LogisticRegression": model = linear_model.LogisticRegression(random_state=0) model.fit(xTrain, yTrain) yTrainPredict = model.predict(xTrain) yTestPredict = model.predict(xTest) errorList = calculate_prediction_error(modelName, yTestPredict, yTest, yTrainPredict, yTrain) if errorList["Test Average Error"][0] < 30 and errorList[ "Train Average Error"][0] < 30: try: modelForConsideration = modelForConsideration.append( errorList) except (Exception) as e: print(e) except (Exception, ArithmeticError) as e: print(f"Error occurred while preparing Model {modelName}") return modelForConsideration
def dataset_path(X, y, dataset_name=''): print("\n\ncoefs path processing... dataset: " + dataset_name) print("\nprocessing Lasso...") eps = 1e-5 alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept=False) plot_coefs_path(dataset_name + ' Lasso coefficient paths', alphas_lasso, coefs_lasso) print("\nprocessing ElasticNet...") eps = 1e-5 alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=ENET_L1_RATIO, fit_intercept=False) plot_coefs_path(dataset_name + ' ElasticNet coefficient paths', alphas_enet, coefs_enet) print("\nprocessing Ridge...") m_ridge = Ridge(fit_intercept=False) alphas = np.logspace(5, -1, 100) ridge_alphas, model_coef = model_path(m_ridge, X, y, alphas) plot_coefs_path(dataset_name +' Ridge coefficient paths', ridge_alphas, model_coef)
def enet_plot(l1_ratio): """Function plotting enet_path for some tuning parameter.""" _, theta_enet, _ = linear_model.enet_path(A, b, alphas=alphas, fit_intercept=False, l1_ratio=l1_ratio, return_models=False) fig1 = plt.figure(figsize=(12, 8)) ax1 = fig1.add_subplot(111) ax1.plot(alphas, np.transpose(theta_enet), linewidth=3) ax1.set_xscale('log') ax1.set_xlabel(r"$\lambda$") ax1.set_ylabel("Coefficient value") ax1.set_ylim([-1, 5]) plt.savefig(save_fig(path, "enet_coeffs", "pdf")) plt.show() return theta_enet
def enetpath_vs_enet(self, drug_name, alphas=None, l1_ratio=0.5, nfeat=5, max_iter=1000, tol=1e-4, selection="cyclic", fit_intercept=False): """ #if X is not scaled, the enetpath and ElasticNet will give slightly # different results #if X is scale using:: from sklearn import preprocessing xscaled = preprocessing.scale(X) xscaled = pd.DataFrame(xscaled, columns=X.columns) """ # 1. use enet to loop over alphas and then plot the coefficients # along alpha for each feature # Get the data for the requested drug X, Y = self._get_one_drug_data(drug_name) # Run elasticnet for a bunch of alphas to get the coefficients alphas, coeffs, _ = enet_path(X, Y, l1_ratio=l1_ratio, alphas=alphas) # estimate the best alpha for later best_alpha = self.runCV(drug_name, verbose=False).alpha # The final data, coeffs is sorted by coefficients on the smallest alpha coeffs = pd.DataFrame(coeffs, index=list(X.columns)) N = len(alphas) coeffs.sort_values(by=N - 1, inplace=True) results = { "alphas": alphas, "coeffs": coeffs, "best_alpha": best_alpha } # the Viewer self._plot_enet(results) return results
def crossValENetRocksMines(xNormalized, labelNormalized, nrows, ncols): nxval = 10 for ixval in range(nxval): idxTest = [a for a in range(nrows) if a%nxval == ixval*nxval] idxTrain = [a for a in range(nrows) if a%nxval != ixval*nxval] xTrain = [xNormalized[r] for r in idxTrain] xTest = [xNormalized[r] for r in idxTest] labelTrain = [labelNormalized[r] for r in idxTrain] labelTest = [labelNormalized[r] for r in idxTest] alphas, coefs,_ = enet_path(xTrain, labelTrain, l1_ratio = 0.8, \ fit_intercept=False, return_models=False) if ixval == 0: pred = numpy.dot(xTest, coefs) yOut = labelTest else: yTemp = numpy.array(yOut) yOut = numpy.concatenate((yTemp,labelTest),axis=0) predTemp = numpy.array(pred) pred = numpy.concatenate((predTemp,numpy.dot(xTest,coefs)),axis=0) misClassRate = [] _,nPred = pred.shape for iPred in range(1,nPred): predList = list(pred[:,iPred]) errCnt = 0.0 for irow in range(nrows): if (predList[irow] < 0.0) and (yOut[irow]>=0.0): errCnt += 1.0 elif (predList[irow] >= 0.0) and (yOut[irow] < 0.0): errCnt += 1.0 misClassRate.append(errCnt/nrows) return (misClassRate, alphas)
def crossValENetRocksMines(xNormalized, labelNormalized, nrows, ncols): nxval = 10 for ixval in range(nxval): idxTest = [a for a in range(nrows) if a % nxval == ixval * nxval] idxTrain = [a for a in range(nrows) if a % nxval != ixval * nxval] xTrain = [xNormalized[r] for r in idxTrain] xTest = [xNormalized[r] for r in idxTest] labelTrain = [labelNormalized[r] for r in idxTrain] labelTest = [labelNormalized[r] for r in idxTest] alphas, coefs,_ = enet_path(xTrain, labelTrain, l1_ratio = 0.8, \ fit_intercept=False, return_models=False) if ixval == 0: pred = numpy.dot(xTest, coefs) yOut = labelTest else: yTemp = numpy.array(yOut) yOut = numpy.concatenate((yTemp, labelTest), axis=0) predTemp = numpy.array(pred) pred = numpy.concatenate((predTemp, numpy.dot(xTest, coefs)), axis=0) misClassRate = [] _, nPred = pred.shape for iPred in range(1, nPred): predList = list(pred[:, iPred]) errCnt = 0.0 for irow in range(nrows): if (predList[irow] < 0.0) and (yOut[irow] >= 0.0): errCnt += 1.0 elif (predList[irow] >= 0.0) and (yOut[irow] < 0.0): errCnt += 1.0 misClassRate.append(errCnt / nrows) return (misClassRate, alphas)
def enet_plot(l1_ratio): """ function plotting enet_path for some tuning parameter """ _, theta_enet, _ = enet_path(X, y, alphas=alphas, fit_intercept=False, l1_ratio=l1_ratio, return_models=False) fig1 = plt.figure(figsize=(12, 8)) plt.title("Enet path: " + r"$p={0}, n={1} $".format(n_features, n_samples), fontsize=16) ax1 = fig1.add_subplot(111) ax1.plot(alphas, np.transpose(theta_enet), linewidth=3) ax1.set_xscale('log') ax1.set_xlabel(r"$\lambda$") ax1.set_ylabel("Coefficient value") ax1.set_ylim([-1, 2]) sns.despine() plt.show() filename = "Enet_path" + str(l1_ratio) filename = filename.replace(".", "") my_saving_display(fig1, dirname, filename, imageformat) return theta_enet
def myenetpath(data, lam, alpha): n_samples, n_features = data.shape sig = np.ones(n_features) tor = np.array([]) for iter1 in range(500): A = np.zeros((n_features * n_samples, n_features * n_features)) for i in range(1, n_features + 1): A[(i - 1) * n_samples:i * n_samples, (i - 1) * n_features:i * n_features] = data * np.tile(np.sqrt(sig / sig[i - 1]), (n_samples, 1)) temp = np.eye(n_features) temp = temp.flatten('F') loc = np.array(np.nonzero(temp == 0)) loc = loc.reshape(loc.shape[1]) A = A[:, loc] y = data.flatten('F') _, coef_path, _ = enet_path(A * np.sqrt(2 * n_samples), y * np.sqrt(2 * n_samples), alphas=lam, l1_ratio=alpha) return coef_path
def main(): path = '/users/davecwright/documents/kaggle/liberty_fire_cost/' path = 'c:\\users\\dwright\\code\\' train_name = path + 'train.csv' test_name = path + 'test.csv' # f = open(path + 'train.csv', 'rb') readRows = 2000 #None for all print 'loading train_data' train_data = pd.read_csv(train_name, nrows=readRows) print 'train_data loaded' y_data = train_data['target'].values train_data.drop('target', 1) train_data = scrub(train_data) y_data = y_data.astype(float) eps = 5e-3 X = train_data y = y_data folds = 10 kf = cross_validation.KFold(y_data.shape[0], n_folds=folds) alphas = range(folds) k = 0 for test, train in kf: penalty = (alphas[k] + 1) * 1/folds print alpha clf = ElasticNet(l1_ratio=penalty, eps=eps) clf.train(X, y) # doing our own cv parametarization print 'computing lasso path' alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept = False) print 'computing enet path' alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False) plt.figure(1) ax = plt.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = plt.plot(-np.log10(alphas_lasso), coefs_lasso.T) l2 = plt.plot(-np.log10(alphas_enet), coefs_enet.T, linestyle='--') plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Lasso and Elastic-Net Paths') plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left') plt.axis('tight') plt.show() sys.exit() for train_index, test_index in kf: #print train_index, test_index print train_data.iloc[train_index], train_data.iloc[test_index] # I am implementing the lasso reduction here... #for train, test in kf: fig = plt.figure(figsize=(12, 9)) #ax = fig.add_subplot(111) #ax.plot(np.sort(y_data)) #plt.xlabel("Number of Features") #plt.ylabel("claims cost") #plt.title("claims_cost") #ax.set_xscale("log") #ax.set_position([box.x0, box.y0 + box.height * 0.3, box.width, box.height * 0.7]) #ax.legend(**_PLT_LEGEND_OPTIONS) #plt.show() train_data = train_data.drop('target', 1) # A - preprocessing # encode the text variables, var1-var9, Z values are NaN # fill in missing values in the text variables and in the continuous variables # skip continuous for now # A3. build new features through the interactions of various items # skip for now # A4. dimensionality reduction to take the feature set back down to something more manageable. est_clf = svm.SVR(kernel='linear', C=1) rks = select_ests(X_train, y_data, 100, est_clf) X_train = X_train[:, rks] clf = svm.SVR(kernel='linear', C=1) acy = cv(X_train, y_data, clf, None, estimator_name(clf)) # the point here is to understand what accuracy is print 'accuracy:', acy #select_model(X_train, y_data) # B. split out test and fit sets test_data = pd.read_csv(test_name, nrows=readRows) test_data = encode_impute(test_data) X_test = scaler.transform(test_data)
eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the lasso...") # The return_models parameter sets that lasso_path will return # the alphas and the coefficients as output, instead of a list # of models as it does by default. Returning the list of models # is deprecated and will eventually be removed in 0.15 alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, return_models=False) print("Computing regularization path using the positive lasso...") alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path(X, y, eps, positive=True, return_models=False) print("Computing regularization path using the elastic net...") alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8, return_models=False) print("Computing regularization path using the positve elastic net...") alphas_positive_enet, coefs_positive_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True, return_models=False) # Display results pl.figure(1) ax = pl.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = pl.plot(-np.log10(alphas_lasso), coefs_lasso.T) l2 = pl.plot(-np.log10(alphas_enet), coefs_enet.T, linestyle='--')
def main(): print(__doc__) # Author: Alexandre Gramfort <*****@*****.**> # License: BSD 3 clause X, y = datasets.load_diabetes(return_X_y=True) X /= X.std( axis=0) # Standardize data (easier to set the l1_ratio parameter) # Compute paths eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the lasso...") alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps=eps, fit_intercept=False) print("Computing regularization path using the positive lasso...") alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path( X, y, eps=eps, positive=True, fit_intercept=False) print("Computing regularization path using the elastic net...") alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False) print("Computing regularization path using the positive elastic net...") alphas_positive_enet, coefs_positive_enet, _ = enet_path( X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) # Display results plt.figure(1) colors = cycle(['b', 'r', 'g', 'c', 'k']) neg_log_alphas_lasso = -np.log10(alphas_lasso) neg_log_alphas_enet = -np.log10(alphas_enet) for coef_l, coef_e, c in zip(coefs_lasso, coefs_enet, colors): l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c) l2 = plt.plot(neg_log_alphas_enet, coef_e, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Lasso and Elastic-Net Paths') plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left') plt.axis('tight') plt.figure(2) neg_log_alphas_positive_lasso = -np.log10(alphas_positive_lasso) for coef_l, coef_pl, c in zip(coefs_lasso, coefs_positive_lasso, colors): l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c) l2 = plt.plot(neg_log_alphas_positive_lasso, coef_pl, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Lasso and positive Lasso') plt.legend((l1[-1], l2[-1]), ('Lasso', 'positive Lasso'), loc='lower left') plt.axis('tight') plt.figure(3) neg_log_alphas_positive_enet = -np.log10(alphas_positive_enet) for (coef_e, coef_pe, c) in zip(coefs_enet, coefs_positive_enet, colors): l1 = plt.plot(neg_log_alphas_enet, coef_e, c=c) l2 = plt.plot(neg_log_alphas_positive_enet, coef_pe, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Elastic-Net and positive Elastic-Net') plt.legend((l1[-1], l2[-1]), ('Elastic-Net', 'positive Elastic-Net'), loc='lower left') plt.axis('tight') plt.show()
idxTrain = [a for a in range(nrow) if a%nxval != ixval%nxval] #Define test and training attribute and label sets xTrain = numpy.array([xNormalized[r] for r in idxTrain]) xTest = numpy.array([xNormalized[r] for r in idxTest]) yTrain = [yNormalized[r] for r in idxTrain] yTest = [yNormalized[r] for r in idxTest] labelsTest = [labels[r] for r in idxTest] #build model for each column in yTrain models = [] lenTrain = len(yTrain) lenTest = nrow - lenTrain for iModel in range(nlabels): yTemp = numpy.array([yTrain[j][iModel] for j in range(lenTrain)]) models.append(enet_path(xTrain, yTemp,l1_ratio=1.0, fit_intercept=False, eps=0.5e-3, n_alphas=nAlphas , return_models=False)) for iStep in range(1,nAlphas): #Assemble the predictions for all the models, find largest prediction and calc error allPredictions = [] for iModel in range(nlabels): _, coefs, _ = models[iModel] predTemp = list(numpy.dot(xTest, coefs[:,iStep])) #un-normalize the prediction for comparison predUnNorm = [(predTemp[j]*ySD[iModel] + yMeans[iModel]) for j in range(len(predTemp))] allPredictions.append(predUnNorm) predictions = [] for i in range(lenTest): listOfPredictions = [allPredictions[j][i] for j in range(nlabels) ] idxMax = listOfPredictions.index(max(listOfPredictions))
y = diabetes.target X /= X.std(0) # Standardize data (easier to set the rho parameter) ################################################################################ # Compute paths eps = 5e-3 # the smaller it is the longer is the path print "Computing regularization path using the lasso..." models = lasso_path(X, y, eps=eps) alphas_lasso = np.array([model.alpha for model in models]) coefs_lasso = np.array([model.coef_ for model in models]) print "Computing regularization path using the elastic net..." models = enet_path(X, y, eps=eps, rho=0.8) alphas_enet = np.array([model.alpha for model in models]) coefs_enet = np.array([model.coef_ for model in models]) ################################################################################ # Display results ax = pl.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = pl.plot(coefs_lasso) l2 = pl.plot(coefs_enet, linestyle='--') pl.xlabel('-Log(lambda)') pl.ylabel('weights') pl.title('Lasso and Elastic-Net Paths') pl.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left')
#> WRAIN 0.000000 #> DEGREES 0.074101 #> HRAIN -0.041159 #> TIME_SV 0.024027 #> dtype: float64 my_test = pd.DataFrame([[500, 17, 120, 2]]) my_pipeline.predict(my_test) #> array([-1.41981616]) ### 8.6.2 ペナルティの強さと係数の関係 As = np.e**np.arange(2, -5.5, -0.1) B = 0.1 _, my_path, _ = enet_path(zscore(X), zscore(y), alphas=As, l1_ratio=B) pd.DataFrame(my_path.T, columns=X.columns, index=np.log(As)).plot(xlabel='log A ( = log alpha)', ylabel='Coefficients') ### 8.6.3 パラメータ$A,\,B$の決定 As = np.linspace(0, 0.1, 21) Bs = np.linspace(0, 0.1, 6) my_pipeline = Pipeline([('sc', StandardScaler()), ('enet', ElasticNet())]) my_search = GridSearchCV(estimator=my_pipeline, param_grid={ 'enet__alpha': As, 'enet__l1_ratio': Bs
def enetpath_vs_enet(self, drug_name, alphas=None, l1_ratio=0.5, nfeat=5, max_iter=1000, tol=1e-4, selection="cyclic", fit_intercept=False): """ #if X is not scaled, the enetpath and ElasticNet will give slightly differen results #if X is scale using:: from sklearn import preprocessing xscaled = preprocessing.scale(X) xscaled = pd.DataFrame(xscaled, columns=X.columns) """ # 1. use enet to loop over alphas and then plot the coefficients along alpha # for each feature # Get the data for the requested drug xscaled, Y = self._get_one_drug_data(drug_name) alphas, coefs1, _ = enet_path(xscaled, Y, l1_ratio=l1_ratio, alphas=alphas) pylab.figure(1) pylab.clf() for this in coefs1: pylab.plot(pylab.log(alphas), this) self.alphas1 = alphas self.coefs1 = coefs1 # Identify the first 5 # 2. should be equivalenet to using ElasticNet for each alphas coefs2 = [] # if alphas is None, it will be created automatically from enet_path for alpha in alphas: # to have same results as in enet_path, normalize must be set to # False when X is scaled. en = sklearn.linear_model.ElasticNet(l1_ratio=l1_ratio, alpha=alpha, max_iter=max_iter, tol=tol, selection=selection, fit_intercept=fit_intercept) res = en.fit(xscaled, Y) coefs2.append(res.coef_) coefs2 = np.array(coefs2).transpose() pylab.figure(2) pylab.clf() for this in coefs2: pylab.plot(pylab.log(alphas), this) self.coefs2 = coefs2 #pylab.plot(-pylab.log(res.coef_)) pylab.figure(3) pylab.clf() self.df1 = pd.DataFrame(coefs1.transpose(), columns=xscaled.columns) self.df2 = pd.DataFrame(coefs2.transpose(), columns=xscaled.columns) (self.df1 == 0).sum().plot() (self.df2 == 0).sum().plot() self.indices1 = (self.df1 == 0).sum().sort_values().ix[0:nfeat] self.indices2 = (self.df2 == 0).sum().sort_values().ix[0:nfeat] names1 = self.indices1.index names2 = self.indices2.index print(names2)
xNormalized.append(rowNormalized) #normalize labels to center meanLabel = sum(labels)/nrow sdLabel = sqrt(sum([(labels[i] - meanLabel) * (labels[i] - meanLabel) for i in range(nrow)])/nrow) labelNormalized = [(labels[i] - meanLabel)/sdLabel for i in range(nrow)] #Convert normalized labels to numpy array Y = numpy.array(labelNormalized) #Convert normalized attributes to numpy array X = numpy.array(xNormalized) alphas, coefs, _ = enet_path(X, Y,l1_ratio=0.8, fit_intercept=False, return_models=False) plot.plot(alphas,coefs.T) plot.xlabel('alpha') plot.ylabel('Coefficients') plot.axis('tight') plot.semilogx() ax = plot.gca() ax.invert_xaxis() plot.show() nattr, nalpha = coefs.shape #find coefficient ordering nzList = []
# number of cross validation folds nxval = 10 for ixval in range(nxval): # Define test and training index sets idxTest = [a for a in range(nrow) if a % nxval == ixval % nxval] idxTrain = [a for a in range(nrow) if a % nxval != ixval % nxval] # Define test and training attribute and label sets xTrain = numpy.array([xNormalized[r] for r in idxTrain]) xTest = numpy.array([xNormalized[r] for r in idxTest]) labelTrain = numpy.array([labelNormalized[r] for r in idxTrain]) labelTest = numpy.array([labelNormalized[r] for r in idxTest]) alphas, coefs, _ = enet_path(xTrain, labelTrain, l1_ratio=0.8, fit_intercept=False, return_models=False) # apply coefs to test data to produce predictions and accumulate if ixval == 0: pred = numpy.dot(xTest, coefs) yOut = labelTest else: # accumulate predictions yTemp = numpy.array(yOut) yOut = numpy.concatenate((yTemp, labelTest), axis=0) # accumulate predictions predTemp = numpy.array(pred) pred = numpy.concatenate((predTemp, numpy.dot(xTest, coefs)), axis=0)
def train_models(train_x, train_y, n_labels, n_alphas): for i_model in range(n_labels): current_y = train_y[:, i_model] yield enet_path(train_x, current_y, l1_ratio=1.0, eps=0.5e-3, n_alphas=n_alphas, return_models=False)
print(" R2: %s" % r2) # Log mlflow attributes for mlflow UI mlflow.log_param("alpha", alpha) mlflow.log_param("l1_ratio", l1_ratio) mlflow.log_metric("rmse", rmse) mlflow.log_metric("r2", r2) mlflow.log_metric("mae", mae) mlflow.sklearn.log_model(lr, "model") # Compute paths eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the elastic net.") alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=l1_ratio, fit_intercept=False) # Display results fig = plt.figure(1) ax = plt.gca() colors = cycle(['b', 'r', 'g', 'c', 'k']) neg_log_alphas_enet = -np.log10(alphas_enet) for coef_e, c in zip(coefs_enet, colors): l2 = plt.plot(neg_log_alphas_enet, coef_e, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') title = 'ElasticNet Path by alpha for l1_ratio = ' + str(l1_ratio) plt.title(title) plt.axis('tight')
y = diabetes.target X /= X.std(axis=0) # Standardize data (easier to set the l1_ratio parameter) # Compute paths eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the lasso...") alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept=False) print("Computing regularization path using the positive lasso...") alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path( X, y, eps, positive=True, fit_intercept=False) print("Computing regularization path using the elastic net...") alphas_enet, coefs_enet, _ = enet_path( X, y, eps=eps, l1_ratio=0.8, fit_intercept=False) print("Computing regularization path using the positve elastic net...") alphas_positive_enet, coefs_positive_enet, _ = enet_path( X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) # Display results plt.figure(1) ax = plt.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = plt.plot(-np.log10(alphas_lasso), coefs_lasso.T) l2 = plt.plot(-np.log10(alphas_enet), coefs_enet.T, linestyle='--') plt.xlabel('-Log(alpha)') plt.ylabel('coefficients')
def path_calc(X, y, X_holdout, y_holdout, alphas, paramgrid, colname='CV', yname='', method='Elastic Net'): #make a copy of the parameters before popping things off copy_params = copy.deepcopy(paramgrid) fit_intercept = copy_params.pop('fit_intercept') precompute = copy_params.pop('precompute') copy_X = copy_params.pop('copy_X') normalize = False # this code adapted from sklearn ElasticNet fit function, which unfortunately doesn't accept multiple alphas at once X, y = check_X_y(X, y, accept_sparse='csc', order='F', dtype=[np.float64, np.float32], copy=copy_X and fit_intercept, multi_output=True, y_numeric=True) y = check_array(y, order='F', copy=False, dtype=X.dtype.type, ensure_2d=False) #this is the step that gives the data to find intercept if fit_intercept is true. X, y, X_offset, y_offset, X_scale, precompute, Xy = _pre_fit(X, y, None, precompute, normalize, fit_intercept, copy=False) y = np.squeeze(y) #do the path calculation, and tell how long it took print('Calculating path...') start_t = time.time() if method == 'Elastic Net': path_alphas, path_coefs, path_gaps, path_iters = enet_path( X, y, alphas=alphas, return_n_iter=True, **copy_params) if method == 'LASSO': path_alphas, path_coefs, path_gaps, path_iters = lasso_path( X, y, alphas=alphas, return_n_iter=True, **copy_params) dt = time.time() - start_t print('Took ' + str(dt) + ' seconds') #create some empty arrays to store the result y_pred_holdouts = np.empty(shape=(len(alphas), len(y_holdout))) intercepts = np.empty(shape=(len(alphas))) rmses = np.empty(shape=(len(alphas))) cvcols = [] for j in list(range(len(path_alphas))): coef_temp = path_coefs[:, j] if fit_intercept: coef_temp = coef_temp / X_scale intercept = y_offset - np.dot(X_offset, coef_temp.T) else: intercept = 0. y_pred_holdouts[j, :] = np.dot(X_holdout, path_coefs[:, j]) + intercept intercepts[j] = intercept rmses[j] = RMSE(y_pred_holdouts[j, :], y_holdout) cvcols.append( ('predict', '"' + method + ' - ' + yname + ' - ' + colname + ' - Alpha:' + str(path_alphas[j]) + ' - ' + str(paramgrid) + '"')) return path_alphas, path_coefs, intercepts, path_iters, y_pred_holdouts, rmses, cvcols
def compute_paths(X, y, eps, path): import os from sklearn.linear_model import lasso_path, enet_path from sklearn.linear_model import lasso_path, enet_path import matplotlib.pyplot as plt import numpy as np from itertools import cycle print("Computing regularization path using the lasso...") alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept=False) print("Computing regularization path using the positive lasso...") alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path( X, y, eps, positive=True, fit_intercept=False) print("Computing regularization path using the elastic net...") alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False) print("Computing regularization path using the positive elastic net...") alphas_positive_enet, coefs_positive_enet, _ = enet_path( X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) # Lasso and Elastic-Net Paths fig1 = plt.figure(1) ax = plt.gca() colors = cycle(['b', 'r', 'g', 'c', 'k']) neg_log_alphas_lasso = -np.log10(alphas_lasso) neg_log_alphas_enet = -np.log10(alphas_enet) for coef_l, coef_e, c in zip(coefs_lasso, coefs_enet, colors): l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c) l2 = plt.plot(neg_log_alphas_enet, coef_e, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Lasso and Elastic-Net Paths') plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left') plt.axis('tight') # Lasso and Positive Lasso fig2 = plt.figure(2) ax = plt.gca() neg_log_alphas_positive_lasso = -np.log10(alphas_positive_lasso) for coef_l, coef_pl, c in zip(coefs_lasso, coefs_positive_lasso, colors): l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c) l2 = plt.plot(neg_log_alphas_positive_lasso, coef_pl, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Lasso and positive Lasso') plt.legend((l1[-1], l2[-1]), ('Lasso', 'positive Lasso'), loc='lower left') plt.axis('tight') # Elastic Net and Positive Elastic Net fig3 = plt.figure(3) ax = plt.gca() neg_log_alphas_positive_enet = -np.log10(alphas_positive_enet) for (coef_e, coef_pe, c) in zip(coefs_enet, coefs_positive_enet, colors): l1 = plt.plot(neg_log_alphas_enet, coef_e, c=c) l2 = plt.plot(neg_log_alphas_positive_enet, coef_pe, linestyle='--', c=c) plt.xlabel('-Log(alpha)') plt.ylabel('coefficients') plt.title('Elastic-Net and positive Elastic-Net') plt.legend((l1[-1], l2[-1]), ('Elastic-Net', 'positive Elastic-Net'), loc='lower left') plt.axis('tight') # Save figures if os.path.isfile(path + "plot1.png"): os.system("rm " + path + "plot1.png") if os.path.isfile(path + "plot2.png"): os.system("rm " + path + "plot2.png") if os.path.isfile(path + "plot3.png"): os.system("rm " + path + "plot3.png") fig1.savefig(path + "plot1.png") fig2.savefig(path + "plot2.png") fig3.savefig(path + "plot3.png")
# Compute paths eps = 5e-3 # the smaller it is the longer is the path print "Computing regularization path using the lasso..." models = lasso_path(X, y, eps=eps) alphas_lasso = np.array([model.alpha for model in models]) coefs_lasso = np.array([model.coef_ for model in models]) print "Computing regularization path using the positive lasso..." models = lasso_path(X, y, eps=eps, positive=True) alphas_positive_lasso = np.array([model.alpha for model in models]) coefs_positive_lasso = np.array([model.coef_ for model in models]) print "Computing regularization path using the elastic net..." models = enet_path(X, y, eps=eps, rho=0.8) alphas_enet = np.array([model.alpha for model in models]) coefs_enet = np.array([model.coef_ for model in models]) print "Computing regularization path using the positve elastic net..." models = enet_path(X, y, eps=eps, rho=0.8, positive=True) alphas_positive_enet = np.array([model.alpha for model in models]) coefs_positive_enet = np.array([model.coef_ for model in models]) ############################################################################### # Display results pl.figure(1) ax = pl.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = pl.plot(coefs_lasso)
print(" MAE: %s" % mae) print(" R2: %s" % r2) # Log mlflow attributes for mlflow UI mlflow.log_param("alpha", alpha) mlflow.log_param("l1_ratio", l1_ratio) mlflow.log_metric("rmse", rmse) mlflow.log_metric("r2", r2) mlflow.log_metric("mae", mae) mlflow.sklearn.log_model(lr, "model") # Compute paths eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the elastic net.") alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=l1_ratio) # Display results fig = plt.figure(1) ax = plt.gca() colors = cycle(["b", "r", "g", "c", "k"]) neg_log_alphas_enet = -np.log10(alphas_enet) for coef_e, c in zip(coefs_enet, colors): l2 = plt.plot(neg_log_alphas_enet, coef_e, linestyle="--", c=c) plt.xlabel("-Log(alpha)") plt.ylabel("coefficients") title = "ElasticNet Path by alpha for l1_ratio = " + str(l1_ratio) plt.title(title) plt.axis("tight")
def train(data_path, alpha, l1_ratio, run_origin="none"): print("run_origin:", run_origin) np.random.seed(40) # Read the wine-quality csv file print("data_path:", data_path) data = pd.read_csv(data_path) # Split the data into training and test sets. (0.75, 0.25) split. train, test = train_test_split(data) # The predicted column is "quality" which is a scalar from [3, 9] train_x = train.drop(["quality"], axis=1) test_x = test.drop(["quality"], axis=1) train_y = train[["quality"]] test_y = test[["quality"]] mlflow.set_experiment(experiment_name) client = mlflow.tracking.MlflowClient() experiment_id = client.get_experiment_by_name( experiment_name).experiment_id print("experiment_id:", experiment_id) current_file = os.path.basename(__file__) with mlflow.start_run(source_name=current_file) as run: run_id = run.info.run_uuid print("run_id:", run_id) clf = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42) clf.fit(train_x, train_y) predicted_qualities = clf.predict(test_x) (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities) print("Elasticnet model (alpha={}, l1_ratio={}):".format( alpha, l1_ratio)) print(" RMSE:", rmse) print(" MAE:", mae) print(" R2:", r2) mlflow.log_param("data_path", data_path) mlflow.log_param("alpha", alpha) mlflow.log_param("l1_ratio", l1_ratio) mlflow.log_param("run_origin", run_origin) mlflow.log_metric("rmse", rmse) mlflow.log_metric("r2", r2) mlflow.log_metric("mae", mae) mlflow.set_tag("platform", platform.system()) mlflow.set_tag("run_origin", run_origin) mlflow.sklearn.log_model(clf, "model") eps = 5e-3 # the smaller it is the longer is the path X = data.drop(["quality"], axis=1).values y = data[["quality"]].values.ravel() alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=l1_ratio, fit_intercept=False) plot_file = "wine_ElasticNet-paths_{}_{}.png".format(alpha, l1_ratio) plot_utils.plot_enet_descent_path(X, y, l1_ratio, alphas_enet, coefs_enet, plot_file) mlflow.log_artifact(plot_file) return (experiment_id, run_id)
def compute_save_multi_task_elastic_net_path(self, phi_tilde, X, max_iter=1e5, tol=1e-12, l1_ratio=0.99): """ computing multi task elastic net and save the coefficient of the path """ num_alpha = len(self.alpha_range) ## 1. normalize the features by making modal amplititute to 1 for all features if self.normalize_phi_tilde: print("EDMD/KDMD dictionary features are normalized....") scaling_transform = np.diag(1. / np.abs(phi_tilde[0, :])) inverse_scaling = np.linalg.inv(scaling_transform) assert np.linalg.norm(scaling_transform @ inverse_scaling - np.eye(scaling_transform.shape[0])) < 1e-6 phi_tilde_scaled = phi_tilde @ scaling_transform print('norm = ', np.linalg.norm(phi_tilde_scaled, axis=0)) print(phi_tilde_scaled.shape) print(phi_tilde_scaled[0, :]) else: phi_tilde_scaled = phi_tilde ## 2. augmenting the complex AX=B problem into a AX=B problem with real entries ## since current package only support real number array # -- note: must run in sequential....to have good convergence property as shown in my github page. a = np.hstack([np.real(phi_tilde_scaled), -np.imag(phi_tilde_scaled)]) b = np.hstack([np.imag(phi_tilde_scaled), np.real(phi_tilde_scaled)]) phi_tilde_aug = np.vstack([a, b]) X_aug = np.vstack([X, np.zeros(X.shape)]) num_data = X.shape[0] num_target = X.shape[1] alphas_enet, coefs_enet_aug, _ = enet_path(phi_tilde_aug, X_aug, max_iter=max_iter, tol=tol, alphas=self.alpha_range, l1_ratio=l1_ratio, fit_intercept=False, check_input=True, verbose=0) num_total_eigen_func = int(coefs_enet_aug.shape[1] / 2) # get the real and image part from the complex solution coefs_enet_real = coefs_enet_aug[:, :num_total_eigen_func, :] coefs_enet_imag = coefs_enet_aug[:, num_total_eigen_func:, :] assert coefs_enet_imag.shape == coefs_enet_real.shape # combine them into complex arrary for final results! coefs_enet_comp = coefs_enet_real + 1j * coefs_enet_imag ## 2.5 remove feature that is smaller than 1e-3 of the max. because most often, for i_alpha in range(coefs_enet_comp.shape[2]): for i_target in range(coefs_enet_comp.shape[0]): coef_cutoff_value = self.truncation_threshold * np.max( abs(coefs_enet_comp[i_target, :, i_alpha])) index_remove = abs( coefs_enet_comp[i_target, :, i_alpha]) < coef_cutoff_value coefs_enet_comp[i_target, index_remove, i_alpha] = 0 + 0j ## 2.7 given features selected, do LS-refit to remove the bias of any kind of regularization for i_alpha in range(coefs_enet_comp.shape[2]): bool_non_zero = np.linalg.norm(coefs_enet_comp[:, :, i_alpha], axis=0) > 0 phi_tilde_scaled_reduced = phi_tilde_scaled[:, bool_non_zero] coef_enet_comp_reduced_i_alpha = np.linalg.lstsq( phi_tilde_scaled_reduced, X)[0] coefs_enet_comp[:, bool_non_zero, i_alpha] = coef_enet_comp_reduced_i_alpha.T coefs_enet_comp[:, np.invert(bool_non_zero), i_alpha] = 0 ## 3. compute residual for parameter sweep. so I can draw the trade off plot between num. non-zero vs rec. resdiual # convert complex array into mag. coefs_enet_mag = np.abs(coefs_enet_comp) def compute_residual_list(i): residual = np.linalg.norm( X - np.matmul(phi_tilde_scaled, coefs_enet_comp[:, :, i].T)[:num_data]) # computed # the augmented but only compare first half rows. residual = residual / np.linalg.norm( X ) # normalize the residual in the same fashion... L2 norm of X return residual residual_array = np.array( joblib.Parallel(n_jobs=N_CPU)( joblib.delayed(compute_residual_list)(i) for i in range(num_alpha))) # finally, compute the complex koopman modes on those kept eigenfunctions if self.normalize_phi_tilde: phi_tilde_scaled = phi_tilde_scaled @ inverse_scaling # save data for trade-off plot np.savez(self.save_dir + 'MultiTaskElasticNet_result.npz', alphas_enet=alphas_enet, coefs_enet=coefs_enet_mag, coefs_enet_comp=coefs_enet_comp, residual_array=residual_array) ## 4. sweep for each alpha # 1. for each alpha, we get the non-zero index of selected eigentj print("evaluation module... start alpha sweeping...") mkdir(self.save_dir + 'sweep') sweep_index_list = [] for ii, alpha in enumerate(alphas_enet): print("current alpha = ", alpha, " index = ", ii) alpha_dir_eval = self.save_dir + 'sweep/sweep_alpha_' + str(alpha) mkdir(alpha_dir_eval) # compute selected index non_zero_index_bool_array = np.linalg.norm( coefs_enet_comp[:, :, ii], axis=0) > 0 sweep_index_list.append(non_zero_index_bool_array) return sweep_index_list
# Compute paths eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the lasso...") models = lasso_path(X, y, eps=eps) alphas_lasso = np.array([model.alpha for model in models]) coefs_lasso = np.array([model.coef_ for model in models]) print("Computing regularization path using the positive lasso...") models = lasso_path(X, y, eps=eps, positive=True) alphas_positive_lasso = np.array([model.alpha for model in models]) coefs_positive_lasso = np.array([model.coef_ for model in models]) print("Computing regularization path using the elastic net...") models = enet_path(X, y, eps=eps, l1_ratio=0.8) alphas_enet = np.array([model.alpha for model in models]) coefs_enet = np.array([model.coef_ for model in models]) print("Computing regularization path using the positve elastic net...") models = enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True) alphas_positive_enet = np.array([model.alpha for model in models]) coefs_positive_enet = np.array([model.coef_ for model in models]) ############################################################################### # Display results pl.figure(1) ax = pl.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = pl.plot(-np.log10(alphas_lasso), coefs_lasso)
eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the lasso...") alphas_lasso, coefs_lasso, _ = lasso_path(X_train, y_train, eps, fit_intercept=False) print("Computing regularization path using the positive lasso...") alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path( X_train, y_train, eps, positive=True, fit_intercept=False) print("Computing regularization path using the elastic net...") alphas_enet, coefs_enet, _ = enet_path(X_train, y_train, eps=eps, l1_ratio=0.8, fit_intercept=False) print("Computing regularization path using the positive elastic net...") alphas_positive_enet, coefs_positive_enet, _ = enet_path(X_train, y_train, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) plt.figure(1) colors = cycle(['b', 'r', 'g', 'c', 'k']) neg_log_alphas_lasso = -np.log10(alphas_lasso) neg_log_alphas_enet = -np.log10(alphas_enet)
coefs = [] for a in alphas: clf.set_params(alpha=a, max_iter=1000) clf.fit(xtrain, ytrain) coefs.append(clf.coef_) # lasso and elastic net path eps = 5e-3 # the smaller it is the longer is the path print("Computing regularization path using the lasso...") alphas_lasso, coefs_lasso, _ = lasso_path(xtrain, ytrain, eps, fit_intercept=False) print("Computing regularization path using the elastic net...") alphas_enet, coefs_enet, _ = enet_path( xtrain, ytrain, eps=eps, l1_ratio=0.8, fit_intercept=False) # Display results plt.figure(1) ax = plt.gca() ax.set_color_cycle(['b', 'r', 'g', 'c', 'k', 'y', 'm']) ax.plot(alphas, coefs) ax.set_xscale('log') ax.set_xlim(ax.get_xlim()[::-1]) # reverse axis plt.xlabel('alpha') plt.ylabel('weights') plt.title('Ridge coefficients as a function of the regularization') plt.axis('tight') plt.show()