lars_test = np.arange(0.0001, 0.002, 0.0001) lars_alpha, lars_err = general_model(LassoLars, lars_test) alpha_list.append(lars_alpha) err_list.append(lars_err) lars_alpha, lars_err ## max_iter = 50000 lasso_model = Lasso(alpha=alpha_list[0], max_iter=max_iter).fit(trainX, trainY) elasticNet_model = ElasticNet(alpha=alpha_list[1], max_iter=max_iter).fit(trainX, trainY) ridge_model = Ridge(alpha=alpha_list[2], max_iter=max_iter).fit(trainX, trainY) lars_model = LassoLars(alpha=alpha_list[3], max_iter=max_iter).fit(trainX, trainY) lasso_pred = np.expm1(lasso_model.predict(raw_test_df)) ridge_pred = np.expm1(ridge_model.predict(raw_test_df)) elasticNet_pred = np.expm1(elasticNet_model.predict(raw_test_df)) lars_pred = np.expm1(lars_model.predict(raw_test_df)) pred_list = np.array( [lasso_pred, ridge_pred, elasticNet_pred, lars_pred, xgb_pred]) # take average of 4 models err_list.append(xgb_err) err_list = np.array(err_list) w_list = 1 / err_list total_w = np.sum(w_list) predictions = np.matmul(w_list / total_w, pred_list)
for expl in format_as_all(res, clf): assert 'Error' in expl assert 'BaseEstimator' in expl with pytest.raises(TypeError): explain_weights(clf, unknown_argument=True) @pytest.mark.parametrize(['reg'], [ [ElasticNet(random_state=42)], [ElasticNetCV(random_state=42)], [HuberRegressor()], [Lars()], [LarsCV(max_n_alphas=10)], [Lasso(random_state=42)], [LassoCV(random_state=42)], [LassoLars(alpha=0.01)], [LassoLarsCV(max_n_alphas=10)], [LassoLarsIC()], [OrthogonalMatchingPursuit(n_nonzero_coefs=10)], [OrthogonalMatchingPursuitCV()], [PassiveAggressiveRegressor(C=0.1, random_state=42)], [Ridge(random_state=42)], [RidgeCV()], [SGDRegressor(random_state=42)], [LinearRegression()], [LinearSVR(random_state=42)], [TheilSenRegressor(random_state=42)], ]) def test_explain_linear_regression(boston_train, reg): assert_explained_weights_linear_regressor(boston_train, reg)
# 从使用默认配置初始化Lasso。 lasso = Lasso(alpha=0.07) scores7 = cross_val_score(lasso, sX, sy, cv=10,scoring='neg_mean_squared_error') print scores7 scores7=scores7.mean() # 从sklearn.linear_model导入Ridge。 from sklearn.linear_model import Ridge # 使用默认配置初始化Riedge。 ridge = Ridge(alpha=1) scores8 = cross_val_score(ridge, sX, sy, cv=10,scoring='neg_mean_squared_error') print scores8 scores8=scores8.mean() from sklearn.linear_model import LassoLars lars=LassoLars(alpha=0.009) scores9 = cross_val_score(lars, sX, sy, cv=10,scoring='neg_mean_squared_error') print scores9 scores9=scores9.mean() from sklearn.linear_model import ElasticNetCV elasticnet=ElasticNetCV(l1_ratio=0.13) scores10 = cross_val_score(elasticnet, sX, sy, cv=10,scoring='neg_mean_squared_error') print scores10 scores10=scores10.mean() from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, GradientBoostingRegressor # 从sklearn导入特征筛选器。 from sklearn import feature_selection fs = feature_selection.SelectPercentile(feature_selection.f_regression, percentile = 100)#30
ll = sum(act * sp.log(pred) + sp.subtract(1, act) * sp.log(sp.subtract(1, pred))) ll = ll * -1.0 / len(act) return ll # add two columns for hour and weekday def dayhour(timestr): d = datetime.strptime(str(x), "%y%m%d%H") return [float(d.weekday()), float(d.hour)] fh = FeatureHasher(n_features=2**20, input_type="string") # Train classifier clf = LassoLars() train = pd.read_csv("train/subtrain.csv", chunksize=100000, iterator=True) all_classes = np.array([0, 1]) for chunk in train: y_train = chunk["click"] chunk = chunk[cols] chunk = chunk.join( pd.DataFrame([dayhour(x) for x in chunk.hour], columns=["wd", "hr"])) chunk.drop(["hour"], axis=1, inplace=True) Xcat = fh.transform(np.asarray(chunk.astype(str))) clf.fit(Xcat, y_train) # Create a submission file usecols = cols + ["id"] X_test = pd.read_csv("test/mtest.csv", usecols=usecols) X_test = X_test.join(
def get_model_from_name(model_name, training_params=None): # For Keras epochs = 250 if 'is_test_suite' in sys.argv: print( 'Heard that this is the test suite. Limiting epochs to 10, which will increase training speed dramatically at the expense of model accuracy' ) epochs = 10 all_model_params = { 'LogisticRegression': { 'n_jobs': -2 }, 'RandomForestClassifier': { 'n_jobs': -2 }, 'ExtraTreesClassifier': { 'n_jobs': -1 }, 'AdaBoostClassifier': { 'n_estimators': 10 }, 'SGDClassifier': { 'n_jobs': -1 }, 'Perceptron': { 'n_jobs': -1 }, 'LinearRegression': { 'n_jobs': -2 }, 'RandomForestRegressor': { 'n_jobs': -2 }, 'ExtraTreesRegressor': { 'n_jobs': -1 }, 'MiniBatchKMeans': { 'n_clusters': 8 }, 'GradientBoostingRegressor': { 'presort': False }, 'SGDRegressor': { 'shuffle': False }, 'PassiveAggressiveRegressor': { 'shuffle': False }, 'AdaBoostRegressor': { 'n_estimators': 10 }, 'XGBRegressor': { 'nthread': -1, 'n_estimators': 200 }, 'XGBClassifier': { 'nthread': -1, 'n_estimators': 200 }, 'LGBMRegressor': {}, 'LGBMClassifier': {}, 'DeepLearningRegressor': { 'epochs': epochs, 'batch_size': 50, 'verbose': 2 }, 'DeepLearningClassifier': { 'epochs': epochs, 'batch_size': 50, 'verbose': 2 } } model_params = all_model_params.get(model_name, None) if model_params is None: model_params = {} if training_params is not None: print('Now using the model training_params that you passed in:') print(training_params) # Overwrite our stock params with what the user passes in (i.e., if the user wants 10,000 trees, we will let them do it) model_params.update(training_params) print( 'After overwriting our defaults with your values, here are the final params that will be used to initialize the model:' ) print(model_params) model_map = { # Classifiers 'LogisticRegression': LogisticRegression(), 'RandomForestClassifier': RandomForestClassifier(), 'RidgeClassifier': RidgeClassifier(), 'GradientBoostingClassifier': GradientBoostingClassifier(), 'ExtraTreesClassifier': ExtraTreesClassifier(), 'AdaBoostClassifier': AdaBoostClassifier(), 'SGDClassifier': SGDClassifier(), 'Perceptron': Perceptron(), 'PassiveAggressiveClassifier': PassiveAggressiveClassifier(), # Regressors 'LinearRegression': LinearRegression(), 'RandomForestRegressor': RandomForestRegressor(), 'Ridge': Ridge(), 'ExtraTreesRegressor': ExtraTreesRegressor(), 'AdaBoostRegressor': AdaBoostRegressor(), 'RANSACRegressor': RANSACRegressor(), 'GradientBoostingRegressor': GradientBoostingRegressor(), 'Lasso': Lasso(), 'ElasticNet': ElasticNet(), 'LassoLars': LassoLars(), 'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(), 'BayesianRidge': BayesianRidge(), 'ARDRegression': ARDRegression(), 'SGDRegressor': SGDRegressor(), 'PassiveAggressiveRegressor': PassiveAggressiveRegressor(), # Clustering 'MiniBatchKMeans': MiniBatchKMeans() } if xgb_installed: model_map['XGBClassifier'] = xgb.XGBClassifier() model_map['XGBRegressor'] = xgb.XGBRegressor() if lgb_installed: model_map['LGBMRegressor'] = lgb.LGBMRegressor() model_map['LGBMClassifier'] = lgb.LGBMClassifier() if keras_installed: model_map['DeepLearningClassifier'] = KerasClassifier( build_fn=make_deep_learning_classifier) model_map['DeepLearningRegressor'] = KerasRegressor( build_fn=make_deep_learning_model) model_without_params = model_map[model_name] model_with_params = model_without_params.set_params(**model_params) return model_with_params
def model_compare(data, target): # mean evaluate(MeanEstimator(), data.values, target.values, "Mean Estimator") # linear param_grid = {"normalize": [True, False], "fit_intercept": [True, False]} evaluate(LinearRegression(), data, target, "Linear", param_grid=param_grid) # poly poly = Pipeline([('poly', PolynomialFeatures(degree=2, interaction_only=True)), ('linear', Lasso(alpha=3))]) evaluate(poly, data, target, "Poly") # decision tree param_grid = {"max_features": ["auto", "sqrt", "log2", None]} evaluate(DecisionTreeRegressor(criterion="mae"), data, target, "Decision Tree", param_grid=param_grid) # elastic param_grid = dict(alpha=10.0**np.arange(-5, 4), l1_ratio=0.1 * np.arange(0, 11), normalize=[True, False], fit_intercept=[True, False]) evaluate(ElasticNet(), data, target, "Elastic", param_grid=param_grid) # ridge param_grid = dict( alpha=10.0**np.arange(-5, 4), normalize=[True, False], fit_intercept=[True, False], solver=["auto", "svd", "cholesky", "lsqr", 'sparse_cg', 'sag']) evaluate(Ridge(), data, target, "Ridge", param_grid=param_grid) # SVR param_grid = dict(C=10.0 * np.arange(50, 70, 10), kernel=['linear', 'poly', 'rbf', 'sigmoid']) evaluate(svm.SVR(), data, target, "SVR", param_grid=param_grid, n_jobs=4) # XGBoost param_grid = {'max_depth': [2, 4, 6], 'n_estimators': [50, 100, 200]} evaluate(xgb.XGBRegressor(), data, target, "XGBoost", param_grid=param_grid) # SGD Regressor() param_grid = { 'loss': [ 'squared_loss', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive' ], 'penalty': ['none', 'l2', 'l1', 'elasticnet'] } evaluate(SGDRegressor(), data, target, "SGDRegressor", param_grid=param_grid) # GradientBoostingRegressor gbr = GradientBoostingRegressor(loss='quantile', criterion="mae") evaluate(gbr, data, target, "GradientBoostingRegressor") # # AdaBoostClassifier # ada = AdaBoostClassifier(n_estimators=100) # evaluate(ada, data, target, "AdaBoostClassifier") # BaggingRegressor evaluate(BaggingRegressor(), data, target, "BaggingRegressor") # KNeighborsRegressor kn = KNeighborsRegressor(n_neighbors=4, weights="distance") evaluate(kn, data, target, "KNeighborsRegressor") # BayesianRidge br = BayesianRidge() evaluate(br, data, target, "BayesianRidge") lasso_lars_ic(data, target) alpha = lassoCV(data, target) print "alpha " + str(alpha) lasso = Lasso(alpha=alpha, normalize=False) evaluate(lasso, data, target, "Lasso") alpha = lasso_lars_cv(data, target) print "alpha " + str(alpha) lasso_lars = LassoLars(alpha=alpha) evaluate(lasso_lars, data, target, "Lasso Lars")
def lassoLars(X, y, value): regressor = LassoLars(alpha=0.3, max_iter=600000) regressor.fit(X, y) y_pred = regressor.predict(value) return y_pred
def lassoCD(X, y, ll, ul, step, state): kf = KFold(n_splits=10, shuffle=True, random_state=state) feature = [] pred = [] true = [] r2 = [] mse = [] ilist = np.linspace(ll, ul, step) pbar = tnrange(step * 10, desc='loop') for i in ilist: r2_single = [] mse_single = [] pred_single = [] true_single = [] feature_single = [] for train_index, test_index in kf.split(X): y_train, y_test = y[train_index], y[test_index] X_train_tmp, X_test_tmp = X[train_index], X[test_index] clf = LassoLars(alpha=i) clf.fit(X_train_tmp, np.ravel(y_train)) feature_index = np.where(clf.coef_ > 0)[0] X_train = X_train_tmp[:, feature_index] X_test = X_test_tmp[:, feature_index] svr = svm.SVR(kernel='linear') svr.fit(X_train, np.ravel(y_train)) y_test_pred = svr.predict(X_test) feature_single.append(feature_index) pred_single.append(y_test_pred) true_single.append(np.ravel(y_test)) r2_single.append(r2_score(y_test, y_test_pred)) mse_single.append(mean_squared_error(y_test, y_test_pred)) pbar.update(1) r2.append(r2_single) r2_single = np.array(r2_single) f = np.where(r2_single == max(r2_single))[0][0] mse.append(mse_single) pred.append(pred_single) true.append(true_single) feature.append(np.array(feature_single[f])) # print(np.array(feature_single)[f]) r2 = np.array(r2) r2_mean = np.average(r2, axis=1, weights=weight) a = np.where(r2_mean == max(r2_mean))[0][0] pred = np.array(pred)[a] true = np.array(true)[a] mse = np.array(mse)[a] feature = np.array(feature)[a] print(feature.shape[0]) alpha = ilist[a] r2 = r2[a] tmp = np.zeros([0, 2]) tmp_ = np.zeros([0, 10]) for i in range(10): p = np.expand_dims(pred[i], axis=1) t = np.expand_dims(true[i], axis=1) tmp1 = np.concatenate([p, t], axis=1) tmp = np.concatenate([tmp, tmp1], axis=0) df1 = pd.DataFrame(tmp, columns=['Predict', 'True']) df2 = pd.DataFrame({'r2': r2, 'mse': mse}) df3 = pd.DataFrame({'features': feature}) pbar.close() plt.figure() plt.plot(ilist, r2_mean) plt.xlabel('$alpha$') plt.ylabel('$R^2$') print('max r2_score=', r2_mean[a], ', corresponding alpha=', alpha, a) print('number of selected features:', feature.shape[0]) return df1, df2, df3
for i in range(result_row): sumsum = sumsum + (y[i] - y_test[i]) * (y[i] - y_test[i]) rank_result['Lars_pca'] = sumsum / float(result_row) rs_score['Lars_pca'] = r2_score(y_test, y) LarsModel = Lars() LarsModel.fit(X_train_std, y_train) y = LarsModel.predict(X_test_std) [result_row] = y.shape sumsum = 0 #print y for i in range(result_row): sumsum = sumsum + (y[i] - y_test[i]) * (y[i] - y_test[i]) rank_result['Lars_std'] = sumsum / float(result_row) rs_score['Lars_std'] = r2_score(y_test, y) LassoLarsModel = LassoLars() LassoLarsModel.fit(X_train_pca, y_train) y = LassoLarsModel.predict(X_test_pca) [result_row] = y.shape sumsum = 0 #print y for i in range(result_row): sumsum = sumsum + (y[i] - y_test[i]) * (y[i] - y_test[i]) rank_result['LassoLars_pca'] = sumsum / float(result_row) rs_score['LassoLars_pca'] = r2_score(y_test, y) LassoLarsModel = LassoLars() LassoLarsModel.fit(X_train_std, y_train) y = LassoLarsModel.predict(X_test_std) [result_row] = y.shape sumsum = 0 #print y
for l1_ratio in l1_ratios: models_eln_l1.append( ("ELN_L1_" + str(l1_ratio), ElasticNet(l1_ratio=l1_ratio, alpha=0.00001))) getCVResult(models_eln_l1, X_learning, Y_learning) #LassoLars tuning alphas = [ 0.000005, 0.00001, 0.00003, 0.000035, 0.000036, 0.000037, 0.000038, 0.00004, 0.00005, 0.00007, 0.0001 ] models_lala = [] for alpha in alphas: models_lala.append(("LaLa_" + str(alpha), LassoLars(alpha=alpha))) getCVResult(models_lala, X_learning2, Y_learning) #XGB model tuning n_estimators = [400, 450, 470, 540, 550, 560] models_xgb = [] for n_estimator in n_estimators: models_xgb.append(("XGB_" + str(n_estimator), xgb.XGBRegressor(n_estimators=n_estimator, max_depth=3, min_child_weight=3))) getCVResult(models_xgb, X_learning, Y_learning)
def all_models_info(): '''takes in data sets baseline sets SSE, MSE, and RMSE returns infor for all 4''' # get data df = acquire.acquire_zillow() df = prepare.clean_zillow(df) df = prepare.focused_zillow(df) # pull from add to trian train = evaluate.add_to_train() X_train, y_train, X_validate, y_validate, X_test, y_test = evaluate.xtrain_xval_xtest( ) #OLS Model lm = LinearRegression(normalize=True) lm.fit(X_train, y_train.appraised_value) y_train['appraised_value_pred_lm'] = lm.predict(X_train) rmse_train_lm = mean_squared_error( y_train.appraised_value, y_train.appraised_value_pred_lm)**(1 / 2) y_validate['appraised_value_pred_lm'] = lm.predict(X_validate) rmse_validate_lm = mean_squared_error( y_validate.appraised_value, y_validate.appraised_value_pred_lm)**(1 / 2) #LARS Model lars = LassoLars(alpha=1.0) lars.fit(X_train, y_train.appraised_value) y_train['appraised_value_pred_lars'] = lars.predict(X_train) rmse_train_lars = mean_squared_error( y_train.appraised_value, y_train.appraised_value_pred_lars)**1 / 2 y_validate['appraised_value_pred_lars'] = lars.predict(X_validate) rmse_validate_lars = mean_squared_error( y_validate.appraised_value, y_validate.appraised_value_pred_lars)**1 / 2 #GLM glm = TweedieRegressor(power=1, alpha=0) glm.fit(X_train, y_train.appraised_value) y_train['appraised_value_pred_glm'] = glm.predict(X_train) rmse_train_glm = mean_squared_error( y_train.appraised_value, y_train.appraised_value_pred_glm)**1 / 2 y_validate['appraised_value_pred_glm'] = glm.predict(X_validate) rmse_validate_glm = mean_squared_error( y_validate.appraised_value, y_validate.appraised_value_pred_glm)**1 / 2 # PF pf = PolynomialFeatures(degree=2) X_train_degree2 = pf.fit_transform(X_train) X_validate_degree2 = pf.transform(X_validate) X_test_degree2 = pf.transform(X_test) # LM2 lm2 = LinearRegression(normalize=True) lm2.fit(X_train_degree2, y_train.appraised_value) y_train['appraised_value_pred_lm2'] = lm2.predict(X_train_degree2) rmse_train_lm2 = mean_squared_error( y_train.appraised_value, y_train.appraised_value_pred_lm2)**1 / 2 y_validate['appraised_value_pred_lm2'] = lm2.predict(X_validate_degree2) rmse_validate_lm2 = mean_squared_error( y_validate.appraised_value, y_validate.appraised_value_pred_lm2)**1 / 2 print("RMSE for OLS using LinearRegression\nTraining/In-Sample: ", rmse_train_lm, "\nValidation/Out-of-Sample: ", rmse_validate_lm) print("--------------------------------------------------------------") print("RMSE for Lasso + Lars\nTraining/In-Sample: ", rmse_train_lars, "\nValidation/Out-of-Sample: ", rmse_validate_lars) print("--------------------------------------------------------------") print( "RMSE for GLM using Tweedie, power=1 & alpha=0\nTraining/In-Sample: ", rmse_train_glm, "\nValidation/Out-of-Sample: ", rmse_validate_glm) print("--------------------------------------------------------------") print("RMSE for Polynomial Model, degrees=2\nTraining/In-Sample: ", rmse_train_lm2, "\nValidation/Out-of-Sample: ", rmse_validate_lm2)
def compute_pruned_kernel( # pylint: disable=too-many-locals,too-many-branches,too-many-statements self, X, W2, Y, alpha=1e-4, c_new=None, tolerance=0.02): """compute which channels to be pruned by lasso""" tf.logging.info('computing pruned kernel') nb_samples = X.shape[0] c_in = X.shape[-1] c_out = W2.shape[-1] samples = np.random.randint(0, nb_samples, min(400, nb_samples // 20)) reshape_X = np.rollaxis( np.transpose(X, (0, 3, 1, 2)).reshape( (nb_samples, c_in, -1))[samples], 1, 0) reshape_W2 = np.transpose( np.transpose(W2, (3, 2, 0, 1)).reshape((c_out, c_in, -1)), [1, 2, 0]) product = np.matmul(reshape_X, reshape_W2).reshape((c_in, -1)).T reshape_Y = Y[samples].reshape(-1) # feature tmp = np.nonzero(np.sum(np.abs(product), 0))[0].size if FLAGS.debug: tf.logging.info('feature num: {}, non zero: {}'.format( product.shape[1], tmp)) solver = LassoLars(alpha=alpha, fit_intercept=False, max_iter=3000) def solve(alpha): """ Solve the Lasso""" solver.alpha = alpha solver.fit(product, reshape_Y) idxs = solver.coef_ != 0. tmp = sum(idxs) return idxs, tmp, solver.coef_ tf.logging.info('pruned channel selecting') start = timer() if c_new == c_in: idxs = np.array([True] * c_new) else: left = 0 right = alpha lbound = c_new - tolerance * c_in / 2 rbound = c_new + tolerance * c_in / 2 while True: _, tmp, coef = solve(right) if tmp < c_new: break else: right *= 2 if FLAGS.debug: tf.logging.debug("relax right to {}".format(right)) tf.logging.debug( "we expect got less than {} channels, but got {} channels" .format(c_new, tmp)) while True: if lbound < 0: lbound = 1 idxs, tmp, coef = solve(alpha) # print loss loss = 1 / (2 * float(product.shape[0])) * \ np.sqrt(np.sum((reshape_Y - np.matmul(product, coef)) ** 2, axis=0)) + \ alpha * np.sum(np.fabs(coef)) if FLAGS.debug: tf.logging.debug( 'loss: {}, alpha: {}, feature nums: {}, left: {}, right: {}, \ left_bound: {}, right_bound: {}'.format(loss, alpha, tmp, left, right, lbound, rbound)) if FLAGS.debug: tf.logging.info( 'tmp {}, lbound {}, rbound {}, alpha {}, left {}, right {}' .format(tmp, lbound, rbound, alpha, left, right)) if FLAGS.cp_quadruple: if tmp % 4 == 0 and abs(tmp - lbound) <= 2: break if lbound <= tmp and tmp <= rbound: if FLAGS.cp_quadruple: if tmp % 4 == 0: break elif tmp % 4 <= 2: rbound = tmp - 1 lbound = lbound - 2 else: lbound = tmp + 1 rbound = rbound + 2 else: break elif abs(left - right) <= right * 0.1: if lbound > 1: lbound = lbound - 1 if rbound < c_in: rbound = rbound + 1 left = left / 1.2 right = right * 1.2 elif tmp > rbound: left = left + (alpha - left) / 2 else: right = right - (right - alpha) / 2 if alpha < 1e-10: break alpha = (left + right) / 2 c_new = tmp tf.logging.info('Channel selection time cost: {}s'.format(timer() - start)) start = timer() tf.logging.info('Feature map reconstructing') newW2, _ = self.featuremap_reconstruction(X[:, :, :, idxs].reshape( (nb_samples, -1)), Y, fit_intercept=False) tf.logging.info( 'Feature map reconstruction time cost: {}s'.format(timer() - start)) return idxs, newW2
def cvx_online_dict_learning(X, y_true, n_hat, k_cluster, T, lmda, eps, flag=True, version = 'Rr'): ''' X: R^(n * m) y_true: str^n W_0: R^(n_hat * k) x_i : R^m alpha: R^k cvx_online problem min||x_i - X.T * W * alpha|| + lambda * ||alpha|| in the online setting, there is no X in (n * m), instead, we need to store a candidate set and solve the subproblem: min ||x_i - X_hat * W_hat * alpha|| + lambda * ||alpha|| X_hat : R^(m * n_hat) W_hat : R^(n_hat * k) version: Rr, restricted, heuristic approach Ru, uniform, random assignment ''' n_dim, m_dim = X.shape A_t = np.zeros((k_cluster, k_cluster)) B_t = np.zeros((m_dim, k_cluster)) x_sum = 0 alpha_sum = 0 # step 1: sample n_hat * k_cluster points as initial X_hat. X_0 = np.zeros((m_dim, n_hat)) for idx in range(n_hat): sample_idx = np.random.randint(0, n_dim) x_sample = X[sample_idx, :] X_0[:, idx] = x_sample # step 1: initialization, get X_hat (including clusters info) # and W_hat from X_0, using same init as in CNMF. # here representative_size_count is the n_1_hat, n_2_hat, ..., n_k_hat. t1 = time.time() X_hat, W_hat, representative_size_count = initialize_X_W_hat(X_0, k_cluster) X_0, W_0 = X_hat.copy(), W_hat.copy() t2 = time.time() # print('init cost {:.4f}'.format(t2 - t1)) # step 2: after initialization of X_hat, update alpha, W_hat and X_hat alternatively. t_start = time.time() print(lmda, _NF, eps) for t in range(T): # t_start_online = time.time() if t % 50 == 0 and flag: D_t = np.matmul(X_hat, W_hat) tmp_assignment = get_clustering_assignment_1(X, D_t, k_cluster) tmp_acc, tmp_AMI = evaluation_clustering(tmp_assignment, y_true) print('1)iteration {}, distance acc = {:.4f}, AMI = {:.4f}'.format(t, tmp_acc, tmp_AMI)) tmp_assignment = get_clustering_assignment_2(X, D_t, k_cluster, lmda) tmp_acc, tmp_AMI = evaluation_clustering(tmp_assignment, y_true) print('2)iteration {}, kmeans of weights acc = {:.4f}, AMI = {:.4f}'.format(t, tmp_acc, tmp_AMI)) t_end = time.time() print('time elapse = {:.4f}s'.format(t_end - t_start)) t_start = t_end print('-' * 7) sample_idx = np.random.randint(0, n_dim) x_sample = X[sample_idx, :] # update alpha t1 = time.time() lars_lasso = LassoLars(alpha = lmda, max_iter = 500) D_t = np.matmul(X_hat, W_hat) lars_lasso.fit(D_t, x_sample) alpha_t = lars_lasso.coef_ t2 = time.time() # print('lasso cost {:.4f}s'.format(t2 - t1)) # using different clustering assignment t1 = time.time() if version == 'Rr': cluster_of_x_i = np.argmax(alpha_t) # elif version == 'Ru': else: cluster_of_x_i = int(np.random.uniform(0, k_cluster)) t2 = time.time() # print('argmax alpha cost {:.4f}s'.format(t2 - t1)) t1 = time.time() A_t += np.matmul(alpha_t.reshape(k_cluster, 1), alpha_t.reshape(1, k_cluster)) B_t += np.matmul(x_sample.reshape(m_dim, 1), alpha_t.reshape(1, k_cluster)) x_sum += (np.linalg.norm(x_sample) ** 2) alpha_sum += lmda * np.linalg.norm(alpha_t, 1) t2 = time.time() # print('update At, Bt cost {:.4f}s'.format(t2 - t1)) # update X_hat t1 = time.time() W_hat, X_hat = update_W_X_hat(W_hat, X_hat, representative_size_count, x_sample, cluster_of_x_i, A_t, B_t, x_sum, alpha_sum, t, eps) t2 = time.time() # print('update X_hat, W_hat cost {:.4f}s'.format(t2 - t1)) print('Dcitionary update done! Time elapse {:.04f}s'.format(time.time() - t_start)) return W_hat, X_hat, representative_size_count, X_0, W_0
LogisticRegression, LassoCV) from sklearn.multioutput import MultiOutputRegressor from sklearn.neighbors import KNeighborsRegressor from sklearn.neural_network import MLPRegressor from sklearn.utils import resample from sklearn.preprocessing import StandardScaler estimators = [ ('RANSACReg', RANSACRegressor(base_estimator=LinearRegression(fit_intercept=False))), ('LinReg', LinearRegression(fit_intercept=False)), ('Theil_Sen', TheilSenRegressor(fit_intercept=False)), ('Ridge', Ridge(fit_intercept=False)), ('HuberRegressor', HuberRegressor(fit_intercept=False)), ('BayesRidge', BayesianRidge(fit_intercept=False)), ('LassoLars', LassoLars(fit_intercept=False, alpha=25)), ('Lasso', Lasso(fit_intercept=False, alpha=25)), ('ElasticNet', ElasticNet(alpha=13, fit_intercept=False)), ('ARDRegression', ARDRegression(fit_intercept=False)) ] #The "environment" is our interface for code competitions # The "environment" is our interface for code competitions env = kagglegym.make() # We get our initial observation by calling "reset" observation = env.reset() # Note that the first observation we get has a "train" dataframe trains = observation.train print("Train has {} rows".format(len(trains)))
def lasso_pruning(X, Y, W, c_new, alpha=1e-4, tolerance=0.02, debug=False): # Conv # Example: B is sample number # : c_in is channel input # : c_out is channel output # : 3x3 is kernel size # X shape: [B, c_in, 3, 3] # Y shape: [B, c_out] # W shape: [c_out, c_in, 3, 3] # Linear # X shape: [B, c_in] # Y shape: [B, c_out] # W shape: [c_out, c_in] if debug: print("input shape: {}".format(X.shape)) print("output shape: {}".format(Y.shape)) print("weight shape: {}".format(W.shape)) print("curr chn: {} target chn: {}".format(W.shape[1], c_new)) num_samples = X.shape[0] # num of training samples c_in = W.shape[1] # num of input channels c_out = W.shape[0] # num of output channels # conv if len(W.shape) == 4: # sample and reshape X to [c_in, B, 9] reshape_X = X.reshape((num_samples, c_in, -1)).transpose((1, 0, 2)) # reshape W to [c_in, 9, c_out] reshape_W = W.reshape((c_out, c_in, -1)).transpose((1, 2, 0)) else: # linear # sample and reshape X to [c_in, B] and expand to [c_in, B, 1] reshape_X = X.transpose((1, 0))[..., np.newaxis] # reshape to [c_in, 1, c_out] reshape_W = W.reshape((c_out, c_in, 1)).transpose((1, 2, 0)) # reshape Y to [B x c_out] reshape_Y = Y.reshape(-1) # product has size [B x c_out, c_in] product = np.matmul(reshape_X, reshape_W).reshape((c_in, -1)).T # use LassoLars because it's more robust than Lasso solver = LassoLars(alpha=alpha, fit_intercept=False, max_iter=3000) # solver = Lasso(alpha=alpha, fit_intercept=False, # max_iter=3000, warm_start=True, selection='random') def solve(alpha): """ Solve the Lasso""" solver.alpha = alpha solver.fit(product, reshape_Y) nonzero_inds = np.where(solver.coef_ != 0.)[0] nonzero_num = sum(solver.coef_ != 0.) return nonzero_inds, nonzero_num, solver.coef_ tic = time.perf_counter() left = 0 # minimum alpha is 0, which means don't use lasso regularizer at all right = alpha # the left bound of num of selected channels lbound = c_new # the right bound of num of selected channels rbound = c_new + tolerance * c_new # increase alpha until the lasso can find a selection with size < c_new while True: _, keep_num, coef = solve(right) if debug: print("relax right to %.6f" % right) print("expected %d channels, but got %d channels" % (c_new, keep_num)) if keep_num < c_new: break else: right *= 2 # shrink the alpha for less aggressive lasso regularization # if the selected num of channels is less than the lbound while True: # binary search alpha = (left + right) / 2 keep_inds, keep_num, coef = solve(alpha) # print loss # product has size [B x c_out, c_in] loss = 1 / (2 * float(product.shape[0])) * \ np.sqrt(np.sum((reshape_Y - np.matmul(product, coef)) ** 2, axis=0)) + \ alpha * np.sum(np.fabs(coef)) if debug: print( 'loss: %.6f, alpha: %.6f, feature nums: %d, ' 'left: %.6f, right: %.6f, left_bound: %.6f, right_bound: %.6f' % (loss, alpha, keep_num, left, right, lbound, rbound)) if keep_num > rbound: left = alpha elif keep_num < lbound: right = alpha else: break if alpha < 1e-10: break toc = time.perf_counter() if debug: print('Lasso Regression time: %.2f s' % (toc - tic)) print('Chn keep idx: {}'.format(keep_inds)) print(c_new, keep_num) print("orig chn num = {} keep chn num = {}".format(c_in, keep_num)) return keep_inds, keep_num
def make_estimator(self, **params): return make_pipeline(*self.preprocessors, LassoLars(normalize=True)).set_params(**params)
def task2(data): df = data dfreg = df.loc[:, ['Adj Close', 'Volume']] dfreg['HL_PCT'] = (df['High'] - df['Low']) / df['Close'] * 100.0 dfreg['PCT_change'] = (df['Close'] - df['Open']) / df['Open'] * 100.0 # Drop missing value dfreg.fillna(value=-99999, inplace=True) # We want to separate 1 percent of the data to forecast forecast_out = int(math.ceil(0.01 * len(dfreg))) # Separating the label here, we want to predict the AdjClose forecast_col = 'Adj Close' dfreg['label'] = dfreg[forecast_col].shift(-forecast_out) X = np.array(dfreg.drop(['label'], 1)) # Scale the X so that everyone can have the same distribution for linear regression X = preprocessing.scale(X) # Finally We want to find Data Series of late X and early X (train) for model generation and evaluation X_lately = X[-forecast_out:] X = X[:-forecast_out] # Separate label and identify it as y y = np.array(dfreg['label']) y = y[:-forecast_out] #Split data X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) ################## ################## ################## # Linear regression clfreg = LinearRegression(n_jobs=-1) clfreg.fit(X_train, y_train) # Quadratic Regression 2 clfpoly2 = make_pipeline(PolynomialFeatures(2), Ridge()) clfpoly2.fit(X_train, y_train) # Quadratic Regression 3 clfpoly3 = make_pipeline(PolynomialFeatures(3), Ridge()) clfpoly3.fit(X_train, y_train) # KNN Regression clfknn = KNeighborsRegressor(n_neighbors=2) clfknn.fit(X_train, y_train) # Lasso Regression clflas = Lasso() clflas.fit(X_train, y_train) # Multitask Lasso Regression # clfmtl = MultiTaskLasso(alpha=1.) # clfmtl.fit(X_train, y_train).coef_ # Bayesian Ridge Regression clfbyr = BayesianRidge() clfbyr.fit(X_train, y_train) # Lasso LARS Regression clflar = LassoLars(alpha=.1) clflar.fit(X_train, y_train) # Orthogonal Matching Pursuit Regression clfomp = OrthogonalMatchingPursuit(n_nonzero_coefs=2) clfomp.fit(X_train, y_train) # Automatic Relevance Determination Regression clfard = ARDRegression(compute_score=True) clfard.fit(X_train, y_train) # Logistic Regression # clflgr = linear_model.LogisticRegression(penalty='l1', solver='saga', tol=1e-6, max_iter=int(1e6), warm_start=True) # coefs_ = [] # for c in cs: # clflgr.set_params(C=c) # clflgr.fit(X_train, y_train) # coefs_.append(clflgr.coef_.ravel().copy()) clfsgd = SGDRegressor(random_state=0, max_iter=1000, tol=1e-3) clfsgd.fit(X_train, y_train) ################## ################## ################## #Create confindence scores confidencereg = clfreg.score(X_test, y_test) confidencepoly2 = clfpoly2.score(X_test, y_test) confidencepoly3 = clfpoly3.score(X_test, y_test) confidenceknn = clfknn.score(X_test, y_test) confidencelas = clflas.score(X_test, y_test) # confidencemtl = clfmtl.score(X_test, y_test) confidencebyr = clfbyr.score(X_test, y_test) confidencelar = clflar.score(X_test, y_test) confidenceomp = clfomp.score(X_test, y_test) confidenceard = clfard.score(X_test, y_test) confidencesgd = clfsgd.score(X_test, y_test) # results print('The linear regression confidence is:', confidencereg * 100) print('The quadratic regression 2 confidence is:', confidencepoly2 * 100) print('The quadratic regression 3 confidence is:', confidencepoly3 * 100) print('The knn regression confidence is:', confidenceknn * 100) print('The lasso regression confidence is:', confidencelas * 100) # print('The lasso regression confidence is:',confidencemtl*100) print('The Bayesian Ridge regression confidence is:', confidencebyr * 100) print('The Lasso LARS regression confidence is:', confidencelar * 100) print('The OMP regression confidence is:', confidenceomp * 100) print('The ARD regression confidence is:', confidenceard * 100) print('The SGD regression confidence is:', confidencesgd * 100) #Create new columns forecast_reg = clfreg.predict(X_lately) forecast_pol2 = clfpoly2.predict(X_lately) forecast_pol3 = clfpoly3.predict(X_lately) forecast_knn = clfknn.predict(X_lately) forecast_las = clflas.predict(X_lately) forecast_byr = clfbyr.predict(X_lately) forecast_lar = clflar.predict(X_lately) forecast_omp = clfomp.predict(X_lately) forecast_ard = clfard.predict(X_lately) forecast_sgd = clfsgd.predict(X_lately) #Process all new columns data dfreg['Forecast_reg'] = np.nan last_date = dfreg.iloc[-1].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_reg: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg.loc[next_date] = [np.nan for _ in range(len(dfreg.columns))] dfreg['Forecast_reg'].loc[next_date] = i dfreg['Forecast_pol2'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_pol2: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_pol2'].loc[next_date] = i dfreg['Forecast_pol3'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_pol3: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_pol3'].loc[next_date] = i dfreg['Forecast_knn'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_knn: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_knn'].loc[next_date] = i dfreg['Forecast_las'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_las: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_las'].loc[next_date] = i dfreg['Forecast_byr'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_byr: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_byr'].loc[next_date] = i dfreg['Forecast_lar'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_lar: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_lar'].loc[next_date] = i dfreg['Forecast_omp'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_omp: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_omp'].loc[next_date] = i dfreg['Forecast_ard'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_ard: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_ard'].loc[next_date] = i dfreg['Forecast_sgd'] = np.nan last_date = dfreg.iloc[-26].name last_unix = last_date next_unix = last_unix + datetime.timedelta(days=1) for i in forecast_sgd: next_date = next_unix next_unix += datetime.timedelta(days=1) dfreg['Forecast_sgd'].loc[next_date] = i return dfreg.index.format(formatter=lambda x: x.strftime( '%Y-%m-%d')), dfreg['Adj Close'].to_list( ), dfreg['Forecast_reg'].to_list(), dfreg['Forecast_pol2'].to_list( ), dfreg['Forecast_pol3'].to_list(), dfreg['Forecast_knn'].to_list( ), dfreg['Forecast_las'].to_list(), dfreg['Forecast_byr'].to_list( ), dfreg['Forecast_lar'].to_list(), dfreg['Forecast_omp'].to_list( ), dfreg['Forecast_ard'].to_list(), dfreg['Forecast_sgd'].to_list()
def make_estimator(self, **params): return make_pipeline(*self.preprocessors, PolynomialFeatures(), StandardScaler(), PCA(), LassoLars(normalize=True)).set_params(**params)
# LassoLars Regression # The Least Angle Regression (LARS) can be used as an alternative method for calculating Least Absolute Shrinkage # and Selection Operator (LASSO) fit. import numpy as np from sklearn import datasets from sklearn.linear_model import LassoLars # load the iris datasets dataset = datasets.load_diabetes() # fit a LASSO using LARS model to the data model = LassoLars(alpha=0.1) model.fit(dataset.data, dataset.target) print(model) # make predictions expected = dataset.target predicted = model.predict(dataset.data) # summarize the fit of the model mse = np.mean((predicted - expected)**2) print(mse) print(model.score(dataset.data, dataset.target))
(numerical_scaler, numerical_cols) ) preprocessor_oe = make_column_transformer( (categorical_encoder_oe, categorical_cols_oe), ) """ #############################################Test All Models##################################### models = { "lr": LinearRegression(), "lasso": Lasso(), "ridge": Ridge(), "elasticnet": ElasticNet(), "lassolars": LassoLars(), "bayridge": BayesianRidge(), "svr": SVR(), "knn": KNeighborsRegressor(), #"gaussianpr" : GaussianProcessRegressor(), mauvais rmse et long à executer "decisiontree": DecisionTreeRegressor(), "rf": RandomForestRegressor(), "extratree": ExtraTreesRegressor(), "adaboost": AdaBoostRegressor(), "gradientboost": GradientBoostingRegressor(), "xgb": xgb.XGBRegressor() } models_todense = ["lassolars", "bayridge", "gaussianpr"] pipelines_oe_std = []
# ************************************************************************************* # ...................................... Modelos .................................... # # ************************************************************************************* #_____________________________________________________# ## ************ a. Integracion temprana ************ ## ### ... Modelos lineales ... ### regressors = { ## modelos 'OLS': LinearRegression(), 'ridge': Ridge(), 'lasso': Lasso(), #'multi-lasso': MultiTaskLasso(), 'elasticnet': ElasticNet(), #'multi-elasticnet': MultiTaskElasticNet(), 'lars': Lars(), 'lassolars': LassoLars(), 'orthogonalmatchingpursuit': OrthogonalMatchingPursuit(), 'bayesianridge': BayesianRidge(), 'passiveaggressivregressor': PassiveAggressiveRegressor(), 'ransacregressor': RANSACRegressor(), 'theilsenregressor': TheilSenRegressor(), 'huberregressor': HuberRegressor() } otra_param_grid = { ## grilla 'OLS': {}, "ridge": {'alpha': [0.01, 0.1, 1, 5, 100]}, "lasso": {'alpha': [0.01, 0.1, 1, 5, 100]}, #"multi-lasso": {'alpha': [0.01, 0.1, 1]}, "elasticnet": {'alpha': [0.001, 0.05, 0.1, 1, 100], 'l1_ratio': [0.001, 0.05, 0.01, 0.1, 1, 100]}, #"multi-elasticnet": {'alpha': [0.01, 0.1, 1], 'l1_ratio': [0.01, 0.1, 1]},
def get_model_from_name(model_name, training_params=None, is_hp_search=False): global keras_imported # For Keras epochs = 1000 # if os.environ.get('is_test_suite', 0) == 'True' and model_name[:12] == 'DeepLearning': # print('Heard that this is the test suite. Limiting number of epochs, which will increase training speed dramatically at the expense of model accuracy') # epochs = 100 all_model_params = { 'LogisticRegression': {}, 'RandomForestClassifier': { 'n_jobs': -2, 'n_estimators': 30 }, 'ExtraTreesClassifier': { 'n_jobs': -1 }, 'AdaBoostClassifier': {}, 'SGDClassifier': { 'n_jobs': -1 }, 'Perceptron': { 'n_jobs': -1 }, 'LinearSVC': { 'dual': False }, 'LinearRegression': { 'n_jobs': -2 }, 'RandomForestRegressor': { 'n_jobs': -2, 'n_estimators': 30 }, 'LinearSVR': { 'dual': False, 'loss': 'squared_epsilon_insensitive' }, 'ExtraTreesRegressor': { 'n_jobs': -1 }, 'MiniBatchKMeans': { 'n_clusters': 8 }, 'GradientBoostingRegressor': { 'presort': False, 'learning_rate': 0.1, 'warm_start': True }, 'GradientBoostingClassifier': { 'presort': False, 'learning_rate': 0.1, 'warm_start': True }, 'SGDRegressor': { 'shuffle': False }, 'PassiveAggressiveRegressor': { 'shuffle': False }, 'AdaBoostRegressor': {}, 'LGBMRegressor': { 'n_estimators': 2000, 'learning_rate': 0.15, 'num_leaves': 8, 'lambda_l2': 0.001, 'histogram_pool_size': 16384 }, 'LGBMClassifier': { 'n_estimators': 2000, 'learning_rate': 0.15, 'num_leaves': 8, 'lambda_l2': 0.001, 'histogram_pool_size': 16384 }, 'DeepLearningRegressor': { 'epochs': epochs, 'batch_size': 50, 'verbose': 2 }, 'DeepLearningClassifier': { 'epochs': epochs, 'batch_size': 50, 'verbose': 2 }, 'CatBoostRegressor': {}, 'CatBoostClassifier': {} } # if os.environ.get('is_test_suite', 0) == 'True': # all_model_params model_params = all_model_params.get(model_name, None) if model_params is None: model_params = {} if is_hp_search == True: if model_name[:12] == 'DeepLearning': model_params['epochs'] = 50 if model_name[:4] == 'LGBM': model_params['n_estimators'] = 500 if training_params is not None: print('Now using the model training_params that you passed in:') print(training_params) # Overwrite our stock params with what the user passes in (i.e., if the user wants 10,000 trees, we will let them do it) model_params.update(training_params) print( 'After overwriting our defaults with your values, here are the final params that will be used to initialize the model:' ) print(model_params) model_map = { # Classifiers 'LogisticRegression': LogisticRegression(), 'RandomForestClassifier': RandomForestClassifier(), 'RidgeClassifier': RidgeClassifier(), 'GradientBoostingClassifier': GradientBoostingClassifier(), 'ExtraTreesClassifier': ExtraTreesClassifier(), 'AdaBoostClassifier': AdaBoostClassifier(), 'LinearSVC': LinearSVC(), # Regressors 'LinearRegression': LinearRegression(), 'RandomForestRegressor': RandomForestRegressor(), 'Ridge': Ridge(), 'LinearSVR': LinearSVR(), 'ExtraTreesRegressor': ExtraTreesRegressor(), 'AdaBoostRegressor': AdaBoostRegressor(), 'RANSACRegressor': RANSACRegressor(), 'GradientBoostingRegressor': GradientBoostingRegressor(), 'Lasso': Lasso(), 'ElasticNet': ElasticNet(), 'LassoLars': LassoLars(), 'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(), 'BayesianRidge': BayesianRidge(), 'ARDRegression': ARDRegression(), # Clustering 'MiniBatchKMeans': MiniBatchKMeans(), } try: model_map['SGDClassifier'] = SGDClassifier(max_iter=1000, tol=0.001) model_map['Perceptron'] = Perceptron(max_iter=1000, tol=0.001) model_map['PassiveAggressiveClassifier'] = PassiveAggressiveClassifier( max_iter=1000, tol=0.001) model_map['SGDRegressor'] = SGDRegressor(max_iter=1000, tol=0.001) model_map['PassiveAggressiveRegressor'] = PassiveAggressiveRegressor( max_iter=1000, tol=0.001) except TypeError: model_map['SGDClassifier'] = SGDClassifier() model_map['Perceptron'] = Perceptron() model_map['PassiveAggressiveClassifier'] = PassiveAggressiveClassifier( ) model_map['SGDRegressor'] = SGDRegressor() model_map['PassiveAggressiveRegressor'] = PassiveAggressiveRegressor() if xgb_installed: model_map['XGBClassifier'] = XGBClassifier() model_map['XGBRegressor'] = XGBRegressor() if lgb_installed: model_map['LGBMRegressor'] = LGBMRegressor() model_map['LGBMClassifier'] = LGBMClassifier() if catboost_installed: model_map['CatBoostRegressor'] = CatBoostRegressor( calc_feature_importance=True) model_map['CatBoostClassifier'] = CatBoostClassifier( calc_feature_importance=True) if model_name[:12] == 'DeepLearning': if keras_imported == False: # Suppress some level of logs if TF is installed (but allow it to not be installed, and use Theano instead) try: os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from tensorflow import logging logging.set_verbosity(logging.INFO) except: pass global maxnorm global Dense, Dropout global LeakyReLU, PReLU, ThresholdedReLU, ELU global Sequential global keras_load_model global regularizers, optimizers global Activation global KerasRegressor, KerasClassifier from keras.constraints import maxnorm from keras.layers import Activation, Dense, Dropout from keras.layers.advanced_activations import LeakyReLU, PReLU, ThresholdedReLU, ELU from keras.models import Sequential from keras.models import load_model as keras_load_model from keras import regularizers, optimizers from keras.wrappers.scikit_learn import KerasRegressor, KerasClassifier keras_imported = True model_map['DeepLearningClassifier'] = KerasClassifier( build_fn=make_deep_learning_classifier) model_map['DeepLearningRegressor'] = KerasRegressor( build_fn=make_deep_learning_model) try: model_without_params = model_map[model_name] except KeyError as e: print( 'It appears you are trying to use a library that is not available when we try to import it, or using a value for model_names that we do not recognize' ) raise (e) if os.environ.get('is_test_suite', False) == 'True': if 'n_jobs' in model_params: model_params['n_jobs'] = 1 model_with_params = model_without_params.set_params(**model_params) return model_with_params
@pytest.mark.parametrize('copy_X', [True, False]) def test_lasso_lars_fit_copyX_behaviour(copy_X): """ Test that user input to .fit for copy_X overrides default __init__ value """ lasso_lars = LassoLarsIC(precompute=False) rng = np.random.RandomState(0) X = rng.normal(0, 1, (100, 5)) X_copy = X.copy() y = X[:, 2] lasso_lars.fit(X, y, copy_X=copy_X) assert copy_X == np.array_equal(X, X_copy) @pytest.mark.parametrize('est', (LassoLars(alpha=1e-3), Lars())) def test_lars_with_jitter(est): # Test that a small amount of jitter helps stability, # using example provided in issue #2746 X = np.array([[0.0, 0.0, 0.0, -1.0, 0.0], [0.0, -1.0, 0.0, 0.0, 0.0]]) y = [-2.5, -2.5] expected_coef = [0, 2.5, 0, 2.5, 0] # set to fit_intercept to False since target is constant and we want check # the value of coef. coef would be all zeros otherwise. est.set_params(fit_intercept=False) est_jitter = clone(est).set_params(jitter=10e-8, random_state=0) est.fit(X, y)
logging.info('Scaling features...') column_name_list = list(feature_df.columns) print(len(column_name_list)) column_name_list.remove('time_to_failure') print(len(column_name_list)) feature_scaler = StandardScaler() feature_df[column_name_list] = feature_scaler.fit_transform(feature_df[column_name_list]) # Initialize models clf_ridg = Ridge(max_iter=5000) clf_laso = Lasso(max_iter=5000) clf_lala = LassoLars(max_iter=5000) clf_enet = ElasticNet(max_iter=5000) clf_xgbr = xgb.XGBRegressor() clf_xgrf = xgb.XGBRFRegressor() clf_rf = RandomForestRegressor(criterion='mae', max_features='sqrt') clf_tree = ExtraTreesRegressor(criterion='mae', max_features='sqrt') clf_ada = AdaBoostRegressor() clf_grad = GradientBoostingRegressor() clf_svr = SVR() # Model parameters # mae 2.160
def GetAllModelsForComparison(X_train, Y_train): models = { 'ARDRegression': ARDRegression(), 'BayesianRidge': BayesianRidge(), 'ElasticNet': ElasticNet(), 'ElasticNetCV': ElasticNetCV(), 'Hinge': Hinge(), #'Huber': Huber(), 'HuberRegressor': HuberRegressor(), 'Lars': Lars(), 'LarsCV': LarsCV(), 'Lasso': Lasso(), 'LassoCV': LassoCV(), 'LassoLars': LassoLars(), 'LassoLarsCV': LassoLarsCV(), 'LinearRegression': LinearRegression(), 'Log': Log(), 'LogisticRegression': LogisticRegression(), 'LogisticRegressionCV': LogisticRegressionCV(), 'ModifiedHuber': ModifiedHuber(), 'MultiTaskElasticNet': MultiTaskElasticNet(), 'MultiTaskElasticNetCV': MultiTaskElasticNetCV(), 'MultiTaskLasso': MultiTaskLasso(), 'MultiTaskLassoCV': MultiTaskLassoCV(), 'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(), 'OrthogonalMatchingPursuitCV': OrthogonalMatchingPursuitCV(), 'PassiveAggressiveClassifier': PassiveAggressiveClassifier(), 'PassiveAggressiveRegressor': PassiveAggressiveRegressor(), 'Perceptron': Perceptron(), 'RANSACRegressor': RANSACRegressor(), #'RandomizedLasso': RandomizedLasso(), #'RandomizedLogisticRegression': RandomizedLogisticRegression(), 'Ridge': Ridge(), 'RidgeCV': RidgeCV(), 'RidgeClassifier': RidgeClassifier(), 'SGDClassifier': SGDClassifier(), 'SGDRegressor': SGDRegressor(), 'SquaredLoss': SquaredLoss(), 'TheilSenRegressor': TheilSenRegressor(), 'BaseEstimator': BaseEstimator(), 'ClassifierMixin': ClassifierMixin(), 'LinearClassifierMixin': LinearClassifierMixin(), 'LinearDiscriminantAnalysis': LinearDiscriminantAnalysis(), 'QuadraticDiscriminantAnalysis': QuadraticDiscriminantAnalysis(), 'StandardScaler': StandardScaler(), 'TransformerMixin': TransformerMixin(), 'BaseEstimator': BaseEstimator(), 'KernelRidge': KernelRidge(), 'RegressorMixin': RegressorMixin(), 'LinearSVC': LinearSVC(), 'LinearSVR': LinearSVR(), 'NuSVC': NuSVC(), 'NuSVR': NuSVR(), 'OneClassSVM': OneClassSVM(), 'SVC': SVC(), 'SVR': SVR(), 'SGDClassifier': SGDClassifier(), 'SGDRegressor': SGDRegressor(), #'BallTree': BallTree(), #'DistanceMetric': DistanceMetric(), #'KDTree': KDTree(), 'KNeighborsClassifier': KNeighborsClassifier(), 'KNeighborsRegressor': KNeighborsRegressor(), 'KernelDensity': KernelDensity(), #'LSHForest': LSHForest(), 'LocalOutlierFactor': LocalOutlierFactor(), 'NearestCentroid': NearestCentroid(), 'NearestNeighbors': NearestNeighbors(), 'RadiusNeighborsClassifier': RadiusNeighborsClassifier(), 'RadiusNeighborsRegressor': RadiusNeighborsRegressor(), #'GaussianProcess': GaussianProcess(), 'GaussianProcessRegressor': GaussianProcessRegressor(), 'GaussianProcessClassifier': GaussianProcessClassifier(), 'CCA': CCA(), 'PLSCanonical': PLSCanonical(), 'PLSRegression': PLSRegression(), 'PLSSVD': PLSSVD(), #'ABCMeta': ABCMeta(), #'BaseDiscreteNB': BaseDiscreteNB(), 'BaseEstimator': BaseEstimator(), #'BaseNB': BaseNB(), 'BernoulliNB': BernoulliNB(), 'ClassifierMixin': ClassifierMixin(), 'GaussianNB': GaussianNB(), 'LabelBinarizer': LabelBinarizer(), 'MultinomialNB': MultinomialNB(), 'DecisionTreeClassifier': DecisionTreeClassifier(), 'DecisionTreeRegressor': DecisionTreeRegressor(), 'ExtraTreeClassifier': ExtraTreeClassifier(), 'AdaBoostClassifier': AdaBoostClassifier(), 'AdaBoostRegressor': AdaBoostRegressor(), 'BaggingClassifier': BaggingClassifier(), 'BaggingRegressor': BaggingRegressor(), #'BaseEnsemble': BaseEnsemble(), 'ExtraTreesClassifier': ExtraTreesClassifier(), 'ExtraTreesRegressor': ExtraTreesRegressor(), 'GradientBoostingClassifier': GradientBoostingClassifier(), 'GradientBoostingRegressor': GradientBoostingRegressor(), 'IsolationForest': IsolationForest(), 'RandomForestClassifier': RandomForestClassifier(), 'RandomForestRegressor': RandomForestRegressor(), 'RandomTreesEmbedding': RandomTreesEmbedding(), #'VotingClassifier': VotingClassifier(), 'BaseEstimator': BaseEstimator(), 'ClassifierMixin': ClassifierMixin(), 'LabelBinarizer': LabelBinarizer(), 'MetaEstimatorMixin': MetaEstimatorMixin(), #'OneVsOneClassifier': OneVsOneClassifier(), #'OneVsRestClassifier': OneVsRestClassifier(), #'OutputCodeClassifier': OutputCodeClassifier(), 'Parallel': Parallel(), #'ABCMeta': ABCMeta(), 'BaseEstimator': BaseEstimator(), #'ClassifierChain': ClassifierChain(), 'ClassifierMixin': ClassifierMixin(), 'MetaEstimatorMixin': MetaEstimatorMixin(), #'MultiOutputClassifier': MultiOutputClassifier(), #'MultiOutputEstimator': MultiOutputEstimator(), #'MultiOutputRegressor': MultiOutputRegressor(), 'Parallel': Parallel(), 'RegressorMixin': RegressorMixin(), 'LabelPropagation': LabelPropagation(), 'LabelSpreading': LabelSpreading(), 'BaseEstimator': BaseEstimator(), 'IsotonicRegression': IsotonicRegression(), 'RegressorMixin': RegressorMixin(), 'TransformerMixin': TransformerMixin(), 'BernoulliRBM': BernoulliRBM(), 'MLPClassifier': MLPClassifier(), 'MLPRegressor': MLPRegressor() } return models
from sklearn.tree import DecisionTreeRegressor from sklearn.ensemble import RandomForestRegressor from sklearn.ensemble import ExtraTreesRegressor from sklearn.ensemble import AdaBoostRegressor from sklearn.ensemble import GradientBoostingRegressor from sklearn.neural_network import MLPRegressor from xgboost import XGBRegressor aml_basic_regressors = [ ('model1', LinearRegression()), ('model2', Lasso()), ('model3', Ridge()), ('model4', ElasticNet()), ('model5', Lars()), ('model6', LassoLars()), ('model7', OrthogonalMatchingPursuit()), ('model8', BayesianRidge()), ('model9', ARDRegression()), ('model10', PassiveAggressiveRegressor()), ('model11', RANSACRegressor()), ('model12', TheilSenRegressor()), ('model13', HuberRegressor()), ('model14', KernelRidge()), ('model15', SVR()), ('model16', KNeighborsRegressor()), ('model17', DecisionTreeRegressor()), ('model18', RandomForestRegressor()), ('model19', ExtraTreesRegressor()), ('model20', AdaBoostRegressor()), ('model21', GradientBoostingRegressor()),
print(len(test_column_name_list)) test_column_name_list.remove('seg_id') print(len(test_column_name_list)) feature_scaler = StandardScaler() feature_df[feat_column_name_list] = feature_scaler.fit_transform( feature_df[feat_column_name_list]) test_x[test_column_name_list] = feature_scaler.transform( test_x[test_column_name_list]) # Initialize models clf_line = LinearRegression() clf_ridg = Ridge(alpha=300, tol=1e-05, solver='sparse_cg', max_iter=5000) clf_laso = Lasso(alpha=0.1, tol=1e-05, max_iter=5000) clf_lala = LassoLars(alpha=0.001, max_iter=5000) clf_enet = ElasticNet(alpha=0.1, tol=0.001, l1_ratio=0.2, max_iter=5000) clf_xgbr = xgb.XGBRegressor() # not yet clf_xgrf = xgb.XGBRFRegressor() # not yet clf_rf = RandomForestRegressor(criterion='mae', max_features='sqrt', n_estimators=200, max_depth=10) clf_tree = ExtraTreesRegressor(criterion='mae', max_features='sqrt', n_estimators=200, max_depth=10) clf_ada = AdaBoostRegressor(n_estimators=3, loss='linear') clf_grad = GradientBoostingRegressor() # not yet
def sparse_encode(X, dictionary, algorithm='mp', fit_tol=None, P_cum=None, l0_sparseness=10, C=0., do_sym=True, verbose=0): """Generic sparse coding Each column of the result is the solution to a sparse coding problem. Parameters ---------- X : array of shape (n_samples, n_pixels) Data matrix. dictionary : array of shape (n_dictionary, n_pixels) The dictionary matrix against which to solve the sparse coding of the data. Some of the algorithms assume normalized rows. algorithm : {'mp', 'lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'} mp : Matching Pursuit lars: uses the least angle regression method (linear_model.lars_path) lasso_lars: uses Lars to compute the Lasso solution lasso_cd: uses the coordinate descent method to compute the Lasso solution (linear_model.Lasso). lasso_lars will be faster if the estimated dictionary are sparse. omp: uses orthogonal matching pursuit to estimate the sparse solution threshold: squashes to zero all coefficients less than regularization from the projection dictionary * data' max_iter : int, 1000 by default Maximum number of iterations to perform if `algorithm='lasso_cd'`. verbose : int Controls the verbosity; the higher, the more messages. Defaults to 0. Returns ------- code : array of shape (n_samples, n_dictionary) The sparse codes """ if X.ndim == 1: X = X[:, np.newaxis] #n_samples, n_pixels = X.shape if algorithm == 'lasso_lars': alpha = float(regularization) / n_pixels # account for scaling from sklearn.linear_model import LassoLars # Not passing in verbose=max(0, verbose-1) because Lars.fit already # corrects the verbosity level. cov = np.dot(dictionary, X.T) lasso_lars = LassoLars(alpha=fit_tol, fit_intercept=False, verbose=verbose, normalize=False, precompute=None, fit_path=False) lasso_lars.fit(dictionary.T, X.T, Xy=cov) sparse_code = lasso_lars.coef_.T elif algorithm == 'lasso_cd': alpha = float(regularization) / n_pixels # account for scaling # TODO: Make verbosity argument for Lasso? # sklearn.linear_model.coordinate_descent.enet_path has a verbosity # argument that we could pass in from Lasso. from sklearn.linear_model import Lasso clf = Lasso(alpha=fit_tol, fit_intercept=False, normalize=False, precompute=None, max_iter=max_iter, warm_start=True) if init is not None: clf.coef_ = init clf.fit(dictionary.T, X.T, check_input=check_input) sparse_code = clf.coef_.T elif algorithm == 'lars': # Not passing in verbose=max(0, verbose-1) because Lars.fit already # corrects the verbosity level. from sklearn.linear_model import Lars cov = np.dot(dictionary, X.T) lars = Lars(fit_intercept=False, verbose=verbose, normalize=False, precompute=None, n_nonzero_coefs=l0_sparseness, fit_path=False) lars.fit(dictionary.T, X.T, Xy=cov) sparse_code = lars.coef_.T elif algorithm == 'threshold': cov = np.dot(dictionary, X.T) sparse_code = ((np.sign(cov) * np.maximum(np.abs(cov) - regularization, 0))).T elif algorithm == 'omp': # TODO: Should verbose argument be passed to this? from sklearn.linear_model import orthogonal_mp_gram from sklearn.utils.extmath import row_norms cov = np.dot(dictionary, X.T) gram = np.dot(dictionary, dictionary.T) sparse_code = orthogonal_mp_gram(Gram=gram, Xy=cov, n_nonzero_coefs=l0_sparseness, tol=None, norms_squared=row_norms(X, squared=True), copy_Xy=False).T elif algorithm == 'mp': sparse_code = mp(X, dictionary, l0_sparseness=l0_sparseness, fit_tol=fit_tol, P_cum=P_cum, C=C, do_sym=do_sym, verbose=verbose) else: raise ValueError( 'Sparse coding method must be "mp", "lasso_lars" ' '"lasso_cd", "lasso", "threshold" or "omp", got %s.' % algorithm) return sparse_code
################################### MODELS ############################################################### ### SGDRegressor from sklearn.linear_model import SGDRegressor regressor_sgd = SGDRegressor() regressor_sgd.fit(X_train,y_train) ### BayesianRidge from sklearn.linear_model import BayesianRidge regressor_br = BayesianRidge() regressor_br.fit(X_train,y_train) ### LassoLars from sklearn.linear_model import LassoLars regressor_ll = LassoLars() regressor_ll.fit(X_train,y_train) from sklearn.linear_model import XGBRegressor from xgboost import XGBRegressor regressor_xgb = XGBRegressor() regressor_xgb.fit(X_train,y_train) # Applying K-fold cross validation from sklearn.model_selection import cross_val_score accuracies_sgd = cross_val_score(estimator = regressor_sgd, X = X_train, y = y_train, cv = 10, n_jobs = -1) accuracies_br = cross_val_score(estimator = regressor_br, X = X_train, y = y_train, cv = 10, n_jobs = -1) accuracies_ll = cross_val_score(estimator = regressor_ll, X = X_train, y = y_train, cv = 5, n_jobs = -1)
x_ts = np.concatenate((x_test, np.square(x_test), np.power(x_test, 3)), axis=1) # print (MSELasso(y_test,pred.reshape((pred.size,1)))) vals = [0.0000001, 0.0001, 1, 10] errors = np.empty(4) for j in range(4): lm = vals[j] k = 4 err = np.empty(k) l = int(np.ma.size(x_train, axis=0) / k) x_cv, x_tr = np.split(x_train.copy(), [l], axis=0) y_cv, y_tr = np.split(y_train.copy(), [l], axis=0) model = LassoLars(alpha=lm) model.fit(x_tr, y_tr.ravel()) pred = model.predict(x_cv) err[0] = MSELasso(y_cv, pred.reshape((pred.size, 1))) for i in range(k - 1): x_tr[i * l:(i + 1) * l], x_cv = x_cv, x_tr[i * l:(i + 1) * l].copy() y_tr[i * l:(i + 1) * l], y_cv = y_cv, y_tr[i * l:(i + 1) * l].copy() model = LassoLars(alpha=lm) model.fit(x_tr, y_tr.ravel()) pred = model.predict(x_cv) err[i + 1] = MSELasso(y_cv, pred.reshape((pred.size, 1))) errors[j] = np.mean(err)