def test_cross_val_criterion(): alpha_min = alpha_max / 10 log_alpha_max = np.log(alpha_max) log_alpha_min = np.log(alpha_min) max_iter = 10000 n_alphas = 10 kf = KFold(n_splits=5, shuffle=True, random_state=56) estimator = sklearn.linear_model.Lasso(fit_intercept=False, max_iter=1000, warm_start=True) monitor_grid = Monitor() criterion = CrossVal(X, y, Lasso, cv=kf, estimator=estimator) algo = Forward() grid_search(algo, criterion, log_alpha_min, log_alpha_max, monitor_grid, max_evals=n_alphas, tol=tol) reg = LassoCV(cv=kf, verbose=True, tol=tol, fit_intercept=False, alphas=np.geomspace(alpha_max, alpha_min, num=n_alphas), max_iter=max_iter).fit(X, y) reg.score(X, y) objs_grid_sk = reg.mse_path_.mean(axis=1) # these 2 value should be the same (objs_grid_sk - np.array(monitor_grid.objs)) assert np.allclose(objs_grid_sk, monitor_grid.objs)
def test_cross_val_criterion(model_name, XX): model = models[model_name] alpha_min = alpha_max / 10 max_iter = 10000 n_alphas = 10 kf = KFold(n_splits=5, shuffle=True, random_state=56) monitor_grid = Monitor() if model_name.startswith("lasso"): sub_crit = HeldOutMSE(None, None) else: sub_crit = HeldOutLogistic(None, None) criterion = CrossVal(sub_crit, cv=kf) grid_search(criterion, model, XX, y, alpha_min, alpha_max, monitor_grid, max_evals=n_alphas, tol=tol) if model_name.startswith("lasso"): reg = linear_model.LassoCV(cv=kf, verbose=True, tol=tol, fit_intercept=False, alphas=np.geomspace(alpha_max, alpha_min, num=n_alphas), max_iter=max_iter).fit(X, y) else: reg = linear_model.LogisticRegressionCV( cv=kf, verbose=True, tol=tol, fit_intercept=False, Cs=len(idx_train) / np.geomspace(alpha_max, alpha_min, num=n_alphas), max_iter=max_iter, penalty='l1', solver='liblinear').fit(X, y) reg.score(XX, y) if model_name.startswith("lasso"): objs_grid_sk = reg.mse_path_.mean(axis=1) else: objs_grid_sk = reg.scores_[1.0].mean(axis=1) # these 2 value should be the same (objs_grid_sk - np.array(monitor_grid.objs)) np.testing.assert_allclose(objs_grid_sk, monitor_grid.objs)
def test_grid_search(): max_evals = 5 monitor_grid = Monitor() model = Lasso(estimator=estimator) criterion = HeldOutMSE(idx_train, idx_train) algo = Forward() log_alpha_opt_grid, _ = grid_search( algo, criterion, model, X, y, log_alpha_min, log_alpha_max, monitor_grid, max_evals=max_evals, tol=1e-5, samp="grid") monitor_random = Monitor() criterion = HeldOutMSE(idx_train, idx_val) algo = Forward() log_alpha_opt_random, _ = grid_search( algo, criterion, model, X, y, log_alpha_min, log_alpha_max, monitor_random, max_evals=max_evals, tol=1e-5, samp="random") assert(monitor_random.log_alphas[ np.argmin(monitor_random.objs)] == log_alpha_opt_random) assert(monitor_grid.log_alphas[ np.argmin(monitor_grid.objs)] == log_alpha_opt_grid) monitor_grid = Monitor() model = Lasso(estimator=estimator) criterion = SmoothedSURE(sigma=sigma_star) algo = Forward() log_alpha_opt_grid, _ = grid_search( algo, criterion, model, X, y, log_alpha_min, log_alpha_max, monitor_grid, max_evals=max_evals, tol=1e-5, samp="grid") monitor_random = Monitor() criterion = SmoothedSURE(sigma=sigma_star) algo = Forward() log_alpha_opt_random, _ = grid_search( algo, criterion, model, X, y, log_alpha_min, log_alpha_max, monitor_random, max_evals=max_evals, tol=1e-5, samp="random") assert(monitor_random.log_alphas[ np.argmin(monitor_random.objs)] == log_alpha_opt_random) assert(monitor_grid.log_alphas[ np.argmin(monitor_grid.objs)] == log_alpha_opt_grid)
estimator = linear_model.Lasso(fit_intercept=False, max_iter=1000, warm_start=True) print('scikit-learn started') t0 = time.time() model = Lasso(estimator=estimator) criterion = HeldOutMSE(idx_train, idx_val) algo = Forward() monitor_grid_sk = Monitor() grid_search(algo, criterion, model, X, y, None, None, monitor_grid_sk, log_alphas=log_alphas, tol=tol) objs = np.array(monitor_grid_sk.objs) t_sk = time.time() - t0 print('scikit-learn finished') ############################################################################## # Grad-search with sparse-ho # -------------------------- print('sparse-ho started')
verbose=True, tol=tol, tol_decrease="geom") else: optimizer = GradientDescent(n_outer=30, p_grad_norm=1., verbose=True, tol=tol) grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor) elif algo_name == 'grid_search': grid_search(algo, criterion, model, X, y, None, None, monitor, max_evals=20, tol=tol, alphas=alphas) elif algo_name == 'random_search' or algo_name == 'bayesian': hyperopt_wrapper(algo, criterion, model, X, y, alpha_min, alpha_max, monitor, max_evals=20,
############################################################################## # Grid-search # ----------- estimator = LogisticRegression(penalty='l1', fit_intercept=False, max_iter=max_iter) model = SparseLogreg(estimator=estimator) criterion = HeldOutLogistic(idx_train, idx_val) monitor_grid = Monitor() grid_search(criterion, model, X, y, alpha_min, alpha_max, monitor_grid, alphas=alphas, tol=tol) objs = np.array(monitor_grid.objs) ############################################################################## # Grad-search # ----------- optimizer_names = ['line-search', 'gradient-descent', 'adam'] optimizers = { 'line-search': LineSearch(n_outer=10, tol=tol), 'gradient-descent': GradientDescent(n_outer=10, step_size=100), 'adam': Adam(n_outer=10, lr=0.11) }
def parallel_function(dataset_name, method, tol=1e-5, n_outer=50, tolerance_decrease='exponential'): # load data X_train, X_val, X_test, y_train, y_val, y_test = get_data(dataset_name) n_samples, n_features = X_train.shape print('n_samples', n_samples) print('n_features', n_features) y_train[y_train == 0.0] = -1.0 y_val[y_val == 0.0] = -1.0 y_test[y_test == 0.0] = -1.0 alpha_max = np.max(np.abs(X_train.T @ y_train)) alpha_max /= X_train.shape[0] log_alpha_max = np.log(alpha_max) alpha_min = alpha_max * 1e-2 # alphas = np.geomspace(alpha_max, alpha_min, 10) # log_alphas = np.log(alphas) log_alpha1_0 = np.log(0.1 * alpha_max) log_alpha2_0 = np.log(0.1 * alpha_max) log_alpha_max = np.log(alpha_max) n_outer = 25 if dataset_name == "rcv1": size_loop = 2 else: size_loop = 2 model = ElasticNet(X_train, y_train, log_alpha1_0, log_alpha2_0, log_alpha_max, max_iter=1000, tol=tol) for i in range(size_loop): monitor = Monitor() if method == "implicit_forward": criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = ImplicitForward(criterion, tol_jac=1e-3, n_iter_jac=100) _, _, _ = grad_search(algo=algo, verbose=False, log_alpha0=np.array( [log_alpha1_0, log_alpha2_0]), tol=tol, n_outer=n_outer, monitor=monitor, t_max=dict_t_max[dataset_name], tolerance_decrease=tolerance_decrease) elif method == "forward": criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) _, _, _ = grad_search(algo=algo, log_alpha0=np.array( [log_alpha1_0, log_alpha2_0]), tol=tol, n_outer=n_outer, monitor=monitor, t_max=dict_t_max[dataset_name], tolerance_decrease=tolerance_decrease) elif method == "implicit": criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Implicit(criterion) _, _, _ = grad_search(algo=algo, log_alpha0=np.array( [log_alpha1_0, log_alpha2_0]), tol=tol, n_outer=n_outer, monitor=monitor, t_max=dict_t_max[dataset_name], tolerance_decrease=tolerance_decrease) elif method == "grid_search": criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) log_alpha_min = np.log(alpha_min) log_alpha_opt, min_g_func = grid_search( algo, log_alpha_min, log_alpha_max, monitor, max_evals=10, tol=tol, samp="grid", t_max=dict_t_max[dataset_name], log_alphas=None, nb_hyperparam=2) print(log_alpha_opt) elif method == "random": criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) log_alpha_min = np.log(alpha_min) log_alpha_opt, min_g_func = grid_search( algo, log_alpha_min, np.log(alpha_max), monitor, max_evals=10, tol=tol, samp="random", t_max=dict_t_max[dataset_name], nb_hyperparam=2) print(log_alpha_opt) elif method == "lhs": criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) log_alpha_min = np.log(alpha_min) log_alpha_opt, min_g_func = grid_search( algo, log_alpha_min, np.log(alpha_max), monitor, max_evals=10, tol=tol, samp="lhs", t_max=dict_t_max[dataset_name]) print(log_alpha_opt) monitor.times = np.array(monitor.times).copy() monitor.objs = np.array(monitor.objs).copy() monitor.objs_test = np.array(monitor.objs_test).copy() monitor.log_alphas = np.array(monitor.log_alphas).copy() return (dataset_name, method, tol, n_outer, tolerance_decrease, monitor.times, monitor.objs, monitor.objs_test, monitor.log_alphas, norm(y_val), norm(y_test), log_alpha_max)
# X_train, X_val, X_test, y_train, y_val, y_test = get_leukemia() n_samples, n_features = X_train.shape print("Starting path computation...") alpha_max = np.max(np.abs(X_train.T @ (-y_train))) alpha_max /= (2 * n_samples) n_alphas = 10 p_alphas = np.geomspace(1, 1e-4, n_alphas) alphas = p_alphas * alpha_max log_alphas = np.log(alphas) tol = 1e-5 # grid search model = SparseLogreg(X_train, y_train, log_alphas[0], max_iter=1000) criterion = HeldOutLogistic(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) monitor_grid_sk = Monitor() grid_search(algo, None, None, monitor_grid_sk, log_alphas=log_alphas, tol=tol) monitor = Monitor() # grad_search( # algo, logCs[0], monitor, n_outer=5, verbose=True, # tolerance_decrease='constant', tol=1e-8, # t_max=10000) plt.figure() plt.plot(monitor_grid_sk.log_alphas, monitor_grid_sk.objs) plt.plot(monitor.log_alphas, monitor.objs, 'bo') plt.show(block=False)
# Grid search with scikit-learn # ----------------------------- estimator = celer.Lasso(fit_intercept=False, warm_start=True) print('Grid search started') t0 = time.time() model = Lasso(estimator=estimator) criterion = HeldOutMSE(idx_train, idx_val) monitor_grid_sk = Monitor() grid_search(criterion, model, X, y, None, None, monitor_grid_sk, alphas=alphas, tol=tol) objs = np.array(monitor_grid_sk.objs) t_sk = time.time() - t0 print('Grid search finished') ############################################################################## # Grad-search with sparse-ho # -------------------------- print('sparse-ho started')
monitor = Monitor() kf = KFold(n_splits=5, shuffle=True, random_state=42) criterion = CrossVal(sub_criterion, cv=kf) algo = ImplicitForward(tol_jac=1e-3) optimizer = GradientDescent(n_outer=30, p_grad_norm=1., verbose=True, tol=tol) if algo_name == 'implicit_forward': grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor) elif algo_name == 'grid_search': grid_search(algo, criterion, model, X, y, alpha_min, alpha_max, monitor, max_evals=20, tol=tol) elif algo_name == 'random_search': hyperopt_wrapper(algo, criterion, model, X, y, alpha_min, alpha_max, monitor, max_evals=20, tol=tol,
def parallel_function(dataset_name, method, tol=1e-5, n_outer=50, tolerance_decrease='constant'): # load data X_train, X_val, X_test, y_train, y_val, y_test = get_data(dataset_name) n_samples, _ = X_train.shape # compute alpha_max alpha_max = np.abs(X_train.T @ y_train).max() / n_samples if model_name == "logreg": alpha_max /= 2 alpha_min = alpha_max / 10_000 log_alpha_max = np.log(alpha_max) log_alpha_min = np.log(alpha_min) log_alpha0 = np.log(0.1 * alpha_max) if model_name == "lasso": model = Lasso(X_train, y_train) elif model_name == "logreg": model = SparseLogreg(X_train, y_train) try: n_outer = dict_n_outers[dataset_name, method] except Exception: n_outer = 20 size_loop = 2 for _ in range(size_loop): if model_name == "lasso": criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) elif model_name == "logreg": criterion = HeldOutLogistic(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = dict_algo[method](criterion) monitor = Monitor() if method == 'grid_search': log_alphas = np.log(np.geomspace(alpha_max, alpha_min, num=100)) grid_search(algo, None, None, monitor, log_alphas=log_alphas, tol=tol) elif method == 'random': grid_search(algo, log_alpha_max, log_alpha_min, monitor, tol=tol, max_evals=n_alphas, t_max=dict_t_max[dataset_name]) elif method in ("bayesian"): hyperopt_wrapper(algo, log_alpha_min, log_alpha_max, monitor, max_evals=n_alphas, tol=tol, method='bayesian', t_max=dict_t_max[dataset_name]) else: # do line search to find the optimal lambda grad_search(algo, log_alpha0, monitor, n_outer=n_outer, tol=tol, tolerance_decrease=tolerance_decrease, t_max=dict_t_max[dataset_name]) monitor.times = np.array(monitor.times) monitor.objs = np.array(monitor.objs) monitor.objs_test = np.array(monitor.objs_test) monitor.log_alphas = np.array(monitor.log_alphas) return (dataset_name, method, tol, n_outer, tolerance_decrease, monitor.times, monitor.objs, monitor.objs_test, monitor.log_alphas, norm(y_val), norm(y_test), log_alpha_max, model_name)
optimizer = LineSearch(n_outer=25, verbose=True, tol=tol) grad_search( algo, cross_val_criterion, model, optimizer, X, y, log_alpha0, monitor) elif algorithm.startswith('grid_search'): if algorithm == 'grid_search10': n_alphas = 5 else: n_alphas = 30 p_alphas = np.geomspace(1, p_alpha_min, n_alphas) alphas = alpha_max * p_alphas log_alphas = np.log(alphas) grid_alphas = [i for i in itertools.product(log_alphas, log_alphas)] grid_search( algo, cross_val_criterion, model, X, y, None, None, monitor, log_alphas=grid_alphas) else: hyperopt_wrapper( algo, cross_val_criterion, model, X, y, log_alpha_min, log_alpha_max, monitor, max_evals=max_evals, method=algorithm, size_space=2) objs = np.array(monitor.objs) log_alphas = np.array(monitor.log_alphas) log_alphas -= np.log(alpha_max) np.save("results/%s_log_alphas_%s_enet" % (dataset, algorithm), log_alphas) np.save("results/%s_objs_%s_enet" % (dataset, algorithm), objs) print('%s finished' % algorithm)
def parallel_function( dataset_name, method, tol=1e-5, n_outer=50, tolerance_decrease='exponential'): # load data X_train, X_val, X_test, y_train, y_val, y_test = get_data(dataset_name, csr=True) n_samples, n_features = X_train.shape print('n_samples', n_samples) print('n_features', n_features) y_train[y_train == 0.0] = -1.0 y_val[y_val == 0.0] = -1.0 y_test[y_test == 0.0] = -1.0 C_max = 100 logC = np.log(1e-2) n_outer = 5 if dataset_name == "rcv1": size_loop = 1 else: size_loop = 1 model = SVM( X_train, y_train, logC, max_iter=10000, tol=tol) for i in range(size_loop): monitor = Monitor() if method == "implicit_forward": criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = ImplicitForward(criterion, tol_jac=1e-3, n_iter_jac=100) _, _, _ = grad_search( algo=algo, verbose=False, log_alpha0=logC, tol=tol, n_outer=n_outer, monitor=monitor, t_max=dict_t_max[dataset_name], tolerance_decrease=tolerance_decrease) elif method == "forward": criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) _, _, _ = grad_search( algo=algo, log_alpha0=logC, tol=tol, n_outer=n_outer, monitor=monitor, t_max=dict_t_max[dataset_name], tolerance_decrease=tolerance_decrease) elif method == "implicit": criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Implicit(criterion) _, _, _ = grad_search( algo=algo, log_alpha0=logC, tol=tol, n_outer=n_outer, monitor=monitor, t_max=dict_t_max[dataset_name], tolerance_decrease=tolerance_decrease) elif method == "grid_search": criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) log_alpha_min = np.log(1e-2) log_alpha_opt, min_g_func = grid_search( algo, log_alpha_min, np.log(C_max), monitor, max_evals=25, tol=tol, samp="grid") print(log_alpha_opt) elif method == "random": criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) log_alpha_min = np.log(1e-2) log_alpha_opt, min_g_func = grid_search( algo, log_alpha_min, np.log(C_max), monitor, max_evals=25, tol=tol, samp="random") print(log_alpha_opt) elif method == "lhs": criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward(criterion) log_alpha_min = np.log(1e-2) log_alpha_opt, min_g_func = grid_search( algo, log_alpha_min, np.log(C_max), monitor, max_evals=25, tol=tol, samp="lhs") print(log_alpha_opt) monitor.times = np.array(monitor.times) monitor.objs = np.array(monitor.objs) monitor.objs_test = np.array(monitor.objs_test) monitor.log_alphas = np.array(monitor.log_alphas) return (dataset_name, method, tol, n_outer, tolerance_decrease, monitor.times, monitor.objs, monitor.objs_test, monitor.log_alphas, norm(y_val), norm(y_test))
def parallel_function(dataset_name, method, tol=1e-5, n_outer=50, tolerance_decrease='constant'): # load data X, y = fetch_libsvm(dataset_name) y -= np.mean(y) # compute alpha_max alpha_max = np.abs(X.T @ y).max() / len(y) if model_name == "logreg": alpha_max /= 2 alpha_min = alpha_max * dict_palphamin[dataset_name] if model_name == "enet": estimator = linear_model.ElasticNet(fit_intercept=False, max_iter=10_000, warm_start=True, tol=tol) model = ElasticNet(estimator=estimator) elif model_name == "logreg": model = SparseLogreg(estimator=estimator) # TODO improve this try: n_outer = dict_n_outers[dataset_name, method] except Exception: n_outer = 20 size_loop = 2 for _ in range(size_loop): if model_name == "lasso" or model_name == "enet": sub_criterion = HeldOutMSE(None, None) elif model_name == "logreg": criterion = HeldOutLogistic(None, None) kf = KFold(n_splits=5, shuffle=True, random_state=42) criterion = CrossVal(sub_criterion, cv=kf) algo = ImplicitForward(tol_jac=1e-3) monitor = Monitor() t_max = dict_t_max[dataset_name] if method == 'grid_search': num1D = dict_point_grid_search[dataset_name] alpha1D = np.geomspace(alpha_max, alpha_min, num=num1D) alphas = [np.array(i) for i in product(alpha1D, alpha1D)] grid_search(algo, criterion, model, X, y, alpha_min, alpha_max, monitor, max_evals=100, tol=tol, alphas=alphas) elif method == 'random' or method == 'bayesian': hyperopt_wrapper(algo, criterion, model, X, y, alpha_min, alpha_max, monitor, max_evals=30, tol=tol, method=method, size_space=2, t_max=t_max) elif method.startswith("implicit_forward"): # do gradient descent to find the optimal lambda alpha0 = np.array([alpha_max / 100, alpha_max / 100]) n_outer = 30 if method == 'implicit_forward': optimizer = GradientDescent(n_outer=n_outer, p_grad_norm=1, verbose=True, tol=tol, t_max=t_max) else: optimizer = GradientDescent(n_outer=n_outer, p_grad_norm=1, verbose=True, tol=tol, t_max=t_max, tol_decrease="geom") grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor) else: raise NotImplementedError monitor.times = np.array(monitor.times) monitor.objs = np.array(monitor.objs) monitor.objs_test = 0 # TODO monitor.alphas = np.array(monitor.alphas) return (dataset_name, method, tol, n_outer, tolerance_decrease, monitor.times, monitor.objs, monitor.objs_test, monitor.alphas, alpha_max, model_name)
def test_grid_search(): max_evals = 5 monitor_grid = Monitor() model = Lasso(X_train, y_train, estimator=estimator) criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward() log_alpha_opt_grid, _ = grid_search(algo, criterion, log_alpha_min, log_alpha_max, monitor_grid, max_evals=max_evals, tol=1e-5, samp="grid") monitor_random = Monitor() criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test) algo = Forward() log_alpha_opt_random, _ = grid_search(algo, criterion, log_alpha_min, log_alpha_max, monitor_random, max_evals=max_evals, tol=1e-5, samp="random") assert (monitor_random.log_alphas[np.argmin( monitor_random.objs)] == log_alpha_opt_random) assert (monitor_grid.log_alphas[np.argmin( monitor_grid.objs)] == log_alpha_opt_grid) monitor_grid = Monitor() model = Lasso(X_train, y_train, estimator=estimator) criterion = SURE(X_train, y_train, model, sigma=sigma_star, X_test=X_test, y_test=y_test) algo = Forward() log_alpha_opt_grid, _ = grid_search(algo, criterion, log_alpha_min, log_alpha_max, monitor_grid, max_evals=max_evals, tol=1e-5, samp="grid") monitor_random = Monitor() criterion = SURE(X_train, y_train, model, sigma=sigma_star, X_test=X_test, y_test=y_test) algo = Forward() log_alpha_opt_random, _ = grid_search(algo, criterion, log_alpha_min, log_alpha_max, monitor_random, max_evals=max_evals, tol=1e-5, samp="random") assert (monitor_random.log_alphas[np.argmin( monitor_random.objs)] == log_alpha_opt_random) assert (monitor_grid.log_alphas[np.argmin( monitor_grid.objs)] == log_alpha_opt_grid)
print('scikit started') t0 = time.time() estimator = LogisticRegression(penalty='l1', fit_intercept=False, max_iter=max_iter) model = SparseLogreg(max_iter=max_iter, estimator=estimator) criterion = HeldOutLogistic(idx_train, idx_val) algo_grid = Forward() monitor_grid = Monitor() grid_search(algo_grid, criterion, model, X, y, log_alpha_min, log_alpha_max, monitor_grid, log_alphas=log_alphas, tol=tol) objs = np.array(monitor_grid.objs) t_sk = time.time() - t0 print('scikit finished') print("Time to compute CV for scikit-learn: %.2f" % t_sk) ############################################################################## # Grad-search # -----------
def test_grid_search(): max_evals = 5 monitor_grid = Monitor() model = Lasso(estimator=estimator) criterion = HeldOutMSE(idx_train, idx_train) alpha_opt_grid, _ = grid_search(criterion, model, X, y, alpha_min, alpha_max, monitor_grid, max_evals=max_evals, tol=1e-5, samp="grid") monitor_random = Monitor() criterion = HeldOutMSE(idx_train, idx_val) alpha_opt_random, _ = grid_search(criterion, model, X, y, alpha_min, alpha_max, monitor_random, max_evals=max_evals, tol=1e-5, samp="random") np.testing.assert_allclose( monitor_random.alphas[np.argmin(monitor_random.objs)], alpha_opt_random) np.testing.assert_allclose( monitor_grid.alphas[np.argmin(monitor_grid.objs)], alpha_opt_grid) monitor_grid = Monitor() model = Lasso(estimator=estimator) criterion = FiniteDiffMonteCarloSure(sigma=sigma_star) alpha_opt_grid, _ = grid_search(criterion, model, X, y, alpha_min, alpha_max, monitor_grid, max_evals=max_evals, tol=1e-5, samp="grid") monitor_random = Monitor() criterion = FiniteDiffMonteCarloSure(sigma=sigma_star) alpha_opt_random, _ = grid_search(criterion, model, X, y, alpha_min, alpha_max, monitor_random, max_evals=max_evals, tol=1e-5, samp="random") np.testing.assert_allclose( monitor_random.alphas[np.argmin(monitor_random.objs)], alpha_opt_random) np.testing.assert_allclose( monitor_grid.alphas[np.argmin(monitor_grid.objs)], alpha_opt_grid)