def test_cross_val_criterion():
alpha_min = alpha_max / 10
log_alpha_max = np.log(alpha_max)
log_alpha_min = np.log(alpha_min)
max_iter = 10000
n_alphas = 10
kf = KFold(n_splits=5, shuffle=True, random_state=56)
estimator = sklearn.linear_model.Lasso(fit_intercept=False,
max_iter=1000,
warm_start=True)
monitor_grid = Monitor()
criterion = CrossVal(X, y, Lasso, cv=kf, estimator=estimator)
algo = Forward()
grid_search(algo,
criterion,
log_alpha_min,
log_alpha_max,
monitor_grid,
max_evals=n_alphas,
tol=tol)
reg = LassoCV(cv=kf,
verbose=True,
tol=tol,
fit_intercept=False,
alphas=np.geomspace(alpha_max, alpha_min, num=n_alphas),
max_iter=max_iter).fit(X, y)
reg.score(X, y)
objs_grid_sk = reg.mse_path_.mean(axis=1)
# these 2 value should be the same
(objs_grid_sk - np.array(monitor_grid.objs))
assert np.allclose(objs_grid_sk, monitor_grid.objs)
def test_cross_val_criterion(model_name, XX):
model = models[model_name]
alpha_min = alpha_max / 10
max_iter = 10000
n_alphas = 10
kf = KFold(n_splits=5, shuffle=True, random_state=56)
monitor_grid = Monitor()
if model_name.startswith("lasso"):
sub_crit = HeldOutMSE(None, None)
else:
sub_crit = HeldOutLogistic(None, None)
criterion = CrossVal(sub_crit, cv=kf)
grid_search(criterion,
model,
XX,
y,
alpha_min,
alpha_max,
monitor_grid,
max_evals=n_alphas,
tol=tol)
if model_name.startswith("lasso"):
reg = linear_model.LassoCV(cv=kf,
verbose=True,
tol=tol,
fit_intercept=False,
alphas=np.geomspace(alpha_max,
alpha_min,
num=n_alphas),
max_iter=max_iter).fit(X, y)
else:
reg = linear_model.LogisticRegressionCV(
cv=kf,
verbose=True,
tol=tol,
fit_intercept=False,
Cs=len(idx_train) /
np.geomspace(alpha_max, alpha_min, num=n_alphas),
max_iter=max_iter,
penalty='l1',
solver='liblinear').fit(X, y)
reg.score(XX, y)
if model_name.startswith("lasso"):
objs_grid_sk = reg.mse_path_.mean(axis=1)
else:
objs_grid_sk = reg.scores_[1.0].mean(axis=1)
# these 2 value should be the same
(objs_grid_sk - np.array(monitor_grid.objs))
np.testing.assert_allclose(objs_grid_sk, monitor_grid.objs)
def test_grid_search():
max_evals = 5
monitor_grid = Monitor()
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_train)
algo = Forward()
log_alpha_opt_grid, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_grid, max_evals=max_evals,
tol=1e-5, samp="grid")
monitor_random = Monitor()
criterion = HeldOutMSE(idx_train, idx_val)
algo = Forward()
log_alpha_opt_random, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_random,
max_evals=max_evals, tol=1e-5, samp="random")
assert(monitor_random.log_alphas[
np.argmin(monitor_random.objs)] == log_alpha_opt_random)
assert(monitor_grid.log_alphas[
np.argmin(monitor_grid.objs)] == log_alpha_opt_grid)
monitor_grid = Monitor()
model = Lasso(estimator=estimator)
criterion = SmoothedSURE(sigma=sigma_star)
algo = Forward()
log_alpha_opt_grid, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_grid, max_evals=max_evals,
tol=1e-5, samp="grid")
monitor_random = Monitor()
criterion = SmoothedSURE(sigma=sigma_star)
algo = Forward()
log_alpha_opt_random, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_random,
max_evals=max_evals, tol=1e-5, samp="random")
assert(monitor_random.log_alphas[
np.argmin(monitor_random.objs)] == log_alpha_opt_random)
assert(monitor_grid.log_alphas[
np.argmin(monitor_grid.objs)] == log_alpha_opt_grid)
estimator = linear_model.Lasso(fit_intercept=False,
max_iter=1000,
warm_start=True)
print('scikit-learn started')
t0 = time.time()
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_val)
algo = Forward()
monitor_grid_sk = Monitor()
grid_search(algo,
criterion,
model,
X,
y,
None,
None,
monitor_grid_sk,
log_alphas=log_alphas,
tol=tol)
objs = np.array(monitor_grid_sk.objs)
t_sk = time.time() - t0
print('scikit-learn finished')
##############################################################################
# Grad-search with sparse-ho
# --------------------------
print('sparse-ho started')
verbose=True,
tol=tol,
tol_decrease="geom")
else:
optimizer = GradientDescent(n_outer=30,
p_grad_norm=1.,
verbose=True,
tol=tol)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor)
elif algo_name == 'grid_search':
grid_search(algo,
criterion,
model,
X,
y,
None,
None,
monitor,
max_evals=20,
tol=tol,
alphas=alphas)
elif algo_name == 'random_search' or algo_name == 'bayesian':
hyperopt_wrapper(algo,
criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor,
max_evals=20,
##############################################################################
# Grid-search
# -----------
estimator = LogisticRegression(penalty='l1',
fit_intercept=False,
max_iter=max_iter)
model = SparseLogreg(estimator=estimator)
criterion = HeldOutLogistic(idx_train, idx_val)
monitor_grid = Monitor()
grid_search(criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor_grid,
alphas=alphas,
tol=tol)
objs = np.array(monitor_grid.objs)
##############################################################################
# Grad-search
# -----------
optimizer_names = ['line-search', 'gradient-descent', 'adam']
optimizers = {
'line-search': LineSearch(n_outer=10, tol=tol),
'gradient-descent': GradientDescent(n_outer=10, step_size=100),
'adam': Adam(n_outer=10, lr=0.11)
}
def parallel_function(dataset_name,
method,
tol=1e-5,
n_outer=50,
tolerance_decrease='exponential'):
# load data
X_train, X_val, X_test, y_train, y_val, y_test = get_data(dataset_name)
n_samples, n_features = X_train.shape
print('n_samples', n_samples)
print('n_features', n_features)
y_train[y_train == 0.0] = -1.0
y_val[y_val == 0.0] = -1.0
y_test[y_test == 0.0] = -1.0
alpha_max = np.max(np.abs(X_train.T @ y_train))
alpha_max /= X_train.shape[0]
log_alpha_max = np.log(alpha_max)
alpha_min = alpha_max * 1e-2
# alphas = np.geomspace(alpha_max, alpha_min, 10)
# log_alphas = np.log(alphas)
log_alpha1_0 = np.log(0.1 * alpha_max)
log_alpha2_0 = np.log(0.1 * alpha_max)
log_alpha_max = np.log(alpha_max)
n_outer = 25
if dataset_name == "rcv1":
size_loop = 2
else:
size_loop = 2
model = ElasticNet(X_train,
y_train,
log_alpha1_0,
log_alpha2_0,
log_alpha_max,
max_iter=1000,
tol=tol)
for i in range(size_loop):
monitor = Monitor()
if method == "implicit_forward":
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
algo = ImplicitForward(criterion, tol_jac=1e-3, n_iter_jac=100)
_, _, _ = grad_search(algo=algo,
verbose=False,
log_alpha0=np.array(
[log_alpha1_0, log_alpha2_0]),
tol=tol,
n_outer=n_outer,
monitor=monitor,
t_max=dict_t_max[dataset_name],
tolerance_decrease=tolerance_decrease)
elif method == "forward":
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
algo = Forward(criterion)
_, _, _ = grad_search(algo=algo,
log_alpha0=np.array(
[log_alpha1_0, log_alpha2_0]),
tol=tol,
n_outer=n_outer,
monitor=monitor,
t_max=dict_t_max[dataset_name],
tolerance_decrease=tolerance_decrease)
elif method == "implicit":
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
algo = Implicit(criterion)
_, _, _ = grad_search(algo=algo,
log_alpha0=np.array(
[log_alpha1_0, log_alpha2_0]),
tol=tol,
n_outer=n_outer,
monitor=monitor,
t_max=dict_t_max[dataset_name],
tolerance_decrease=tolerance_decrease)
elif method == "grid_search":
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
algo = Forward(criterion)
log_alpha_min = np.log(alpha_min)
log_alpha_opt, min_g_func = grid_search(
algo,
log_alpha_min,
log_alpha_max,
monitor,
max_evals=10,
tol=tol,
samp="grid",
t_max=dict_t_max[dataset_name],
log_alphas=None,
nb_hyperparam=2)
print(log_alpha_opt)
elif method == "random":
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
algo = Forward(criterion)
log_alpha_min = np.log(alpha_min)
log_alpha_opt, min_g_func = grid_search(
algo,
log_alpha_min,
np.log(alpha_max),
monitor,
max_evals=10,
tol=tol,
samp="random",
t_max=dict_t_max[dataset_name],
nb_hyperparam=2)
print(log_alpha_opt)
elif method == "lhs":
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
algo = Forward(criterion)
log_alpha_min = np.log(alpha_min)
log_alpha_opt, min_g_func = grid_search(
algo,
log_alpha_min,
np.log(alpha_max),
monitor,
max_evals=10,
tol=tol,
samp="lhs",
t_max=dict_t_max[dataset_name])
print(log_alpha_opt)
monitor.times = np.array(monitor.times).copy()
monitor.objs = np.array(monitor.objs).copy()
monitor.objs_test = np.array(monitor.objs_test).copy()
monitor.log_alphas = np.array(monitor.log_alphas).copy()
return (dataset_name, method, tol, n_outer, tolerance_decrease,
monitor.times, monitor.objs, monitor.objs_test, monitor.log_alphas,
norm(y_val), norm(y_test), log_alpha_max)
# X_train, X_val, X_test, y_train, y_val, y_test = get_leukemia()
n_samples, n_features = X_train.shape
print("Starting path computation...")
alpha_max = np.max(np.abs(X_train.T @ (-y_train)))
alpha_max /= (2 * n_samples)
n_alphas = 10
p_alphas = np.geomspace(1, 1e-4, n_alphas)
alphas = p_alphas * alpha_max
log_alphas = np.log(alphas)
tol = 1e-5
# grid search
model = SparseLogreg(X_train, y_train, log_alphas[0], max_iter=1000)
criterion = HeldOutLogistic(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Forward(criterion)
monitor_grid_sk = Monitor()
grid_search(algo, None, None, monitor_grid_sk, log_alphas=log_alphas, tol=tol)
monitor = Monitor()
# grad_search(
# algo, logCs[0], monitor, n_outer=5, verbose=True,
# tolerance_decrease='constant', tol=1e-8,
# t_max=10000)
plt.figure()
plt.plot(monitor_grid_sk.log_alphas, monitor_grid_sk.objs)
plt.plot(monitor.log_alphas, monitor.objs, 'bo')
plt.show(block=False)
# Grid search with scikit-learn
# -----------------------------
estimator = celer.Lasso(fit_intercept=False, warm_start=True)
print('Grid search started')
t0 = time.time()
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_val)
monitor_grid_sk = Monitor()
grid_search(criterion,
model,
X,
y,
None,
None,
monitor_grid_sk,
alphas=alphas,
tol=tol)
objs = np.array(monitor_grid_sk.objs)
t_sk = time.time() - t0
print('Grid search finished')
##############################################################################
# Grad-search with sparse-ho
# --------------------------
print('sparse-ho started')
monitor = Monitor()
kf = KFold(n_splits=5, shuffle=True, random_state=42)
criterion = CrossVal(sub_criterion, cv=kf)
algo = ImplicitForward(tol_jac=1e-3)
optimizer = GradientDescent(n_outer=30,
p_grad_norm=1.,
verbose=True,
tol=tol)
if algo_name == 'implicit_forward':
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor)
elif algo_name == 'grid_search':
grid_search(algo,
criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor,
max_evals=20,
tol=tol)
elif algo_name == 'random_search':
hyperopt_wrapper(algo,
criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor,
max_evals=20,
tol=tol,
def parallel_function(dataset_name,
method,
tol=1e-5,
n_outer=50,
tolerance_decrease='constant'):
# load data
X_train, X_val, X_test, y_train, y_val, y_test = get_data(dataset_name)
n_samples, _ = X_train.shape
# compute alpha_max
alpha_max = np.abs(X_train.T @ y_train).max() / n_samples
if model_name == "logreg":
alpha_max /= 2
alpha_min = alpha_max / 10_000
log_alpha_max = np.log(alpha_max)
log_alpha_min = np.log(alpha_min)
log_alpha0 = np.log(0.1 * alpha_max)
if model_name == "lasso":
model = Lasso(X_train, y_train)
elif model_name == "logreg":
model = SparseLogreg(X_train, y_train)
try:
n_outer = dict_n_outers[dataset_name, method]
except Exception:
n_outer = 20
size_loop = 2
for _ in range(size_loop):
if model_name == "lasso":
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
elif model_name == "logreg":
criterion = HeldOutLogistic(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
algo = dict_algo[method](criterion)
monitor = Monitor()
if method == 'grid_search':
log_alphas = np.log(np.geomspace(alpha_max, alpha_min, num=100))
grid_search(algo,
None,
None,
monitor,
log_alphas=log_alphas,
tol=tol)
elif method == 'random':
grid_search(algo,
log_alpha_max,
log_alpha_min,
monitor,
tol=tol,
max_evals=n_alphas,
t_max=dict_t_max[dataset_name])
elif method in ("bayesian"):
hyperopt_wrapper(algo,
log_alpha_min,
log_alpha_max,
monitor,
max_evals=n_alphas,
tol=tol,
method='bayesian',
t_max=dict_t_max[dataset_name])
else:
# do line search to find the optimal lambda
grad_search(algo,
log_alpha0,
monitor,
n_outer=n_outer,
tol=tol,
tolerance_decrease=tolerance_decrease,
t_max=dict_t_max[dataset_name])
monitor.times = np.array(monitor.times)
monitor.objs = np.array(monitor.objs)
monitor.objs_test = np.array(monitor.objs_test)
monitor.log_alphas = np.array(monitor.log_alphas)
return (dataset_name, method, tol, n_outer, tolerance_decrease,
monitor.times, monitor.objs, monitor.objs_test, monitor.log_alphas,
norm(y_val), norm(y_test), log_alpha_max, model_name)
optimizer = LineSearch(n_outer=25, verbose=True, tol=tol)
grad_search(
algo, cross_val_criterion, model, optimizer, X, y, log_alpha0,
monitor)
elif algorithm.startswith('grid_search'):
if algorithm == 'grid_search10':
n_alphas = 5
else:
n_alphas = 30
p_alphas = np.geomspace(1, p_alpha_min, n_alphas)
alphas = alpha_max * p_alphas
log_alphas = np.log(alphas)
grid_alphas = [i for i in itertools.product(log_alphas, log_alphas)]
grid_search(
algo, cross_val_criterion, model, X, y, None, None, monitor,
log_alphas=grid_alphas)
else:
hyperopt_wrapper(
algo, cross_val_criterion, model, X, y, log_alpha_min,
log_alpha_max, monitor, max_evals=max_evals,
method=algorithm, size_space=2)
objs = np.array(monitor.objs)
log_alphas = np.array(monitor.log_alphas)
log_alphas -= np.log(alpha_max)
np.save("results/%s_log_alphas_%s_enet" % (dataset, algorithm), log_alphas)
np.save("results/%s_objs_%s_enet" % (dataset, algorithm), objs)
print('%s finished' % algorithm)
def parallel_function(
dataset_name, method, tol=1e-5, n_outer=50,
tolerance_decrease='exponential'):
# load data
X_train, X_val, X_test, y_train, y_val, y_test = get_data(dataset_name, csr=True)
n_samples, n_features = X_train.shape
print('n_samples', n_samples)
print('n_features', n_features)
y_train[y_train == 0.0] = -1.0
y_val[y_val == 0.0] = -1.0
y_test[y_test == 0.0] = -1.0
C_max = 100
logC = np.log(1e-2)
n_outer = 5
if dataset_name == "rcv1":
size_loop = 1
else:
size_loop = 1
model = SVM(
X_train, y_train, logC, max_iter=10000, tol=tol)
for i in range(size_loop):
monitor = Monitor()
if method == "implicit_forward":
criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = ImplicitForward(criterion, tol_jac=1e-3, n_iter_jac=100)
_, _, _ = grad_search(
algo=algo, verbose=False,
log_alpha0=logC, tol=tol,
n_outer=n_outer, monitor=monitor,
t_max=dict_t_max[dataset_name],
tolerance_decrease=tolerance_decrease)
elif method == "forward":
criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Forward(criterion)
_, _, _ = grad_search(
algo=algo,
log_alpha0=logC, tol=tol,
n_outer=n_outer, monitor=monitor,
t_max=dict_t_max[dataset_name],
tolerance_decrease=tolerance_decrease)
elif method == "implicit":
criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Implicit(criterion)
_, _, _ = grad_search(
algo=algo,
log_alpha0=logC, tol=tol,
n_outer=n_outer, monitor=monitor,
t_max=dict_t_max[dataset_name],
tolerance_decrease=tolerance_decrease)
elif method == "grid_search":
criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Forward(criterion)
log_alpha_min = np.log(1e-2)
log_alpha_opt, min_g_func = grid_search(
algo, log_alpha_min, np.log(C_max), monitor, max_evals=25,
tol=tol, samp="grid")
print(log_alpha_opt)
elif method == "random":
criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Forward(criterion)
log_alpha_min = np.log(1e-2)
log_alpha_opt, min_g_func = grid_search(
algo, log_alpha_min, np.log(C_max), monitor, max_evals=25,
tol=tol, samp="random")
print(log_alpha_opt)
elif method == "lhs":
criterion = HeldOutSmoothedHinge(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Forward(criterion)
log_alpha_min = np.log(1e-2)
log_alpha_opt, min_g_func = grid_search(
algo, log_alpha_min, np.log(C_max), monitor, max_evals=25,
tol=tol, samp="lhs")
print(log_alpha_opt)
monitor.times = np.array(monitor.times)
monitor.objs = np.array(monitor.objs)
monitor.objs_test = np.array(monitor.objs_test)
monitor.log_alphas = np.array(monitor.log_alphas)
return (dataset_name, method, tol, n_outer, tolerance_decrease,
monitor.times, monitor.objs, monitor.objs_test,
monitor.log_alphas, norm(y_val), norm(y_test))
def parallel_function(dataset_name,
method,
tol=1e-5,
n_outer=50,
tolerance_decrease='constant'):
# load data
X, y = fetch_libsvm(dataset_name)
y -= np.mean(y)
# compute alpha_max
alpha_max = np.abs(X.T @ y).max() / len(y)
if model_name == "logreg":
alpha_max /= 2
alpha_min = alpha_max * dict_palphamin[dataset_name]
if model_name == "enet":
estimator = linear_model.ElasticNet(fit_intercept=False,
max_iter=10_000,
warm_start=True,
tol=tol)
model = ElasticNet(estimator=estimator)
elif model_name == "logreg":
model = SparseLogreg(estimator=estimator)
# TODO improve this
try:
n_outer = dict_n_outers[dataset_name, method]
except Exception:
n_outer = 20
size_loop = 2
for _ in range(size_loop):
if model_name == "lasso" or model_name == "enet":
sub_criterion = HeldOutMSE(None, None)
elif model_name == "logreg":
criterion = HeldOutLogistic(None, None)
kf = KFold(n_splits=5, shuffle=True, random_state=42)
criterion = CrossVal(sub_criterion, cv=kf)
algo = ImplicitForward(tol_jac=1e-3)
monitor = Monitor()
t_max = dict_t_max[dataset_name]
if method == 'grid_search':
num1D = dict_point_grid_search[dataset_name]
alpha1D = np.geomspace(alpha_max, alpha_min, num=num1D)
alphas = [np.array(i) for i in product(alpha1D, alpha1D)]
grid_search(algo,
criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor,
max_evals=100,
tol=tol,
alphas=alphas)
elif method == 'random' or method == 'bayesian':
hyperopt_wrapper(algo,
criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor,
max_evals=30,
tol=tol,
method=method,
size_space=2,
t_max=t_max)
elif method.startswith("implicit_forward"):
# do gradient descent to find the optimal lambda
alpha0 = np.array([alpha_max / 100, alpha_max / 100])
n_outer = 30
if method == 'implicit_forward':
optimizer = GradientDescent(n_outer=n_outer,
p_grad_norm=1,
verbose=True,
tol=tol,
t_max=t_max)
else:
optimizer = GradientDescent(n_outer=n_outer,
p_grad_norm=1,
verbose=True,
tol=tol,
t_max=t_max,
tol_decrease="geom")
grad_search(algo, criterion, model, optimizer, X, y, alpha0,
monitor)
else:
raise NotImplementedError
monitor.times = np.array(monitor.times)
monitor.objs = np.array(monitor.objs)
monitor.objs_test = 0 # TODO
monitor.alphas = np.array(monitor.alphas)
return (dataset_name, method, tol, n_outer, tolerance_decrease,
monitor.times, monitor.objs, monitor.objs_test, monitor.alphas,
alpha_max, model_name)
def test_grid_search():
max_evals = 5
monitor_grid = Monitor()
model = Lasso(X_train, y_train, estimator=estimator)
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Forward()
log_alpha_opt_grid, _ = grid_search(algo,
criterion,
log_alpha_min,
log_alpha_max,
monitor_grid,
max_evals=max_evals,
tol=1e-5,
samp="grid")
monitor_random = Monitor()
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = Forward()
log_alpha_opt_random, _ = grid_search(algo,
criterion,
log_alpha_min,
log_alpha_max,
monitor_random,
max_evals=max_evals,
tol=1e-5,
samp="random")
assert (monitor_random.log_alphas[np.argmin(
monitor_random.objs)] == log_alpha_opt_random)
assert (monitor_grid.log_alphas[np.argmin(
monitor_grid.objs)] == log_alpha_opt_grid)
monitor_grid = Monitor()
model = Lasso(X_train, y_train, estimator=estimator)
criterion = SURE(X_train,
y_train,
model,
sigma=sigma_star,
X_test=X_test,
y_test=y_test)
algo = Forward()
log_alpha_opt_grid, _ = grid_search(algo,
criterion,
log_alpha_min,
log_alpha_max,
monitor_grid,
max_evals=max_evals,
tol=1e-5,
samp="grid")
monitor_random = Monitor()
criterion = SURE(X_train,
y_train,
model,
sigma=sigma_star,
X_test=X_test,
y_test=y_test)
algo = Forward()
log_alpha_opt_random, _ = grid_search(algo,
criterion,
log_alpha_min,
log_alpha_max,
monitor_random,
max_evals=max_evals,
tol=1e-5,
samp="random")
assert (monitor_random.log_alphas[np.argmin(
monitor_random.objs)] == log_alpha_opt_random)
assert (monitor_grid.log_alphas[np.argmin(
monitor_grid.objs)] == log_alpha_opt_grid)
print('scikit started')
t0 = time.time()
estimator = LogisticRegression(penalty='l1',
fit_intercept=False,
max_iter=max_iter)
model = SparseLogreg(max_iter=max_iter, estimator=estimator)
criterion = HeldOutLogistic(idx_train, idx_val)
algo_grid = Forward()
monitor_grid = Monitor()
grid_search(algo_grid,
criterion,
model,
X,
y,
log_alpha_min,
log_alpha_max,
monitor_grid,
log_alphas=log_alphas,
tol=tol)
objs = np.array(monitor_grid.objs)
t_sk = time.time() - t0
print('scikit finished')
print("Time to compute CV for scikit-learn: %.2f" % t_sk)
##############################################################################
# Grad-search
# -----------
def test_grid_search():
max_evals = 5
monitor_grid = Monitor()
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_train)
alpha_opt_grid, _ = grid_search(criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor_grid,
max_evals=max_evals,
tol=1e-5,
samp="grid")
monitor_random = Monitor()
criterion = HeldOutMSE(idx_train, idx_val)
alpha_opt_random, _ = grid_search(criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor_random,
max_evals=max_evals,
tol=1e-5,
samp="random")
np.testing.assert_allclose(
monitor_random.alphas[np.argmin(monitor_random.objs)],
alpha_opt_random)
np.testing.assert_allclose(
monitor_grid.alphas[np.argmin(monitor_grid.objs)], alpha_opt_grid)
monitor_grid = Monitor()
model = Lasso(estimator=estimator)
criterion = FiniteDiffMonteCarloSure(sigma=sigma_star)
alpha_opt_grid, _ = grid_search(criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor_grid,
max_evals=max_evals,
tol=1e-5,
samp="grid")
monitor_random = Monitor()
criterion = FiniteDiffMonteCarloSure(sigma=sigma_star)
alpha_opt_random, _ = grid_search(criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor_random,
max_evals=max_evals,
tol=1e-5,
samp="random")
np.testing.assert_allclose(
monitor_random.alphas[np.argmin(monitor_random.objs)],
alpha_opt_random)
np.testing.assert_allclose(
monitor_grid.alphas[np.argmin(monitor_grid.objs)], alpha_opt_grid)
