def test_cross_val_criterion(model_name, criterion):
# verify dtype from criterion, and the good shape
algo = Forward()
monitor_get_val = Monitor()
monitor_get_val_grad = Monitor()
model = models[model_name]
for log_alpha in dict_list_log_alphas[model_name]:
criterion.get_val(model,
X,
y,
log_alpha,
tol=tol,
monitor=monitor_get_val)
criterion.get_val_grad(model,
X,
y,
log_alpha,
algo.get_beta_jac_v,
tol=tol,
monitor=monitor_get_val_grad)
obj_val = np.array(monitor_get_val.objs)
obj_val_grad = np.array(monitor_get_val_grad.objs)
np.testing.assert_allclose(obj_val, obj_val_grad)
def test_grad_search(model, crit):
"""check that the paths are the same in the line search"""
n_outer = 2
criterion = HeldOutLogistic(X_val, y_val, model)
monitor1 = Monitor()
algo = Forward()
grad_search(algo, criterion, log_alpha, monitor1, n_outer=n_outer,
tol=tol)
criterion = HeldOutLogistic(X_val, y_val, model)
monitor2 = Monitor()
algo = Implicit()
grad_search(algo, criterion, log_alpha, monitor2, n_outer=n_outer,
tol=tol)
criterion = HeldOutLogistic(X_val, y_val, model)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=tol, n_iter_jac=5000)
grad_search(algo, criterion, log_alpha, monitor3, n_outer=n_outer,
tol=tol)
assert np.allclose(
np.array(monitor1.log_alphas), np.array(monitor3.log_alphas))
assert np.allclose(
np.array(monitor1.grads), np.array(monitor3.grads), atol=1e-4)
assert np.allclose(
np.array(monitor1.objs), np.array(monitor3.objs))
assert not np.allclose(
np.array(monitor1.times), np.array(monitor3.times))
def test_grid_search():
monitor = Monitor()
grid_searchCV(X_train,
y_train,
log_alphas,
X_test,
y_test,
X_test,
y_test,
tol,
monitor,
sk=False)
monitor_sparse = Monitor()
grid_searchCV(X_train_s,
y_train,
log_alphas,
X_test_s,
y_test,
X_test_s,
y_test,
tol,
monitor_sparse,
sk=False)
assert np.allclose(monitor.objs, monitor_sparse.objs)
def test_grad_search(Optimizer, model, crit):
"""check that the paths are the same in the line search"""
n_outer = 2
criterion = HeldOutMSE(idx_train, idx_val)
monitor1 = Monitor()
algo = Forward()
optimizer = Optimizer(n_outer=n_outer, tol=1e-16)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor1)
criterion = HeldOutMSE(idx_train, idx_val)
monitor2 = Monitor()
algo = Implicit()
optimizer = Optimizer(n_outer=n_outer, tol=1e-16)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor2)
criterion = HeldOutMSE(idx_train, idx_val)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=1e-8, n_iter_jac=5000)
optimizer = Optimizer(n_outer=n_outer, tol=1e-16)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor3)
np.testing.assert_allclose(np.array(monitor1.alphas),
np.array(monitor3.alphas))
np.testing.assert_allclose(np.array(monitor1.grads),
np.array(monitor3.grads),
rtol=1e-5)
np.testing.assert_allclose(np.array(monitor1.objs),
np.array(monitor3.objs))
assert not np.allclose(np.array(monitor1.times), np.array(monitor3.times))
def test_grad_search(model):
# criterion = SURE(
# X_train, y_train, model, sigma=sigma_star, X_test=X_test,
# y_test=y_test)
n_outer = 3
criterion = HeldOutSmoothedHinge(X_val,
y_val,
model,
X_test=None,
y_test=None)
monitor1 = Monitor()
algo = Forward()
grad_search(algo,
criterion,
np.log(1e-3),
monitor1,
n_outer=n_outer,
tol=1e-13)
# criterion = SURE(
# X_train, y_train, model, sigma=sigma_star, X_test=X_test,
# y_test=y_test)
criterion = HeldOutSmoothedHinge(X_val,
y_val,
model,
X_test=None,
y_test=None)
monitor2 = Monitor()
algo = Implicit()
grad_search(algo,
criterion,
np.log(1e-3),
monitor2,
n_outer=n_outer,
tol=1e-13)
# criterion = SURE(
# X_train, y_train, model, sigma=sigma_star, X_test=X_test,
# y_test=y_test)
criterion = HeldOutSmoothedHinge(X_val,
y_val,
model,
X_test=None,
y_test=None)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=1e-6, n_iter_jac=100)
grad_search(algo,
criterion,
np.log(1e-3),
monitor3,
n_outer=n_outer,
tol=1e-13)
assert np.allclose(np.array(monitor1.log_alphas),
np.array(monitor3.log_alphas))
assert np.allclose(np.array(monitor1.grads), np.array(monitor3.grads))
assert np.allclose(np.array(monitor1.objs), np.array(monitor3.objs))
# assert np.allclose(
# np.array(monitor1.objs_test), np.array(monitor3.objs_test))
assert not np.allclose(np.array(monitor1.times), np.array(monitor3.times))
def test_our_vs_sklearn():
monitor_grid = Monitor()
monitor_grid_sk = Monitor()
for i in range(n_alphas):
# one versus all (ovr) logreg from scikit learn
p_alpha = p_alphas[:, i]
lr = LogisticRegression(solver='saga',
multi_class='ovr',
penalty='l1',
max_iter=max_iter,
random_state=42,
fit_intercept=False,
warm_start=True,
C=1 /
(alpha_max * p_alpha[0] * len(idx_train)),
tol=tol)
lr.fit(X[idx_train, :], y[idx_train])
y_pred_val = lr.predict(X[idx_val, :])
accuracy_val = sklearn.metrics.accuracy_score(y_pred_val, y[idx_val])
print("accuracy validation (scikit) %f " % accuracy_val)
monitor_grid_sk(None, None, acc_val=accuracy_val)
log_alpha_i = np.log(alpha_max * p_alpha)
# our one verus all
val, grad = logit_multiclass.get_val_grad(model, X, y, log_alpha_i,
None, monitor_grid, tol)
print("accuracy validation (our) %f " % monitor_grid.acc_vals[-1])
np.testing.assert_allclose(np.array(monitor_grid.acc_vals),
np.array(monitor_grid_sk.acc_vals))
def test_grad_search(model, crit):
"""check that the paths are the same in the line search"""
if crit == 'cv':
n_outer = 2
criterion = HeldOutMSE(idx_train, idx_val)
else:
n_outer = 2
criterion = SmoothedSURE(sigma_star)
# TODO MM@QBE if else scheme surprising
criterion = HeldOutMSE(idx_train, idx_val)
monitor1 = Monitor()
algo = Forward()
grad_search(algo,
criterion,
model,
X,
y,
log_alpha,
monitor1,
n_outer=n_outer,
tol=1e-16)
criterion = HeldOutMSE(idx_train, idx_val)
monitor2 = Monitor()
algo = Implicit()
grad_search(algo,
criterion,
model,
X,
y,
log_alpha,
monitor2,
n_outer=n_outer,
tol=1e-16)
criterion = HeldOutMSE(idx_train, idx_val)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=1e-8, n_iter_jac=5000)
grad_search(algo,
criterion,
model,
X,
y,
log_alpha,
monitor3,
n_outer=n_outer,
tol=1e-16)
np.testing.assert_allclose(np.array(monitor1.log_alphas),
np.array(monitor3.log_alphas))
np.testing.assert_allclose(np.array(monitor1.grads),
np.array(monitor3.grads),
atol=1e-8)
np.testing.assert_allclose(np.array(monitor1.objs),
np.array(monitor3.objs))
assert not np.allclose(np.array(monitor1.times), np.array(monitor3.times))
def test_grad_search(model, crit):
"""check that the paths are the same in the line search"""
if crit == 'cv':
n_outer = 2
criterion = HeldOutMSE(X_val,
y_val,
model,
X_test=X_test,
y_test=y_test)
else:
n_outer = 2
criterion = SURE(X_train,
y_train,
model,
sigma=sigma_star,
X_test=X_test,
y_test=y_test)
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
monitor1 = Monitor()
algo = Forward()
grad_search(algo,
criterion,
log_alpha,
monitor1,
n_outer=n_outer,
tol=1e-16)
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
monitor2 = Monitor()
algo = Implicit()
grad_search(algo,
criterion,
log_alpha,
monitor2,
n_outer=n_outer,
tol=1e-16)
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=1e-8, n_iter_jac=5000)
grad_search(algo,
criterion,
log_alpha,
monitor3,
n_outer=n_outer,
tol=1e-16)
assert np.allclose(np.array(monitor1.log_alphas),
np.array(monitor3.log_alphas))
assert np.allclose(np.array(monitor1.grads), np.array(monitor3.grads))
assert np.allclose(np.array(monitor1.objs), np.array(monitor3.objs))
assert np.allclose(np.array(monitor1.objs_test),
np.array(monitor3.objs_test))
assert not np.allclose(np.array(monitor1.times), np.array(monitor3.times))
def test_grad_search():
n_outer = 3
criterion = HeldOutMSE(X_val, y_val, model, X_test=None, y_test=None)
monitor1 = Monitor()
algo = Forward()
grad_search(algo,
criterion,
np.array([log_alpha1, log_alpha2]),
monitor1,
n_outer=n_outer,
tol=1e-16)
criterion = HeldOutMSE(X_val, y_val, model, X_test=None, y_test=None)
monitor2 = Monitor()
algo = Implicit()
grad_search(algo,
criterion,
np.array([log_alpha1, log_alpha2]),
monitor2,
n_outer=n_outer,
tol=1e-16)
criterion = HeldOutMSE(X_val, y_val, model, X_test=None, y_test=None)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=1e-3, n_iter_jac=1000)
grad_search(algo,
criterion,
np.array([log_alpha1, log_alpha2]),
monitor3,
n_outer=n_outer,
tol=1e-16)
[np.linalg.norm(grad) for grad in monitor1.grads]
[np.exp(alpha) for alpha in monitor1.log_alphas]
assert np.allclose(np.array(monitor1.log_alphas),
np.array(monitor3.log_alphas))
assert np.allclose(np.array(monitor1.grads), np.array(monitor3.grads))
assert np.allclose(np.array(monitor1.objs), np.array(monitor3.objs))
assert not np.allclose(np.array(monitor1.times), np.array(monitor3.times))
assert np.allclose(np.array(monitor1.log_alphas),
np.array(monitor2.log_alphas),
atol=1e-2)
assert np.allclose(np.array(monitor1.grads),
np.array(monitor2.grads),
atol=1e-2)
assert np.allclose(np.array(monitor1.objs),
np.array(monitor2.objs),
atol=1e-2)
assert not np.allclose(np.array(monitor1.times), np.array(monitor2.times))
def test_grad_search(model):
n_outer = 3
criterion = HeldOutSmoothedHinge(idx_train, idx_val)
monitor1 = Monitor()
algo = Forward()
grad_search(algo,
criterion,
model,
X,
y,
np.log(1e-3),
monitor1,
n_outer=n_outer,
tol=1e-13)
criterion = HeldOutSmoothedHinge(idx_train, idx_val)
monitor2 = Monitor()
algo = Implicit()
grad_search(algo,
criterion,
model,
X,
y,
np.log(1e-3),
monitor2,
n_outer=n_outer,
tol=1e-13)
criterion = HeldOutSmoothedHinge(idx_train, idx_val)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=1e-6, n_iter_jac=100)
grad_search(algo,
criterion,
model,
X,
y,
np.log(1e-3),
monitor3,
n_outer=n_outer,
tol=1e-13)
assert np.allclose(np.array(monitor1.log_alphas),
np.array(monitor3.log_alphas))
assert np.allclose(np.array(monitor1.grads), np.array(monitor3.grads))
assert np.allclose(np.array(monitor1.objs), np.array(monitor3.objs))
# assert np.allclose(
# np.array(monitor1.objs_test), np.array(monitor3.objs_test))
assert not np.allclose(np.array(monitor1.times), np.array(monitor3.times))
def test_cross_val_criterion():
alpha_min = alpha_max / 10
log_alpha_max = np.log(alpha_max)
log_alpha_min = np.log(alpha_min)
max_iter = 10000
n_alphas = 10
kf = KFold(n_splits=5, shuffle=True, random_state=56)
estimator = sklearn.linear_model.Lasso(fit_intercept=False,
max_iter=1000,
warm_start=True)
monitor_grid = Monitor()
criterion = CrossVal(X, y, Lasso, cv=kf, estimator=estimator)
algo = Forward()
grid_search(algo,
criterion,
log_alpha_min,
log_alpha_max,
monitor_grid,
max_evals=n_alphas,
tol=tol)
reg = LassoCV(cv=kf,
verbose=True,
tol=tol,
fit_intercept=False,
alphas=np.geomspace(alpha_max, alpha_min, num=n_alphas),
max_iter=max_iter).fit(X, y)
reg.score(X, y)
objs_grid_sk = reg.mse_path_.mean(axis=1)
# these 2 value should be the same
(objs_grid_sk - np.array(monitor_grid.objs))
assert np.allclose(objs_grid_sk, monitor_grid.objs)
def hyperopt_lasso(
X_train, y_train, log_alpha, X_val, y_val, X_test, y_test, tol,
maxit=1000, max_evals=30, method="bayesian", criterion="cv", sigma=1.0,
beta_star=None):
n_samples, n_features = X_train.shape
alpha_max = np.abs((X_train.T @ y_train)).max() / n_samples
space = hp.uniform(
'log_alpha', np.log(alpha_max / 1000), np.log(alpha_max))
monitor = Monitor()
warm_start = WarmStart()
if criterion == "cv":
def objective(log_alpha):
value = get_val_grad(
X_train, y_train, log_alpha, X_val, y_val, X_test, y_test, tol,
monitor, warm_start, method="hyperopt", maxit=1000,
model="lasso", beta_star=beta_star)
return value
elif criterion == "sure":
def objective(log_alpha):
value = get_val_grad(
X_train, y_train, log_alpha, X_val, y_val, X_test, y_test, tol,
monitor, warm_start, method="hyperopt", maxit=1000,
model="lasso", criterion="sure", sigma=sigma,
beta_star=beta_star)
return value
if method == "bayesian":
best = fmin(objective, space, algo=tpe.suggest, max_evals=max_evals)
elif method == "random":
best = fmin(objective, space, algo=rand.suggest, max_evals=max_evals)
return monitor
def test_grid_search():
max_evals = 5
monitor_grid = Monitor()
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_train)
algo = Forward()
log_alpha_opt_grid, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_grid, max_evals=max_evals,
tol=1e-5, samp="grid")
monitor_random = Monitor()
criterion = HeldOutMSE(idx_train, idx_val)
algo = Forward()
log_alpha_opt_random, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_random,
max_evals=max_evals, tol=1e-5, samp="random")
assert(monitor_random.log_alphas[
np.argmin(monitor_random.objs)] == log_alpha_opt_random)
assert(monitor_grid.log_alphas[
np.argmin(monitor_grid.objs)] == log_alpha_opt_grid)
monitor_grid = Monitor()
model = Lasso(estimator=estimator)
criterion = SmoothedSURE(sigma=sigma_star)
algo = Forward()
log_alpha_opt_grid, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_grid, max_evals=max_evals,
tol=1e-5, samp="grid")
monitor_random = Monitor()
criterion = SmoothedSURE(sigma=sigma_star)
algo = Forward()
log_alpha_opt_random, _ = grid_search(
algo, criterion, model, X, y, log_alpha_min, log_alpha_max,
monitor_random,
max_evals=max_evals, tol=1e-5, samp="random")
assert(monitor_random.log_alphas[
np.argmin(monitor_random.objs)] == log_alpha_opt_random)
assert(monitor_grid.log_alphas[
np.argmin(monitor_grid.objs)] == log_alpha_opt_grid)
def test_monitor():
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_val)
algo = ImplicitForward()
monitor = Monitor(callback=callback)
optimizer = LineSearch(n_outer=10, tol=tol)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor)
np.testing.assert_allclose(np.array(monitor.objs), np.array(objs))
def test_grad_search_custom(model, model_custom, crit):
"""check that the paths are the same in the line search"""
n_outer = 5
criterion = HeldOutLogistic(idx_val, idx_val)
monitor = Monitor()
algo = ImplicitForward(tol_jac=tol, n_iter_jac=5000)
grad_search(algo,
criterion,
model,
X,
y,
log_alpha,
monitor,
n_outer=n_outer,
tol=tol)
criterion = HeldOutLogistic(idx_val, idx_val)
monitor_custom = Monitor()
algo = ImplicitForward(tol_jac=tol, n_iter_jac=5000)
grad_search(algo,
criterion,
model_custom,
X,
y,
log_alpha,
monitor_custom,
n_outer=n_outer,
tol=tol)
np.testing.assert_allclose(np.array(monitor.log_alphas),
np.array(monitor_custom.log_alphas),
atol=1e-3)
np.testing.assert_allclose(np.array(monitor.grads),
np.array(monitor_custom.grads),
atol=1e-4)
np.testing.assert_allclose(np.array(monitor.objs),
np.array(monitor_custom.objs),
atol=1e-5)
assert not np.allclose(np.array(monitor.times),
np.array(monitor_custom.times))
def parallel_function(
dataset_name, method, tol=1e-8, n_outer=15):
# load data
X, y = fetch_libsvm(dataset_name)
# subsample the samples and the features
n_samples, n_features = dict_subsampling[dataset_name]
t_max = dict_t_max[dataset_name]
# t_max = 3600
X, y = clean_dataset(X, y, n_samples, n_features)
alpha_max, n_classes = get_alpha_max(X, y)
log_alpha_max = np.log(alpha_max) # maybe to change alpha max value
algo = ImplicitForward(None, n_iter_jac=2000)
estimator = LogisticRegression(
C=1, fit_intercept=False, warm_start=True, max_iter=30, verbose=False)
model = SparseLogreg(estimator=estimator)
idx_train, idx_val, idx_test = get_splits(X, y)
logit_multiclass = LogisticMulticlass(
idx_train, idx_val, algo, idx_test=idx_test)
monitor = Monitor()
if method == "implicit_forward":
log_alpha0 = np.ones(n_classes) * np.log(0.1 * alpha_max)
optimizer = LineSearch(n_outer=100)
grad_search(
algo, logit_multiclass, model, optimizer, X, y, log_alpha0,
monitor)
elif method.startswith(('random', 'bayesian')):
max_evals = dict_max_eval[dataset_name]
log_alpha_min = np.log(alpha_max) - 7
hyperopt_wrapper(
algo, logit_multiclass, model, X, y, log_alpha_min, log_alpha_max,
monitor, max_evals=max_evals, tol=tol, t_max=t_max, method=method,
size_space=n_classes)
elif method == 'grid_search':
n_alphas = 20
p_alphas = np.geomspace(1, 0.001, n_alphas)
p_alphas = np.tile(p_alphas, (n_classes, 1))
for i in range(n_alphas):
log_alpha_i = np.log(alpha_max * p_alphas[:, i])
logit_multiclass.get_val(
model, X, y, log_alpha_i, None, monitor, tol)
monitor.times = np.array(monitor.times).copy()
monitor.objs = np.array(monitor.objs).copy()
monitor.acc_vals = np.array(monitor.acc_vals).copy()
monitor.acc_tests = np.array(monitor.acc_tests).copy()
monitor.log_alphas = np.array(monitor.log_alphas).copy()
return (
dataset_name, method, tol, n_outer, monitor.times, monitor.objs,
monitor.acc_vals, monitor.acc_tests, monitor.log_alphas, log_alpha_max,
n_samples, n_features, n_classes)
def test_cross_val_criterion(model_name, XX):
model = models[model_name]
alpha_min = alpha_max / 10
max_iter = 10000
n_alphas = 10
kf = KFold(n_splits=5, shuffle=True, random_state=56)
monitor_grid = Monitor()
if model_name.startswith("lasso"):
sub_crit = HeldOutMSE(None, None)
else:
sub_crit = HeldOutLogistic(None, None)
criterion = CrossVal(sub_crit, cv=kf)
grid_search(criterion,
model,
XX,
y,
alpha_min,
alpha_max,
monitor_grid,
max_evals=n_alphas,
tol=tol)
if model_name.startswith("lasso"):
reg = linear_model.LassoCV(cv=kf,
verbose=True,
tol=tol,
fit_intercept=False,
alphas=np.geomspace(alpha_max,
alpha_min,
num=n_alphas),
max_iter=max_iter).fit(X, y)
else:
reg = linear_model.LogisticRegressionCV(
cv=kf,
verbose=True,
tol=tol,
fit_intercept=False,
Cs=len(idx_train) /
np.geomspace(alpha_max, alpha_min, num=n_alphas),
max_iter=max_iter,
penalty='l1',
solver='liblinear').fit(X, y)
reg.score(XX, y)
if model_name.startswith("lasso"):
objs_grid_sk = reg.mse_path_.mean(axis=1)
else:
objs_grid_sk = reg.scores_[1.0].mean(axis=1)
# these 2 value should be the same
(objs_grid_sk - np.array(monitor_grid.objs))
np.testing.assert_allclose(objs_grid_sk, monitor_grid.objs)
def test_grad_search(model, crit):
"""check that the paths are the same in the line search"""
if crit == 'MSE':
n_outer = 2
criterion = HeldOutMSE(idx_train, idx_val)
else:
n_outer = 2
criterion = FiniteDiffMonteCarloSure(sigma_star)
# TODO MM@QBE if else scheme surprising
criterion = HeldOutMSE(idx_train, idx_val)
monitor1 = Monitor()
algo = Forward()
optimizer = LineSearch(n_outer=n_outer, tol=1e-16)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor1)
criterion = HeldOutMSE(idx_train, idx_val)
monitor2 = Monitor()
algo = Implicit()
optimizer = LineSearch(n_outer=n_outer, tol=1e-16)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor2)
criterion = HeldOutMSE(idx_train, idx_val)
monitor3 = Monitor()
algo = ImplicitForward(tol_jac=1e-8, n_iter_jac=5000)
optimizer = LineSearch(n_outer=n_outer, tol=1e-16)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor3)
np.testing.assert_allclose(np.array(monitor1.alphas),
np.array(monitor3.alphas))
np.testing.assert_allclose(np.array(monitor1.grads),
np.array(monitor3.grads),
rtol=1e-5)
np.testing.assert_allclose(np.array(monitor1.objs),
np.array(monitor3.objs))
assert not np.allclose(np.array(monitor1.times), np.array(monitor3.times))
def test_monitor():
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_val)
algo = ImplicitForward()
monitor = Monitor(callback=callback)
grad_search(algo,
criterion,
model,
X,
y,
np.log(alpha_max / 10),
monitor,
n_outer=10,
tol=tol)
np.testing.assert_allclose(np.array(monitor.objs), np.array(objs))
model = SparseLogreg(estimator=estimator)
logit_multiclass = LogisticMulticlass(
idx_train, idx_val, algo, idx_test=idx_test)
alpha_max, n_classes = alpha_max_multiclass(X, y)
tol = 1e-5
n_alphas = 10
p_alphas = np.geomspace(1, 0.001, n_alphas)
p_alphas = np.tile(p_alphas, (n_classes, 1))
print("###################### GRID SEARCH ###################")
monitor_grid = Monitor()
for i in range(n_alphas):
log_alpha_i = np.log(alpha_max * p_alphas[:, i])
logit_multiclass.get_val(
model, X, y, log_alpha_i, None, monitor_grid, tol)
1/0
print("###################### GRAD SEARCH LS ###################")
n_outer = 100
model = SparseLogreg(estimator=estimator)
logit_multiclass = LogisticMulticlass(idx_train, idx_val, idx_test, algo)
monitor = Monitor()
log_alpha0 = np.ones(n_classes) * np.log(0.1 * alpha_max)
idx_min = np.argmin(np.array(monitor_grid.objs))
n_samples = len(y)
sss1 = StratifiedShuffleSplit(n_splits=2, test_size=0.3333, random_state=0)
idx_train, idx_val = sss1.split(X, y)
idx_train = idx_train[0]
idx_val = idx_val[0]
dict_res = {}
for maxit in maxits:
for method in methods:
print("Dataset %s, maxit %i" % (method, maxit))
for i in range(2):
alpha_max = np.max(np.abs(X.T.dot(y))) / n_samples
log_alpha = np.log(alpha_max * p_alpha_max)
monitor = Monitor()
if method == "celer":
clf = Lasso_celer(alpha=np.exp(log_alpha),
fit_intercept=False,
tol=1e-12,
max_iter=maxit)
model = Lasso(estimator=clf, max_iter=maxit)
criterion = HeldOutMSE(idx_train, idx_val)
algo = ImplicitForward(tol_jac=1e-32,
n_iter_jac=maxit,
use_stop_crit=False)
algo.max_iter = maxit
val, grad = criterion.get_val_grad(model,
X,
y,
log_alpha,
log_alphas = np.log(alphas)
##############################################################################
# Grid-search
# -----------
print('scikit started')
t0 = time.time()
estimator = LogisticRegression(penalty='l1',
fit_intercept=False,
max_iter=max_iter)
model = SparseLogreg(max_iter=max_iter, estimator=estimator)
criterion = HeldOutLogistic(idx_train, idx_val)
algo_grid = Forward()
monitor_grid = Monitor()
grid_search(algo_grid,
criterion,
model,
X,
y,
log_alpha_min,
log_alpha_max,
monitor_grid,
log_alphas=log_alphas,
tol=tol)
objs = np.array(monitor_grid.objs)
t_sk = time.time() - t0
print('scikit finished')
def parallel_function(dataset_name,
method,
tol=1e-5,
n_outer=50,
tolerance_decrease='constant'):
# load data
X, y = fetch_libsvm(dataset_name)
y -= np.mean(y)
# compute alpha_max
alpha_max = np.abs(X.T @ y).max() / len(y)
if model_name == "logreg":
alpha_max /= 2
alpha_min = alpha_max * dict_palphamin[dataset_name]
if model_name == "enet":
estimator = linear_model.ElasticNet(fit_intercept=False,
max_iter=10_000,
warm_start=True,
tol=tol)
model = ElasticNet(estimator=estimator)
elif model_name == "logreg":
model = SparseLogreg(estimator=estimator)
# TODO improve this
try:
n_outer = dict_n_outers[dataset_name, method]
except Exception:
n_outer = 20
size_loop = 2
for _ in range(size_loop):
if model_name == "lasso" or model_name == "enet":
sub_criterion = HeldOutMSE(None, None)
elif model_name == "logreg":
criterion = HeldOutLogistic(None, None)
kf = KFold(n_splits=5, shuffle=True, random_state=42)
criterion = CrossVal(sub_criterion, cv=kf)
algo = ImplicitForward(tol_jac=1e-3)
monitor = Monitor()
t_max = dict_t_max[dataset_name]
if method == 'grid_search':
num1D = dict_point_grid_search[dataset_name]
alpha1D = np.geomspace(alpha_max, alpha_min, num=num1D)
alphas = [np.array(i) for i in product(alpha1D, alpha1D)]
grid_search(algo,
criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor,
max_evals=100,
tol=tol,
alphas=alphas)
elif method == 'random' or method == 'bayesian':
hyperopt_wrapper(algo,
criterion,
model,
X,
y,
alpha_min,
alpha_max,
monitor,
max_evals=30,
tol=tol,
method=method,
size_space=2,
t_max=t_max)
elif method.startswith("implicit_forward"):
# do gradient descent to find the optimal lambda
alpha0 = np.array([alpha_max / 100, alpha_max / 100])
n_outer = 30
if method == 'implicit_forward':
optimizer = GradientDescent(n_outer=n_outer,
p_grad_norm=1,
verbose=True,
tol=tol,
t_max=t_max)
else:
optimizer = GradientDescent(n_outer=n_outer,
p_grad_norm=1,
verbose=True,
tol=tol,
t_max=t_max,
tol_decrease="geom")
grad_search(algo, criterion, model, optimizer, X, y, alpha0,
monitor)
else:
raise NotImplementedError
monitor.times = np.array(monitor.times)
monitor.objs = np.array(monitor.objs)
monitor.objs_test = 0 # TODO
monitor.alphas = np.array(monitor.alphas)
return (dataset_name, method, tol, n_outer, tolerance_decrease,
monitor.times, monitor.objs, monitor.objs_test, monitor.alphas,
alpha_max, model_name)
def parallel_function(dataset_name,
div_alpha,
method,
ind_rep,
random_state=10):
maxit = dict_maxits[(dataset_name, div_alpha)][ind_rep]
print("Dataset %s, algo %s, maxit %i" % (dataset_name, method, maxit))
X, y = fetch_libsvm(dataset_name)
n_samples = len(y)
kf = KFold(n_splits=5, random_state=random_state, shuffle=True)
for i in range(2):
alpha_max = np.max(np.abs(X.T.dot(y))) / n_samples
log_alpha = np.log(alpha_max / div_alpha)
monitor = Monitor()
if method == "celer":
clf = Lasso_celer(
alpha=np.exp(log_alpha),
fit_intercept=False,
# TODO maybe change this tol
tol=1e-8,
max_iter=maxit)
model = Lasso(estimator=clf, max_iter=maxit)
criterion = HeldOutMSE(None, None)
cross_val = CrossVal(cv=kf, criterion=criterion)
algo = ImplicitForward(tol_jac=1e-8,
n_iter_jac=maxit,
use_stop_crit=False)
algo.max_iter = maxit
val, grad = cross_val.get_val_grad(model,
X,
y,
log_alpha,
algo.get_beta_jac_v,
tol=tol,
monitor=monitor,
max_iter=maxit)
elif method == "ground_truth":
for file in os.listdir("results/"):
if file.startswith("hypergradient_%s_%i_%s" %
(dataset_name, div_alpha, method)):
return
else:
clf = Lasso_celer(alpha=np.exp(log_alpha),
fit_intercept=False,
warm_start=True,
tol=1e-13,
max_iter=10000)
criterion = HeldOutMSE(None, None)
cross_val = CrossVal(cv=kf, criterion=criterion)
algo = Implicit(criterion)
model = Lasso(estimator=clf, max_iter=10000)
val, grad = cross_val.get_val_grad(model,
X,
y,
log_alpha,
algo.get_beta_jac_v,
tol=1e-13,
monitor=monitor)
else:
model = Lasso(max_iter=maxit)
criterion = HeldOutMSE(None, None)
cross_val = CrossVal(cv=kf, criterion=criterion)
if method == "forward":
algo = Forward(use_stop_crit=False)
elif method == "implicit_forward":
algo = ImplicitForward(use_stop_crit=False,
tol_jac=1e-8,
n_iter_jac=maxit,
max_iter=1000)
elif method == "implicit":
algo = Implicit(use_stop_crit=False, max_iter=1000)
elif method == "backward":
algo = Backward()
else:
1 / 0
algo.max_iter = maxit
algo.use_stop_crit = False
val, grad = cross_val.get_val_grad(model,
X,
y,
log_alpha,
algo.get_beta_jac_v,
tol=tol,
monitor=monitor,
max_iter=maxit)
results = (dataset_name, div_alpha, method, maxit, val, grad,
monitor.times[0])
df = pandas.DataFrame(results).transpose()
df.columns = [
'dataset', 'div_alpha', 'method', 'maxit', 'val', 'grad', 'time'
]
str_results = "results/hypergradient_%s_%i_%s_%i.pkl" % (
dataset_name, div_alpha, method, maxit)
df.to_pickle(str_results)
# Measure mse on test
mse_cv = mean_squared_error(y_test, model_cv.predict(X_test))
print("Vanilla LassoCV: Mean-squared error on test data %f" % mse_cv)
##############################################################################
##############################################################################
# Weighted Lasso with sparse-ho.
# We use the vanilla lassoCV coefficients as a starting point
alpha0 = np.log(model_cv.alpha_) * np.ones(X_train.shape[1])
# Weighted Lasso: Sparse-ho: 1 param per feature
estimator = Lasso(fit_intercept=False, max_iter=10, warm_start=True)
model = WeightedLasso(X_train, y_train, estimator=estimator)
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = ImplicitForward()
monitor = Monitor()
grad_search(algo, criterion, alpha0, monitor, n_outer=20, tol=1e-6)
##############################################################################
##############################################################################
# MSE on validation set
mse_sho_val = mean_squared_error(y_val, estimator.predict(X_val))
# MSE on test set, ie unseen data
mse_sho_test = mean_squared_error(y_test, estimator.predict(X_test))
print("Sparse-ho: Mean-squared error on validation data %f" % mse_sho_val)
print("Sparse-ho: Mean-squared error on test (unseen) data %f" % mse_sho_test)
labels = ['WeightedLasso val', 'WeightedLasso test', 'Lasso CV']
def parallel_function(dataset_name, method):
X, y = fetch_libsvm(dataset_name)
X, y = fetch_libsvm(dataset_name)
if dataset_name == "real-sim":
X = X[:, :2000]
X = csr_matrix(X) # very important for SVM
my_bool = norm(X, axis=1) != 0
X = X[my_bool, :]
y = y[my_bool]
logC = dict_logC[dataset_name]
for max_iter in dict_max_iter[dataset_name]:
print("Dataset %s, max iter %i" % (method, max_iter))
for i in range(2): # TODO change this
sss1 = StratifiedShuffleSplit(n_splits=2,
test_size=0.3333,
random_state=0)
idx_train, idx_val = sss1.split(X, y)
idx_train = idx_train[0]
idx_val = idx_val[0]
monitor = Monitor()
criterion = HeldOutSmoothedHinge(idx_train, idx_val)
model = SVM(estimator=None, max_iter=10_000)
if method == "ground_truth":
for file in os.listdir("results_svm/"):
if file.startswith("hypergradient_svm_%s_%s" %
(dataset_name, method)):
return
clf = LinearSVC(C=np.exp(logC),
tol=1e-32,
max_iter=10_000,
loss='hinge',
permute=False)
algo = Implicit(criterion)
model.estimator = clf
val, grad = criterion.get_val_grad(model,
X,
y,
logC,
algo.compute_beta_grad,
tol=1e-14,
monitor=monitor)
else:
if method == "sota":
clf = LinearSVC(C=np.exp(logC),
loss='hinge',
max_iter=max_iter,
tol=1e-32,
permute=False)
model.estimator = clf
algo = ImplicitForward(tol_jac=1e-32,
n_iter_jac=max_iter,
use_stop_crit=False)
elif method == "forward":
algo = Forward(use_stop_crit=False)
elif method == "implicit_forward":
algo = ImplicitForward(tol_jac=1e-8,
n_iter_jac=max_iter,
use_stop_crit=False)
else:
raise NotImplementedError
algo.max_iter = max_iter
algo.use_stop_crit = False
val, grad = criterion.get_val_grad(model,
X,
y,
logC,
algo.compute_beta_grad,
tol=tol,
monitor=monitor,
max_iter=max_iter)
results = (dataset_name, method, max_iter, val, grad, monitor.times[0])
df = pandas.DataFrame(results).transpose()
df.columns = ['dataset', 'method', 'maxit', 'val', 'grad', 'time']
str_results = "results_svm/hypergradient_svm_%s_%s_%i.pkl" % (
dataset_name, method, max_iter)
df.to_pickle(str_results)
##############################################################################
# Grid-search with scikit-learn
# -----------------------------
estimator = linear_model.Lasso(fit_intercept=False,
max_iter=1000,
warm_start=True)
print('scikit-learn started')
t0 = time.time()
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_val)
algo = Forward()
monitor_grid_sk = Monitor()
grid_search(algo,
criterion,
model,
X,
y,
None,
None,
monitor_grid_sk,
log_alphas=log_alphas,
tol=tol)
objs = np.array(monitor_grid_sk.objs)
t_sk = time.time() - t0
print('scikit-learn finished')
def callback(val, grad, mask, dense, alpha):
# The custom quantity is added at each outer iteration:
# here the prediction MSE on test data
objs_test.append(mean_squared_error(X_test[:, mask] @ dense, y_test))
##############################################################################
# Grad-search with sparse-ho and callback
# ---------------------------------------
model = Lasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_val)
algo = ImplicitForward()
# use Monitor(callback) with your custom callback
monitor = Monitor(callback=callback)
optimizer = LineSearch(n_outer=30)
grad_search(algo, criterion, model, optimizer, X, y, alpha0, monitor)
##############################################################################
# Plot results
# ------------
plt.figure(figsize=(5, 3))
plt.plot(monitor.times, objs_test)
plt.tick_params(width=5)
plt.xlabel("Times (s)")
plt.ylabel(r"$\|y^{\rm{test}} - X^{\rm{test}} \hat \beta^{(\lambda)} \|^2$")
plt.tight_layout()
plt.show(block=False)
def parallel_function(name_model, div_alpha):
index_col = np.arange(10)
alpha_max = (np.abs(X[np.ix_(idx_train, index_col)].T
@ y[idx_train])).max() / len(idx_train)
if name_model == "lasso":
log_alpha = np.log(alpha_max / div_alpha)
elif name_model == "enet":
alpha0 = alpha_max / div_alpha
alpha1 = (1 - l1_ratio) * alpha0 / l1_ratio
log_alpha = np.log(np.array([alpha0, alpha1]))
criterion = HeldOutMSE(idx_train, idx_val)
algo = Forward()
monitor = Monitor()
val, grad = criterion.get_val_grad(dict_models[name_model],
X[:, index_col],
y,
log_alpha,
algo.compute_beta_grad,
tol=tol,
monitor=monitor)
criterion = HeldOutMSE(idx_train, idx_val)
algo = Backward()
monitor = Monitor()
val, grad = criterion.get_val_grad(dict_models[name_model],
X[:, index_col],
y,
log_alpha,
algo.compute_beta_grad,
tol=tol,
monitor=monitor)
val_cvxpy, grad_cvxpy = dict_cvxpy[name_model](X[:, index_col], y,
np.exp(log_alpha),
idx_train, idx_val)
list_times_fwd = []
list_times_bwd = []
list_times_cvxpy = []
for n_col in dict_ncols[div_alpha]:
temp_fwd = []
temp_bwd = []
temp_cvxpy = []
for i in range(repeat):
rng = np.random.RandomState(i)
index_col = rng.choice(n_features, n_col, replace=False)
alpha_max = (np.abs(X[np.ix_(idx_train, index_col)].T
@ y[idx_train])).max() / len(idx_train)
if name_model == "lasso":
log_alpha = np.log(alpha_max / div_alpha)
elif name_model == "enet":
alpha0 = alpha_max / div_alpha
alpha1 = (1 - l1_ratio) * alpha0 / l1_ratio
log_alpha = np.log(np.array([alpha0, alpha1]))
criterion = HeldOutMSE(idx_train, idx_val)
algo = Forward()
monitor = Monitor()
val, grad = criterion.get_val_grad(dict_models[name_model],
X[:, index_col],
y,
log_alpha,
algo.compute_beta_grad,
tol=tol,
monitor=monitor)
temp_fwd.append(monitor.times)
criterion = HeldOutMSE(idx_train, idx_val)
algo = Backward()
monitor = Monitor()
val, grad = criterion.get_val_grad(dict_models[name_model],
X[:, index_col],
y,
log_alpha,
algo.compute_beta_grad,
tol=tol,
monitor=monitor)
temp_bwd.append(monitor.times)
t0 = time.time()
val_cvxpy, grad_cvxpy = dict_cvxpy[name_model](X[:, index_col], y,
np.exp(log_alpha),
idx_train, idx_val)
temp_cvxpy.append(time.time() - t0)
print(np.abs(grad - grad_cvxpy * np.exp(log_alpha)))
list_times_fwd.append(np.mean(np.array(temp_fwd)))
list_times_bwd.append(np.mean(np.array(temp_bwd)))
list_times_cvxpy.append(np.mean(np.array(temp_cvxpy)))
np.save("results/times_%s_forward_%s" % (name_model, div_alpha),
list_times_fwd)
np.save("results/times_%s_backward_%s" % (name_model, div_alpha),
list_times_bwd)
np.save("results/times_%s_cvxpy_%s" % (name_model, div_alpha),
list_times_cvxpy)
np.save("results/nfeatures_%s_%s" % (name_model, div_alpha),
dict_ncols[div_alpha])
max_iter=1000,
warm_start=True,
tol=tol)
dict_monitor = {}
all_algo_name = ['grid_search']
# , 'implicit_forward', "implicit_forward_approx", 'bayesian']
# , 'random_search']
# all_algo_name = ['random_search']
for algo_name in all_algo_name:
model = ElasticNet(estimator=estimator)
sub_criterion = HeldOutMSE(None, None)
alpha0 = np.array([alpha_max / 10, alpha_max / 10])
monitor = Monitor()
kf = KFold(n_splits=5, shuffle=True, random_state=42)
criterion = CrossVal(sub_criterion, cv=kf)
algo = ImplicitForward(tol_jac=1e-3)
# optimizer = LineSearch(n_outer=10, tol=tol)
if algo_name.startswith('implicit_forward'):
if algo_name == "implicit_forward_approx":
optimizer = GradientDescent(n_outer=30,
p_grad_norm=1.,
verbose=True,
tol=tol,
tol_decrease="geom")
else:
optimizer = GradientDescent(n_outer=30,
p_grad_norm=1.,
verbose=True,