def test_get_max_squared_sum():
n_samples = 100
n_features = 10
rng = np.random.RandomState(42)
X = rng.randn(n_samples, n_features).astype(np.float64)
mask = rng.randn(n_samples, n_features)
X[mask > 0] = 0.
X_csr = sp.csr_matrix(X)
X[0, 3] = 0.
X_csr[0, 3] = 0.
sum_X = get_max_squared_sum(X)
sum_X_csr = get_max_squared_sum(X_csr)
assert_almost_equal(sum_X, sum_X_csr)
def test_get_max_squared_sum():
n_samples = 100
n_features = 10
rng = np.random.RandomState(42)
X = rng.randn(n_samples, n_features).astype(np.float64)
mask = rng.randn(n_samples, n_features)
X[mask > 0] = 0.
X_csr = sp.csr_matrix(X)
X[0, 3] = 0.
X_csr[0, 3] = 0.
sum_X = get_max_squared_sum(X)
sum_X_csr = get_max_squared_sum(X_csr)
assert_almost_equal(sum_X, sum_X_csr)
def test_get_auto_step_size():
X = np.array([[1, 2, 3], [2, 3, 4], [2, 3, 2]], dtype=np.float64)
alpha = 1.2
fit_intercept = False
# sum the squares of the second sample because that's the largest
max_squared_sum = 4 + 9 + 16
max_squared_sum_ = get_max_squared_sum(X)
assert_almost_equal(max_squared_sum, max_squared_sum_, decimal=4)
for fit_intercept in (True, False):
step_size_sqr = 1.0 / (max_squared_sum + alpha + int(fit_intercept))
step_size_log = 4.0 / (max_squared_sum + 4.0 * alpha +
int(fit_intercept))
step_size_sqr_ = get_auto_step_size(max_squared_sum_, alpha, "squared",
fit_intercept)
step_size_log_ = get_auto_step_size(max_squared_sum_, alpha, "log",
fit_intercept)
assert_almost_equal(step_size_sqr, step_size_sqr_, decimal=4)
assert_almost_equal(step_size_log, step_size_log_, decimal=4)
msg = 'Unknown loss function for SAG solver, got wrong instead of'
assert_raise_message(ValueError, msg, get_auto_step_size, max_squared_sum_,
alpha, "wrong", fit_intercept)
def test_get_auto_step_size():
X = np.array([[1, 2, 3], [2, 3, 4], [2, 3, 2]], dtype=np.float64)
alpha = 1.2
fit_intercept = False
# sum the squares of the second sample because that's the largest
max_squared_sum = 4 + 9 + 16
max_squared_sum_ = get_max_squared_sum(X)
assert_almost_equal(max_squared_sum, max_squared_sum_, decimal=4)
for fit_intercept in (True, False):
step_size_sqr = 1.0 / (max_squared_sum + alpha + int(fit_intercept))
step_size_log = 4.0 / (max_squared_sum + 4.0 * alpha +
int(fit_intercept))
step_size_sqr_ = get_auto_step_size(max_squared_sum_, alpha, "squared",
fit_intercept)
step_size_log_ = get_auto_step_size(max_squared_sum_, alpha, "log",
fit_intercept)
assert_almost_equal(step_size_sqr, step_size_sqr_, decimal=4)
assert_almost_equal(step_size_log, step_size_log_, decimal=4)
msg = 'Unknown loss function for SAG solver, got wrong instead of'
assert_raise_message(ValueError, msg, get_auto_step_size,
max_squared_sum_, alpha, "wrong", fit_intercept)
def get_auto_step_size(X, alpha, loss, gamma):
"""Compute automatic step size for SAG solver
Stepsize computed using the following objective:
minimize_w 1 / n_samples * \sum_i loss(w^T x_i, y_i)
+ alpha * 0.5 * ||w||^2_2
Parameters
----------
X : ndarray
Array of samples x_i.
alpha : float
Constant that multiplies the l2 penalty term.
loss : string, in {"log", "squared"}
The loss function used in SAG solver.
Returns
-------
step_size : float
Step size used in SAG/SAGA solver.
"""
L = get_max_squared_sum(X)
if loss == 'log':
# inverse Lipschitz constant for log loss
stepsize = 4.0 / (L + 4.0 * alpha)
elif loss == 'squared':
# inverse Lipschitz constant for squared loss
stepsize = 1.0 / (L + alpha)
elif loss == 'modified_huber':
stepsize = 1.0 / (2 * L + alpha)
elif loss == 'smooth_hinge':
stepsize = gamma / (L + gamma * alpha)
elif loss == 'squared_hinge':
stepsize = 1.0 / (2 * L + alpha)
else:
raise ValueError("`auto` stepsize is only available for `squared` or "
"`log` losses (got `%s` loss). Please specify a "
"stepsize." % loss)
return stepsize
def get_auto_step_size(X, alpha, loss, gamma):
"""Compute automatic step size for SAG solver
Stepsize computed using the following objective:
minimize_w 1 / n_samples * \sum_i loss(w^T x_i, y_i)
+ alpha * 0.5 * ||w||^2_2
Parameters
----------
X : ndarray
Array of samples x_i.
alpha : float
Constant that multiplies the l2 penalty term.
loss : string, in {"log", "squared"}
The loss function used in SAG solver.
Returns
-------
step_size : float
Step size used in SAG/SAGA solver.
"""
L = get_max_squared_sum(X)
if loss == 'log':
# inverse Lipschitz constant for log loss
stepsize = 4.0 / (L + 4.0 * alpha)
elif loss == 'squared':
# inverse Lipschitz constant for squared loss
stepsize = 1.0 / (L + alpha)
elif loss == 'modified_huber':
stepsize = 1.0 / (2 * L + alpha)
elif loss == 'smooth_hinge':
stepsize = gamma / (L + gamma * alpha)
elif loss == 'squared_hinge':
stepsize = 1.0 / (2 * L + alpha)
else:
raise ValueError("`auto` stepsize is only available for `squared` or "
"`log` losses (got `%s` loss). Please specify a "
"stepsize." % loss)
return stepsize
fit_intercept=fit_intercept),
newton_iter_range, [], [], [], []),
("LR-lbfgs",
LogisticRegression(C=C, tol=tol,
solver="lbfgs", fit_intercept=fit_intercept),
lbfgs_iter_range, [], [], [], []),
("SGD",
SGDClassifier(alpha=1.0 / C / n_samples, penalty='l2', loss='log',
fit_intercept=fit_intercept, verbose=0),
sgd_iter_range, [], [], [], [])]
if lightning_clf is not None and not fit_intercept:
alpha = 1. / C / n_samples
# compute the same step_size than in LR-sag
max_squared_sum = get_max_squared_sum(X)
step_size = get_auto_step_size(max_squared_sum, alpha, "log",
fit_intercept)
clfs.append(
("Lightning-SVRG",
lightning_clf.SVRGClassifier(alpha=alpha, eta=step_size,
tol=tol, loss="log"),
sag_iter_range, [], [], [], []))
clfs.append(
("Lightning-SAG",
lightning_clf.SAGClassifier(alpha=alpha, eta=step_size,
tol=tol, loss="log"),
sag_iter_range, [], [], [], []))
# We keep only 200 features, to have a dense dataset,
LogisticRegression(C=C,
tol=tol,
solver="lbfgs",
fit_intercept=fit_intercept), lbfgs_iter_range, [],
[], [], []),
("SGD",
SGDClassifier(alpha=1.0 / C / n_samples,
penalty='l2',
loss='log',
fit_intercept=fit_intercept,
verbose=0), sgd_iter_range, [], [], [], [])]
if lightning_clf is not None and not fit_intercept:
alpha = 1. / C / n_samples
# compute the same step_size than in LR-sag
max_squared_sum = get_max_squared_sum(X)
step_size = get_auto_step_size(max_squared_sum, alpha, "log",
fit_intercept)
clfs.append(("Lightning-SVRG",
lightning_clf.SVRGClassifier(alpha=alpha,
eta=step_size,
tol=tol,
loss="log"), sag_iter_range, [],
[], [], []))
clfs.append(("Lightning-SAG",
lightning_clf.SAGClassifier(alpha=alpha,
eta=step_size,
tol=tol,
loss="log"), sag_iter_range, [],
[], [], []))
