def test_get_max_squared_sum(): n_samples = 100 n_features = 10 rng = np.random.RandomState(42) X = rng.randn(n_samples, n_features).astype(np.float64) mask = rng.randn(n_samples, n_features) X[mask > 0] = 0. X_csr = sp.csr_matrix(X) X[0, 3] = 0. X_csr[0, 3] = 0. sum_X = get_max_squared_sum(X) sum_X_csr = get_max_squared_sum(X_csr) assert_almost_equal(sum_X, sum_X_csr)
def test_get_auto_step_size(): X = np.array([[1, 2, 3], [2, 3, 4], [2, 3, 2]], dtype=np.float64) alpha = 1.2 fit_intercept = False # sum the squares of the second sample because that's the largest max_squared_sum = 4 + 9 + 16 max_squared_sum_ = get_max_squared_sum(X) assert_almost_equal(max_squared_sum, max_squared_sum_, decimal=4) for fit_intercept in (True, False): step_size_sqr = 1.0 / (max_squared_sum + alpha + int(fit_intercept)) step_size_log = 4.0 / (max_squared_sum + 4.0 * alpha + int(fit_intercept)) step_size_sqr_ = get_auto_step_size(max_squared_sum_, alpha, "squared", fit_intercept) step_size_log_ = get_auto_step_size(max_squared_sum_, alpha, "log", fit_intercept) assert_almost_equal(step_size_sqr, step_size_sqr_, decimal=4) assert_almost_equal(step_size_log, step_size_log_, decimal=4) msg = 'Unknown loss function for SAG solver, got wrong instead of' assert_raise_message(ValueError, msg, get_auto_step_size, max_squared_sum_, alpha, "wrong", fit_intercept)
def get_auto_step_size(X, alpha, loss, gamma): """Compute automatic step size for SAG solver Stepsize computed using the following objective: minimize_w 1 / n_samples * \sum_i loss(w^T x_i, y_i) + alpha * 0.5 * ||w||^2_2 Parameters ---------- X : ndarray Array of samples x_i. alpha : float Constant that multiplies the l2 penalty term. loss : string, in {"log", "squared"} The loss function used in SAG solver. Returns ------- step_size : float Step size used in SAG/SAGA solver. """ L = get_max_squared_sum(X) if loss == 'log': # inverse Lipschitz constant for log loss stepsize = 4.0 / (L + 4.0 * alpha) elif loss == 'squared': # inverse Lipschitz constant for squared loss stepsize = 1.0 / (L + alpha) elif loss == 'modified_huber': stepsize = 1.0 / (2 * L + alpha) elif loss == 'smooth_hinge': stepsize = gamma / (L + gamma * alpha) elif loss == 'squared_hinge': stepsize = 1.0 / (2 * L + alpha) else: raise ValueError("`auto` stepsize is only available for `squared` or " "`log` losses (got `%s` loss). Please specify a " "stepsize." % loss) return stepsize
fit_intercept=fit_intercept), newton_iter_range, [], [], [], []), ("LR-lbfgs", LogisticRegression(C=C, tol=tol, solver="lbfgs", fit_intercept=fit_intercept), lbfgs_iter_range, [], [], [], []), ("SGD", SGDClassifier(alpha=1.0 / C / n_samples, penalty='l2', loss='log', fit_intercept=fit_intercept, verbose=0), sgd_iter_range, [], [], [], [])] if lightning_clf is not None and not fit_intercept: alpha = 1. / C / n_samples # compute the same step_size than in LR-sag max_squared_sum = get_max_squared_sum(X) step_size = get_auto_step_size(max_squared_sum, alpha, "log", fit_intercept) clfs.append( ("Lightning-SVRG", lightning_clf.SVRGClassifier(alpha=alpha, eta=step_size, tol=tol, loss="log"), sag_iter_range, [], [], [], [])) clfs.append( ("Lightning-SAG", lightning_clf.SAGClassifier(alpha=alpha, eta=step_size, tol=tol, loss="log"), sag_iter_range, [], [], [], [])) # We keep only 200 features, to have a dense dataset,
LogisticRegression(C=C, tol=tol, solver="lbfgs", fit_intercept=fit_intercept), lbfgs_iter_range, [], [], [], []), ("SGD", SGDClassifier(alpha=1.0 / C / n_samples, penalty='l2', loss='log', fit_intercept=fit_intercept, verbose=0), sgd_iter_range, [], [], [], [])] if lightning_clf is not None and not fit_intercept: alpha = 1. / C / n_samples # compute the same step_size than in LR-sag max_squared_sum = get_max_squared_sum(X) step_size = get_auto_step_size(max_squared_sum, alpha, "log", fit_intercept) clfs.append(("Lightning-SVRG", lightning_clf.SVRGClassifier(alpha=alpha, eta=step_size, tol=tol, loss="log"), sag_iter_range, [], [], [], [])) clfs.append(("Lightning-SAG", lightning_clf.SAGClassifier(alpha=alpha, eta=step_size, tol=tol, loss="log"), sag_iter_range, [], [], [], []))