def fit(self, X, y):
try:
#TODO remove svrg, performance no good with same parameter space with sgd
if self.solver == 'svrg':
self.estimator = SVRGClassifier(loss=self.loss,
alpha=self.alpha,
tol=self.tol,
eta=self.eta,
gamma=self.gamma,
max_iter=self.max_iter,
verbose=self.verbose)
elif self.solver == 'sgd':
self.estimator = SGDClassifier(
loss=self.loss,
alpha=self.alpha,
penalty=self.penalty,
eta0=self.eta,
max_iter=self.max_iter,
multiclass=self.multiclass,
learning_rate=self.learning_rate,
verbose=self.verbose)
else:
raise NotImplementedError
print('No impelement solver of {}'.format(self.solver))
except Exception as e:
print(self.solver, e)
self.estimator.fit(X, y)
return self
def test_svrg_callback():
class Callback(object):
def __init__(self, X, y):
self.X = X
self.y = y
self.obj = []
def __call__(self, clf):
clf._finalize_coef()
y_pred = clf.decision_function(self.X).ravel()
loss = (np.maximum(1 - self.y * y_pred, 0) ** 2).mean()
coef = clf.coef_.ravel()
regul = 0.5 * clf.alpha * np.dot(coef, coef)
self.obj.append(loss + regul)
cb = Callback(X_bin, y_bin)
clf = SVRGClassifier(loss="squared_hinge", eta=1e-3, max_iter=20,
random_state=0, callback=cb)
clf.fit(X_bin, y_bin)
assert_true(np.all(np.diff(cb.obj) <= 0))
X, y = make_classification(n_samples=10000,
n_features=100,
n_classes=2,
random_state=0)
etas = (1e-3, 1e-4, 1e-5)
n_inners = (0.25, 0.5, 1.0, 1.5)
y = y * 2 - 1
plt.figure()
for eta in etas:
print "eta =", eta
cb = Callback(X, y)
clf = SVRGClassifier(loss="squared_hinge", alpha=1e-5, eta=eta,
n_inner=1.0, max_iter=20, random_state=0, callback=cb)
clf.fit(X, y)
plt.plot(cb.times, cb.obj, label="eta=" + str(eta))
plt.xlabel("CPU time")
plt.ylabel("Objective value")
plt.legend()
plt.figure()
for n_inner in n_inners:
print "n_inner =", n_inner
cb = Callback(X, y)
clf = SVRGClassifier(loss="squared_hinge", alpha=1e-5, eta=1e-4,
n_inner=n_inner, max_iter=1000, random_state=0,
callback=cb, verbose=1)
n_classes=2,
random_state=0)
etas = (1e-3, 1e-4, 1e-5)
n_inners = (0.25, 0.5, 1.0, 1.5)
y = y * 2 - 1
plt.figure()
for eta in etas:
print "eta =", eta
cb = Callback(X, y)
clf = SVRGClassifier(loss="squared_hinge",
alpha=1e-5,
eta=eta,
n_inner=1.0,
max_iter=20,
random_state=0,
callback=cb)
clf.fit(X, y)
plt.plot(cb.times, cb.obj, label="eta=" + str(eta))
plt.xlabel("CPU time")
plt.ylabel("Objective value")
plt.legend()
plt.figure()
for n_inner in n_inners:
print "n_inner =", n_inner
cb = Callback(X, y)
random_state=0)
alpha = 1e-2
eta_svrg = 1e-3
eta_adagrad = 1e-2
xlim = [0, 2]
y = y * 2 - 1
# make sure the method does not stop prematurely, we want to see
# the full convergence path
tol = 1e-24
clf1 = SVRGClassifier(loss="squared_hinge",
alpha=alpha,
eta=eta_svrg,
n_inner=1.0,
max_iter=100,
random_state=0,
tol=1e-24)
clf2 = SDCAClassifier(loss="squared_hinge",
alpha=alpha,
max_iter=100,
n_calls=X.shape[0] / 2,
random_state=0,
tol=tol)
clf3 = CDClassifier(loss="squared_hinge",
alpha=alpha,
C=1.0 / X.shape[0],
max_iter=50,
n_calls=X.shape[1] / 3,
random_state=0,
def test_multiclass_classes():
clf = SVRGClassifier()
clf.fit(X, y)
assert_equal(list(clf.classes_), [0, 1, 2])
def test_bin_classes():
clf = SVRGClassifier()
clf.fit(X_bin, y_bin)
assert_equal(list(clf.classes_), [-1, 1])
def test_svrg():
clf = SVRGClassifier(eta=1e-3, max_iter=20, random_state=0, verbose=0)
clf.fit(X_bin, y_bin)
assert not hasattr(clf, 'predict_proba')
assert_equal(clf.score(X_bin, y_bin), 1.0)
def test_multiclass_classes():
clf = SVRGClassifier()
clf.fit(X, y)
assert list(clf.classes_) == [0, 1, 2]
def test_bin_classes():
clf = SVRGClassifier()
clf.fit(X_bin, y_bin)
assert list(clf.classes_) == [-1, 1]
import time
import numpy as np
from sklearn.datasets import fetch_20newsgroups_vectorized
from lightning.classification import SVRGClassifier
bunch = fetch_20newsgroups_vectorized(subset="all")
X = bunch.data
y = bunch.target
y[y >= 1] = 1
clf = SVRGClassifier(eta=0.1, alpha=1e-5, tol=1e-3, max_iter=20, verbose=1,
random_state=0)
start = time.time()
clf.fit(X, y)
print "Training time", time.time() - start
print "Accuracy", np.mean(clf.predict(X) == y)
print "% non-zero", clf.n_nonzero(percentage=True)
import time
import numpy as np
from sklearn.datasets import fetch_20newsgroups_vectorized
from lightning.classification import SVRGClassifier
bunch = fetch_20newsgroups_vectorized(subset="all")
X = bunch.data
y = bunch.target
y[y >= 1] = 1
clf = SVRGClassifier(eta=0.1,
alpha=1e-5,
tol=1e-3,
max_iter=20,
verbose=1,
random_state=0)
start = time.time()
clf.fit(X, y)
print "Training time", time.time() - start
print "Accuracy", np.mean(clf.predict(X) == y)
print "% non-zero", clf.n_nonzero(percentage=True)