def test_precomputed():
"""
SVC with a precomputed kernel.
We test it with a toy dataset and with iris.
"""
clf = svm.SVC(kernel='precomputed')
# just a linear kernel
K = np.dot(X, np.array(X).T)
clf.fit(K, Y)
# gram matrix
KT = np.dot(T, np.array(X).T)
pred = clf.predict(KT)
assert_array_equal(clf.dual_coef_, [[0.25, -.25]])
assert_array_equal(clf.intercept_, [0])
assert_array_almost_equal(clf.support_, [[2], [4]])
assert_array_equal(pred, true_result)
# same as before, but using function instead of the kernel
# matrix. kernel is just a linear kernel
kfunc = lambda x, y: np.dot(x, y.T)
clf = svm.SVC(kernel=kfunc)
clf.fit(X, Y)
pred = clf.predict(T)
assert_array_equal(clf.dual_coef_, [[0.25, -.25]])
assert_array_equal(clf.intercept_, [0])
assert_array_almost_equal(clf.support_, [[2], [4]])
assert_array_equal(pred, true_result)
def train_svm(DATA, LABELS):
""" train_svm(DATA, LABELS)
Function that applies scikits.learn LIBSVM Python bindings to train a linear SVM classifier on labeled data.
inputs: DATA -- A NumPy matrix where each row is a feature vector.
LABELS -- A NumPy matrix where each row is a singleton value (+1 or -1) that labels the corresponding row of DATA.
outputs: clf -- A scikits.learn native object / data structure containing the parameters for the trained SVM. Use pickling to
save this for persistence across different Python sessions.
"""
# Run the scikits.learn setup and training commands; return the result.
parameters = {'gamma': np.arange(0.1, 2.0, 0.1), 'C': np.arange(1, 20, 1)}
clf = gs.GridSearchCV(svm.SVC(kernel='rbf', probability=True), parameters)
clf.fit(np.asarray(DATA), np.asarray(LABELS))
best_parameters, score = max(clf.grid_scores_, key=lambda x: x[1])
new_gamma = best_parameters['gamma']
new_C = best_parameters['C']
print "Optimal parameters found: gamma %f and C %f" % (new_gamma, new_C)
clf1 = svm.SVC(kernel='rbf', gamma=new_gamma, C=new_C, probability=True)
clf1.fit(DATA, LABELS, class_weight='auto')
return clf1
def test_auto_weight():
"""Test class weights for imbalanced data"""
# compute reference metrics on iris dataset that is quite balanced by
# default
X, y = iris.data, iris.target
clf = svm.SVC(kernel="linear").fit(X, y)
assert_almost_equal(metrics.f1_score(y, clf.predict(X)), 0.99, 2)
# make the same prediction using automated class_weight
clf_auto = svm.SVC(kernel="linear").fit(X, y, class_weight="auto")
assert_almost_equal(metrics.f1_score(y, clf_auto.predict(X)), 0.99, 2)
# Make sure that in the balanced case it does not change anything
# to use "auto"
assert_array_almost_equal(clf.coef_, clf_auto.coef_, 6)
# build an very very imbalanced dataset out of iris data
X_0 = X[y == 0, :]
y_0 = y[y == 0]
X_imbalanced = np.vstack([X] + [X_0] * 10)
y_imbalanced = np.concatenate([y] + [y_0] * 10)
# fit a model on the imbalanced data without class weight info
y_pred = svm.SVC().fit(X_imbalanced, y_imbalanced).predict(X)
assert_almost_equal(metrics.f1_score(y, y_pred), 0.88, 2)
# fit a model with auto class_weight enabled
clf = svm.SVC().fit(X_imbalanced, y_imbalanced, class_weight="auto")
y_pred = clf.predict(X)
assert_almost_equal(metrics.f1_score(y, y_pred), 0.92, 2)
def test_bad_input():
"""
Test that it gives proper exception on deficient input
"""
# impossible value of C
assert_raises(ValueError, svm.SVC(C=-1).fit, X, Y)
# impossible value of nu
clf = svm.NuSVC(nu=0.0)
assert_raises(ValueError, clf.fit, X, Y)
Y2 = Y[:-1] # wrong dimensions for labels
assert_raises(ValueError, clf.fit, X, Y2)
# Test with arrays that are non-contiguous.
Xf = np.asfortranarray(X)
clf = svm.SVC()
clf.fit(Xf, Y)
assert_array_equal(clf.predict(T), true_result)
# error for precomputed kernelsx
clf = svm.SVC(kernel='precomputed')
assert_raises(ValueError, clf.fit, X, Y)
Xt = np.array(X).T
clf = svm.SVC(kernel='precomputed')
clf.fit(np.dot(X, Xt), Y)
assert_raises(ValueError, clf.predict, X)
clf = svm.SVC()
clf.fit(X, Y)
assert_raises(ValueError, clf.predict, Xt)
def test_sanity_checks_predict():
Xt = np.array(X).T
clf = svm.SVC(kernel='precomputed')
clf.fit(np.dot(X, Xt), Y)
assert_raises(ValueError, clf.predict, X)
clf = svm.SVC()
clf.fit(X, Y)
assert_raises(ValueError, clf.predict, Xt)
def test_probability():
"""
Predict probabilities using SVC
This uses cross validation, so we use a slightly bigger testing
set.
"""
from scikits.learn import datasets
iris = datasets.load_iris()
clf = svm.SVC(probability=True)
clf.fit(iris.data, iris.target)
# predict on a simple dataset
T = [[0, 0, 0, 0],
[2, 2, 2, 2]]
assert_array_almost_equal(clf.predict_proba(T),
[[ 0.993, 0.003, 0.002],
[ 0.740, 0.223 , 0.035]],
decimal=2)
# make sure probabilities sum to one
pprob = clf.predict_proba(X)
assert_array_almost_equal( pprob.sum(axis=1),
np.ones(len(X)))
def test_decision_function():
"""
Test decision_function
Sanity check, test that decision_function implemented in python
returns the same as the one in libsvm
TODO: proabably could be simplified
"""
clf = svm.SVC(kernel='linear').fit(iris.data, iris.target)
data = iris.data[0]
sv_start = np.r_[0, np.cumsum(clf.n_support_)]
n_class = 3
kvalue = np.dot(data, clf.support_vectors_.T)
dec = np.empty(n_class * (n_class - 1) / 2)
p = 0
for i in range(n_class):
for j in range(i + 1, n_class):
coef1 = clf.dual_coef_[j - 1]
coef2 = clf.dual_coef_[i]
idx1 = slice(sv_start[i], sv_start[i + 1])
idx2 = slice(sv_start[j], sv_start[j + 1])
s = np.dot(coef1[idx1], kvalue[idx1]) + \
np.dot(coef2[idx2], kvalue[idx2]) + \
clf.intercept_[p]
dec[p] = s
p += 1
assert_array_almost_equal(-dec, np.ravel(clf.decision_function(data)))
def test_weight():
"""
Test class weights
"""
clf = svm.SVC()
# we give a small weights to class 1
clf.fit(X, Y, {1: 0.1})
# so all predicted values belong to class 2
assert_array_almost_equal(clf.predict(X), [2] * 6)
X_, y_ = test_dataset_classif(n_samples=200, n_features=100, param=[5, 1],
seed=0)
for clf in (linear_model.LogisticRegression(), svm.LinearSVC(), svm.SVC()):
clf.fit(X_[: 180], y_[: 180], class_weight={0: 5})
y_pred = clf.predict(X_[180:])
assert np.sum(y_pred == y_[180:]) >= 11
def test_probability():
"""
Predict probabilities using SVC
<<<<<<< HEAD:scikits/learn/tests/test_svm.py
FIXME: is it harmless that we obtain slightly different results on
different operating systems ? (that is why we only check for 1
decimal precission)
TODO: test also on an example with intercept != 0
=======
This uses cross validation, so we use a slightly bigger testing
set.
>>>>>>> 2f4844ca54934b842fd76a5fe38b2c4de090b666:scikits/learn/tests/test_svm.py
"""
from scikits.learn import datasets
iris = datasets.load_iris()
clf = svm.SVC(probability=True)
clf.fit(iris.data, iris.target)
# predict on a simple dataset
T = [[0, 0, 0, 0], [2, 2, 2, 2]]
assert_array_almost_equal(clf.predict_proba(T),
[[0.993, 0.003, 0.002], [0.740, 0.223, 0.035]],
decimal=2)
# make sure probabilities sum to one
pprob = clf.predict_proba(X)
assert_array_almost_equal(pprob.sum(axis=1), np.ones(len(X)))
def test_libsvm_iris():
"""
Check consistency on dataset iris.
"""
# shuffle the dataset so that labels are not ordered
for k in ('linear', 'rbf'):
clf = svm.SVC(kernel=k).fit(iris.data, iris.target)
assert np.mean(clf.predict(iris.data) == iris.target) > 0.9
assert_array_equal(clf.label_, np.sort(clf.label_))
# check also the low-level API
model = svm.libsvm.fit(iris.data, iris.target.astype(np.float64))
pred = svm.libsvm.predict(iris.data, *model)
assert np.mean(pred == iris.target) > .95
model = svm.libsvm.fit(iris.data,
iris.target.astype(np.float64),
kernel='linear')
pred = svm.libsvm.predict(iris.data, *model, **{'kernel': 'linear'})
assert np.mean(pred == iris.target) > .95
pred = svm.libsvm.cross_validation(iris.data,
iris.target.astype(np.float64),
5,
kernel='linear')
assert np.mean(pred == iris.target) > .95
def test_sparse_realdata():
"""
Test on a subset from the 20newsgroups dataset.
This catchs some bugs if input is not correctly converted into
sparse format or weights are not correctly initialized.
"""
data = np.array([0.03771744, 0.1003567, 0.01174647, 0.027069])
indices = np.array([6, 5, 35, 31])
indptr = np.array([
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 4, 4, 4
])
X = scipy.sparse.csr_matrix((data, indices, indptr))
y = np.array([
1., 0., 2., 2., 1., 1., 1., 2., 2., 0., 1., 2., 2., 0., 2., 0., 3., 0.,
3., 0., 1., 1., 3., 2., 3., 2., 0., 3., 1., 0., 2., 1., 2., 0., 1., 0.,
2., 3., 1., 3., 0., 1., 0., 0., 2., 0., 1., 2., 2., 2., 3., 2., 0., 3.,
2., 1., 2., 3., 2., 2., 0., 1., 0., 1., 2., 3., 0., 0., 2., 2., 1., 3.,
1., 1., 0., 1., 2., 1., 1., 3.
])
clf = svm.SVC(kernel='linear').fit(X.todense(), y)
sp_clf = svm.sparse.SVC(kernel='linear').fit(X, y)
assert_array_equal(clf.support_vectors_, sp_clf.support_vectors_.todense())
assert_array_equal(clf.dual_coef_, sp_clf.dual_coef_.todense())
def test_SVC():
"""Check that sparse SVC gives the same result as SVC"""
clf = svm.SVC(kernel='linear').fit(X, Y)
sp_clf = svm.sparse.SVC(kernel='linear').fit(X, Y)
assert_array_equal(sp_clf.predict(T), true_result)
assert scipy.sparse.issparse(sp_clf.support_vectors_)
assert_array_almost_equal(clf.support_vectors_,
sp_clf.support_vectors_.todense())
assert scipy.sparse.issparse(sp_clf.dual_coef_)
assert_array_almost_equal(clf.dual_coef_, sp_clf.dual_coef_.todense())
assert scipy.sparse.issparse(sp_clf.coef_)
assert_array_almost_equal(clf.coef_, sp_clf.coef_.todense())
assert_array_almost_equal(clf.predict(T), sp_clf.predict(T))
# refit with a different dataset
clf.fit(X2, Y2)
sp_clf.fit(X2, Y2)
assert_array_almost_equal(clf.support_vectors_,
sp_clf.support_vectors_.todense())
assert_array_almost_equal(clf.dual_coef_, sp_clf.dual_coef_.todense())
assert_array_almost_equal(clf.coef_, sp_clf.coef_.todense())
assert_array_almost_equal(clf.predict(T2), sp_clf.predict(T2))
def fit(self):
print "fit the model"
train = np.array(self.model.data)
X = train[:, :2]
y = train[:, 2]
C = float(self.complexity.get())
gamma = float(self.gamma.get())
coef0 = float(self.coef0.get())
degree = int(self.degree.get())
kernel_map = {0: "linear", 1: "rbf", 2: "poly"}
if len(np.unique(y)) == 1:
clf = svm.OneClassSVM(kernel=kernel_map[self.kernel.get()],
gamma=gamma,
coef0=coef0,
degree=degree)
clf.fit(X)
else:
clf = svm.SVC(kernel=kernel_map[self.kernel.get()],
C=C,
gamma=gamma,
coef0=coef0,
degree=degree)
clf.fit(X, y)
if hasattr(clf, 'score'):
print "Accuracy:", clf.score(X, y) * 100
X1, X2, Z = self.decision_surface(clf)
self.model.clf = clf
self.model.set_surface((X1, X2, Z))
self.model.surface_type = self.surface_type.get()
self.fitted = True
self.model.changed("surface")
def test_CSVC():
"""
C_SVC algorithm and linear kernel.
We test this on two datasets, the first one with two classes and
the second one with three classes. We check for predicted values
and estimated parameters.
TODO: check with different parameters of C, nonlinear kernel
"""
clf = svm.SVC(kernel='linear')
clf.fit(X, Y)
pred = clf.predict(T)
assert_array_equal(clf.dual_coef_, [[ 0.25, -.25]])
assert_array_equal(clf.support_, [[-1,-1], [1, 1]])
assert_array_equal(clf.intercept_, [0.])
assert_array_equal(pred, true_result)
# the same with other dataset
clf.fit(X2, Y2)
pred = clf.predict(T2)
assert_array_almost_equal(clf.dual_coef_,
[[ .99, -.006, -.49, -.49, -.07],
[ .072, .16, 0, 0, -.16]], decimal=2)
# TODO: why are we getting all the dataset as support vectors
assert_array_equal(clf.support_,
[[ 0., 0., 0.],
[ 1., 1., 1.],
[ 2., 0., 0.],
[ 0., 0., 2.],
[ 3., 3., 3.]])
assert_array_equal(pred, true_result2)
def train_liblinear_classifier_core(trainXy,
classifier_type="liblinear",
trace_normalize=False,
**kwargs):
""" Classifier training using SVMs
Input:
train_features = training features (both positive and negative)
train_labels = corresponding label vector
svm_eps = eps of svm
svm_C = C parameter of svm
classifier_type = liblinear or libsvm"""
#do normalization
(train_features, train_labels), train_mean, train_std, trace = normalize(
[trainXy], trace_normalize=trace_normalize)
if classifier_type == 'liblinear':
clf = sklearn_svm.LinearSVC(**kwargs)
if classifier_type == 'libSVM':
clf = sklearn_svm.SVC(**kwargs)
elif classifier_type == 'LRL':
clf = LogisticRegression(**kwargs)
elif classifier_type == 'MCC':
clf = CorrelationClassifier(**kwargs)
elif classifier_type.startswith('svm.'):
ct = classifier_type.split('.')[-1]
clf = getattr(sklearn_svm, ct)(**kwargs)
elif classifier_type.startswith('linear_model.'):
ct = classifier_type.split('.')[-1]
clf = getattr(sklearn_linear_model, ct)(**kwargs)
clf.fit(train_features, train_labels)
return clf, train_mean, train_std, trace
def train_scikit_svm():
# NOTE FROM SCIT KIT : All classifiers in scikit-learn do multiclass classification out-of-the-box.
# You don’t need to use the sklearn.multiclass module unless you want to experiment with different multiclass strategies.
clf = svm.SVC(decision_function_shape='ovo')
SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
decision_function_shape='ovo', degree=3, gamma='auto', kernel='rbf',
max_iter=-1, probability=False, random_state=None, shrinking=True,
tol=0.001, verbose=False)
def get_svn_params(X, y):
clf = svm.SVC(kernel='linear')
clf.fit(X, y)
if len(clf.intercept_) <> 1:
print "ERROR in get_hyperplane(X, y). more than 1 coefficient??"
return
return clf.coef_[0], clf.intercept_[0], clf
def test_error():
"""
Test that it gives proper exception on deficient input
"""
# impossible value of nu
clf = svm.SVC(impl='nu_svc', kernel='linear', nu=0.0)
assert_raises(ValueError, clf.fit, X, Y)
Y2 = Y[:-1] # wrong dimensions for labels
assert_raises(ValueError, svm.SVC, X, Y2)
# Test with arrays that are non-contiguous.
Xt = np.array(X).transpose()
Yt = [1, 2]
clf = svm.SVC()
clf.fit(Xt, Yt)
assert_array_equal(clf.predict(T), [1, 2, 2])
def svm_classification(table):
'''
'''
from scikits.learn import svm
X = table[:, 1:]
Y = table[:, 0]
clf = svm.SVC()
clf.fit(X, Y)
print clf.support_
def fitting(D, cp, ca, kwargs={'kernel': 'linear', 'C': 1.0}):
#clfP = svm.SVC(kernel='rbf', gamma=0.7, C=1.0)
#clfA = svm.SVC(kernel='rbf', gamma=0.7, C=1.0)
clfP = svm.SVC(**kwargs)
clfA = svm.SVC(**kwargs)
log('fitting plastic')
t = time.time()
clfP.fit(D, cp)
ttime = time.time() - t
log('in %f sec' % (time.time() - t))
log('fitting animals')
t = time.time()
clfA.fit(D, ca)
ttime += (time.time() - t)
log('in %f sec' % (time.time() - t))
return clfP, clfA, ttime
def test_weight():
"""
Test class weights
"""
clf = svm.SVC()
# we give a small weights to class 1
clf.fit(X, Y, {1: 0.1})
# so all predicted values belong to class 2
assert_array_almost_equal(clf.predict(X), [2]*6)
def train(*filenames):
"""Returns a classifier that """
data = None
answers = None
all_images = []
for filename in filenames:
print filename
if not os.path.exists(filename) and os.path.exists(filename + '.code'):
return False
keys = getTrainingKey(filename + '.code')
images = getImDictFromImage(filename)
this_data = images.reshape(images.shape[0], -1)
this_answers = numpy.array(keys)
all_images.extend(images)
if data is None:
answers = this_answers
data = this_data
else:
data = numpy.concatenate([data, this_data], 0)
answers = numpy.concatenate([answers, this_answers], 0)
print 'image shape', images.shape
print 'data shape', data.shape
print 'answers shape', answers.shape
from scikits.learn import svm
from scikits.learn.metrics import classification_report
from scikits.learn.metrics import confusion_matrix
classifier = svm.SVC()
divider = 400
classifier.fit(data[:divider], answers[:divider])
expected = answers[divider:]
predicted = classifier.predict(data[divider:])
print "check:"
print classifier
print 'predicted', predicted
print
print classification_report(expected, predicted)
print confusion_matrix(expected, predicted)
print 'len of all_images:', len(all_images)
for index, (image, prediction) in enumerate(
zip(all_images[divider:], predicted)[:25]):
#for index, (image, prediction) in enumerate(zip(all_images, answers)[50:75]):
print index, prediction
pylab.subplot(5, 5, index + 1)
pylab.imshow(image, cmap=pylab.cm.gray_r)
pylab.title('Prediction: ' + numToTile(prediction))
pylab.show()
def test_margin():
"""
Test predict_margin
TODO: more tests
"""
clf = svm.SVC()
clf.fit(X, Y)
assert_array_almost_equal(clf.predict_margin(T),
[[0.976], [-0.939], [-0.619]],
decimal=3)
def test_coef_and_intercept_SVC_vs_LinearSVC():
"""
Test that SVC and LinearSVC return the same coef_ and intercept_
"""
svc = svm.SVC(kernel='linear', C=1).fit(X, Y)
linsvc = svm.LinearSVC(C=1, penalty='l2', loss='l1', dual=True).fit(X, Y)
assert_array_equal(linsvc.coef_.shape, svc.coef_.shape)
assert_array_almost_equal(linsvc.coef_, svc.coef_, decimal=5)
assert_array_almost_equal(linsvc.intercept_, svc.intercept_, decimal=5)
def test_error():
"""
Test that it gives proper exception on deficient input
"""
# impossible value of C
assert_raises(ValueError, svm.SVC(C=-1).fit, X, Y)
# impossible value of nu
clf = svm.NuSVC(nu=0.0)
assert_raises(ValueError, clf.fit, X, Y)
Y2 = Y[:-1] # wrong dimensions for labels
assert_raises(ValueError, clf.fit, X, Y2)
assert_raises(ValueError, svm.SVC, X, Y2)
# Test with arrays that are non-contiguous.
Xf = np.asfortranarray(X)
clf = svm.SVC()
clf.fit(Xf, Y)
assert_array_equal(clf.predict(T), true_result)
def test_sample_weights():
"""
Test weights on individual samples
"""
clf = svm.SVC()
clf.fit(X, Y)
assert_array_equal(clf.predict(X[2]), [1.])
sample_weight = [.1] * 3 + [10] * 3
clf.fit(X, Y, sample_weight=sample_weight)
assert_array_equal(clf.predict(X[2]), [2.])
def test_libsvm_parameters():
"""
Test parameters on classes that make use of libsvm.
"""
clf = svm.SVC(kernel='linear').fit(X, Y)
assert_array_equal(clf.dual_coef_, [[0.25, -.25]])
assert_array_equal(clf.support_, [1, 3])
assert_array_equal(clf.support_vectors_, (X[1], X[3]))
assert_array_equal(clf.intercept_, [0.])
assert_array_equal(clf.predict(X), Y)
def demo(num=10, kwargs={'kernel': 'rbf', 'gamma': 0.7, 'C': 1.0}):
D, cp, ca = loadData()
Dr = D[0:num, [689, 3659]] #,1444]]
cpr, car = cp[0:num], ca[0:num]
clfP = svm.SVC(**kwargs)
log('fitting plastic')
t = time.time()
clfP.fit(Dr, cpr)
log('in %f sec' % (time.time() - t))
plotClf(clfP, Dr, cpr)
return clfP, Dr, cpr
def test_libsvm_iris():
"""
Check consistency on dataset iris.
"""
# shuffle the dataset so that labels are not ordered
for k in ('linear', 'rbf'):
clf = svm.SVC(kernel=k).fit(iris.data, iris.target)
assert np.mean(clf.predict(iris.data) == iris.target) > 0.9
assert_array_equal(clf.label_, np.sort(clf.label_))
def test_sample_weights():
"""
Test weights on individual samples
"""
# TODO: check on NuSVR, OneClass, etc.
clf = svm.SVC()
clf.fit(X, Y)
assert_array_equal(clf.predict(X[2]), [1.])
sample_weight = [.1] * 3 + [10] * 3
clf.fit(X, Y, sample_weight=sample_weight)
assert_array_equal(clf.predict(X[2]), [2.])
