def test_kernel_normalization(self):
K = self.X @ self.X.T
Kn_torch = preprocessing.kernel_normalization(K)
Kn_numpy = preprocessing.kernel_normalization(K.numpy())
self.assertAlmostEqual(Kn_torch.max().item(), 1., places=6)
self.assertAlmostEqual(Kn_torch.diag().min().item(), 1., places=6)
self.assertEqual(Kn_torch.shape, (5, 5))
self.assertTrue(matNear(Kn_torch, Kn_numpy))
self.assertEqual(type(Kn_torch), torch.Tensor)
self.assertEqual(type(Kn_numpy), torch.Tensor)
linear = pairwise_mk.linear_kernel(preprocessing.normalization(self.X))
self.assertTrue(matNear(Kn_torch, linear, eps=1e-7))
def kernel_evaluation(K):
kernel_results_dict = {}
K = kernel_normalization(K) # normalize the kernel K (useless in the case of HPK computed on normalized data)
kernel_results_dict['score_margin'] = margin(K,
Ytr) # the distance between the positive and negative classes in the kernel space
kernel_results_dict['score_radius'] = radius(
K) # the radius of the Einimum Enclosing Ball containing data in the kernel space
kernel_results_dict['score_ratio'] = ratio(K,
Ytr) # the radius/margin ratio defined as (radius**2/margin**2)/n_examples
kernel_results_dict['score_froben'] = frobenius(K) # the Frobenius norm of a kernel matrix
kernel_results_dict['score_trace'] = trace(K) # the trace of the kernel matrix
return kernel_results_dict
def MKL():
fname, pv, tv, org_metrics = experiment_setting()
print(fname, pv, tv)
list_pair_metrics = [["l1", "l2"]]
for metrics in list_pair_metrics:
X, y, sim_matrices = get_s_metric(fname=fname,
tv=tv,
pv=pv,
metrics=metrics)
# # from similarity to kernel matrix
KL = [np.exp(s) / 0.01 for s in sim_matrices]
KL_norm = [kernel_normalization(K) for K in KL]
print(KL_norm, sim_matrices)
# KLtr, KLte, Ytr, Yte = train_test_split(KL, Y, random_state=42, shuffle=True, test_size=.3)
print(y)
# # polynomial kernel
# KL_norm = [hpk(X, degree=d) for d in range(1,11)]
gamma_values = [0.001, 0.01, 0.1, 1, 10]
lam_values = [0, 0.1, 0.2, 1]
C_values = [0.01, 1, 100]
# for lam in lam_values:
# for gamma, C in product(gamma_values, C_values):
# svm = SVR(kernel="rbf", C=C, gamma=gamma)
# mkl = EasyMKL(lam=lam, learner=svm)
# scores = cross_val_score(KL_norm, y, mkl, n_folds=3, scoring='mae')
# print (lam, C, scores)
for lam, C in product(lam_values, C_values):
svm = SVC(C=C)
mkl = EasyMKL(lam=lam, learner=svm)
# # add into MKL sources
scores = cross_val_score(KL_norm, y, mkl, n_folds=3, scoring='mae')
print(lam, C, scores)
print('preprocessing data...', end='')
#boolean kernels can be applied on binary-valued data, i.e. {0,1}.
#in this example, we binarize a real-valued dataset
from MKLpy.preprocessing import binarization
binarizer = binarization.AverageBinarizer()
Xbin = binarizer.fit_transform(X, Y)
print('done')
#compute normalized homogeneous polynomial kernels with degrees 0,1,2,...,10.
print('computing monotone Conjunctive Kernels...', end='')
from MKLpy.metrics import pairwise
from MKLpy.preprocessing import kernel_normalization
#WARNING: the maximum arity of the conjunctive kernel depends on the number of active variables for each example,
# that is 4 in the case of iris dataset binarized
KL = [
kernel_normalization(pairwise.monotone_conjunctive_kernel(Xbin, c=c))
for c in range(5)
]
print('done')
#train/test KL split (N.B. here we split a kernel list directly)
from MKLpy.model_selection import train_test_split
KLtr, KLte, Ytr, Yte = train_test_split(KL, Y, test_size=.3, random_state=42)
#MKL algorithms
from MKLpy.algorithms import EasyMKL, KOMD #KOMD is not a MKL algorithm but a simple kernel machine like the SVM
from MKLpy.model_selection import cross_val_score, cross_val_predict
from sklearn.svm import SVC
import numpy as np
print('tuning lambda for EasyMKL...', end='')
base_learner = SVC(C=10000) #simil hard-margin svm
#preprocess data
print ('preprocessing data...', end='')
#boolean kernels can be applied on binary-valued data, i.e. {0,1}.
#in this example, we binarize a real-valued dataset
from MKLpy.preprocessing import binarization
binarizer = binarization.AverageBinarizer()
Xbin = binarizer.fit_transform(X,Y)
print ('done')
#compute normalized homogeneous polynomial kernels with degrees 0,1,2,...,10.
print ('computing monotone Conjunctive Kernels...', end='')
from MKLpy.metrics import pairwise
from MKLpy.preprocessing import kernel_normalization
#WARNING: the maximum arity of the conjunctive kernel depends on the number of active variables for each example,
# that is 4 in the case of iris dataset binarized
KL = [kernel_normalization(pairwise.monotone_conjunctive_kernel(Xbin, c=c)) for c in range(5)]
print ('done')
#train/test KL split (N.B. here we split a kernel list directly)
from MKLpy.model_selection import train_test_split
KLtr,KLte,Ytr,Yte = train_test_split(KL, Y, test_size=.3, random_state=42)
#MKL algorithms
from MKLpy.algorithms import EasyMKL, KOMD #KOMD is not a MKL algorithm but a simple kernel machine like the SVM
from MKLpy.model_selection import cross_val_score, cross_val_predict
from sklearn.svm import SVC
import numpy as np
print ('tuning lambda for EasyMKL...', end='')
base_learner = SVC(C=10000) #simil hard-margin svm
best_results = {}
for lam in [0, 0.01, 0.1, 0.2, 0.9, 1]: #possible lambda values for the EasyMKL algorithm
#compute homogeneous polynomial kernels with degrees 0,1,2,...,10.
print ('computing Homogeneous Polynomial Kernels...', end='')
from MKLpy.metrics import pairwise
KLtr = [pairwise.homogeneous_polynomial_kernel(Xtr, degree=d) for d in range(11)]
KLte = [pairwise.homogeneous_polynomial_kernel(Xte,Xtr, degree=d) for d in range(11)]
print ('done')
#evaluate kernels in terms of margin, radius etc...
print ('evaluating metrics...', end='')
from MKLpy.metrics import margin, radius, ratio, trace, frobenius
from MKLpy.preprocessing import kernel_normalization
deg = 5
K = KLtr[deg] #the HPK with degree 5
K = kernel_normalization(K) #normalize the kernel K (useless in the case of HPK computed on normalized data)
score_margin = margin(K,Ytr) #the distance between the positive and negative classes in the kernel space
score_radius = radius(K) #the radius of the Einimum Enclosing Ball containing data in the kernel space
score_ratio = ratio (K,Ytr) #the radius/margin ratio defined as (radius**2/margin**2)/n_examples
#the ratio can be also computed as score_radius**2/score_margin**2/len(Ytr)
score_trace = trace (K) #the trace of the kernel matrix
score_froben = frobenius(K) #the Frobenius norm of a kernel matrix
print ('done')
print ('results of the %d-degree HP kernel:' % deg)
print ('margin: %.4f, radius: %.4f, radiu-margin ratio: %.4f,' % (score_margin, score_radius, score_ratio))
print ('trace: %.4f, frobenius norm: %.4f' % (score_trace, score_froben))
#evaluate the empirical complexity of the kernel matrix, i.e. the Spectral Ratio
# Michele Donini, Fabio Aiolli: "Learning deep kernels in the space of dot-product polynomials". Machine Learning (2017)
# Ivano Lauriola, Mirko Polato, Fabio Aiolli: "The Minimum Effort Maximum Output principle applied to Multiple Kernel Learning". ESANN (2018)
#preprocess data
print ('preprocessing data...', end='')
#boolean kernels can be applied on binary-valued data, i.e. {0,1}.
#in this example, we binarize a real-valued dataset
from MKLpy.preprocessing import binarization
binarizer = binarization.AverageBinarizer()
Xbin = binarizer.fit_transform(X,Y)
print ('done')
#compute normalized homogeneous polynomial kernels with degrees 0,1,2,...,10.
print ('computing monotone Conjunctive Kernels...', end='')
from MKLpy.metrics import pairwise
from MKLpy.preprocessing import kernel_normalization
#WARNING: the maximum arity of the conjunctive kernel depends on the number of active variables for each example,
# that is 4 in the case of iris dataset binarized
KL = [kernel_normalization(pairwise.monotone_conjunctive_kernel(Xbin, c=c)) for c in range(5)]
print ('done')
#train/test KL split (N.B. here we split a kernel list directly)
from MKLpy.model_selection import train_test_split
KLtr,KLte,Ytr,Yte = train_test_split(KL, Y, test_size=.3, random_state=42)
#MKL algorithms
from MKLpy.algorithms import EasyMKL, KOMD #KOMD is not a MKL algorithm but a simple kernel machine like the SVM
from MKLpy.model_selection import cross_val_score, cross_val_predict
from sklearn.svm import SVC
import numpy as np
print ('tuning lambda for EasyMKL...', end='')
base_learner = SVC(C=10000) #simil hard-margin svm
best_results = {}
for lam in [0, 0.01, 0.1, 0.2, 0.9, 1]: #possible lambda values for the EasyMKL algorithm