def test_kernel_normalization(self): K = self.X @ self.X.T Kn_torch = preprocessing.kernel_normalization(K) Kn_numpy = preprocessing.kernel_normalization(K.numpy()) self.assertAlmostEqual(Kn_torch.max().item(), 1., places=6) self.assertAlmostEqual(Kn_torch.diag().min().item(), 1., places=6) self.assertEqual(Kn_torch.shape, (5, 5)) self.assertTrue(matNear(Kn_torch, Kn_numpy)) self.assertEqual(type(Kn_torch), torch.Tensor) self.assertEqual(type(Kn_numpy), torch.Tensor) linear = pairwise_mk.linear_kernel(preprocessing.normalization(self.X)) self.assertTrue(matNear(Kn_torch, linear, eps=1e-7))
def kernel_evaluation(K): kernel_results_dict = {} K = kernel_normalization(K) # normalize the kernel K (useless in the case of HPK computed on normalized data) kernel_results_dict['score_margin'] = margin(K, Ytr) # the distance between the positive and negative classes in the kernel space kernel_results_dict['score_radius'] = radius( K) # the radius of the Einimum Enclosing Ball containing data in the kernel space kernel_results_dict['score_ratio'] = ratio(K, Ytr) # the radius/margin ratio defined as (radius**2/margin**2)/n_examples kernel_results_dict['score_froben'] = frobenius(K) # the Frobenius norm of a kernel matrix kernel_results_dict['score_trace'] = trace(K) # the trace of the kernel matrix return kernel_results_dict
def MKL(): fname, pv, tv, org_metrics = experiment_setting() print(fname, pv, tv) list_pair_metrics = [["l1", "l2"]] for metrics in list_pair_metrics: X, y, sim_matrices = get_s_metric(fname=fname, tv=tv, pv=pv, metrics=metrics) # # from similarity to kernel matrix KL = [np.exp(s) / 0.01 for s in sim_matrices] KL_norm = [kernel_normalization(K) for K in KL] print(KL_norm, sim_matrices) # KLtr, KLte, Ytr, Yte = train_test_split(KL, Y, random_state=42, shuffle=True, test_size=.3) print(y) # # polynomial kernel # KL_norm = [hpk(X, degree=d) for d in range(1,11)] gamma_values = [0.001, 0.01, 0.1, 1, 10] lam_values = [0, 0.1, 0.2, 1] C_values = [0.01, 1, 100] # for lam in lam_values: # for gamma, C in product(gamma_values, C_values): # svm = SVR(kernel="rbf", C=C, gamma=gamma) # mkl = EasyMKL(lam=lam, learner=svm) # scores = cross_val_score(KL_norm, y, mkl, n_folds=3, scoring='mae') # print (lam, C, scores) for lam, C in product(lam_values, C_values): svm = SVC(C=C) mkl = EasyMKL(lam=lam, learner=svm) # # add into MKL sources scores = cross_val_score(KL_norm, y, mkl, n_folds=3, scoring='mae') print(lam, C, scores)
print('preprocessing data...', end='') #boolean kernels can be applied on binary-valued data, i.e. {0,1}. #in this example, we binarize a real-valued dataset from MKLpy.preprocessing import binarization binarizer = binarization.AverageBinarizer() Xbin = binarizer.fit_transform(X, Y) print('done') #compute normalized homogeneous polynomial kernels with degrees 0,1,2,...,10. print('computing monotone Conjunctive Kernels...', end='') from MKLpy.metrics import pairwise from MKLpy.preprocessing import kernel_normalization #WARNING: the maximum arity of the conjunctive kernel depends on the number of active variables for each example, # that is 4 in the case of iris dataset binarized KL = [ kernel_normalization(pairwise.monotone_conjunctive_kernel(Xbin, c=c)) for c in range(5) ] print('done') #train/test KL split (N.B. here we split a kernel list directly) from MKLpy.model_selection import train_test_split KLtr, KLte, Ytr, Yte = train_test_split(KL, Y, test_size=.3, random_state=42) #MKL algorithms from MKLpy.algorithms import EasyMKL, KOMD #KOMD is not a MKL algorithm but a simple kernel machine like the SVM from MKLpy.model_selection import cross_val_score, cross_val_predict from sklearn.svm import SVC import numpy as np print('tuning lambda for EasyMKL...', end='') base_learner = SVC(C=10000) #simil hard-margin svm
#preprocess data print ('preprocessing data...', end='') #boolean kernels can be applied on binary-valued data, i.e. {0,1}. #in this example, we binarize a real-valued dataset from MKLpy.preprocessing import binarization binarizer = binarization.AverageBinarizer() Xbin = binarizer.fit_transform(X,Y) print ('done') #compute normalized homogeneous polynomial kernels with degrees 0,1,2,...,10. print ('computing monotone Conjunctive Kernels...', end='') from MKLpy.metrics import pairwise from MKLpy.preprocessing import kernel_normalization #WARNING: the maximum arity of the conjunctive kernel depends on the number of active variables for each example, # that is 4 in the case of iris dataset binarized KL = [kernel_normalization(pairwise.monotone_conjunctive_kernel(Xbin, c=c)) for c in range(5)] print ('done') #train/test KL split (N.B. here we split a kernel list directly) from MKLpy.model_selection import train_test_split KLtr,KLte,Ytr,Yte = train_test_split(KL, Y, test_size=.3, random_state=42) #MKL algorithms from MKLpy.algorithms import EasyMKL, KOMD #KOMD is not a MKL algorithm but a simple kernel machine like the SVM from MKLpy.model_selection import cross_val_score, cross_val_predict from sklearn.svm import SVC import numpy as np print ('tuning lambda for EasyMKL...', end='') base_learner = SVC(C=10000) #simil hard-margin svm best_results = {} for lam in [0, 0.01, 0.1, 0.2, 0.9, 1]: #possible lambda values for the EasyMKL algorithm
#compute homogeneous polynomial kernels with degrees 0,1,2,...,10. print ('computing Homogeneous Polynomial Kernels...', end='') from MKLpy.metrics import pairwise KLtr = [pairwise.homogeneous_polynomial_kernel(Xtr, degree=d) for d in range(11)] KLte = [pairwise.homogeneous_polynomial_kernel(Xte,Xtr, degree=d) for d in range(11)] print ('done') #evaluate kernels in terms of margin, radius etc... print ('evaluating metrics...', end='') from MKLpy.metrics import margin, radius, ratio, trace, frobenius from MKLpy.preprocessing import kernel_normalization deg = 5 K = KLtr[deg] #the HPK with degree 5 K = kernel_normalization(K) #normalize the kernel K (useless in the case of HPK computed on normalized data) score_margin = margin(K,Ytr) #the distance between the positive and negative classes in the kernel space score_radius = radius(K) #the radius of the Einimum Enclosing Ball containing data in the kernel space score_ratio = ratio (K,Ytr) #the radius/margin ratio defined as (radius**2/margin**2)/n_examples #the ratio can be also computed as score_radius**2/score_margin**2/len(Ytr) score_trace = trace (K) #the trace of the kernel matrix score_froben = frobenius(K) #the Frobenius norm of a kernel matrix print ('done') print ('results of the %d-degree HP kernel:' % deg) print ('margin: %.4f, radius: %.4f, radiu-margin ratio: %.4f,' % (score_margin, score_radius, score_ratio)) print ('trace: %.4f, frobenius norm: %.4f' % (score_trace, score_froben)) #evaluate the empirical complexity of the kernel matrix, i.e. the Spectral Ratio # Michele Donini, Fabio Aiolli: "Learning deep kernels in the space of dot-product polynomials". Machine Learning (2017) # Ivano Lauriola, Mirko Polato, Fabio Aiolli: "The Minimum Effort Maximum Output principle applied to Multiple Kernel Learning". ESANN (2018)
#preprocess data print ('preprocessing data...', end='') #boolean kernels can be applied on binary-valued data, i.e. {0,1}. #in this example, we binarize a real-valued dataset from MKLpy.preprocessing import binarization binarizer = binarization.AverageBinarizer() Xbin = binarizer.fit_transform(X,Y) print ('done') #compute normalized homogeneous polynomial kernels with degrees 0,1,2,...,10. print ('computing monotone Conjunctive Kernels...', end='') from MKLpy.metrics import pairwise from MKLpy.preprocessing import kernel_normalization #WARNING: the maximum arity of the conjunctive kernel depends on the number of active variables for each example, # that is 4 in the case of iris dataset binarized KL = [kernel_normalization(pairwise.monotone_conjunctive_kernel(Xbin, c=c)) for c in range(5)] print ('done') #train/test KL split (N.B. here we split a kernel list directly) from MKLpy.model_selection import train_test_split KLtr,KLte,Ytr,Yte = train_test_split(KL, Y, test_size=.3, random_state=42) #MKL algorithms from MKLpy.algorithms import EasyMKL, KOMD #KOMD is not a MKL algorithm but a simple kernel machine like the SVM from MKLpy.model_selection import cross_val_score, cross_val_predict from sklearn.svm import SVC import numpy as np print ('tuning lambda for EasyMKL...', end='') base_learner = SVC(C=10000) #simil hard-margin svm best_results = {} for lam in [0, 0.01, 0.1, 0.2, 0.9, 1]: #possible lambda values for the EasyMKL algorithm