def __call__(self, X1, X2):
rows = []
for key_1, value_1 in X1.iteritems():
if self.xstats == 1:
x1 = np.array([centroid(e) for e in value_1]).flatten()
elif self.xtats == 2:
x1 = np.array([dispersion(e) for e in value_1]).flatten()
else:
x1_cen = np.array([centroid(e) for e in value_1]).flatten()
x1_dis = np.array([dispersion(e) for e in value_1]).flatten()
columns = []
for key_2, value_2 in X2.iteritems():
if self.xstats == 1:
x2 = np.array([centroid(e) for e in value_2]).flatten()
elif self.xtats == 2:
x2 = np.array([dispersion(e) for e in value_2]).flatten()
else:
x2_cen = np.array([centroid(e) for e in value_2]).flatten()
x2_dis = np.array([dispersion(e) for e in value_2]).flatten()
if self.similarity == 1:
if self.xstats == 3:
value_cen = polynomial_kernel(x1_cen,x2_cen).flatten()[0]
value_dis = polynomial_kernel(x1_dis,x2_dis).flatten()[0]
value = (value_cen + value_dis)/2
else:
value = polynomial_kernel(x1,x2).flatten()[0]
if self.domain_adapt:
if (key_1 < 10500 and key_2 < 10500) or ((key_1 > 10500 and key_2 > 10500)):
columns.append(value)
else:
columns.append(2*value)
else:
columns.append(value)
else:
if self.xstats == 3:
value_cen = cosine_similarity(x1_cen,x2_cen).flatten()[0]
value_dis = cosine_similarity(x1_dis,x2_dis).flatten()[0]
value = (value_cen + value_dis)/2
else:
value = cosine_similarity(x1,x2).flatten()[0]
if self.domain_adapt:
if (key_1 < 10500 and key_2 < 10500) or ((key_1 > 10500 and key_2 > 10500)):
columns.append(value)
else:
columns.append(2*value)
else:
columns.append(value)
rows.append(columns)
m = np.asarray(rows)
print m.shape
return m
def __kernel_definition__(self):
if self.Kf == 'rbf':
return lambda X,Y : rbf_kernel(X,Y,self.rbf_gamma)
if self.Kf == 'poly':
return lambda X,Y : polynomial_kernel(X, Y, degree=self.poly_deg, gamma=None, coef0=self.poly_coeff)
if self.Kf == None or self.Kf == 'linear':
return lambda X,Y : linear_kernel(X,Y)
def _apply_kernel(self, X, y=None):
"""Apply the selected kernel function to the data."""
if self.kernel == 'linear':
phi = linear_kernel(X, y)
elif self.kernel == 'rbf':
phi = rbf_kernel(X, y, gamma=self.gamma)
elif self.kernel == 'poly':
phi = polynomial_kernel(X, y, degree=self.degree)
elif callable(self.kernel):
phi = self.kernel(X, y)
if len(phi.shape) != 2:
raise ValueError(
"Custom kernel function did not return 2D matrix"
)
if phi.shape[0] != X.shape[0]:
raise ValueError(
"Custom kernel function did not return matrix with rows"
" equal to number of data points."""
)
else:
raise ValueError("Kernel selection is invalid.")
phi = phi.T
if self.bias_used:
phi = np.hstack((np.ones((phi.shape[0], 1)), phi))
return phi
def _apply_kernel(self, x, y):
"""Apply the selected kernel function to the data."""
if self.kernel == 'linear':
phi = linear_kernel(x, y)
elif self.kernel == 'rbf':
phi = rbf_kernel(x, y, self.coef1)
elif self.kernel == 'poly':
phi = polynomial_kernel(x, y, self.degree, self.coef1, self.coef0)
elif callable(self.kernel):
phi = self.kernel(x, y)
if len(phi.shape) != 2:
raise ValueError(
"Custom kernel function did not return 2D matrix"
)
if phi.shape[0] != x.shape[0]:
raise ValueError(
"Custom kernel function did not return matrix with rows"
" equal to number of data points."""
)
else:
raise ValueError("Kernel selection is invalid.")
if self.bias_used:
phi = np.append(phi, np.ones((phi.shape[0], 1)), axis=1)
return phi
def __init__(self, *args, **kwargs):
super(QUIRE, self).__init__(*args, **kwargs)
self.Uindex = [idx for idx, _ in self.dataset.get_unlabeled_entries()]
self.Lindex = [idx for idx in range(len(self.dataset)) if idx not in self.Uindex]
self.lmbda = kwargs.pop("lambda", 1.0)
X, self.y = zip(*self.dataset.get_entries())
self.y = list(self.y)
self.kernel = kwargs.pop("kernel", "rbf")
if self.kernel == "rbf":
self.K = rbf_kernel(X=X, Y=X, gamma=kwargs.pop("gamma", 1.0))
elif self.kernel == "poly":
self.K = polynomial_kernel(
X=X, Y=X, coef0=kwargs.pop("coef0", 1), degree=kwargs.pop("degree", 3), gamma=kwargs.pop("gamma", 1.0)
)
elif self.kernel == "linear":
self.K = linear_kernel(X=X, Y=X)
elif hasattr(self.kernel, "__call__"):
self.K = self.kernel(X=np.array(X), Y=np.array(X))
else:
raise NotImplementedError
if not isinstance(self.K, np.ndarray):
raise TypeError("K should be an ndarray")
if self.K.shape != (len(X), len(X)):
raise ValueError("kernel should have size (%d, %d)" % (len(X), len(X)))
self.L = np.linalg.inv(self.K + self.lmbda * np.eye(len(X)))
def test_nystroem_poly_kernel_params():
"""Non-regression: Nystroem should pass other parameters beside gamma."""
rnd = np.random.RandomState(37)
X = rnd.uniform(size=(10, 4))
K = polynomial_kernel(X, degree=3.1, coef0=.1)
nystroem = Nystroem(kernel="polynomial", n_components=X.shape[0],
degree=3.1, coef0=.1)
X_transformed = nystroem.fit_transform(X)
assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)
def test_nystroem_poly_kernel_params():
# Non-regression: Nystroem should pass other parameters beside gamma.
rnd = np.random.RandomState(37)
X = rnd.uniform(size=(10, 4))
K = polynomial_kernel(X, degree=3.1, coef0=.1)
nystroem = Nystroem(kernel="polynomial",
n_components=X.shape[0],
degree=3.1,
coef0=.1)
X_transformed = nystroem.fit_transform(X)
assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)
def polynomial_mmd(codes_g,
codes_r,
degree=3,
gamma=None,
coef0=1,
var_at_m=None,
ret_var=True):
# use k(x, y) = (gamma <x, y> + coef0)^degree
# default gamma is 1 / dim
X = codes_g
Y = codes_r
K_XX = polynomial_kernel(X, degree=degree, gamma=gamma, coef0=coef0)
K_YY = polynomial_kernel(Y, degree=degree, gamma=gamma, coef0=coef0)
K_XY = polynomial_kernel(X, Y, degree=degree, gamma=gamma, coef0=coef0)
return _mmd2_and_variance(K_XX,
K_XY,
K_YY,
var_at_m=var_at_m,
ret_var=ret_var)
def kernel_mean_matching(self, X, Z, kern='lin', B=1.0, eps=None):
nx = X.shape[0]
nz = Z.shape[0]
print("X.shape: ", X.shape, "Z.shape: ", Z.shape)
print("nx: ", nx, " nz: ", nz)
if eps == None:
eps = B/math.sqrt(nz)
if kern == 'lin':
K = np.dot(Z, Z.T) #+ self.lin_c
kappa = np.sum(np.dot(Z, X.T)*float(nz)/float(nx),axis=1)
elif kern == 'rbf':
K= sk.rbf_kernel(Z, Z)
kappa = np.sum(sk.rbf_kernel(Z, X), axis=1)*float(nz)/float(nx)
elif kern == 'poly':
K=sk.polynomial_kernel(Z, Z)
kappa = np.sum(sk.polynomial_kernel(Z, X), axis=1)*float(nz)/float(nx)
elif kern == 'laplacian':
K=sk.laplacian_kernel(Z, Z)
kappa = np.sum(sk.laplacian_kernel(Z, X), axis=1)*float(nz)/float(nx)
elif kern == 'sigmoid':
K=sk.sigmoid_kernel(Z, Z)
kappa = np.sum(sk.sigmoid_kernel(Z, X), axis=1)*float(nz)/float(nx)
else:
raise ValueError('unknown kernel')
K = K.astype(np.double)
K = matrix(K)
kappa = matrix(kappa)
G = matrix(np.r_[np.ones((1,nz)), -np.ones((1,nz)), np.eye(nz), -np.eye(nz)])
h = matrix(np.r_[nz*(1+eps), nz*(eps-1), B*np.ones((nz,)), np.zeros((nz,))])
solvers.options['show_progress'] = False
print("starting solver")
sol = solvers.qp(K, -kappa, G, h)
print(sol)
coef = np.array(sol['x'])
return coef
def test_HPK_train(self):
Ktr = self.Xtr.dot(self.Xtr.T)
self.assertTrue(matNear(Ktr, linear_kernel(self.Xtr)))
self.assertTrue(
matNear(pairwise.homogeneous_polynomial_kernel(self.Xtr, degree=4),
polynomial_kernel(self.Xtr, degree=4, gamma=1, coef0=0)))
self.assertTrue(
matNear(pairwise.homogeneous_polynomial_kernel(self.Xtr, degree=5),
polynomial_kernel(self.Xtr, degree=5, gamma=1, coef0=0)))
self.assertTrue(
matNear(Ktr**3,
polynomial_kernel(self.Xtr, degree=3, gamma=1, coef0=0)))
self.assertTrue(
matNear(
pairwise.homogeneous_polynomial_kernel(self.Xtr,
self.Xtr,
degree=3),
polynomial_kernel(self.Xtr,
self.Xtr,
degree=3,
gamma=1,
coef0=0)))
def _apply_kernel(self, X, Y):
if self.kernel == "rbf":
return rbf_kernel(X, Y, self.gamma)
elif self.kernel == "sigmoid":
return sigmoid_kernel(X, Y, self.gamma, self.coef0)
elif self.kernel == "poly":
return polynomial_kernel(X, Y, self.degree, self.gamma, self.coef0)
elif self.kernel == "linear":
return linear_kernel(X, Y)
elif callable(self.kernel):
return self.kernel(X, Y)
else:
raise ValueError("Unknown kernel: " + str(self.kernel))
def calculate_gram_matrix(x, kernel='linear', gamma=0, degree=0, coef0=0):
if kernel == 'linear':
gram = linear_kernel(x, x)
elif kernel == 'poly':
gram = polynomial_kernel(x, x, degree=degree, gamma=gamma, coef0=coef0)
elif kernel == 'sigmoid':
gram = sigmoid_kernel(x, x, gamma=gamma, coef0=coef0)
elif kernel == 'rbf':
gram = rbf_kernel(x, x, gamma=gamma)
else:
raise ValueError
return gram
def compare_to_sklearn(self, x, y):
"""Helper for comparing the three kernels to avoid copy-pasting """
sk_lk = linear_kernel(x, y)
my_lk = self.lk.gram_matrix(x, y)
self.assertTrue(np.allclose(my_lk, sk_lk))
sk_pk = polynomial_kernel(x, y, self.pd, self.pa, self.pc)
my_pk = self.pk.gram_matrix(x, y)
self.assertTrue(np.allclose(my_pk, sk_pk))
sk_gk = rbf_kernel(x, y, self.g)
my_gk = self.gk.gram_matrix(x, y)
self.assertTrue(np.allclose(my_gk, sk_gk))
def predict_quadratic(self, X, P=None, lams=None):
"""Prediction from the quadratic term of the factorization machine.
Returns <Z, XX'>.
"""
if P is None:
P = self.P_
lams = self.lams_
if not len(lams):
return 0
K = polynomial_kernel(X, np.array(P), degree=2, gamma=1, coef0=0)
return np.dot(K, lams)
def calc_kernel(kernel_type, params, X):
if kernel_type == "linear":
K = linear_kernel(X)
elif kernel_type == "poly":
if np.isnan(X).any():
print("X trouble before kernel")
K = polynomial_kernel(X, degree=params['degree'], coef0=params['coef0'])
if np.isnan(K).any():
print("K trouble once kernel calculated")
elif kernel_type == "rbf":
K = rbf_kernel(X, gamma=params['gamma'])
else:
print("error in calc_kernel : kernel_type unrecognized!")
return K
def predict(self, xtest):
if (self.kernel == 'poly'):
kfunc = lambda x, y: polynomial_kernel(
x, y, degree=self.degree, gamma=self.gamma)
elif (self.kernel == 'rbf'):
kfunc = lambda x, y: rbf_kernel(x, y, self.gamma)
else:
kfunc = self.kernel
KC = kfunc(xtest, self.X[self.blocks[self.b]])
pred = (np.floor(
(KC.dot(self.alpha[self.blocks[self.b]])) >= self.separatrice)
).reshape(len(xtest))
return np.array([self.values[int(p)] for p in pred])
def my_kernel(X, K):
# Creating the RBF kernel
bfrKernel = rbf_kernel(X, K, 0.1)
bfrKernel = bfrKernel.astype(np.double)
# Creating the poly kernel
polyKernel = polynomial_kernel(X, K, degree=2.0, gamma=0.1, coef0=0.0)
polyKernel = polyKernel.astype(np.double)
# Creating the Linear kernel
linearKernel = linear_kernel(X, K)
linearKernel = linearKernel.astype(np.double)
return (linearKernel + bfrKernel + polyKernel) / 3
def __init__(self, X, T=None, func='rbf', params={}, n_feature=2):
self.X = X
self.T = T
self.n_feature = n_feature
self.f_max = len(X[0])
if hasattr(func, '__call__'):
self.func = lambda X, Y: func(X, Y, **params)
elif func == 'linear':
self.func = lambda X, Y: linear_kernel(X, Y, **params)
elif func == 'poly':
self.func = lambda X, Y: polynomial_kernel(X, Y, **params)
elif func == 'rbf' or True:
self.func = lambda X, Y: rbf_kernel(X, Y, **params)
def ployFunc():
if self.parameters["kernel"].__contains__("degree"):
d = self.parameters["kernel"]["degree"]
else:
d = 2
if self.parameters["kernel"].__contains__("offset"):
c = self.parameters["kernel"]["offset"]
else:
c = 0
K = smp.polynomial_kernel(X, Y, degree=d, gamma=None, coef0=c)
return K
def train_and_test (self, dataTest, realOutput=None, aval=False, reg=0.01, deg=3, gamm=None, coef=1):
if self.kernelType == 'rbf':
K = rbf_kernel(self.inTrain, self.inTrain, gamm)
Ktest = rbf_kernel(dataTest, self.inTrain, gamm)
elif self.kernelType == 'pol':
K = polynomial_kernel(self.inTrain, self.inTrain, deg, gamm, coef)
Ktest = polynomial_kernel(dataTest, self.inTrain, deg, gamm, coef)
elif self.kernelType == 'sig':
K = sigmoid_kernel(self.inTrain, self.inTrain, gamm, coef)
Ktest = sigmoid_kernel(dataTest, self.inTrain, gamm, coef)
I = np.eye(self.inTrain.shape[0])
outNet = np.dot (np.dot(Ktest, np.linalg.inv(K + reg*I)), self.outTrain)
if aval:
miss = float(cont_error (realOutput, outNet))
si = float(outNet.shape[0])
acc = (1-miss/si)*100
print 'Miss classification on the test: ', miss, ' of ', si, ' - Accuracy: ',acc , '%'
return outNet, acc
return outNet, None
def predict_quadratic(self, X, P=None, lams=None):
"""Prediction from the quadratic term of the factorization machine.
Returns <Z, XX'>.
"""
if P is None:
P = self.P_
lams = self.lams_
if not len(lams):
return 0
K = polynomial_kernel(X, np.array(P), degree=2, gamma=1, coef0=0)
return np.dot(K, lams)
def calculateMultipleKernel(x, y):
theta = random.sample(range(1,47),46) # given a random theta for now
# Convert our 2d arrays to numpy arrays
x = np.array(x)
y = np.array(y)
# Reshape the array-like input vectors since we only have one sample
x = x.reshape(1,-1)
y = y.reshape(1,-1)
# Variables to aggregate the kernel result
kernelResult = 0;
index = 0;
for i in range(0,3):
kernelResult += theta[index] * additive_chi2_kernel(x,y)
index += 1
for i in range(0,3):
kernelResult += theta[index] * chi2_kernel(x,y,theta[index+1])
index += 2
for i in range(0,3):
kernelResult += theta[index] * cosine_similarity(x,y)
index += 1
for i in range(0,3):
kernelResult += theta[index] * linear_kernel(x,y)
index += 1
for i in range(0,3):
kernelResult += theta[index] * polynomial_kernel(
x,y,theta[index+1],theta[index+2], theta[index+3])
index += 4
for i in range(0,3):
kernelResult += theta[index] * rbf_kernel(x,y,theta[index+1])
index += 2
for i in range(0,3):
kernelResult += theta[index] * laplacian_kernel(x,y,theta[index+1])
index += 2
for i in range(0,3):
kernelResult += theta[index] * sigmoid_kernel(x,y,theta[index+1])
index += 2
return kernelResult
def calculateMultipleKernel(x, y):
theta = random.sample(range(1, 47), 46) # given a random theta for now
# Convert our 2d arrays to numpy arrays
x = np.array(x)
y = np.array(y)
# Reshape the array-like input vectors since we only have one sample
x = x.reshape(1, -1)
y = y.reshape(1, -1)
# Variables to aggregate the kernel result
kernelResult = 0
index = 0
for i in range(0, 3):
kernelResult += theta[index] * additive_chi2_kernel(x, y)
index += 1
for i in range(0, 3):
kernelResult += theta[index] * chi2_kernel(x, y, theta[index + 1])
index += 2
for i in range(0, 3):
kernelResult += theta[index] * cosine_similarity(x, y)
index += 1
for i in range(0, 3):
kernelResult += theta[index] * linear_kernel(x, y)
index += 1
for i in range(0, 3):
kernelResult += theta[index] * polynomial_kernel(
x, y, theta[index + 1], theta[index + 2], theta[index + 3])
index += 4
for i in range(0, 3):
kernelResult += theta[index] * rbf_kernel(x, y, theta[index + 1])
index += 2
for i in range(0, 3):
kernelResult += theta[index] * laplacian_kernel(x, y, theta[index + 1])
index += 2
for i in range(0, 3):
kernelResult += theta[index] * sigmoid_kernel(x, y, theta[index + 1])
index += 2
return kernelResult
def make_kernel(self, X: np.array, Y: np.array):
"""
:param X:
:param Y:
:return: Kernel matrix
"""
if self.kernel == 'linear':
kernel = X @ Y.T
elif self.kernel == 'rbf':
kernel = rbf_kernel(X, Y=Y, gamma=(1 / (2 * self.sigma)))
elif self.kernel == 'poly':
kernel = polynomial_kernel(X, Y=Y, degree=self.degree)
else:
print('invalid kernel: choose linear, rbf, poly')
return kernel
def chooseKernel(data, kerneltype='euclidean'):
if kerneltype == 'euclidean':
K = np.divide(1, (1 + pairwise_distances(data, metric="euclidean")))
elif kerneltype == 'cosine':
K = (pairwise.cosine_kernel(data))
elif kerneltype == 'laplacian':
K = (pairwise.laplacian_kernel(data))
elif kerneltype == 'linear':
K = (pairwise.linear_kernel(data))
elif kerneltype == 'polynomial_kernel':
K = (pairwise.polynomial_kernel(data))
elif kerneltype == 'jaccard':
K = 1 - distance.cdist(data, data, metric='jaccard')
scaler = KernelCenterer().fit(K)
return (scaler.transform(K))
def polynomial_kernel(subsequences_1, subsequences_2, p=2, c=1.0):
'''
Calculates the linear kernel between two time series using their
corresponding set of subsequences.
'''
K_poly = pairwise.polynomial_kernel(subsequences_1,
subsequences_2,
degree=p,
coef0=c,
gamma=1.0)
n = subsequences_1.shape[0]
m = subsequences_2.shape[0]
return np.sum(K_poly) / (n * m)
def polynomial_mmd(codes_g,
codes_r,
degree=2,
gamma=None,
coef0=1,
var_at_m=None,
ret_var=True,
sample=10000):
# use k(x, y) = (gamma <x, y> + coef0)^degree
# default gamma is 1 / dim
sample_g = np.random.choice(codes_g.shape[0], sample)
sample_r = np.random.choice(codes_r.shape[0], sample)
X = codes_g[sample_g]
Y = codes_r[sample_r]
K_XX = polynomial_kernel(X, degree=degree, gamma=gamma, coef0=coef0)
K_YY = polynomial_kernel(Y, degree=degree, gamma=gamma, coef0=coef0)
K_XY = polynomial_kernel(X, Y, degree=degree, gamma=gamma, coef0=coef0)
return _mmd2_and_variance(K_XX,
K_XY,
K_YY,
var_at_m=var_at_m,
ret_var=ret_var)
def PolynomialKernel(Xrows, Xcols, sf, offset, degree):
"""
Computes the polynomial kernel matrix K_{ij} = (sf<x_i, x_j> + offset)^degree
Inputs:
Xrows: array [n_samples_row, n_features]
Xcols: array [n_samples_col, n_features]
Output:
K: array of kernel evaluations [n_samples_row, n_samples_col]
"""
return polynomial_kernel(Xrows,
Xcols,
degree=degree,
gamma=sf,
coef0=offset)
def calc_kernel(kernel_type, params, X):
if kernel_type == "linear":
K = linear_kernel(X)
elif kernel_type == "poly":
if np.isnan(X).any():
print("X trouble before kernel")
K = polynomial_kernel(X,
degree=params['degree'],
coef0=params['coef0'])
if np.isnan(K).any():
print("K trouble once kernel calculated")
elif kernel_type == "rbf":
K = rbf_kernel(X, gamma=params['gamma'])
else:
print("error in calc_kernel : kernel_type unrecognized!")
return K
def __kernel_definition__(self):
"""Select the kernel function
Returns
-------
kernel : a callable relative to selected kernel
"""
if hasattr(self.kernel, '__call__'):
return self.kernel
if self.kernel == 'rbf' or self.kernel == None:
return lambda X,Y : rbf_kernel(X,Y,self.rbf_gamma)
if self.kernel == 'poly':
return lambda X,Y : polynomial_kernel(X, Y, degree=self.degree, gamma=self.rbf_gamma, coef0=self.coef0)
if self.kernel == 'linear':
return lambda X,Y : linear_kernel(X,Y)
if self.kernel == 'precomputed':
return lambda X,Y : X
def polynomial_kernel(X, Y, c, p):
"""
Compute the polynomial kernel between two matrices X and Y::
K(x, y) = (<x, y> + c)^p
for each pair of rows x in X and y in Y.
Args:
X - (n, d) NumPy array (n datapoints each with d features)
Y - (m, d) NumPy array (m datapoints each with d features)
c - a coefficient to trade off high-order and low-order terms (scalar)
p - the degree of the polynomial kernel
Returns:
kernel_matrix - (n, m) Numpy array containing the kernel matrix
"""
# YOUR CODE HERE
return polynomial_kernel(X, Y, p, gamma=None, coef0=c)
def kernel_function(self, x1, x2):
features = []
# linear kernel:
# Cosine distance
features += np.squeeze(1 -
pairwise.paired_cosine_distances(x1, x2)[0]),
# Manhanttan distance
features += pairwise.paired_manhattan_distances(x1, x2)[0],
# Euclidean distance
features += pairwise.paired_euclidean_distances(x1, x2)[0],
# Chebyshev distance
features += pairwise.pairwise_distances(x1, x2,
metric="chebyshev")[0][0],
# stat kernel:
# Pearson coefficient
pearson = stats.pearsonr(np.squeeze(np.asarray(x1)),
np.squeeze(np.asarray(x2)))[0]
features += 0 if np.isnan(pearson) else pearson,
# Spearman coefficient
spearman = stats.spearmanr(x1, x2, axis=1).correlation
features += 0 if np.isnan(spearman) else spearman,
# Kendall tau coefficient
kendall = stats.kendalltau(x1, x2).correlation
features += 0 if np.isnan(kendall) else kendall,
# non-linear kernel:
# polynomial
features += pairwise.polynomial_kernel(x1, x2, degree=2)[0][0],
# rbf
features += pairwise.rbf_kernel(x1, x2)[0][0],
# laplacian
features += pairwise.laplacian_kernel(x1, x2)[0][0],
# sigmoid
features += pairwise.sigmoid_kernel(x1, x2)[0][0],
return features
def predict(self, X):
'''
Classifies new samples using the SVM2+ coefficients.
Parameters
----------
X : array-like, shape (n_samples, n_features)
New samples to classify.
dual_coef : array-like, shape (n_support_vectors, )
Dual coefficients of support vectors in trained model.
In the literature, this is alpha * y.
support_vectors : array-like, shape (n_support_vectors, n_features)
Support vectors.
'''
X = X.reshape(X.shape[0], -1)
if self.decision_kernel == 'rbf':
kernel = rbf_kernel(X, self.support_vectors_, gamma=self.gamma)
elif self.decision_kernel == 'linear':
kernel = linear_kernel(X, self.support_vectors_)
elif self.decision_kernel == 'poly':
kernel = polynomial_kernel(X,
self.support_vectors_,
degree=self.degree,
gamma=self.gamma,
coef0=self.coef0)
elif self.decision_kernel == 'sigmoid':
kernel = sigmoid_kernel(X,
self.support_vectors_,
gamma=self.gamma,
coef0=self.coef0)
else:
raise ValueError('''Kernel in decision space must be one of 'rbf',
'linear', 'poly' or 'sigmoid'.''')
predictions = np.dot(self.dual_coef_, kernel.T) + self.intercept_
predictions = np.sign(predictions).flatten()
predictions = utils.unbinarize_targets(predictions,
self._positive_class_target,
self._negative_class_target)
return predictions
def test_affinity_mat_poly(data):
v1_data = data['fit_data'][0]
distances = cdist(v1_data, v1_data)
gamma = 1 / (2 * np.median(distances)**2)
true_kernel = polynomial_kernel(v1_data, gamma=gamma)
spectral = MultiviewCoRegSpectralClustering(random_state=RANDOM_STATE,
affinity='poly')
p_kernel = spectral._affinity_mat(v1_data)
assert (p_kernel.shape[0] == data['n_fit'])
assert (p_kernel.shape[1] == data['n_fit'])
for ind1 in range(p_kernel.shape[0]):
for ind2 in range(p_kernel.shape[1]):
assert np.abs(true_kernel[ind1][ind2] -
p_kernel[ind1][ind2]) < 0.000001
def homogeneous_kernel(X, P, degree=2):
"""Convenience alias for homogeneous polynomial kernel between X and P::
K_P(x, p) = <x, p> ^ degree
Parameters
----------
X : ndarray of shape (n_samples_1, n_features)
Y : ndarray of shape (n_samples_2, n_features)
degree : int, default 2
Returns
-------
Gram matrix : array of shape (n_samples_1, n_samples_2)
"""
return polynomial_kernel(X, P, degree=degree, gamma=1, coef0=0)
def fit(self, X, y):
self.X = X
perm = np.array([])
if (self.kernel == 'poly'):
kfunc = lambda x, y: polynomial_kernel(
x, y, degree=self.degree, gamma=self.gamma)
elif (self.kernel == 'rbf'):
kfunc = lambda x, y: rbf_kernel(x, y, self.gamma)
else:
kfunc = self.kernel
Kernel = kfunc(np.array(X), np.array(X))
for f in range(self.augmenter):
perm = np.hstack([perm, np.random.permutation(len(X))])
self.perm = perm.astype(np.int64)
self.clf.fit(Kernel[self.perm][:, self.perm], y[self.perm])
return self
def test_nystroem_precomputed_kernel():
# Non-regression: test Nystroem on precomputed kernel.
# PR - 14706
rnd = np.random.RandomState(12)
X = rnd.uniform(size=(10, 4))
K = polynomial_kernel(X, degree=2, coef0=0.1)
nystroem = Nystroem(kernel="precomputed", n_components=X.shape[0])
X_transformed = nystroem.fit_transform(K)
assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)
# if degree, gamma or coef0 is passed, we raise a ValueError
msg = "Don't pass gamma, coef0 or degree to Nystroem"
params = ({"gamma": 1}, {"coef0": 1}, {"degree": 2})
for param in params:
ny = Nystroem(kernel="precomputed", n_components=X.shape[0], **param)
with pytest.raises(ValueError, match=msg):
ny.fit(K)
def homogeneous_kernel(X, P, degree=2):
"""Convenience alias for homogeneous polynomial kernel between X and P::
K_P(x, p) = <x, p> ^ degree
Parameters
----------
X : ndarray of shape (n_samples_1, n_features)
Y : ndarray of shape (n_samples_2, n_features)
degree : int, default 2
Returns
-------
Gram matrix : array of shape (n_samples_1, n_samples_2)
"""
return polynomial_kernel(X, P, degree=degree, gamma=1, coef0=0)
def calc_kernel_matrix(X, kernel, **kwargs):
"""calculate kernel matrix between X and X.
Parameters
----------
kernel : {'linear', 'poly', 'rbf', callable}, optional (default='rbf')
Specifies the kernel type to be used in the algorithm.
It must be one of 'linear', 'poly', 'rbf', or a callable.
If a callable is given it is used to pre-compute the kernel matrix
from data matrices; that matrix should be an array of shape
``(n_samples, n_samples)``.
degree : int, optional (default=3)
Degree of the polynomial kernel function ('poly').
Ignored by all other kernels.
gamma : float, optional (default=1.)
Kernel coefficient for 'rbf', 'poly'.
coef0 : float, optional (default=1.)
Independent term in kernel function.
It is only significant in 'poly'.
Returns
-------
kernel-matrix: array of shape (n_samples_1, n_samples_2)
kernel matrix between X and X.
"""
if kernel == 'rbf':
K = rbf_kernel(X=X, Y=X, gamma=kwargs.pop('gamma', 1.))
elif kernel == 'poly':
K = polynomial_kernel(X=X,
Y=X,
coef0=kwargs.pop('coef0', 1),
degree=kwargs.pop('degree', 3),
gamma=kwargs.pop('gamma', 1.))
elif kernel == 'linear':
K = linear_kernel(X=X, Y=X)
elif hasattr(kernel, '__call__'):
K = kernel(X=np.array(X), Y=np.array(X))
else:
raise NotImplementedError
return K
def evaluate_sklearn_kernel(kernel_name, kernel_param, X, Y=None):
# These names are consistent with sklearn's
if kernel_name not in ['linear', 'polynomial', 'rbf', 'laplacian']:
raise Exception('Unrecognised kernel name \'' + kernel_name + '\'!')
if kernel_name == 'linear':
return linear_kernel(X=X, Y=Y)
elif kernel_name == 'polynomial':
(degree_param, gamma_param,
coef0_param) = get_polynomial_kernel_params(kernel_param=kernel_param)
return polynomial_kernel(X=X,
Y=Y,
degree=degree_param,
gamma=gamma_param,
coef0=coef0_param)
elif kernel_name == 'rbf':
return rbf_kernel(X=X, Y=Y, gamma=kernel_param)
else: # Laplacian
return laplacian_kernel(X=X, Y=Y, gamma=kernel_param)
def _compute_parameter(self, X, y):
if self.kernel == "polynomial":
# K(x, y) = (x^T * y + 1)^n ('*' is dot product)
X_ = polynomial_kernel(X, X, degree=self.degree)
polynomial_matrix = PolynomialFeatures(degree = self.degree)
X = polynomial_matrix.fit_transform(X)
# X_ = np.dot(X, X.T)
P = y.T * X_ * y
elif self.kernel == "radial":
X_ = rbf_kernel(X, X, gamma=self.gamma)
P = y.T * X_ * y
else:
X_ = np.dot(X, X.T)
y_ = np.dot(y, y.T)
P = X_ * y_
m, n = X.shape
minus_i = -1* np.identity(m)
i = np.identity(m)
G = np.concatenate((minus_i, i), axis=0).T
q = -1 * np.ones((1, m))
c = self.c * (np.ones((m, 1)))
h = np.concatenate((np.zeros((m, 1)), c), axis=0).T
b = np.array([0])
A = y
self.alpha = np.array(self._qp_solver(P, q, G, h ,A, b)['x'])
w = np.dot((y * X).T, self.alpha)
self.w = w
sp_idx = []
for idx, val in enumerate(self.alpha):
if val[0] > 0.001:
sp_idx.append(idx)
else:
val[0] = 0
X_sp = X[sp_idx]
y_sp = y[sp_idx]
w_0 = np.mean(y_sp - np.dot(X_sp, w))
self.w_0 = w_0
self.coef_ = (w, w_0, sp_idx)
def update_Z(self, X, y, verbose=False, sample_weight=None):
"""Greedy CD solver for the quadratic term of a factorization machine.
Solves 0.5 ||y - <Z, XX'>||^2_2 + ||Z||_*
Z implicitly stored as P'ΛP
"""
n_samples, n_features = X.shape
rng = check_random_state(self.random_state)
P = self.P_
lams = self.lams_
old_loss = np.inf
max_rank = self.max_rank
if max_rank is None:
max_rank = n_features
##
#residual = self.predict_quadratic(X) - y # could optimize
#loss = self._loss(residual, sample_weight=sample_weight)
#rms = np.sqrt(np.mean((residual) ** 2))
#print("rank={} loss={}, RMSE={}".format(0, loss, rms))
##
for _ in range(self.max_iter_inner):
if self.rank_ >= max_rank:
break
residual = self.predict_quadratic(X) - y # could optimize
if sample_weight is not None:
residual *= sample_weight
p = _find_basis(X, residual, **self.eigsh_kwargs)
P.append(p)
lams.append(0.)
# refit
refit_target = y.copy()
K = polynomial_kernel(X, np.array(P), degree=2, gamma=1, coef0=0)
if sample_weight is not None:
refit_target *= np.sqrt(sample_weight)
K *= np.sqrt(sample_weight)[:, np.newaxis]
K = np.asfortranarray(K)
lams_init = np.array(lams, dtype=np.double)
# minimizes 0.5 * ||y - K * lams||_2^2 + beta * ||w||_1
lams, _, _, _ = enet_coordinate_descent(
lams_init, self.beta, 0, K, refit_target,
max_iter=self.refit_iter, tol=self.tol, rng=rng, random=0,
positive=0)
P = [p for p, lam in zip(P, lams) if np.abs(lam) > 0]
lams = [lam for lam in lams if np.abs(lam) > 0]
self.rank_ = len(lams)
self.quadratic_trace_ = np.sum(np.abs(lams))
predict_quadratic = self.predict_quadratic(X, P, lams)
residual = y - predict_quadratic # y is already shifted
loss = self._loss(residual, sample_weight=sample_weight)
if verbose > 0:
rms = np.sqrt(np.mean((residual) ** 2))
print("rank={} loss={}, RMSE={}".format(self.rank_, loss, rms))
if np.abs(old_loss - loss) < self.tol:
break
old_loss = loss
self.P_ = P
self.lams_ = lams
return X,y
def plotScatter(X0, X1, y):
for x0, x1, cls in zip(X0, X1, y):
color = 'blue' if cls == 1 else 'red'
marker = 'x' if cls == 1 else 'o'
pl.scatter(x0, x1, marker=marker, color=color)
X,y = genMultinomialData(100, 2, 2)
models = [LogisticRegressionCV(),
LogisticRegressionCV(),
LogisticRegressionCV(),
KNeighborsClassifier(n_neighbors=10)]
kernels = [lambda X0, X1: X0, # No Kernel
lambda X0, X1: polynomial_kernel(X0, X1, degree=2),
lambda X0, X1: rbf_kernel(X0, X1, gamma=50), # sigma = .1
lambda X0, X1: X0]
names = ['Linear Logistic Regression',
'Quadratic Logistic Regression',
'RBF Logistic Regression',
'KNN with K=10']
file_names = ['Linear', 'Quad', 'Rbf', 'KNN10']
for i in range(len(models)):
transX = kernels[i](X, X)
model = models[i].fit(transX, y)
xx, yy = np.meshgrid(np.linspace(-1, 1, 250), np.linspace(-1, 1, 250))
Z = model.predict(kernels[i](np.c_[xx.ravel(), yy.ravel()], X)).reshape(xx.shape)
pl.pcolormesh(xx, yy, Z, cmap=pl.cm.coolwarm)
def start_classifier(data_X, data_Label, dataset_name, method):
# plot the Original Data, with 2-D plots
ax_1 = plt.subplot(211)
ax_1.scatter(data_X[:,0], data_X[:,1], color=set_color_points(data_Label))
plt.xlabel("X")
plt.ylabel("Y")
plt.title(dataset_name+" Original Dataset Plot")
#plt.show()
# Compute the parameters
if method == "linear":
marginOptn = input("Which Margin to Choose ? (soft/hard)")
if marginOptn == "soft":
w, w0, support_index = compute_parameters(data_X, data_Label, c=1)
elif marginOptn == "hard":
w, w0, support_index = compute_parameters(data_X, data_Label, c=1000)
else:
print("Error please check the Input (Soft/Hard)")
exit(0)
elif method == "radial":
X = rbf_kernel(data_X, data_X, gamma=0.9)
#print("X has shape",X.shape, data_X.shape)
w, w0, support_index = compute_parameters(X, data_Label, c=1)
elif method == "polynomial":
degree = int(input("Enter the Degree ?"))
if dataset_name == "miss-value":
degree = 1
data_X = polynomial_kernel(data_X, data_X, degree=degree)
else:
data_X = polynomial_kernel(data_X, data_X, degree=degree)
w, w0, support_index = compute_parameters(data_X, data_Label, c=1)
else:
print("ERROR Please Check inputs given")
exit(0)
# plot the support vectors
X_supp_vector = data_X[support_index]
y_supp_vector = np.array(data_Label)[support_index]
ax = plt.subplot(212)
plt.scatter(data_X[:,0], data_X[:,1], color=set_color_points(data_Label))
plt.scatter(X_supp_vector[:,0], X_supp_vector[:,1], c="yellow",label="support vectors")
plt.title("Support Vector Plots on "+dataset_name+ " Dataset")
#plt.show()
# mark the margins
# for plot purpose the line segment's start and end points
if method == "linear":
if marginOptn == "soft":
x_min, x_max = int(min(data_X[:,0]))-1, int(max(data_X[:,0]))+2
x_points = np.array([i for i in range(x_min, x_max)])
y_on_margin, y_for_c1, y_for_c2 = (-w0 - x_points*w[0,0]) / w[1, 0], (1-w0 - x_points*w[0,0]) / w[1, 0], (-1-w0 - x_points*w[0,0]) / w[1, 0]
plt.plot(x_points, y_on_margin,'b',label="When, Wx + W0 = 0")
plt.plot(x_points, y_for_c1,'r--' ,label="When, Wx + W0 = 1")
plt.plot(x_points, y_for_c2,'g--' ,label="When, Wx + W0 = -1")
yHat = make_prediction(w, w0, data_X)
print("Evaluation result for Own Implementation")
evaluate_performance(data_Label,yHat)
clf = SVC(gamma=0.9,kernel="linear",C=1.0)
clf.fit(data_X,data_Label)
Z = clf.predict(data_X)
print("Evaluation result for python: ")
evaluate_performance(data_Label, Z)
elif marginOptn == "hard":
x_min, x_max = int(min(data_X[:,0])), int(max(data_X[:,0]))+2
x_points = np.array([i for i in range(x_min, x_max)])
y_on_margin, y_for_c1, y_for_c2 = (-w0 - x_points*w[0,0]) / w[1, 0], (1-w0 - x_points*w[0,0]) / w[1, 0], (-1-w0 - x_points*w[0,0]) / w[1, 0]
plt.plot(x_points, y_on_margin,'b',label="When, Wx + W0 = 0")
yHat = make_prediction(w, w0, data_X)
print("Evaluation result for Own Implementation")
evaluate_performance(data_Label,yHat)
clf = SVC(gamma=0.9, C=1000, kernel="linear")
clf.fit(data_X,data_Label)
Z = clf.predict(data_X)
print("Evaluation result for python: ")
evaluate_performance(data_Label, Z)
# http://stackoverflow.com/questions/19054923/plot-decision-boundary-matplotlib
elif method == "radial":
# Own Implementation
yHat = make_prediction(w, w0, X)
print("Evaluation result for Own Implementation")
evaluate_performance(data_Label, yHat)
# Python Implementation
clf = SVC(gamma=0.9)
clf.fit(data_X,data_Label)
Z = clf.predict(data_X)
print("Evaluation result for python: ")
evaluate_performance(data_Label, Z)
elif method == "polynomial":
yHat = make_prediction(w, w0, data_X)
evaluate_performance(data_Label, yHat)
if dataset_name == "miss-value":
degree = 1
clf = SVC(kernel="poly",gamma=0.9, degree=degree)
else:
clf = SVC(kernel="poly",gamma=0.9, degree=degree)
Z = clf.fit(data_X, data_Label).predict(data_X)
sup = clf.support_vectors_
print("Evaluation result for python: ")
evaluate_performance(data_Label, Z)
plt.legend(loc="lower right")
manager = plt.get_current_fig_manager()
manager.window.showMaximized()
#plt.savefig(dataset_name+" "+method+".png", bbox_inches='tight')
plt.show()
b = yg * kermat * yg.T
self.weights[ik] = b[0]
norm2 = sum([w for w in self.weights.values()])
for iw in self.weights:
self.weights[iw] = self.weights[iw] / norm2
if self.tracenorm:
for iw in self.weights:
self.weights[iw] = self.weights[iw] / self.traces[iw]
return self.weights
if __name__ == "__main__":
from sklearn.datasets import load_iris
from sklearn.metrics.pairwise import rbf_kernel, polynomial_kernel
data = load_iris()
X = data.data
y = data.target
# REMARK: labels and kernels are "cvxopt.matrix"
y = matrix([-1.0 if i == 0 else 1.0 for i in y])
K_dict = {('rbf', g): matrix(rbf_kernel(X, gamma=g))
for g in [2**i for i in range(-5, 2)]}
K_dict.update({('poly', d): matrix(polynomial_kernel(X, degree=d))
for d in range(1, 5)})
easy = GEasyMKL()
weights_dict = easy.train(K_dict, y)
print weights_dict
def inner(self, x, y):
return polynomial_kernel(to2d(x), to2d(y), degree=self.degree, coef0=self.coef0)
import Kernels as k
from sklearn.metrics.pairwise import euclidean_distances, cosine_distances, rbf_kernel, polynomial_kernel, cosine_similarity
import numpy as np
import Consts as c
a = np.array([[1, 2, 3], [1, 2, 3]]) / 10
aa = np.array([[1, 2, 3]]) / 10
b = np.array([[4, 5, 6]]) / 10
x = [1, 2]
y = [1, 2]
kern = k.Kernels(c.POLY)
res1 = kern.get_value([x], [y])
res2 = polynomial_kernel(x, y, degree=2, gamma=1, coef0=0)
# res2 = euclidean_distances(a, b)
print('{} \n {}'.format(res1, res2))
