def predict(self,X,return_prob=False):
"""predict
Args:
X (2-D arrray) : shape = (N_samples,N_dims)
return_prob (bool) : if True, return probability
Returns:
1-D array or 2-D array: if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. This depends on parameter y when fitting.
If return_prob == True, always return probability of belonging to class1 in each record
"""
gram_mat = np.zeros((self.X.shape[0],X.shape[0]))
for i in range(self.X.shape[0]):
gram_mat[i] = np.array([self.kernel_func(self.X[i],X[j]) for j in range(X.shape[0])])
sig = sigmoid(self.a)
logit = (gram_mat.T@(self.y - sig)).ravel()
sig = sig.ravel()
if return_prob:
c = np.array([self.kernel_func(X[i],X[i]) for i in range(X.shape[0])]) + self.gamma
W = np.diag(1/(sig*(1 - sig)))
sigma = c - np.diag([email protected](W + self.C)@gram_mat)
prob = sigmoid(kappa(sigma)*logit)
return prob
else:
y = np.zeros(X.shape[0])
y[logit >= 0] = 1
return self._inverse_transform(y)
def fit(self, X, y):
"""fit
Args:
X (2-D array) : data, shape = (N_samples,N_dims)
y (1-D array or 2-D array) : if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. should be 2-class data.
Note:
optimizing parameters in gradient descent method
"""
y = self._onehot_to_label(y)
y = y.reshape(-1, 1)
design_mat = self.make_design_mat(X)
self.weight = np.random.randn(design_mat.shape[1]).reshape(-1, 1)
for _ in range(self.max_iter):
probability = sigmoid(design_mat @ self.weight)
loss = binary_cross_entropy(y, probability)
if loss < self.threshold:
break
self.weight -= self.learning_rate * (
self.alpha * self.weight + design_mat.T @ (probability - y))
R = (probability * (1.0 - probability)).ravel()
self.S = 1 / self.alpha * np.eye(
self.weight.shape[1]) + (R * design_mat.T) @ design_mat
def fit(self, X, y):
"""fit
Args:
X (2-D array) : data, shape = (N_samples,N_dims)
y (1-D array or 2-D array) : if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. should be 2-class data.
Note:
optimizing parameters in IRLS
"""
target = self._onehot_to_label(y)
target = target.reshape(-1, 1)
design_mat = self.make_design_mat(X)
self.weight = np.random.randn(design_mat.shape[1]).reshape(-1, 1)
for _ in range(self.max_iter):
y = sigmoid(design_mat @ self.weight)
if binary_cross_entropy(target, y) < self.threshold:
break
R = y * (1.0 - y)
if np.any(abs(R) < 1e-20): # prevent overflow and error
R += 1e-10
z = design_mat @ self.weight - (y - target) / R
self.weight = np.linalg.pinv(
design_mat.T @ (R * design_mat)) @ design_mat.T @ (R * z)
def fit(self,X,y):
"""fit
Args:
X (2-D array) : explanatory variable, shape = (N_samples,N_dims)
y (1-D array or 2-D array) : if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. should be 2-class data.
"""
y = self._onehot_to_label(y)
y = y.reshape(-1,1)
self.X = X
self.y = y
C = self.gram_func(X)/self.alpha + np.eye(X.shape[0])*self.gamma
C_inv = np.linalg.inv(C)
# search mode using Neweton-method
a = np.random.randn(X.shape[0],1)
for _ in range(self.max_iter):
sig = sigmoid(a)
W = sig*(1 - sig)
a = [email protected](np.eye(X.shape[0]) + W*C)@(y - sig + W*a)
da = y - sig - C_inv@a
if np.dot(da.ravel(),da.ravel())**0.5 < self.threshold:
break
self.C = C
self.a = a
def predict(self,X,return_prob=False):
"""predict
Args:
X (2-D arrray) : shape = (N_samples,N_dims)
return_prob (bool) : if True, return probability
Returns:
1-D array or 2-D array: if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. This depends on parameter y when fitting.
If return_prob == True, always return probability of belonging to class1 in each record
"""
design_mat = np.zeros((self.relevance_vector_X.shape[0],X.shape[0]))
for i in range(self.relevance_vector_X.shape[0]):
design_mat[i] = np.array([self.kernel_func(self.relevance_vector_X[i],X[j]) for j in range(X.shape[0])])
logit = ([email protected]).ravel()
if return_prob:
sigma = np.diag([email protected]@design_mat)
prob = sigmoid(kappa(sigma)*logit)
return prob
else:
y = np.zeros(X.shape[0])
y[logit >= 0] = 1
return self._inverse_transform(y)
def fit(self, X, y):
"""fit
Args:
X (2-D array): shape = (N_samples,N_dim),
y (1-D array or 2-D array) : if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. should be 2-class data.
"""
t = self._onehot_to_label(y)
t = t.reshape(-1, 1)
design_mat = self.make_design_mat(X)
N = design_mat.shape[0]
gamma = np.random.rand(N, self.K) + 1
gamma /= gamma.sum(axis=1, keepdims=True)
weight = None
for _ in range(self.max_iter):
# M step
pi = gamma.mean(axis=0)
weight = self._Mstep(design_mat, t, gamma, weight)
y = sigmoid(design_mat @ weight.T)
# E step
prob = pi * (y**t) * ((1 - y)**(1 - t)) + 1e-10
new_gamma = prob / prob.sum(axis=1, keepdims=True)
if np.mean((new_gamma - gamma)**2)**0.5 < self.threshold:
gamma = new_gamma
break
gamma = new_gamma
self.pi = pi
self.weight = weight
def predict(self, X):
"""predict
Args:
X (2-D array) : shape = (N_samples,N_dims)
Returns:
1-D array or 2-D array: if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. This depends on parameter y when fitting.
"""
logit = X @ self.weight + self.b
prob = sigmoid(logit)
y = np.zeros(X.shape[0])
y[prob > 0.5] = 1
return self._inverse_transform(y)
def _optimize_weight(self,design_mat,y,max_iter):
"""_optimize_weight
Args:
design_mat (array): design matrix
y (array): target class
max_iter (int): max iteration when model optimize parameters
Returns:
array: new covariance of post distribution
"""
for _ in range(max_iter):
y_pred = sigmoid([email protected])
grad = self.alpha*self.weight - design_mat.T@(y - y_pred)
B = y_pred*(1 - y_pred)
H_inv = np.linalg.pinv(design_mat.T@(B*design_mat) + np.diag(self.alpha.ravel()))
self.weight -= H_inv@grad
return H_inv
def predict(self, X, return_prob=False):
"""predict
Args:
X (2-D arrray) : shape = (N_samples,N_dims)
return_prob (bool) : if True, return probability
Returns:
1-D array or 2-D array: if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. This depends on parameter y when fitting.
If return_prob == True, always return probability of belonging to class1 in each record
"""
design_mat = self.make_design_mat(X)
y = np.dot(sigmoid(design_mat @ self.weight.T), self.pi)
if return_prob:
return y
else:
y = np.zeros(X.shape[0])
y[y >= 0.5] = 1
return self._inverse_transform(y)
def _Mstep(self, design_mat, t, gamma, weight):
M = design_mat.shape[1]
if weight is None:
weight = np.random.randn(self.K, M)
for _ in range(4):
y = sigmoid(design_mat @ weight.T)
# gradient
tmp = gamma * (y - t)
dQ = tmp.T @ design_mat
# hessian
tmp1 = gamma * y * (1 - y)
tmp2 = design_mat.reshape(-1, M, 1) * design_mat.reshape(-1, 1, M)
H = np.sum(tmp1.T.reshape(self.K, -1, 1, 1) * tmp2, axis=1)
print(H)
dweight = np.squeeze(np.linalg.inv(H) @ dQ.reshape(self.K, M, 1))
if np.mean(np.abs(dweight)) < self.threshold:
weight -= dweight
return weight
weight -= dweight
return weight
def predict(self, X, return_prob=False):
"""predict
Args:
X (2-D arrray) : shape = (N_samples,N_dims)
return_prob (bool) : if True, return probability
Returns:
1-D array or 2-D array: if 1-D array, y should be label-encoded, but 2-D arrray, y should be one-hot-encoded. This depends on parameter y when fitting.
If return_prob == True, always return probability of belonging to class1 in each record
"""
design_mat = self.make_design_mat(X)
logit = (design_mat @ self.weight).ravel()
if return_prob:
sigma = np.diag(design_mat @ self.S @ design_mat.T)
prob = sigmoid(kappa(sigma) * logit)
return prob
else:
y = np.zeros(X.shape[0])
y[logit >= 0] = 1
return self._inverse_transform(y)
# if v < 0, what does (2*pi*v)**(D/2) mean? this cause error
# class EP_for_noisy_data():
# """EP for noisy data
# Attributes:
# D (int): data dimension
# m (1-D array): shape = (D), mean of prior distribution
# v (float): std of prior distribution
# Ss (1-D array): param of each factor
# Ms (2-D array): mean of each factor
# max_iter (int) : max iteration for parameter optimization
# threshold (float) : threshold for optimizint parameters
# """
# def __init__(self,max_iter=100,threshold=1e-4):
# self.max_iter = max_iter
# self.threshold = threshold
# def fit(self,X,a=10,b=100,w=0.5,m=None,v=None):
# """fit
# Args:
# X (2-D array): shape = (N,D), data
# a,b,w (float): param of noisy data
# m (1-D array): shape = (D),initial param for prior distribution
# v (float): initial param for prior distribution
# """
# N = X.shape[0]
# D = X.shape[1]
# if m is None:
# m = np.random.randn(D)
# elif m.shape[0] != D:
# raise ValueError("m.shape[0] != X.shape[1]")
# if v is None:
# v = 1
# Ss = np.random.randn(N)
# Ms = np.random.randn(N,D)
# Vs = np.ones(N)
# def inv(a):
# if a > 0:
# return 1/(a + 1e-5)
# else:
# return 1/(a - 1e-5)
# def gauss(x,m,v):
# return inv(2*np.pi*v)**(D/2)*np.exp(-np.sum((x-m)**2)*inv(v))
# for _ in range(self.max_iter):
# param_diff_max = -1e20
# for i in range(N):
# v_n = inv(inv(v) - inv(Vs[i]))
# m_n = m + v_n*inv(Vs[i])*(m - Ms[i])
# Z_n = (1 - w)*gauss(X[i],m_n,v_n+1) + w*gauss(X[i],0,a)
# pho_n = 1 - w*inv(Z_n)*gauss(X[i],0,a)
# new_m = m_n + pho_n*v_n*inv(v_n + 1)*(X[i] - m_n)
# new_v = v_n - pho_n*(v_n)**2*inv(v_n + 1) + pho_n*(1 - pho_n)*(v_n)**2*np.sum((X[i] - m_n)**2)*inv(D*(v_n + 1)**2)
# new_Vi = inv(inv(new_v) - inv(v_n))
# new_Mi = m_n + (Vs[i] + v_n)*inv(v_n)*(new_m - m_n)
# new_Si = Z_n*inv((2*np.pi*Vs[i])**(D/2)*gauss(Ms[i],m_n,Vs[i]+v_n))
# param_diff_max = max(param_diff_max,np.abs(new_m - m).max())
# param_diff_max = max(param_diff_max,abs(new_v - v))
# param_diff_max = max(param_diff_max,abs(new_Vi - Vs[i]))
# param_diff_max = max(param_diff_max,np.abs(new_Mi - Ms[i]).max())
# param_diff_max = max(param_diff_max,abs(new_Si - Ss[i]))
# m = new_m
# v = new_v
# Vs[i] = new_Vi
# Ms[i] = new_Mi
# Ss[i] = new_Si
# if param_diff_max < self.threshold:
# break
# self.D = D
# self.m = m
# self.v = v
# self.Ss = Ss
# self.Ms = Ms
# self.Vs = Vs
# return m,v,Ss,Ms,Vs
# def calc_posterior_mean(self):
# """calc_posterior_mean
# Returns:
# mean (1-D array): mean of posterior density for theta
# """
# return self.m
# def calc_evidence(self):
# """calc_evidence
# Returns:
# evidence (float): model evidence
# """
# B = np.dot(self.m,self.m)/self.v - np.diag([email protected])/self.Vs
# return (2*np.pi*self.v)**(self.D/2)*np.exp(B/2)*np.prod(self.Ss*(2*np.pi*self.Vs)**(-self.D/2))
def _lamda(self, xi):
return (sigmoid(xi) - 0.5) / (2 * xi)