def steepest_gradient_descent(weights,
inputs,
labels,
iterations=1e6,
epsilon=1e-4,
learning_rate=1e-3):
""" steepest gradient descent
to classify data into two classes (Bernoulli Distribution)
(gradient in matrix form)
A : inputs, shape=(n,k=3)
yi : labels[i,0], shape=()
xi : inputs[i:i+1], shape=(1,k=3)
W : weights, shape=(k=3,1)
gradients : A.t()*(yi-1/(1+e**(-xi*W)), shape=(k,1)
W(n+1) = W(n) + learning_rate*gradients
"""
for idx in range(int(iterations)):
print('iteration index: ', idx)
old_weights = Mat(weights)
gradients = []
for i in range(labels.shape[0]):
exp_term = (-1 * inputs[i:i + 1] * old_weights)[0, 0]
gradients.append([labels[i, 0] - 1.0 / (1 + math.exp(exp_term))])
gradients = learning_rate * inputs.t() * Mat(gradients)
weights = old_weights + gradients
print(weights)
if mean_abs_diff(old_weights, weights) < epsilon:
break
return weights
def __init__(self, file_path, num_bases, converge_epsilon, init_weights):
""" Get polynomail regression result by Newton's Method Optimization
Args
----
file_path : str,
path to dataset
num_bases : int,
degree of polynomial
converge_epsilon : float,
the condition of convergence
init_weights : list, shape=[num_bases,]
the parameters of polynomial
"""
assert num_bases == len(init_weights)
self._num_bases = int(num_bases)
self._converge_epsilon = float(converge_epsilon)
self._init_weights = Mat([[float(i)] for i in init_weights])
self._data = []
self._input = []
self._label = []
with open(file_path, 'r') as file_:
for line in file_.readlines():
# 1 for x^0 (bias)
self._data.append([1, int(line.strip().split(',')[0])])
self._label.append([int(line.strip().split(',')[1])])
for v in self._data:
for i in range(2, num_bases):
v += [v[1]**i]
self._input.append(v)
self._input = Mat(self._input)
self._label = Mat(self._label)
self._weights = self._fit()
def __init__(self, file_path, num_bases, lambda_):
""" Get polynomail regression result by least square error
Args
----
file_path : str,
path to dataset
num_bases : int,
degree of polynomial
lambda_ : float,
parameters of L2 regularizar term
"""
# data IO
assert num_bases >= 2
self._num_bases = int(num_bases)
self._lambda = float(lambda_)
self._data = []
self._input = []
self._label = []
with open(file_path, 'r') as file_:
for line in file_.readlines():
# 1 for x^0 (bias)
self._data.append([1, int(line.strip().split(',')[0])])
self._label.append([int(line.strip().split(',')[1])])
for v in self._data:
for i in range(2, num_bases):
v += [v[1]**i]
self._input.append(v)
self._input = Mat(self._input)
self._label = Mat(self._label)
self._weights = self._fit()
self._error = self._mse()
def logistic_regression(n,
mx1,
vx1,
my1,
vy1,
mx2,
vx2,
my2,
vy2,
optimizer='SGD'):
"""
weights : shape=(k=3,1)
Input
-----
optimizer : 'SGD' or 'NTM'
'SGD' == 'Steepest Gradient Descent'
'NTM' == 'Newton's Method'
"""
inputs = []
labels = []
D1_label = 0.0
D2_label = 1.0
bias_term = 1.0
for _ in range(n):
# Data 1
D1x = Generator.univariate_gaussian(mx1, vx1)
D1y = Generator.univariate_gaussian(my1, vy1)
inputs.append([bias_term, D1x, D1y])
labels.append([D1_label])
# Data 2
D2x = Generator.univariate_gaussian(mx2, vx2)
D2y = Generator.univariate_gaussian(my2, vy2)
inputs.append([bias_term, D2x, D2y])
labels.append([D2_label])
inputs = Mat(inputs)
labels = Mat(labels)
# init weights
weights = Mat([[-6.0], [1.0], [-0.1]])
print('inputs shape:\t', inputs.shape)
print('labels shape:\t', labels.shape)
print('weights shape:\t', weights.shape)
# optimization
if optimizer == 'SGD':
weights = steepest_gradient_descent(weights, inputs, labels)
elif optimizer == 'NTM':
weights = newton_method(weights, inputs, labels)
else:
raise AttributeError('{} is not a valid optimizor'.format(optimizer))
# inference
logits = inference(weights, inputs)
# evaluate model
CM = ConfusionMatrix(logits, labels)
CM.show_matrix()
CM.show_accuracy()
CM.show_sensitivity()
CM.show_specificity()
def newton_method(weights,
inputs,
labels,
iterations=1e6,
epsilon=1e-4,
learning_rate=1e-3):
""" newton's method
to classify data into two classes (Bernoulli Distribution)
(gradient in matrix form)
A : inputs, shape=(n,k=3)
yi : labels[i,0], shape=()
xi : inputs[i:i+1], shape=(1,k=3)
W : weights, shape=(k=3,1)
gradients : A.t()*(yi-1/(1+e**(-xi*W)), shape=(k,1)
D : diagonal matrix, shape=(n,n)
trace(i)=e**(-xi*W)/(1+e**(-xi*W))**2
hessian : A.t()*D*A
W(n+1) = W(n) + gradients
"""
for idx in range(int(iterations)):
print('iteration index: ', idx)
old_weights = Mat(weights)
gradients = []
for i in range(labels.shape[0]):
exp_term = (-1 * inputs[i:i + 1] * weights)[0, 0]
gradients.append([labels[i, 0] - 1.0 / (1 + math.exp(exp_term))])
D = Mat.identity(dims=labels.shape[0])
for i in range(labels.shape[0]):
exp_term = (-1 * inputs[i:i + 1] * weights)[0, 0]
D[i, i] = math.exp(exp_term) / ((1 + math.exp(exp_term))**2)
# D[i, i] = math.exp(exp_term-2*math.log(1+math.exp(exp_term)))
gradients = inputs.t() * Mat(gradients)
hessian = inputs.t() * D * inputs
if is_33_singullar(hessian):
print('Singular')
break
grad = learning_rate * hessian.inv() * gradients
# clipping ?!
# for i in range(grad.shape[0]):
# if grad[i, 0] > 0 and grad[i, 0] > 10:
# grad[i, 0] = 10
# if grad[i, 0] < 0 and grad[i, 0] < -10:
# grad[i, 0] = -10
weights = old_weights - grad
print(weights)
if mean_abs_diff(old_weights, weights) < epsilon:
break
return weights
def main():
print('Start data I/O...')
train_images, train_labels = load_mnist(
dataset='training', fetch_size=100)
# test_images, test_labels = load_mnist(dataset='testing', fetch_size=10000)
train_images = preprocessing(train_images)
for i in range(28):
print(train_images[0, i])
print(train_labels[0])
init_theta = Mat([[np.random.uniform(0.0, 1.0) for _ in range(train_images.shape[1]
* train_images.shape[2])] for _ in range(10)])
# init_theta = Mat([[0.5 for _ in range(train_images.shape[1]
# * train_images.shape[2])] for _ in range(10)])
init_lambda = [0.05, 0.10, 0.15, 0.10, 0.05, 0.10, 0.15, 0.10, 0.10, 0.10]
# init_lambda = [0.1 for _ in range(10)]
logits, group_labels = em_algorithm(train_images, train_labels, init_theta, init_lambda)
prediction = []
for i in range(logits.shape[0]):
largest_idx = 0
for z in range(logits.shape[1]):
# print(logits[i, z])
if logits[i, z] > logits[i, largest_idx]:
largest_idx = z
# input()
prediction.append([group_labels[largest_idx]])
prediction = Mat(prediction)
train_labels = Mat([train_labels]).t()
# confusion matrix
for z in range(10):
print('digit {}'.format(z))
tmp_logits = Mat(prediction)
tmp_labels = Mat(train_labels)
for i in range(tmp_logits.shape[0]):
if tmp_logits[i, 0] == tmp_labels[i, 0]:
tmp_logits[i, 0] = 1
else:
tmp_logits[i, 0] = 0
if tmp_labels[i, 0] == z:
tmp_labels[i, 0] = 1
else:
tmp_labels[i, 0] = 0
CM = ConfusionMatrix(tmp_logits, tmp_labels)
CM.show_matrix()
CM.show_accuracy()
CM.show_sensitivity()
CM.show_specificity()
def inference(weights, inputs):
logits = []
for i in range(inputs.shape[0]):
exp_term = (-1 * inputs[i:i + 1] * weights)[0, 0]
output = 1.0 / (1 + math.exp(exp_term))
# print(output)
# decision boundary
if output > 0.5:
output = 1
else:
output = 0
logits.append([output])
return Mat(logits)
def _fit(self):
""" W = (A.t() * A + lambda * I).inv() * A.t() * b
A : input
b : label
"""
I_mat = []
for i in range(self._num_bases):
temp = []
for j in range(self._num_bases):
v = 1.0 if i == j else 0.0
temp.append(v)
I_mat.append(temp)
I_mat = Mat(I_mat)
return (self._input.t() * self._input + self._lambda * I_mat).inv() * self._input.t() * self._label
def em_algorithm(train_images, train_labels, theta, lambda_, iterations=1e6, epsilon=1e-15):
"""
Expectation step:
theta : shape=[10, image_size]
the probability of class i showing 1 for each pixels
lambda = [L0,L1,L2,L3,L4,L5,L6,L7,L8,L9]
M : number of pixel=1 in each pixels
image_size : total number of pixels = 28*28
N : number of data
mu_pixel : mu of each pixel showing 1 for each class
P(Zi=z,Xi|theta) = lambda(z) * theta[z]**M[i] * (1-theta[z])**(N-M)
all(z) = sum_over_i(P(Zi=z,Xi|theta))
W(i,z) = P(Zi=z,Xi|theta)/all(z)
Maximization step:
lambda(z) = sum_over_i(W(i,z))/N
theta(z) = sum_over_i(W(i,z)*M[i])/sum_over_i(W(i,z))
"""
N = train_images.shape[0]
image_size = train_images.shape[1] * train_images.shape[2]
for iter_idx in range(int(3)):
print('iteration index: ', iter_idx)
old_theta = copy.deepcopy(theta)
theta = Mat.zeros(shape=[10, image_size]) + 1e-8 # avoid zeros
sum_w = [1e-8 for _ in range(10)]
majority_vote = Mat([[0 for _ in range(10)] for _ in range(10)])
# Expectation Step
for i in range(N):
weights = Mat.zeros([train_images.shape[0], 10]) + 1e-8
for z in range(10):
logprob = 0.0
for row_idx in range(train_images.shape[1]):
for col_idx in range(train_images.shape[2]):
logprob += math.log(old_theta[z, row_idx*28+col_idx])*train_images[i, row_idx, col_idx] + math.log(
1-old_theta[z, row_idx*28+col_idx])*(1-train_images[i, row_idx, col_idx])
logprob += math.log(lambda_[z])
weights[i, z] = logprob
# input(weights[i, z])
max_weight = [-1e10 for _ in range(N)]
for z in range(10):
if max_weight[i] < weights[i, z]:
max_weight[i] = weights[i, z]
sum_ = 0.0
for z in range(10):
weights[i, z] -= max_weight[i] # normalized
weights[i, z] = math.exp(weights[i, z])
# input(weights[i,z])
sum_ += weights[i, z]
for z in range(10):
weights[i, z] /= sum_
for z in range(10):
sum_w[z] += weights[i, z]
for pixels in range(image_size):
theta[z, pixels] += weights[i, z] * \
train_images[i, pixels//28, pixels % 28]
# label groups by majority vote
predict_result = -1
max_weight = -1e10
for z in range(10):
if weights[i, z] > max_weight:
max_weight = weights[i, z]
predict_result = z
for z in range(10):
majority_vote[predict_result, train_labels[i]] += 1
# Maximization Step
for z in range(10):
lambda_[z] = sum_w[z]/N
for pixels in range(image_size):
theta[z, pixels] /= sum_w[z]
if theta[z, pixels] >= 1.0:
theta[z, pixels] = 1.0-1e-8
# show mean
for z in range(10):
print()
for row_idx in range(28):
str_ = ''
for col_idx in range(28):
if int(theta[z,row_idx*28+col_idx] > 0.5) == 0:
str_ += '0 '
else:
str_ += ' '
print(str_)
# if mean_abs_diff(old_theta, theta) < epsilon:
# break
group_labels = [-1 for _ in range(10)]
for z in range(10):
label = -1
max_vote = -1e10
for l in range(10):
if majority_vote[z, l]> max_vote:
max_vote = majority_vote[z, l]
label = l
group_labels[z] = label
return weights, group_labels
def baysian_linear_regression(b, num_basis, error_variance):
"""
Prior
b: precision of prior weights (b*I is linear independent matrix), shape=(num_basis, num_basis)
S: inverse of prior corvariance matrix (S is a covariance matrix), shape=(num_basis, num_basis)
m: mean of prior, shape=(num_basis, 1)
Posterior
X: design matrix, shape=(n, num_basis)
y: all data samples, shape=(n, 1)
a: precision of data distribution, shape=(num_basis, num_basis)
COV = a*X.t()*X + b*I, shape=(num_basis, num_basis)
mean = COV.inv()*(a*X.t()*y+Sm), shape=(num_basis, 1)
Predictive Distribution
mean = X*mu
variance = 1/a+X*COV.inv()*X.t()
"""
prior_weight_var = b * Mat.identity(dims=num_basis)
weights = []
for i in range(num_basis):
weights.append(Generator.univariate_gaussian(
mean=0, variance=prior_weight_var[i, i]))
prior_mean_mat = Mat([weights]).t()
prior_cov_mat = b.inv()
posterior_mean_mat = math.inf
posterior_cov_mat = math.inf
diff = math.inf
X = None
y = None
iteration_idx = 0
while diff > 1e-2:
iteration_idx += 1
new_y, new_x = Generator.polynomial(num_basis, error_variance, weights)
if iteration_idx == 1:
y = Mat([[new_y]])
X = Mat([[new_x**i for i in range(num_basis)]])
else:
y.append([new_y])
X.append([new_x**i for i in range(num_basis)])
# change annotation
a = error_variance
S = prior_cov_mat.inv()
m = prior_mean_mat
# posterior
posterior_cov_mat = a*X.t()*X + S
COV = posterior_cov_mat
posterior_mean_mat = COV.inv()*(a*X.t()*y+S*m)
# check converge
if iteration_idx>1:
sub_mat = prior_mean_mat - posterior_mean_mat
diff = 0.0
for i in range(sub_mat.shape[0]):
diff += abs(sub_mat[i, 0])
diff /= sub_mat.shape[0]
# predictive distribution
predictive_mean = X*posterior_mean_mat
predictive_var= 1/a+X*posterior_cov_mat.inv()*X.t()
# update prior
prior_mean_mat = posterior_mean_mat
prior_cov_mat = posterior_cov_mat
print('Iteration: {}'.format(iteration_idx))
print('New Data Point: x:{}, y:{}'.format(new_x, new_y))
print('Posterior Mean:\n{}'.format(posterior_mean_mat))
print('Posterior Variance:\n{}'.format(posterior_cov_mat))
print('Predictive Distribution Mean:\n{}'.format(predictive_mean))
print('Predictive Distribution Variance:\n{}'.format(predictive_var))
if iteration_idx % 5 == 0:
input()
def main():
num_basis = 2
b_matrix = Mat.identity(num_basis)
baysian_linear_regression(b_matrix, num_basis, 3)
class NewtonMethod(object):
""" Newton's Method Optimization
"""
def __init__(self, file_path, num_bases, converge_epsilon, init_weights):
""" Get polynomail regression result by Newton's Method Optimization
Args
----
file_path : str,
path to dataset
num_bases : int,
degree of polynomial
converge_epsilon : float,
the condition of convergence
init_weights : list, shape=[num_bases,]
the parameters of polynomial
"""
assert num_bases == len(init_weights)
self._num_bases = int(num_bases)
self._converge_epsilon = float(converge_epsilon)
self._init_weights = Mat([[float(i)] for i in init_weights])
self._data = []
self._input = []
self._label = []
with open(file_path, 'r') as file_:
for line in file_.readlines():
# 1 for x^0 (bias)
self._data.append([1, int(line.strip().split(',')[0])])
self._label.append([int(line.strip().split(',')[1])])
for v in self._data:
for i in range(2, num_bases):
v += [v[1]**i]
self._input.append(v)
self._input = Mat(self._input)
self._label = Mat(self._label)
self._weights = self._fit()
# print('input shape = {}'.format(self._input.shape))
# print('label shape = {}'.format(self._label.shape))
def _fit(self):
"""
d_F = 2 * (A.t() * A * X - A.t() * b)
dd_F = 2 * A.t() * A
"""
loss = 1e10
weights = self._init_weights
while loss > self._converge_epsilon:
d_F = 2 * (self._input.t() * self._input * weights -
self._input.t() * self._label)
dd_F = 2 * self._input.t() * self._input
weights = weights - dd_F.inv() * d_F
loss = self._mse(weights)
print('Error : {}'.format(loss))
return weights
def _mse(self, weights):
""" mean square error
Retrun
------
mean square error : float, shape=()
"""
error = self._input * weights - self._label
sum_ = 0.0
for i in range(self._input.shape[0]):
sum_ += error[i, 0]**2
return sum_ / self._input.shape[0]
def show_formula(self):
""" to print out result of NewtonMethod as formula:
y = Wn * x^n + W(n-1) * x^(n-1) + ... + W1 * x^1 + W0
"""
formula = "y = "
for base in range(self._num_bases - 1, -1, -1):
if base == 0:
formula += "{}".format(self._weights[base, 0])
else:
formula += "{} * x^{} + ".format(self._weights[base, 0], base)
print(formula)
class rLSE(object):
""" regularized Least Square Error
"""
def __init__(self, file_path, num_bases, lambda_):
""" Get polynomail regression result by least square error
Args
----
file_path : str,
path to dataset
num_bases : int,
degree of polynomial
lambda_ : float,
parameters of L2 regularizar term
"""
# data IO
assert num_bases >= 2
self._num_bases = int(num_bases)
self._lambda = float(lambda_)
self._data = []
self._input = []
self._label = []
with open(file_path, 'r') as file_:
for line in file_.readlines():
# 1 for x^0 (bias)
self._data.append([1, int(line.strip().split(',')[0])])
self._label.append([int(line.strip().split(',')[1])])
for v in self._data:
for i in range(2, num_bases):
v += [v[1]**i]
self._input.append(v)
self._input = Mat(self._input)
self._label = Mat(self._label)
self._weights = self._fit()
self._error = self._mse()
# print('input shape = {}'.format(self._input.shape))
# print('label shape = {}'.format(self._label.shape))
def _fit(self):
""" W = (A.t() * A + lambda * I).inv() * A.t() * b
A : input
b : label
"""
I_mat = []
for i in range(self._num_bases):
temp = []
for j in range(self._num_bases):
v = 1.0 if i == j else 0.0
temp.append(v)
I_mat.append(temp)
I_mat = Mat(I_mat)
return (self._input.t() * self._input + self._lambda * I_mat).inv() * self._input.t() * self._label
def _mse(self):
""" mean square error
Retrun
------
mean square error : float, shape=()
"""
error = self._input * self._weights - self._label
sum_ = 0.0
for i in range(self._input.shape[0]):
sum_ += error[i, 0]**2
return sum_/self._input.shape[0]
def show_formula(self):
""" to print out result of LSE as formula:
y = Wn * x^n + W(n-1) * x^(n-1) + ... + W1 * x^1 + W0
"""
formula = "LSE Result : y = "
for base in range(self._num_bases-1, -1, -1):
if base == 0:
formula += "{}".format(self._weights[base, 0])
else:
formula += "{} * x^{} + ".format(self._weights[base, 0], base)
print(formula)
def show_error(self):
""" to print out error of LSE
"""
print('LSE Error : {}'.format(self._error))