def reg_logistic_regression(y,
tx,
lambda_,
initial_w,
max_iters,
gamma,
all_losses=False):
"""
#Regularized logistic regression using gradient descent
:param y: labels
:type y: numpy 1D array
:param lambda_: trade-off parameter (how big part of the loss/cost function to give to regularization function)
:type lambda_: float64
:param tx: extended (contains bias column) feature matrix, where each row is a datapoint and each column a feature
:type tx: numpy 2D array
:param initial_w: initial value of weights
:type initial_w: numpy 1D array
:param max_iters: the number of maximal iterations
:type max_iters: int
:param gamma: learning rate
:type gamma: float64
if all_losses == False
:return: trained weights , value of the loss/cost function at the last step (the value of the loss function at the end of learning)
:rtype: numpy array, float64
if all_losses == True and return_total_number_of_iterations == False
:return: trained weights , all values of the loss/cost function per step/iteration of training before the update rule is applied to the weights
:rtype: numpy array, numpy array
"""
# Define parameters to store weights and the value of the loss(cost) funcion
loss = 0.0
w = initial_w
losses = []
for n_iter in range(max_iters):
# compute the gradient at the given point
gradient = gradient_of_logistic_regression(tx, y, w, lambda_)
# update weights accordint to the BGD
w = w - gamma * gradient
losses.append(loss_of_logistic_regression(tx, y, w))
# calculate loss function
loss = loss_of_logistic_regression(tx, y, w, lambda_)
if all_losses:
return w, losses
return w, loss
def reg_batch_logistic_regression(y, tx, lambda_, initial_w, max_iters, gamma,
batch_size):
"""
#Regularized logistic regression using gradient descent
:param y: labels
:type y: numpy 1D array
:param lambda_: trade-off parameter (how big part of the loss/cost function to give to regularization function)
:type lambda_: float64
:param tx: extended (contains bias column) feature matrix, where each row is a datapoint and each column a feature
:type tx: numpy 2D array
:param initial_w: initial value of weights
:type initial_w: numpy 1D array
:param max_iters: the number of maximal iterations
:type max_iters: int
:param gamma: learning rate
:type gamma: float64
:return: trained weights , value of the loss/cost function at the last step (the value of the loss function at the end of learning)
:rtype: numpy array, float64
"""
n = len(y)
# Define parameters to store weights and the value of the loss(cost) funcion
loss = 0.0
w = initial_w
for n_iter in range(max_iters):
batch_indices = np.random.randint(n, size=batch_size)
batch_y = y[batch_indices]
batch_tx = tx[batch_indices]
# compute the gradient at the given point
gradient = gradient_of_logistic_regression(batch_tx, batch_y, w,
lambda_)
# update weights accordint to the BGD
w = w - gamma * gradient
# calculate loss function
loss = loss_of_logistic_regression(tx, y, w, lambda_)
return w, loss
def logistic_regression(y, tx, initial_w, max_iters, gamma, all_losses=False):
"""
#Logistic regression using gradient descent
:param y: labels
:type y: numpy 1D array
:param tx: extended (contains bias column) feature matrix, where each row is a datapoint and each column a feature
:type tx: numpy 2D array
:param initial_w: initial value of weights
:type initial_w: numpy 1D array
:param max_iters: the number of maximal iterations
:type max_iters: int
:param gamma: learning rate
:type gamma: float64
if all_losses == False
:return: trained weights , value of the loss/cost function at the last step (the value of the loss function at the end of learning)
:rtype: numpy array, float64
if all_losses == True
:return: trained weights , all values of the loss/cost function per step/iteration of training before the update rule is applied to the weights
:rtype: numpy array, numpy array
"""
loss = 0.0
w = initial_w
losses = []
for n_iter in range(max_iters):
# compute the gradient at the given point
gradient = gradient_of_logistic_regression(tx, y, w)
if all_losses:
losses.append(loss_of_logistic_regression(tx, y, w))
w = w - gamma * gradient
# calculate loss function
loss = loss_of_logistic_regression(tx, y, w)
if all_losses:
return w, losses
return w, loss
def reg_logistic_regression_smart(y,
tx,
lambda_,
initial_w,
max_iters,
gamma,
epsilon,
all_losses=False,
return_total_number_of_iterations=False):
"""
#Reg. Logistic regression using gradient descent with smart feature,
that will stop if the absolute difference of two consecutive values
of the loss function is smaller than desired argument called epsilon
:param y: labels
:type y: numpy 1D array
:param lambda_: trade-off parameter (how big part of the loss/cost function to give to regularization function)
:type lambda_: float64
:param tx: extended (contains bias column) feature matrix, where each row is a datapoint and each column a feature
:type tx: numpy 2D array
:param initial_w: initial value of weights
:type initial_w: numpy 1D array
:param max_iters: the number of maximal iterations
:type max_iters: int
:param gamma: learning rate
:type gamma: float64
if all_losses == False
:return: trained weights , value of the loss/cost function at the last step (the value of the loss function at the end of learning)
:rtype: numpy array, float64
if all_losses == True and return_total_number_of_iterations == False
:return: trained weights , all values of the loss/cost function per step/iteration of training before the update rule is applied to the weights
:rtype: numpy array, numpy array
if all_losses == True and return_total_number_of_iterations == True
:return: trained weights , all values of the loss/cost function per step/iteration of training before the update rule is applied to the weights,
total number of steps before achieving the desired predefined EPSILON precision
:rtype: numpy array, numpy array, int
"""
loss = 0.0
w = initial_w
losses = []
prev_loss = loss_of_logistic_regression(tx, y, w, lambda_)
total_number_of_iteratiorns = max_iters
for n_iter in range(max_iters):
# compute the gradient at the given point
gradient = gradient_of_logistic_regression(tx, y, w, lambda_)
# update weights
w = w - gamma * gradient
# calculate the current value of the loss function
current_loss = loss_of_logistic_regression(tx, y, w, lambda_)
# save the current value of the loss function
losses.append(current_loss)
# calculate the differences between two consecutive values
# of the loss functions, and if it is less than defined
# epsilon then break because we reached desired precision
if (abs(current_loss - prev_loss) <= epsilon):
total_number_of_iteratiorns = n_iter
break
prev_loss = current_loss
if return_total_number_of_iterations and all_losses:
return w, losses, total_number_of_iteratiorns
if all_losses:
return w, losses
else:
# calculate the final loss function
loss = loss_of_logistic_regression(tx, y, w, lambda_)
return w, loss