def train_adaline(x,
y,
num_epochs=100,
alpha=0.001,
w_initial=None,
build_regressors=True):
if build_regressors:
x_matrix = lin.build_poly_regressors(x)
else:
x_matrix = x
if len(y.shape) == 1:
y = y[:, None]
if w_initial is None:
w = np.zeros((x_matrix.shape[1], y.shape[1]))
else:
w = w_initial.copy()
loss_history = []
for epoch in range(num_epochs):
random_permutation = np.random.permutation(y.shape[0])
for xi, yi in zip(x_matrix[random_permutation], y[random_permutation]):
error = yi - xi @ w
w += alpha * xi[:, None] @ error[:, None].T
loss_history.append(np.mean((y - x_matrix @ w)**2))
return {'w': w, 'loss_history': loss_history}
def train_perceptron(x,
y,
num_epochs=100,
alpha=1,
w_initial=None,
build_regressors=True):
if build_regressors:
x_matrix = lin.build_poly_regressors(x)
else:
x_matrix = x
y = y.copy()
y[y == 0] = -1
if len(y.shape) == 1:
y = y[:, None]
if w_initial is None:
w = np.zeros((x_matrix.shape[1], y.shape[1]))
else:
w = w_initial.copy()
loss_history = []
for epoch in range(num_epochs):
random_permutation = np.random.permutation(y.shape[0])
for xi, yi in zip(x_matrix[random_permutation], y[random_permutation]):
error = yi - np.sign(xi @ w)
w += alpha * xi[:, None] @ error[:, None].T
loss_history.append(np.sum(np.maximum(0,
-(y * np.sign(x_matrix @ w)))))
return {'w': w, 'loss_history': loss_history}
def linear_class_predic(model, x, build_regressors=True):
if build_regressors:
x_matrix = lin.build_poly_regressors(x)
else:
x_matrix = x
if model['w'].shape[1] > 1:
return np.argmax(x_matrix @ model['w'], axis=1)
else:
return np.maximum(0, np.sign(x_matrix @ model['w']))
def predict(w, x, y=None, poly_order=1, build_regressors=True):
if build_regressors:
x_matrix = lin.build_poly_regressors(x, poly_order=poly_order)
else:
x_matrix = x
output_function = logistic_function
loss = logistic_loss
if w.shape[1] > 1:
output_function = softmax_function
loss = softmax_loss
if y is None:
return output_function(w, x_matrix)
else:
pred = output_function(w, x_matrix)
return {'pred': pred, 'loss': loss(y, output_function(w, x_matrix))}
def lms(x,
y,
num_epochs=100,
alpha=0.001,
w_initial=None,
poly_order=1,
build_regressors=True,
compute_loss=True):
if build_regressors:
x_matrix = lin.build_poly_regressors(x, poly_order=poly_order)
else:
x_matrix = x
if len(y.shape) == 1:
y = y[:, None]
if w_initial is None:
w = np.zeros((x_matrix.shape[1], y.shape[1]))
else:
w = w_initial
output_function = logistic_function
loss = logistic_loss
if w.shape[1] > 1:
output_function = softmax_function
loss = softmax_loss
loss_history = []
for epoch in range(num_epochs):
random_permutation = np.random.permutation(y.shape[0])
for xi, yi in zip(x_matrix[random_permutation], y[random_permutation]):
error = yi - output_function(w, xi)
#for k in range(w.shape[1]):
# w[:,k] += alpha * error[k] * xi
w += alpha * xi[:, None] @ error[:, None].T
if compute_loss:
loss_history.append(loss(y, output_function(w, x_matrix)))
if compute_loss:
return {'w': w, 'loss_history': loss_history}
else:
return {'w': w}
def mlp_predict(model, x, build_regressors=True, return_class=True):
if build_regressors:
x_matrix = lin.build_poly_regressors(x)
else:
x_matrix = x
model_output = mlp_forward(model['W_list'],
x_matrix.T,
model['activation_function'],
model['out_activation_function'],
output_only=True)
if return_class and model['output'] != 'regression':
if model_output.shape[0] == 1:
return np.maximum(0, np.sign(model_output[0] - 0.5))
else:
return np.argmax(model_output, axis=0)
else:
return model_output
def gd(x,
y,
num_epochs=100,
alpha=0.001,
w_initial=None,
poly_order=1,
build_regressors=True,
compute_loss=True):
if build_regressors:
x_matrix = lin.build_poly_regressors(x, poly_order=poly_order)
else:
x_matrix = x
if len(y.shape) == 1:
y = y[:, None]
if w_initial is None:
w = np.zeros((x_matrix.shape[1], y.shape[1]))
else:
w = w_initial
output_function = logistic_function
loss = logistic_loss
if w.shape[1] > 1:
output_function = softmax_function
loss = softmax_loss
loss_history = []
for epoch in range(num_epochs):
error = y - output_function(w, x_matrix)
#for k in range(w.shape[1]):
# w[:,k] += alpha * np.mean(error[:,k:k+1] * x_matrix, axis=0)
w += alpha * x_matrix.T @ error / x_matrix.shape[0]
if compute_loss:
loss_history.append(loss(y, output_function(w, x_matrix)))
if compute_loss:
return {'w': w, 'loss_history': loss_history}
else:
return {'w': w}
def mlp_train(x,
y,
x_validation=None,
y_validation=None,
num_hidden_nodes=10,
activation="relu",
output="regression",
num_epochs=100,
alpha=1,
mini_batch_size=1,
momentum=0,
weight_decay=0,
build_regressors=True,
compute_loss=True):
# Preprocess the input and output data
if build_regressors:
x_matrix = lin.build_poly_regressors(x)
if x_validation is not None:
x_validation = lin.build_poly_regressors(x_validation)
else:
x_matrix = x
y = y.copy()
if len(y.shape) == 1:
y = y[:, None]
if y_validation is not None:
y_validation = y_validation.copy()
if len(y_validation.shape) == 1:
y_validation = y_validation[:, None]
# Select the hidden and output activation functions
activation_function, grad_activation_function = select_activation_function(
activation)
out_activation_function, out_grad_activation_function, loss = select_output_type(
output, y)
if type(num_hidden_nodes) is not list:
num_hidden_nodes = [num_hidden_nodes]
num_hidden_layers = len(num_hidden_nodes)
# Initialize the weights
W_list = initialize_weights(x_matrix, y, num_hidden_nodes, activation,
output)
loss_history = []
validation_loss_history = []
past_updates = [0] * (num_hidden_layers + 1)
for epoch in range(num_epochs):
random_permutation = np.array_split(
np.random.permutation(y.shape[0]),
np.ceil(y.shape[0] / mini_batch_size))
for i in random_permutation:
xi = x_matrix[i].T
yi = y[i].T
# Forward pass
layer_u_list, layer_z_list = mlp_forward(W_list,
xi,
activation_function,
out_activation_function,
output_only=False)
# Backward pass
layer_delta_list = mlp_backward(W_list, yi,
grad_activation_function,
out_grad_activation_function,
layer_u_list, layer_z_list)
# Update the weights
for h in range(num_hidden_layers + 1):
if h == 0:
layer_input = xi.T
else:
layer_input = layer_z_list[h - 1].T
delta_weight = (alpha / xi.shape[1]) * layer_delta_list[h] @ layer_input + \
momentum * past_updates[h] - alpha * weight_decay * W_list[h]
W_list[h] += delta_weight
past_updates[h] = delta_weight
if compute_loss:
model_output = mlp_forward(W_list,
x_matrix.T,
activation_function,
out_activation_function,
output_only=True)
loss_history.append(
loss(y.T, model_output) +
0.5 * weight_decay * np.array([(W**2).sum()
for W in W_list]).sum())
if x_validation is not None:
model_output_validation = mlp_forward(W_list,
x_validation.T,
activation_function,
out_activation_function,
output_only=True)
validation_loss_history.append(loss(y_validation.T, model_output_validation) + \
0.5 * weight_decay * np.array([(W**2).sum() for W in W_list]).sum())
return {
'W_list': W_list,
'loss_history': loss_history,
'validation_loss_history': validation_loss_history,
'output': output,
'activation_function': activation_function,
'out_activation_function': out_activation_function
}