def train_adaline(x, y, num_epochs=100, alpha=0.001, w_initial=None, build_regressors=True): if build_regressors: x_matrix = lin.build_poly_regressors(x) else: x_matrix = x if len(y.shape) == 1: y = y[:, None] if w_initial is None: w = np.zeros((x_matrix.shape[1], y.shape[1])) else: w = w_initial.copy() loss_history = [] for epoch in range(num_epochs): random_permutation = np.random.permutation(y.shape[0]) for xi, yi in zip(x_matrix[random_permutation], y[random_permutation]): error = yi - xi @ w w += alpha * xi[:, None] @ error[:, None].T loss_history.append(np.mean((y - x_matrix @ w)**2)) return {'w': w, 'loss_history': loss_history}
def train_perceptron(x, y, num_epochs=100, alpha=1, w_initial=None, build_regressors=True): if build_regressors: x_matrix = lin.build_poly_regressors(x) else: x_matrix = x y = y.copy() y[y == 0] = -1 if len(y.shape) == 1: y = y[:, None] if w_initial is None: w = np.zeros((x_matrix.shape[1], y.shape[1])) else: w = w_initial.copy() loss_history = [] for epoch in range(num_epochs): random_permutation = np.random.permutation(y.shape[0]) for xi, yi in zip(x_matrix[random_permutation], y[random_permutation]): error = yi - np.sign(xi @ w) w += alpha * xi[:, None] @ error[:, None].T loss_history.append(np.sum(np.maximum(0, -(y * np.sign(x_matrix @ w))))) return {'w': w, 'loss_history': loss_history}
def linear_class_predic(model, x, build_regressors=True): if build_regressors: x_matrix = lin.build_poly_regressors(x) else: x_matrix = x if model['w'].shape[1] > 1: return np.argmax(x_matrix @ model['w'], axis=1) else: return np.maximum(0, np.sign(x_matrix @ model['w']))
def predict(w, x, y=None, poly_order=1, build_regressors=True): if build_regressors: x_matrix = lin.build_poly_regressors(x, poly_order=poly_order) else: x_matrix = x output_function = logistic_function loss = logistic_loss if w.shape[1] > 1: output_function = softmax_function loss = softmax_loss if y is None: return output_function(w, x_matrix) else: pred = output_function(w, x_matrix) return {'pred': pred, 'loss': loss(y, output_function(w, x_matrix))}
def lms(x, y, num_epochs=100, alpha=0.001, w_initial=None, poly_order=1, build_regressors=True, compute_loss=True): if build_regressors: x_matrix = lin.build_poly_regressors(x, poly_order=poly_order) else: x_matrix = x if len(y.shape) == 1: y = y[:, None] if w_initial is None: w = np.zeros((x_matrix.shape[1], y.shape[1])) else: w = w_initial output_function = logistic_function loss = logistic_loss if w.shape[1] > 1: output_function = softmax_function loss = softmax_loss loss_history = [] for epoch in range(num_epochs): random_permutation = np.random.permutation(y.shape[0]) for xi, yi in zip(x_matrix[random_permutation], y[random_permutation]): error = yi - output_function(w, xi) #for k in range(w.shape[1]): # w[:,k] += alpha * error[k] * xi w += alpha * xi[:, None] @ error[:, None].T if compute_loss: loss_history.append(loss(y, output_function(w, x_matrix))) if compute_loss: return {'w': w, 'loss_history': loss_history} else: return {'w': w}
def mlp_predict(model, x, build_regressors=True, return_class=True): if build_regressors: x_matrix = lin.build_poly_regressors(x) else: x_matrix = x model_output = mlp_forward(model['W_list'], x_matrix.T, model['activation_function'], model['out_activation_function'], output_only=True) if return_class and model['output'] != 'regression': if model_output.shape[0] == 1: return np.maximum(0, np.sign(model_output[0] - 0.5)) else: return np.argmax(model_output, axis=0) else: return model_output
def gd(x, y, num_epochs=100, alpha=0.001, w_initial=None, poly_order=1, build_regressors=True, compute_loss=True): if build_regressors: x_matrix = lin.build_poly_regressors(x, poly_order=poly_order) else: x_matrix = x if len(y.shape) == 1: y = y[:, None] if w_initial is None: w = np.zeros((x_matrix.shape[1], y.shape[1])) else: w = w_initial output_function = logistic_function loss = logistic_loss if w.shape[1] > 1: output_function = softmax_function loss = softmax_loss loss_history = [] for epoch in range(num_epochs): error = y - output_function(w, x_matrix) #for k in range(w.shape[1]): # w[:,k] += alpha * np.mean(error[:,k:k+1] * x_matrix, axis=0) w += alpha * x_matrix.T @ error / x_matrix.shape[0] if compute_loss: loss_history.append(loss(y, output_function(w, x_matrix))) if compute_loss: return {'w': w, 'loss_history': loss_history} else: return {'w': w}
def mlp_train(x, y, x_validation=None, y_validation=None, num_hidden_nodes=10, activation="relu", output="regression", num_epochs=100, alpha=1, mini_batch_size=1, momentum=0, weight_decay=0, build_regressors=True, compute_loss=True): # Preprocess the input and output data if build_regressors: x_matrix = lin.build_poly_regressors(x) if x_validation is not None: x_validation = lin.build_poly_regressors(x_validation) else: x_matrix = x y = y.copy() if len(y.shape) == 1: y = y[:, None] if y_validation is not None: y_validation = y_validation.copy() if len(y_validation.shape) == 1: y_validation = y_validation[:, None] # Select the hidden and output activation functions activation_function, grad_activation_function = select_activation_function( activation) out_activation_function, out_grad_activation_function, loss = select_output_type( output, y) if type(num_hidden_nodes) is not list: num_hidden_nodes = [num_hidden_nodes] num_hidden_layers = len(num_hidden_nodes) # Initialize the weights W_list = initialize_weights(x_matrix, y, num_hidden_nodes, activation, output) loss_history = [] validation_loss_history = [] past_updates = [0] * (num_hidden_layers + 1) for epoch in range(num_epochs): random_permutation = np.array_split( np.random.permutation(y.shape[0]), np.ceil(y.shape[0] / mini_batch_size)) for i in random_permutation: xi = x_matrix[i].T yi = y[i].T # Forward pass layer_u_list, layer_z_list = mlp_forward(W_list, xi, activation_function, out_activation_function, output_only=False) # Backward pass layer_delta_list = mlp_backward(W_list, yi, grad_activation_function, out_grad_activation_function, layer_u_list, layer_z_list) # Update the weights for h in range(num_hidden_layers + 1): if h == 0: layer_input = xi.T else: layer_input = layer_z_list[h - 1].T delta_weight = (alpha / xi.shape[1]) * layer_delta_list[h] @ layer_input + \ momentum * past_updates[h] - alpha * weight_decay * W_list[h] W_list[h] += delta_weight past_updates[h] = delta_weight if compute_loss: model_output = mlp_forward(W_list, x_matrix.T, activation_function, out_activation_function, output_only=True) loss_history.append( loss(y.T, model_output) + 0.5 * weight_decay * np.array([(W**2).sum() for W in W_list]).sum()) if x_validation is not None: model_output_validation = mlp_forward(W_list, x_validation.T, activation_function, out_activation_function, output_only=True) validation_loss_history.append(loss(y_validation.T, model_output_validation) + \ 0.5 * weight_decay * np.array([(W**2).sum() for W in W_list]).sum()) return { 'W_list': W_list, 'loss_history': loss_history, 'validation_loss_history': validation_loss_history, 'output': output, 'activation_function': activation_function, 'out_activation_function': out_activation_function }