def remove_component(X): """Remove components of already obtained eigen vectors from X.""" X_m = X[:m] X_n = X[m:] for eivec in eivec_m: coeff = dotproduct(X_m, eivec) X_m = [x1 - coeff * x2 for x1, x2 in zip(X_m, eivec)] for eivec in eivec_n: coeff = dotproduct(X_n, eivec) X_n = [x1 - coeff * x2 for x1, x2 in zip(X_n, eivec)] return X_m + X_n
def predict(example): o_nodes = learned_net[1] # Forward pass for node in o_nodes: in_val = dotproduct(example, node.weights) node.value = node.activation(in_val) # Hypothesis return find_max_node(o_nodes)
def LinearLearner(dataset, learning_rate=0.01, epochs=100): """Define with learner = LinearLearner(data); infer with learner(x).""" idx_i = dataset.inputs idx_t = dataset.target # As of now, dataset.target gives only one index. examples = dataset.examples num_examples = len(examples) # X transpose X_col = [dataset.values[i] for i in idx_i] # vertical columns of X # Add dummy ones = [1 for _ in range(len(examples))] X_col = [ones] + X_col # Initialize random weigts num_weights = len(idx_i) + 1 w = random_weights(min_value=-0.5, max_value=0.5, num_weights=num_weights) for epoch in range(epochs): err = [] # Pass over all examples for example in examples: x = [1] + example y = dotproduct(w, x) t = example[idx_t] err.append(t - y) # update weights for i in range(len(w)): w[i] = w[i] + learning_rate * (dotproduct(err, X_col[i]) / num_examples) def predict(example): x = [1] + example return dotproduct(w, x) return predict
def predict(example): # Input nodes i_nodes = learned_net[0] # Activate input layer for v, n in zip(example, i_nodes): n.value = v # Forward pass for layer in learned_net[1:]: for node in layer: inc = [n.value for n in node.inputs] in_val = dotproduct(inc, node.weights) node.value = node.activation(in_val) # Hypothesis o_nodes = learned_net[-1] prediction = find_max_node(o_nodes) return prediction
def predict(example): x = [1] + example return dotproduct(w, x)
def BackPropagationLearner(dataset, net, learning_rate, epochs, activation=sigmoid): """[Figure 18.23] The back-propagation algorithm for multilayer networks""" # Initialise weights for layer in net: for node in layer: node.weights = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(node.weights)) examples = dataset.examples ''' As of now dataset.target gives an int instead of list, Changing dataset class will have effect on all the learners. Will be taken care of later. ''' o_nodes = net[-1] i_nodes = net[0] o_units = len(o_nodes) idx_t = dataset.target idx_i = dataset.inputs n_layers = len(net) inputs, targets = init_examples(examples, idx_i, idx_t, o_units) for epoch in range(epochs): # Iterate over each example for e in range(len(examples)): i_val = inputs[e] t_val = targets[e] # Activate input layer for v, n in zip(i_val, i_nodes): n.value = v # Forward pass for layer in net[1:]: for node in layer: inc = [n.value for n in node.inputs] in_val = dotproduct(inc, node.weights) node.value = node.activation(in_val) # Initialize delta delta = [[] for _ in range(n_layers)] # Compute outer layer delta # Error for the MSE cost function err = [t_val[i] - o_nodes[i].value for i in range(o_units)] # The activation function used is relu or sigmoid function if node.activation == sigmoid: delta[-1] = [ sigmoid_derivative(o_nodes[i].value) * err[i] for i in range(o_units) ] elif node.activation == relu: delta[-1] = [ relu_derivative(o_nodes[i].value) * err[i] for i in range(o_units) ] elif node.activation == tanh: delta[-1] = [ tanh_derivative(o_nodes[i].value) * err[i] for i in range(o_units) ] elif node.activation == elu: delta[-1] = [ elu_derivative(o_nodes[i].value) * err[i] for i in range(o_units) ] else: delta[-1] = [ leaky_relu_derivative(o_nodes[i].value) * err[i] for i in range(o_units) ] # Backward pass h_layers = n_layers - 2 for i in range(h_layers, 0, -1): layer = net[i] h_units = len(layer) nx_layer = net[i + 1] # weights from each ith layer node to each i + 1th layer node w = [[node.weights[k] for node in nx_layer] for k in range(h_units)] if activation == sigmoid: delta[i] = [ sigmoid_derivative(layer[j].value) * dotproduct(w[j], delta[i + 1]) for j in range(h_units) ] elif activation == relu: delta[i] = [ relu_derivative(layer[j].value) * dotproduct(w[j], delta[i + 1]) for j in range(h_units) ] elif activation == tanh: delta[i] = [ tanh_derivative(layer[j].value) * dotproduct(w[j], delta[i + 1]) for j in range(h_units) ] elif activation == elu: delta[i] = [ elu_derivative(layer[j].value) * dotproduct(w[j], delta[i + 1]) for j in range(h_units) ] else: delta[i] = [ leaky_relu_derivative(layer[j].value) * dotproduct(w[j], delta[i + 1]) for j in range(h_units) ] # Update weights for i in range(1, n_layers): layer = net[i] inc = [node.value for node in net[i - 1]] units = len(layer) for j in range(units): layer[j].weights = vector_add( layer[j].weights, scalar_vector_product(learning_rate * delta[i][j], inc)) return net