def backpropagate(network, input_vector, targets):
hidden_outputs, outputs = feed_forward(network, input_vector)
# the output * (1 - output) is from the derivative of sigmoid
output_deltas = [
output * (1 - output) * (output - target)
for output, target in zip(outputs, targets)
]
# adjust weights for output layer, one neuron at a time
for i, output_neuron in enumerate(network[-1]):
# focus on the i-th output layer neuron
for j, hidden_output in enumerate(hidden_outputs + [1]):
# adjust the j-th weight
# based on both this neuron's delta and its j-th input
output_neuron[j] -= output_deltas[i] * hidden_output
# back-propagate errors to hidden layer
hidden_deltas = [
hidden_output * (1 - hidden_output) *
la.dot(output_deltas, [n[i] for n in network[-1]])
for i, hidden_output in enumerate(hidden_outputs)
]
# adjust weights for hidden layer, one neuron at a time
for i, hidden_neuron in enumerate(network[0]):
for j, input in enumerate(input_vector + [1]):
hidden_neuron[j] -= hidden_deltas[i] * input
def logistic_log_partial_ij(x_i, y_i, beta, j):
"""here i is the index of the data point,
j the index of the derivative"""
return (y_i - logistic(la.dot(x_i, beta))) * x_i[j]
def logistic_log_likelihood_i(x_i, y_i, beta):
if y_i == 1:
return math.log(logistic(la.dot(x_i, beta)))
else:
return math.log(1 - logistic(la.dot(x_i, beta)))
def transform_vector(v, components):
return [la.dot(v, w) for w in components]
def project(v, w):
"""return the projection of v onto the direction w"""
projection_length = la.dot(v, w)
return [projection_length * wi for wi in w]
def perceptron_output(weights, bias, x):
calculation = la.dot(weights, x) + bias
return step_function(calculation)
def neuron_output(weights, inputs):
return sigmoid(la.dot(weights, inputs))
def predict(x_i, beta):
"""assumes that the first element of each x_i is 1"""
return la.dot(x_i, beta)
def ridge_penalty(beta, alpha):
return alpha * la.dot(beta[1:], beta[1:])