def backpropagate(network, input_vector, target): hidden_outputs, outputs = feed_forward(network, input_vector) # the output * (1 - output) is from the derivative of sigmoid output_deltas = [ output * (1 - output) * (output - target[i]) for i, output in enumerate(outputs) ] # adjust weights for output layer (network[-1]) for i, output_neuron in enumerate(network[-1]): for j, hidden_output in enumerate(hidden_outputs + [1]): output_neuron[j] -= output_deltas[i] * hidden_output # back-propagate errors to hidden layer hidden_deltas = [ hidden_output * (1 - hidden_output) * dot(output_deltas, [n[i] for n in network[-1]]) for i, hidden_output in enumerate(hidden_outputs) ] # adjust weights for hidden layer (network[0]) for i, hidden_neuron in enumerate(network[0]): for j, input_j in enumerate(input_vector + [1]): hidden_neuron[j] -= hidden_deltas[i] * input_j
def make_graph_dot_product_as_vector_projection(plt): v = [2, 1] w = [math.sqrt(0.25), math.sqrt(0.75)] c = dot(v, w) vonw = scalar_multiply(c, w) o = [0, 0] plt.arrow(0, 0, v[0], v[1], width=0.002, head_width=0.1, length_includes_head=True) plt.annotate("v", v, xytext=[v[0] + 0.1, v[1]]) plt.arrow(0, 0, w[0], w[1], width=0.002, head_width=0.1, length_includes_head=True) plt.annotate("w", w, xytext=[w[0] - 0.1, w[1]]) plt.arrow(0, 0, vonw[0], vonw[1], length_includes_head=True) plt.annotate(u"(vâ¢w)w", vonw, xytext=[vonw[0] - 0.1, vonw[1] + 0.1]) plt.arrow( v[0], v[1], vonw[0] - v[0], vonw[1] - v[1], linestyle="dotted", length_includes_head=True, ) plt.scatter(*zip(v, w, o), marker=".") plt.axis("equal") plt.show()
def neuron_output(weights, inputs): return sigmoid(dot(weights, inputs))
def perceptron_output(weights, bias, x): """returns 1 if the perceptron 'fires', 0 if not""" return step_function(dot(weights, x) + bias)
def ridge_penalty(beta, alpha): # alpha is a *hyperparameter* controlling how harsh the penalty is # sometimes it's called "lambda" but that already means something in Python return alpha * dot(beta[1:], beta[1:])
def main(): _x = [ [1, 49, 4, 0], [1, 41, 9, 0], [1, 40, 8, 0], [1, 25, 6, 0], [1, 21, 1, 0], [1, 21, 0, 0], [1, 19, 3, 0], [1, 19, 0, 0], [1, 18, 9, 0], [1, 18, 8, 0], [1, 16, 4, 0], [1, 15, 3, 0], [1, 15, 0, 0], [1, 15, 2, 0], [1, 15, 7, 0], [1, 14, 0, 0], [1, 14, 1, 0], [1, 13, 1, 0], [1, 13, 7, 0], [1, 13, 4, 0], [1, 13, 2, 0], [1, 12, 5, 0], [1, 12, 0, 0], [1, 11, 9, 0], [1, 10, 9, 0], [1, 10, 1, 0], [1, 10, 1, 0], [1, 10, 7, 0], [1, 10, 9, 0], [1, 10, 1, 0], [1, 10, 6, 0], [1, 10, 6, 0], [1, 10, 8, 0], [1, 10, 10, 0], [1, 10, 6, 0], [1, 10, 0, 0], [1, 10, 5, 0], [1, 10, 3, 0], [1, 10, 4, 0], [1, 9, 9, 0], [1, 9, 9, 0], [1, 9, 0, 0], [1, 9, 0, 0], [1, 9, 6, 0], [1, 9, 10, 0], [1, 9, 8, 0], [1, 9, 5, 0], [1, 9, 2, 0], [1, 9, 9, 0], [1, 9, 10, 0], [1, 9, 7, 0], [1, 9, 2, 0], [1, 9, 0, 0], [1, 9, 4, 0], [1, 9, 6, 0], [1, 9, 4, 0], [1, 9, 7, 0], [1, 8, 3, 0], [1, 8, 2, 0], [1, 8, 4, 0], [1, 8, 9, 0], [1, 8, 2, 0], [1, 8, 3, 0], [1, 8, 5, 0], [1, 8, 8, 0], [1, 8, 0, 0], [1, 8, 9, 0], [1, 8, 10, 0], [1, 8, 5, 0], [1, 8, 5, 0], [1, 7, 5, 0], [1, 7, 5, 0], [1, 7, 0, 0], [1, 7, 2, 0], [1, 7, 8, 0], [1, 7, 10, 0], [1, 7, 5, 0], [1, 7, 3, 0], [1, 7, 3, 0], [1, 7, 6, 0], [1, 7, 7, 0], [1, 7, 7, 0], [1, 7, 9, 0], [1, 7, 3, 0], [1, 7, 8, 0], [1, 6, 4, 0], [1, 6, 6, 0], [1, 6, 4, 0], [1, 6, 9, 0], [1, 6, 0, 0], [1, 6, 1, 0], [1, 6, 4, 0], [1, 6, 1, 0], [1, 6, 0, 0], [1, 6, 7, 0], [1, 6, 0, 0], [1, 6, 8, 0], [1, 6, 4, 0], [1, 6, 2, 1], [1, 6, 1, 1], [1, 6, 3, 1], [1, 6, 6, 1], [1, 6, 4, 1], [1, 6, 4, 1], [1, 6, 1, 1], [1, 6, 3, 1], [1, 6, 4, 1], [1, 5, 1, 1], [1, 5, 9, 1], [1, 5, 4, 1], [1, 5, 6, 1], [1, 5, 4, 1], [1, 5, 4, 1], [1, 5, 10, 1], [1, 5, 5, 1], [1, 5, 2, 1], [1, 5, 4, 1], [1, 5, 4, 1], [1, 5, 9, 1], [1, 5, 3, 1], [1, 5, 10, 1], [1, 5, 2, 1], [1, 5, 2, 1], [1, 5, 9, 1], [1, 4, 8, 1], [1, 4, 6, 1], [1, 4, 0, 1], [1, 4, 10, 1], [1, 4, 5, 1], [1, 4, 10, 1], [1, 4, 9, 1], [1, 4, 1, 1], [1, 4, 4, 1], [1, 4, 4, 1], [1, 4, 0, 1], [1, 4, 3, 1], [1, 4, 1, 1], [1, 4, 3, 1], [1, 4, 2, 1], [1, 4, 4, 1], [1, 4, 4, 1], [1, 4, 8, 1], [1, 4, 2, 1], [1, 4, 4, 1], [1, 3, 2, 1], [1, 3, 6, 1], [1, 3, 4, 1], [1, 3, 7, 1], [1, 3, 4, 1], [1, 3, 1, 1], [1, 3, 10, 1], [1, 3, 3, 1], [1, 3, 4, 1], [1, 3, 7, 1], [1, 3, 5, 1], [1, 3, 6, 1], [1, 3, 1, 1], [1, 3, 6, 1], [1, 3, 10, 1], [1, 3, 2, 1], [1, 3, 4, 1], [1, 3, 2, 1], [1, 3, 1, 1], [1, 3, 5, 1], [1, 2, 4, 1], [1, 2, 2, 1], [1, 2, 8, 1], [1, 2, 3, 1], [1, 2, 1, 1], [1, 2, 9, 1], [1, 2, 10, 1], [1, 2, 9, 1], [1, 2, 4, 1], [1, 2, 5, 1], [1, 2, 0, 1], [1, 2, 9, 1], [1, 2, 9, 1], [1, 2, 0, 1], [1, 2, 1, 1], [1, 2, 1, 1], [1, 2, 4, 1], [1, 1, 0, 1], [1, 1, 2, 1], [1, 1, 2, 1], [1, 1, 5, 1], [1, 1, 3, 1], [1, 1, 10, 1], [1, 1, 6, 1], [1, 1, 0, 1], [1, 1, 8, 1], [1, 1, 6, 1], [1, 1, 4, 1], [1, 1, 9, 1], [1, 1, 9, 1], [1, 1, 4, 1], [1, 1, 2, 1], [1, 1, 9, 1], [1, 1, 0, 1], [1, 1, 8, 1], [1, 1, 6, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 5, 1], ] daily_minutes_good = [ 68.77, 51.25, 52.08, 38.36, 44.54, 57.13, 51.4, 41.42, 31.22, 34.76, 54.01, 38.79, 47.59, 49.1, 27.66, 41.03, 36.73, 48.65, 28.12, 46.62, 35.57, 32.98, 35, 26.07, 23.77, 39.73, 40.57, 31.65, 31.21, 36.32, 20.45, 21.93, 26.02, 27.34, 23.49, 46.94, 30.5, 33.8, 24.23, 21.4, 27.94, 32.24, 40.57, 25.07, 19.42, 22.39, 18.42, 46.96, 23.72, 26.41, 26.97, 36.76, 40.32, 35.02, 29.47, 30.2, 31, 38.11, 38.18, 36.31, 21.03, 30.86, 36.07, 28.66, 29.08, 37.28, 15.28, 24.17, 22.31, 30.17, 25.53, 19.85, 35.37, 44.6, 17.23, 13.47, 26.33, 35.02, 32.09, 24.81, 19.33, 28.77, 24.26, 31.98, 25.73, 24.86, 16.28, 34.51, 15.23, 39.72, 40.8, 26.06, 35.76, 34.76, 16.13, 44.04, 18.03, 19.65, 32.62, 35.59, 39.43, 14.18, 35.24, 40.13, 41.82, 35.45, 36.07, 43.67, 24.61, 20.9, 21.9, 18.79, 27.61, 27.21, 26.61, 29.77, 20.59, 27.53, 13.82, 33.2, 25, 33.1, 36.65, 18.63, 14.87, 22.2, 36.81, 25.53, 24.62, 26.25, 18.21, 28.08, 19.42, 29.79, 32.8, 35.99, 28.32, 27.79, 35.88, 29.06, 36.28, 14.1, 36.63, 37.49, 26.9, 18.58, 38.48, 24.48, 18.95, 33.55, 14.24, 29.04, 32.51, 25.63, 22.22, 19, 32.73, 15.16, 13.9, 27.2, 32.01, 29.27, 33, 13.74, 20.42, 27.32, 18.23, 35.35, 28.48, 9.08, 24.62, 20.12, 35.26, 19.92, 31.02, 16.49, 12.16, 30.7, 31.22, 34.65, 13.13, 27.51, 33.2, 31.57, 14.1, 33.42, 17.44, 10.12, 24.42, 9.82, 23.39, 30.93, 15.03, 21.67, 31.09, 33.29, 22.61, 26.89, 23.48, 8.38, 27.81, 32.35, 23.84, ] random.seed(0) _beta = estimate_beta(_x, daily_minutes_good) # [30.63, 0.972, -1.868, 0.911] logging.info("%r", "beta {}".format(_beta)) logging.info( "%r", "r-squared {}".format(multiple_r_squared(_x, daily_minutes_good, _beta))) logging.info("digression: the bootstrap") # 101 points all very close to 100 close_to_100 = [99.5 + random.random() for _ in range(101)] # 101 points, 50 of them near 0, 50 of them near 200 far_from_100 = ([99.5 + random.random()] + [random.random() for _ in range(50)] + [200 + random.random() for _ in range(50)]) logging.info( "%r", "bootstrap_statistic(close_to_100, median, 100): {}".format( bootstrap_statistic(close_to_100, median, 100))) logging.info( "%r", "bootstrap_statistic(far_from_100, median, 100): {}".format( bootstrap_statistic(far_from_100, median, 100))) random.seed(0) # so that you get the same results as me bootstrap_betas = bootstrap_statistic(list(zip(_x, daily_minutes_good)), estimate_sample_beta, 100) bootstrap_standard_errors = [ standard_deviation([beta[i] for beta in bootstrap_betas]) for i in range(4) ] logging.info( "%r", "bootstrap standard errors {}".format(bootstrap_standard_errors)) logging.info("%r", "p_value(30.63, 1.174) {}".format(p_value(30.63, 1.174))) logging.info("%r", "p_value(0.972, 0.079) {}".format(p_value(0.972, 0.079))) logging.info("%r", "p_value(-1.868, 0.131) {}".format(p_value(-1.868, 0.131))) logging.info("%r", "p_value(0.911, 0.990) {}".format(p_value(0.911, 0.990))) logging.info("regularization") random.seed(0) for _alpha in [0.0, 0.01, 0.1, 1, 10]: _beta = estimate_beta_ridge(_x, daily_minutes_good, alpha=_alpha) logging.info("%r", "alpha {}".format(_alpha)) logging.info("%r", "beta {}".format(_beta)) logging.info( "%r", "dot(beta[1:],beta[1:]) {}".format(dot(_beta[1:], _beta[1:]))) logging.info( "%r", "r-squared {}".format( multiple_r_squared(_x, daily_minutes_good, _beta)), )
def predict(x_i, beta): return dot(x_i, beta)
def matrix_product_entry(a_matrix, b_matrix, i, j): return dot(get_row(a_matrix, i), get_column(b_matrix, j))
def covariance(x, y): n = len(x) return dot(de_mean(x), de_mean(y)) / (n - 1)
def project(v, w): """return the projection of v onto w""" coefficient = dot(v, w) return scalar_multiply(coefficient, w)
def transform_vector(v, components): return [dot(v, w) for w in components]
def directional_variance_gradient_i(x_i, w): """the contribution of row x_i to the gradient of the direction-w variance""" projection_length = dot(x_i, direction(w)) return [2 * projection_length * x_ij for x_ij in x_i]
def directional_variance_i(x_i, w): """the variance of the row x_i in the direction w""" return dot(x_i, direction(w))**2
def main(): data = [ (0.7, 48000, 1), (1.9, 48000, 0), (2.5, 60000, 1), (4.2, 63000, 0), (6, 76000, 0), (6.5, 69000, 0), (7.5, 76000, 0), (8.1, 88000, 0), (8.7, 83000, 1), (10, 83000, 1), (0.8, 43000, 0), (1.8, 60000, 0), (10, 79000, 1), (6.1, 76000, 0), (1.4, 50000, 0), (9.1, 92000, 0), (5.8, 75000, 0), (5.2, 69000, 0), (1, 56000, 0), (6, 67000, 0), (4.9, 74000, 0), (6.4, 63000, 1), (6.2, 82000, 0), (3.3, 58000, 0), (9.3, 90000, 1), (5.5, 57000, 1), (9.1, 102000, 0), (2.4, 54000, 0), (8.2, 65000, 1), (5.3, 82000, 0), (9.8, 107000, 0), (1.8, 64000, 0), (0.6, 46000, 1), (0.8, 48000, 0), (8.6, 84000, 1), (0.6, 45000, 0), (0.5, 30000, 1), (7.3, 89000, 0), (2.5, 48000, 1), (5.6, 76000, 0), (7.4, 77000, 0), (2.7, 56000, 0), (0.7, 48000, 0), (1.2, 42000, 0), (0.2, 32000, 1), (4.7, 56000, 1), (2.8, 44000, 1), (7.6, 78000, 0), (1.1, 63000, 0), (8, 79000, 1), (2.7, 56000, 0), (6, 52000, 1), (4.6, 56000, 0), (2.5, 51000, 0), (5.7, 71000, 0), (2.9, 65000, 0), (1.1, 33000, 1), (3, 62000, 0), (4, 71000, 0), (2.4, 61000, 0), (7.5, 75000, 0), (9.7, 81000, 1), (3.2, 62000, 0), (7.9, 88000, 0), (4.7, 44000, 1), (2.5, 55000, 0), (1.6, 41000, 0), (6.7, 64000, 1), (6.9, 66000, 1), (7.9, 78000, 1), (8.1, 102000, 0), (5.3, 48000, 1), (8.5, 66000, 1), (0.2, 56000, 0), (6, 69000, 0), (7.5, 77000, 0), (8, 86000, 0), (4.4, 68000, 0), (4.9, 75000, 0), (1.5, 60000, 0), (2.2, 50000, 0), (3.4, 49000, 1), (4.2, 70000, 0), (7.7, 98000, 0), (8.2, 85000, 0), (5.4, 88000, 0), (0.1, 46000, 0), (1.5, 37000, 0), (6.3, 86000, 0), (3.7, 57000, 0), (8.4, 85000, 0), (2, 42000, 0), (5.8, 69000, 1), (2.7, 64000, 0), (3.1, 63000, 0), (1.9, 48000, 0), (10, 72000, 1), (0.2, 45000, 0), (8.6, 95000, 0), (1.5, 64000, 0), (9.8, 95000, 0), (5.3, 65000, 0), (7.5, 80000, 0), (9.9, 91000, 0), (9.7, 50000, 1), (2.8, 68000, 0), (3.6, 58000, 0), (3.9, 74000, 0), (4.4, 76000, 0), (2.5, 49000, 0), (7.2, 81000, 0), (5.2, 60000, 1), (2.4, 62000, 0), (8.9, 94000, 0), (2.4, 63000, 0), (6.8, 69000, 1), (6.5, 77000, 0), (7, 86000, 0), (9.4, 94000, 0), (7.8, 72000, 1), (0.2, 53000, 0), (10, 97000, 0), (5.5, 65000, 0), (7.7, 71000, 1), (8.1, 66000, 1), (9.8, 91000, 0), (8, 84000, 0), (2.7, 55000, 0), (2.8, 62000, 0), (9.4, 79000, 0), (2.5, 57000, 0), (7.4, 70000, 1), (2.1, 47000, 0), (5.3, 62000, 1), (6.3, 79000, 0), (6.8, 58000, 1), (5.7, 80000, 0), (2.2, 61000, 0), (4.8, 62000, 0), (3.7, 64000, 0), (4.1, 85000, 0), (2.3, 51000, 0), (3.5, 58000, 0), (0.9, 43000, 0), (0.9, 54000, 0), (4.5, 74000, 0), (6.5, 55000, 1), (4.1, 41000, 1), (7.1, 73000, 0), (1.1, 66000, 0), (9.1, 81000, 1), (8, 69000, 1), (7.3, 72000, 1), (3.3, 50000, 0), (3.9, 58000, 0), (2.6, 49000, 0), (1.6, 78000, 0), (0.7, 56000, 0), (2.1, 36000, 1), (7.5, 90000, 0), (4.8, 59000, 1), (8.9, 95000, 0), (6.2, 72000, 0), (6.3, 63000, 0), (9.1, 100000, 0), (7.3, 61000, 1), (5.6, 74000, 0), (0.5, 66000, 0), (1.1, 59000, 0), (5.1, 61000, 0), (6.2, 70000, 0), (6.6, 56000, 1), (6.3, 76000, 0), (6.5, 78000, 0), (5.1, 59000, 0), (9.5, 74000, 1), (4.5, 64000, 0), (2, 54000, 0), (1, 52000, 0), (4, 69000, 0), (6.5, 76000, 0), (3, 60000, 0), (4.5, 63000, 0), (7.8, 70000, 0), (3.9, 60000, 1), (0.8, 51000, 0), (4.2, 78000, 0), (1.1, 54000, 0), (6.2, 60000, 0), (2.9, 59000, 0), (2.1, 52000, 0), (8.2, 87000, 0), (4.8, 73000, 0), (2.2, 42000, 1), (9.1, 98000, 0), (6.5, 84000, 0), (6.9, 73000, 0), (5.1, 72000, 0), (9.1, 69000, 1), (9.8, 79000, 1), ] data = list(map(list, data)) # change tuples to lists _x = [[1] + row[:2] for row in data] # each element is [1, experience, salary] _y = [row[2] for row in data] # each element is paid_account logging.info("linear regression:") rescaled_x = rescale(_x) _beta = estimate_beta(rescaled_x, _y) logging.info("%r", "beta = {}".format(_beta)) logging.info("logistic regression") random.seed(0) x_train, x_test, y_train, y_test = train_test_split(rescaled_x, _y, 0.33) # want to maximize log likelihood on the training data fn = partial(logistic_log_likelihood, x_train, y_train) gradient_fn = partial(logistic_log_gradient, x_train, y_train) # pick a random starting point beta_0 = [1, 1, 1] # and maximize using gradient descent beta_hat = maximize_batch(fn, gradient_fn, beta_0) logging.info("%r", "beta_batch {}".format(beta_hat)) beta_0 = [1, 1, 1] beta_hat = maximize_stochastic(logistic_log_likelihood_i, logistic_log_gradient_i, x_train, y_train, beta_0) logging.info("%r", "beta stochastic {}".format(beta_hat)) true_positives = false_positives = true_negatives = false_negatives = 0 for _x_i, _y_i in zip(x_test, y_test): predict = logistic(dot(beta_hat, _x_i)) if _y_i == 1 and predict >= 0.5: # TP: paid and we predict paid true_positives += 1 elif _y_i == 1: # FN: paid and we predict unpaid false_negatives += 1 elif predict >= 0.5: # FP: unpaid and we predict paid false_positives += 1 else: # TN: unpaid and we predict unpaid true_negatives += 1 precision = true_positives / (true_positives + false_positives) recall = true_positives / (true_positives + false_negatives) logging.info("%r", "precision {}".format(precision)) logging.info("%r", "recall {}".format(recall))
def logistic_log_partial_ij(x_i, y_i, beta, j): """here i is the index of the data point, j the index of the derivative""" return (y_i - logistic(dot(x_i, beta))) * x_i[j]
def logistic_log_likelihood_i(x_i, y_i, beta): if y_i == 1: return math.log(logistic(dot(x_i, beta))) else: return math.log(1 - logistic(dot(x_i, beta)))