def minimize_stochastic(target_fn, gradient_fn, x, y, theta_0, alpha_0=.01): data = zip(x,y) theta = theta_0 alpha = alpha_0 min_theta, min_value = None, float("inf") iterations_with_no_improvement = 0 # If we ever go to 100 iterations with no improvement, stop while iterations_with_no_improvement < 100: value = sum(target_fn(x_i, y_i, theta) for x_i, y_i in data) if value < min_value: # if we've found a new minimum, remember it # and go back to the original step size min_theta, min_value = theta, value iterations_with_no_improvement = 0 alpha = alpha_0 else: # otherwise we're no improving, so we should try shrinking the step size iterations_with_no_improvement += 1 alpha *= .9 # ad take a gradient step for each of the datapoints for x_i, y_i in in_random_order(data): gradient_i = gradient_fn(x_i, y_i, theta) theta = Ch4.vector_subtract(theta, Ch4.scalar_multiply(alpha, gradient_i)) return min_theta
def variance(x): """Assumes x has at least two elements""" n = len(x) deviations = de_mean(x) return Ch4.sum_of_squares(deviations) / (n-1)
return [v_i + step_size * direction_i for v_i, direction_i in zip(v, direction)] def sum_of_squares_gradient(v): return [2 * v_i for v_i in v] if isMain: # pick a random starting point v = [random.randint(-10, 10) for _ in range(3)] tolerance = .00000001 while True: gradient = sum_of_squares_gradient(v) next_v = step(v, gradient, -0.01) if Ch4.distance(next_v, v) < tolerance: break v = next_v print("Approx solutions: ", v) # But what should the step size be? # This is a safe apply function, just in case we try a bad step size def safe(f): """ Return a new function that is the same as f, except that it outputs infinity whenever f produces an error """ def safe_f(*args, **kwargs): try:
def covariance(x, y): n = len(x) return Ch4.dot(de_mean(x), de_mean(y))/(n-1)