def train(self, inputs): self.means = random.sample(inputs, self.k) assignments = None its = 0 print(self.k) while True: # Find new assignments new_assignments = list(map(self.classify, inputs)) #print('KMeans.. ', its) #its += 1 # If no assignments have changed, we're done. if assignments == new_assignments: self.last_assignments = assignments return # Otherwise keep the new assignments, assignments = new_assignments for i in range(self.k): i_points = [p for p, a in zip(inputs, assignments) if a == i] # avoid divide-by-zero if i_points is empty if i_points: self.means[i] = vector_mean(i_points)
def train(self, input): #choose k random points as the initial means self.means = random.sample(inputs, self.k) assignments = None while True: #find new assigments new_assignments = list(map(self.classify, inputs)) print(new_assignments) #if not assignments have change, we're done if assignments == new_assignments: return #otherwise keep the new assignments assignments = new_assignments #only to plot image self.assignments_cluster = new_assignments #and compute new means based on the new assignments for i in range(self.k): #find all points assignments to cluster i i_points = [p for p, a in zip(inputs, assignments) if a == i] #make sure i_points is not empty so don't divide by 0 if i_points: self.means[i] = vector_mean(i_points)
def ols_fit(xs: List[Vector], ys: List[float], learning_rate: float = 0.001, num_steps: int = 1000, batch_size: int = 1) -> Vector: """ Find the vector (of coefficients) that minimizes the sum of squared errors """ # Initialise random guess guess = [random.random() for _ in xs[0]] # iterate 'num_steps' number of times for _ in range(num_steps): #iterate through data with steps of 'batch_size' for start in range(0, len(xs), batch_size): # get number of rows according to batch_size batch_xs = xs[start:start + batch_size] batch_ys = ys[start:start + batch_size] # calculate 'mean' gradient across these points gradient = vector_mean([ squared_error_gradient(x, y, guess) for x, y in zip(batch_xs, batch_ys) ]) # update the 'guess' using gradient times the learning rate guess = gradient_step(guess, gradient, -learning_rate) return guess
def train(self, inputs): # choose k random points as the initial means self.means = random.sample(inputs, self.k) assignments = None while True: # print(assignments) # Find new assignments new_assignments = list(map(self.classify, inputs)) # If no assignments have changed, we're done if assignments == new_assignments: return # Otherwise keep the new assignments assignments = new_assignments # And compute new means based on the new assignments for i in range(self.k): # find all the points assigned to cluster i i_points = [p for p, a in zip(inputs, assignments) if a == i] # make sure i_points is not empty so don't divide by 0 if i_points: self.means[i] = vector_mean(i_points)
def train(self, inputs): self.means = random.sample(inputs, self.k) assignments = None progress = 0 while progress < 2: progress = progress + 1 print "iteration : {0}".format(progress) print "--a--" # Find new assignments new_assignments = map(self.classify, inputs) print "--b--" # If no assignments have changed, we're done. if assignments == new_assignments: return # Otherwise keep the new assignments, assignments = new_assignments print "--c--" for i in range(self.k): i_points = [p for p, a in zip(inputs, assignments) if a == i] # avoid divide-by-zero if i_points is empty if i_points: self.means[i] = vector_mean(i_points)
def scale(data: List[Vector]) -> Tuple[Vector, Vector]: "Return the column-wise mean and standard deviation of a dataset" dim = len(data[0]) means = vector_mean(data) std_devs = [ standard_deviation([vector[i] for vector in data]) for i in range(dim) ] return means, std_devs
def fit_model(parameters): theta = [random.uniform(-1, 1), random.uniform(-1, 1)] learning_rate = 0.001 for epoch in range(5000): grad = vector_mean([linear_gradient(x, y, theta) for x, y in parameters]) theta = gradient_step(theta, grad, -learning_rate) #take step in direction of gradient print(epoch, theta) slope, intercept = theta assert 19.9 < slope < 20.1 # slope should be around 20 assert 4.9 < intercept < 5.1 # intercept should be around 5
def train(self, vectors): self.means = random.sample(vectors, self.k) assignments = None while True: new_assignments = [self.classify(v) for v in vectors] if assignments == new_assignments: return assignments assignments = new_assignments for i in range(self.k): i_points = [v for v, a in zip(vectors, assignments) if i == a] if i_points: self.means[i] = vector_mean(i_points)
def train(self, inputs): self.means = random.sample(inputs, self.k) assignments = None while True: new_assignments = [self.classify(i) for i in inputs] if assignments == new_assignments: return assignments assignments = new_assignments for i in range(self.k): i_points = [p for p, a in zip(inputs, assignments) if a == i] if i_points: self.means[i] = vector_mean(i_points)
def train(self, inputs): self.means = random.sample(inputs, self.k) self.assignments = None while True: new_assignments = list(map(self.classify, inputs)) if self.assignments == new_assignments: return self.assignments = new_assignments for i in range(self.k): i_points = [ p for p, a in zip(inputs, self.assignments) if a == i ] if i_points: self.means[i] = vector_mean(i_points)
def train(self, inputs): self.means = random.sample(inputs, self.k) assignments = None while True: # 요소들이 어느 (변경된)중심점에 가까이 있는지 매핑 [1,1,0,2,1, ... 이런식 new_assignments = map(self.classify, inputs) # 기존 매핑과 동일하다면 더 이상 프로세싱 할 필요 없음. 리턴 ~ if assignments == new_assignments: return # 현재 매핑을 저장 assignments = new_assignments # 현재 클러스터링의 중심점 재 계산 for i in range(self.k): i_points = [p for p, a in zip(inputs, assignments) if a == i] # i 번 요소들의 리스트 if i_points: self.means[i] = vector_mean(i_points) # 그것들의 평균 ( 중심점 )
def train(self, inputs): # choose k random points from inputs as the initial means self.means = random.sample(inputs, self.k) count = 0 while count < self.iters: # find new assignments; run classify method on all inputs and return a list assignments = map(self.classify, inputs) # and compute new means based on the new assignments for i in range(self.k): # find all points assigned to cluster i i_points = [p for p, a in zip(inputs, assignments) if a == i] # make sure i_points is not empty so don't divide by 0 if i_points: self.means[i] = vector_mean(i_points) count += 1
def run(): print("computing random values for theta") time.sleep(2) theta = [random.uniform(-1, 1), random.uniform(-1, 1)] print("Generating learning rate at 0.001") time.sleep(2) learning_rate = 0.001 for epoch in range(5000): grad = vector_mean([linear_gradient(x, y, theta) for x, y in inputs]) theta = gradient_step(theta, grad, -learning_rate) print(f"epoch: {epoch}, theta: {theta}") slope, intercept = theta print( f"Final slope at {slope} and intercept at {intercept}.\n Expected --> slope: 10 intercept: 7" ) assert 9.99 < slope < 10.11, "slope should be about 10" assert 6.9 < intercept < 7.1, "intercept should be about 7"
def train(self, inputs): # choose k random points as the initial means self.means = random.sample(inputs, self.k) assignments = None while True: # find new assignments new_assignments = map(self.classify, inputs) # if no assignments have changed, we're done if assignments == new_assignments: return # otherwise, keep the new assignments assignments = new_assignments # and compute new means based on the new assignments for i in range(self.k): # find all the points assigned to cluster i i_points = [p for p, a in zip(inputs, assignments) if a == 1] # make sure i_points is not empty so don't divide by zero if i_points: self.means[i] = vector_mean(i_points)
def train(self, inputs): self.means = random.sample(inputs, self.k) assignments = None while True: # Find new assignments new_assignments = map(self.classify, inputs) # If no assignments have changed, we're done. if assignments == new_assignments: return # Otherwise keep the new assignments, assignments = new_assignments for i in range(self.k): i_points = [p for p, a in zip(inputs, assignments) if a == i] # avoid divide-by-zero if i_points is empty if i_points: self.means[i] = vector_mean(i_points)
def gradient_descent_linear_model(X: Vector, y: Vector, learning_rate: float = 0.001, iterations: int = 5000, show_case: bool = True) -> None: theta = [random.uniform(-1, 1), random.uniform(-1, 1)] # Random start point. for epoch in range(iterations): # we gather the gradient ME error for every case. (which direction we can go to minimize error.) # remember that the values in the list are not SE error, but rather the dirrection # at which we should move to minimize wathever the error was. we are concerned with # this since we are TRAINING not TESTING! fit_se_err = [linear_gradient(data, actual, theta) for data, actual in zip(X, y)] # calculate the average dirrection where we can go to minimize the error MSE # here, this is not the calculated MSE, but just the average of all of # the gradients! mse_gradient = vector_mean(fit_se_err) # We calculate the step at which we want to move, following the gradient starting # at point theta and at the step size of learning rate in the direction of minimization # hence the negative. theta = gradient_step(theta, mse_gradient, -learning_rate) # this is just to show the progress in the training. if(show_case): print(f"{str(epoch)} - parameters: {str(theta)} -- mse average minimization gradient: {str(mse_gradient)}")
print("vector A = ", A) print("vector B = ", B) C = la.vector_add(A, B) print("A + B = ", C) C = la.vector_subtract(A, B) print("A - B = ", C) C = la.vector_sum([A, B]) print("A and B summary = ", C) C = la.scalar_multiply(10, A) print("10 * A = ", C) C = la.vector_mean([A, B]) print("A and B mean = ", C) C = la.dot(A, B) print("A dot B = ", C) C = la.sum_of_squares(A) print("A^2's summary = ", C) C = la.magnitude(A) print("A's magnitude = ", C) C = la.distance(A, B) print("A's distance = ", C) print()
def de_mean(data: List[Vector]) -> List[Vector]: """Recenter data to 0""" mean = vector_mean(data) return [subtract(vector, mean) for vector in data]
def scale(data: List[Vector]) -> Tuple[Vector, Vector]: dim = len(data[0]) means = vector_mean(data) stdevs = [standard_deviation([vector[i] for vector in data]) for i in range(dim)] return means, stdevs
but it deviates in that instead of calculating the gradient on the whole dataset, we instead split the dataset into pieces (batches) from which we calculate the gradient. the advantages to this approach is mainly visible when working with large datasets. while the computation seems to follow a O(n^2), the speed with which it moves towards the optimal training parameters is faster since steps towards it are more common than when having to compute the gradient for every single point in the dataset. Therefore, with this approach we can approach the optimal parameters for out models with less epochs than with regular gradient descent. """ theta = [random.uniform(-1, 1), random.uniform(-1, 1)] for epoch in range(iterations): for batch in minibatch([(data, actual) for data, actual in zip(X, y)], batch_size): gradient_mu = vector_mean([linear_gradient(data, actual, theta) for data, actual in batch]) theta = gradient_step(theta, gradient_mu, -learning_rate) if(show_case): print(f"{str(epoch)} - parameters: {str(theta)} -- mse average minimization gradient: {str(gradient_mu)}") def stochastic_gradient_descent(X: List[T], y: List[T], batch_size: int=20, learning_rate: float=0.001, iterations: float=1_000, show_case: bool=True) -> None: """ Stochastic gradient descent (SGD) This technique of training is very similar to that of minibatch gradient descent. Many of its benefits are shared, but again, the time it takes to approach the optimal parameters for a model are smaller due to taking steps for every training example. A large trade off is that while we can find the optimal parameters much faster, the
# inputs = [(x,x) for x in range(-50, 50)] # inputs = [(x,10*x) for x in range(-50, 50)] # inputs = [(x,x*x) for x in range(-50, 50)] # inputs = [(x,x/10) for x in range(-50, 50)] def linear_gradient(x: float, y: float, theta: Vector) -> Vector: slope, intercept = theta predicted = slope * x + intercept # The prediction of the model. error = (predicted - y) # error is (predicted - actual). squared_error = error ** 2 # We'll minimize squared error grad = [2 * error * x, 2 * error] # using its gradient. return grad def gradient_step(v: Vector, gradient: Vector, step_size: float) -> Vector: """Moves `step_size` in the `gradient` direction from `v`""" assert len(v) == len(gradient) step = scalar_multiply(step_size, gradient) return add(v, step) def sum_of_squares_gradient(v: Vector) -> Vector: return [2 * v_i for v_i in v] # pick a random starting point learning_rate = 0.001 theta = [random.uniform(-1, 1), random.uniform(-1, 1)] for epoch in range(5000): # Compute the mean of the gradients grad = vector_mean([linear_gradient(x, y, theta) for x, y in inputs]) # Take a step in that direction theta = gradient_step(theta, grad, -learning_rate) print(epoch, theta) slope, intercept = theta # assert 19.9 < slope < 20.1, "slope should be about 20" # assert 4.9 < intercept < 5.1, "intercept should be about 5"
def test_vector_mean(self): result = vector_mean([[2, 3, 1], [1, 1, 2], [3, 1, 1]]) self.assertAlmostEqual(6 / 3, result[0]) self.assertAlmostEqual(5 / 3, result[1]) self.assertAlmostEqual(4 / 3, result[2])