def check_logreg(tracker): import models print("Sanity checking logistic regression...") np_random = np.random.RandomState(0) pts = 0 # Check that run returns a node, and that the score in the node is correct for dimensions in range(1, 11): p = models.LogisticRegressionModel(dimensions) p_weights = np.array(p.get_weights()) if (len(p_weights) == dimensions): pts = pts + 0.1 else: print('Weight List Initialized Does Not Have Correct Dimensions') point = np_random.uniform(-10, 10, (1, dimensions)) score = p.DotProduct(p_weights.flatten(), point.flatten()) expected_score = float(np.dot(point.flatten(), p_weights.flatten())) if np.isclose(score, expected_score): pts = pts + 0.1 else: print( "The score computed by LogisticRegressionModel.run() ({:.4f}) does not match the expected score ({:.4f})" .format(score, expected_score)) tracker.add_points(pts) # Partial credit for passing sanity checks print("Sanity checking complete. Now training perceptron") model = models.LogisticRegressionModel(3) dataset = backend.LogisticRegressionDataset(model) model.train(dataset) backend.maybe_sleep_and_close(2) assert dataset.epoch != 0, "LogisticRegressionModel code never iterated over the training data" accuracy = np.mean( np.where( np.dot(dataset.x, np.array(model.get_weights()).flatten()) >= 0.0, 1.0, -1.0) == dataset.y.T) if accuracy < 1.0: print( "The weights learned by your perceptron correctly classified {:.2%} of training examples" .format(accuracy)) print( "To receive full points for this question, your perceptron must converge to 100% accuracy" ) return tracker.add_points(4)
def check_perceptron(tracker): import perceptron, backend print("Sanity checking perceptron...") np_random = np.random.RandomState(0) with no_graphics(): # Check that the perceptron weights are initialized to a zero vector of `dimensions` entries. for _ in range(10): dimensions = np_random.randint(1, 10) p = perceptron.Perceptron(dimensions) p_weights = p.get_weights() assert p_weights is not None,\ "Perceptron.get_weights() should return weights, not None" p_shape = np.asarray(p_weights).shape assert p_shape == (dimensions,),\ "Perceptron weights had shape {}, expected {}".format(p_shape, (dimensions,)) assert np.count_nonzero(p.get_weights( )) == 0, "Perceptron weights should be initialized to zero." # Check that the untrained perceptron predicts 1 on any point for _ in range(10): dimensions = np_random.randint(1, 10) p = perceptron.Perceptron(dimensions) point = np_random.uniform(-10, 10, dimensions) pred = p.predict(point) assert pred == 1, "Untrained perceptron should predict 1 instead of {} for {}.".format( pred, pred) # Check that a correctly classified point does not update the perceptron. for _ in range(10): dimensions = np_random.randint(1, 10) p = perceptron.Perceptron(dimensions) point = np_random.uniform(-10, 10, dimensions) old_weights = p.get_weights().copy() # All points should be classified as 1 at this point updated = p.update(point, 1) assert updated is not None, "Perceptron.update should return True or False, not None" assert not updated, "Updating with a correctly classified point ({}) should return `False`.".format( point) new_weights = p.get_weights() assert np.allclose(new_weights, old_weights),\ "Updating with a correctly classified point ({}) should not change weights from {} to {}".format(point, new_weights, old_weights) # Check that the perceptron weight updates are correct xs = np.array([[0., -1., 1., 1.8], [-1., 0., -2., 2.6]]).T ys = np.array([-1, -1, -1, -1], dtype=int) expected_weights = np.array([[0., 1., 1., -0.8], [1., 1., 1., -1.6]]).T expected_returns = [True, True, False, True] p = perceptron.Perceptron(2) for i in range(xs.shape[0]): old_weights = p.get_weights().copy() res = p.update(xs[i], ys[i]) assert res == expected_returns[i],\ """Perceptron.update returned {}. Expected: {} Old weights: {} x: {} y: {}""".format(res, expected_returns[i], old_weights, xs[i], ys[i]) assert np.allclose(p.get_weights(), expected_weights[i]),\ """Perceptron weights are {}. Expected: {} Old weights: {} x: {} y: {}""".format(p.get_weights(), expected_weights[i], old_weights, xs[i], ys[i]) print("Sanity checking complete. Now training perceptron") p = perceptron.Perceptron(3) p.train() backend.maybe_sleep_and_close(1) stats = backend.get_stats(p) if stats is None: print("Your perceptron never trained for a full epoch!") return if stats["accuracy"] < 1.0: print( "The weights learned by your perceptron correctly classified {:.2%} of training examples" .format(stats['accuracy'])) print( "To receive points for this question, your perceptron must converge to 100% accuracy" ) else: tracker.add_points(3)
def check_polyregression(tracker): import models model = models.PolyRegressionModel(2) dataset = backend.PolyRegressionDataset(model, 2) detected_parameters = None for batch_size in (1, 1, 1): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) #print(nn.as_scalar(inp_x)) output_node = model.run(inp_x) verify_node(output_node, 'node', (batch_size, 1), "PolyRegressionModel.run()") trace = trace_node(output_node) if detected_parameters is None: detected_parameters = [ node for node in trace if isinstance(node, nn.Parameter) ] for node in trace: assert not isinstance( node, nn.Parameter ) or node in detected_parameters, ( "Calling PolyRegressionModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected" ) for batch_size in (1, 1, 1): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) loss_node = model.get_loss(inp_x, inp_y) verify_node(loss_node, 'loss', None, "PolyRegressionModel.get_loss()") trace = trace_node(loss_node) #assert inp_x in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)" assert inp_y in trace, "Node returned fromPolyRegressionModel.get_loss() does not depend on the provided labels (y)" for node in trace: assert not isinstance( node, nn.Parameter ) or node in detected_parameters, ( "PolyRegressionModel.get_loss() should not use additional parameters not used by RegressionModel.run()" ) tracker.add_points(2) # Partial credit for passing sanity checks model.train(dataset) backend.maybe_sleep_and_close(1) train_loss = model.get_loss(nn.Constant(dataset.x), nn.Constant(dataset.y)) verify_node(train_loss, 'loss', None, "PolyRegressionModel.get_loss()") train_loss = nn.as_scalar(train_loss) # Re-compute the loss ourselves: otherwise get_loss() could be hard-coded # to always return zero train_predicted = model.run(nn.Constant(dataset.x)) verify_node(train_predicted, 'node', (dataset.x.shape[0], 1), "PolyRegressionModel.run()") sanity_loss = 0.5 * np.mean((train_predicted.data - dataset.y)**2) assert np.isclose(train_loss, sanity_loss), ( "PolyRegressionModel.get_loss() returned a loss of {:.4f}, " "but the autograder computed a loss of {:.4f} " "based on the output of PolyRegressionModel.run()".format( train_loss, sanity_loss)) loss_threshold = 0.1 if train_loss <= loss_threshold: print("Your final loss is: {:f}".format(train_loss)) tracker.add_points(4) else: print( "Your final loss ({:f}) must be no more than {:.4f} to receive full points for this question" .format(train_loss, loss_threshold))
def check_perceptron(tracker): import models print("Sanity checking perceptron...") np_random = np.random.RandomState(0) # Check that the perceptron weights are initialized to a vector with `dimensions` entries. for dimensions in range(1, 10): p = models.PerceptronModel(dimensions) p_weights = p.get_weights() verify_node(p_weights, 'parameter', (1, dimensions), "PerceptronModel.get_weights()") # Check that run returns a node, and that the score in the node is correct for dimensions in range(1, 10): p = models.PerceptronModel(dimensions) p_weights = p.get_weights() verify_node(p_weights, 'parameter', (1, dimensions), "PerceptronModel.get_weights()") point = np_random.uniform(-10, 10, (1, dimensions)) score = p.run(nn.Constant(point)) verify_node(score, 'node', (1, 1), "PerceptronModel.run()") calculated_score = nn.as_scalar(score) expected_score = float( np.dot(point.flatten(), p_weights.data.flatten())) assert np.isclose(calculated_score, expected_score), ( "The score computed by PerceptronModel.run() ({:.4f}) does not match the expected score ({:.4f})" .format(calculated_score, expected_score)) # Check that get_prediction returns the correct values, including the # case when a point lies exactly on the decision boundary for dimensions in range(1, 10): p = models.PerceptronModel(dimensions) random_point = np_random.uniform(-10, 10, (1, dimensions)) for point in (random_point, np.zeros_like(random_point)): prediction = p.get_prediction(nn.Constant(point)) assert prediction == 1 or prediction == -1, ( "PerceptronModel.get_prediction() should return 1 or -1, not {}" .format(prediction)) expected_prediction = np.asscalar( np.where(np.dot(point, p.get_weights().data.T) >= 0, 1, -1)) assert prediction == expected_prediction, ( "PerceptronModel.get_prediction() returned {}; expected {}". format(prediction, expected_prediction)) tracker.add_points(2) # Partial credit for passing sanity checks print("Sanity checking perceptron weight updates...") # Test weight updates. This involves constructing a dataset that # requires 0 or 1 updates before convergence, and testing that weight # values change as expected. Note that (multiplier < -1 or multiplier > 1) # must be true for the testing code to be correct. dimensions = 2 for multiplier in (-5, -2, 2, 5): p = models.PerceptronModel(dimensions) orig_weights = p.get_weights().data.reshape((1, dimensions)).copy() if np.abs(orig_weights).sum() == 0.0: # This autograder test doesn't work when weights are exactly zero continue point = multiplier * orig_weights sanity_dataset = backend.Dataset(x=np.tile(point, (500, 1)), y=np.ones((500, 1)) * -1.0) p.train(sanity_dataset) new_weights = p.get_weights().data.reshape((1, dimensions)) if multiplier < 0: expected_weights = orig_weights else: expected_weights = orig_weights - point if not np.all(new_weights == expected_weights): print() print("Initial perceptron weights were: [{:.4f}, {:.4f}]".format( orig_weights[0, 0], orig_weights[0, 1])) print("All data points in the dataset were identical and had:") print(" x = [{:.4f}, {:.4f}]".format(point[0, 0], point[0, 1])) print(" y = -1") print("Your trained weights were: [{:.4f}, {:.4f}]".format( new_weights[0, 0], new_weights[0, 1])) print("Expected weights after training: [{:.4f}, {:.4f}]".format( expected_weights[0, 0], expected_weights[0, 1])) print() assert False, "Weight update sanity check failed" print("Sanity checking complete. Now training perceptron") model = models.PerceptronModel(3) dataset = backend.PerceptronDataset(model) model.train(dataset) backend.maybe_sleep_and_close(1) assert dataset.epoch != 0, "Perceptron code never iterated over the training data" accuracy = np.mean( np.where( np.dot(dataset.x, model.get_weights().data.T) >= 0.0, 1.0, -1.0) == dataset.y) if accuracy < 1.0: print( "The weights learned by your perceptron correctly classified {:.2%} of training examples" .format(accuracy)) print( "To receive full points for this question, your perceptron must converge to 100% accuracy" ) return tracker.add_points(4)
def check_regression(tracker): import models model = models.RegressionModel() dataset = backend.RegressionDataset(model) detected_parameters = None for batch_size in (1, 2, 4): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) output_node = model.run(inp_x) verify_node(output_node, 'node', (batch_size, 1), "RegressionModel.run()") trace = trace_node(output_node) assert inp_x in trace, "Node returned from RegressionModel.run() does not depend on the provided input (x)" if detected_parameters is None: detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)] for node in trace: assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( "Calling RegressionModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected") for batch_size in (1, 2, 4): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) loss_node = model.get_loss(inp_x, inp_y) verify_node(loss_node, 'loss', None, "RegressionModel.get_loss()") trace = trace_node(loss_node) assert inp_x in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)" assert inp_y in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)" for node in trace: assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( "RegressionModel.get_loss() should not use additional parameters not used by RegressionModel.run()") tracker.add_points(2) # Partial credit for passing sanity checks time_out = False if platform == "linux" or platform == "linux2" or platform == "darwin": # https://stackoverflow.com/questions/492519/timeout-on-a-function-call # linux # OS X # print("Using MacOS") signal.signal(signal.SIGALRM, handler) signal.alarm(240) try: model.train(dataset) except Exception as exc: print(exc) time_out = True signal.alarm(0) elif platform == "win32": # Windows... print("Using Windows no automatic timeout included") model.train(dataset) if time_out == False: model.train(dataset) backend.maybe_sleep_and_close(1) train_loss = model.get_loss(nn.Constant(dataset.x), nn.Constant(dataset.y)) verify_node(train_loss, 'loss', None, "RegressionModel.get_loss()") train_loss = nn.as_scalar(train_loss) # Re-compute the loss ourselves: otherwise get_loss() could be hard-coded # to always return zero train_predicted = model.run(nn.Constant(dataset.x)) verify_node(train_predicted, 'node', (dataset.x.shape[0], 1), "RegressionModel.run()") sanity_loss = 0.5 * np.mean((train_predicted.data - dataset.y)**2) assert np.isclose(train_loss, sanity_loss), ( "RegressionModel.get_loss() returned a loss of {:.4f}, " "but the autograder computed a loss of {:.4f} " "based on the output of RegressionModel.run()".format( train_loss, sanity_loss)) loss_threshold = 0.02 if train_loss <= loss_threshold: print("Your final loss is: {:f}".format(train_loss)) tracker.add_points(4) else: print("Your final loss ({:f}) must be no more than {:.4f} to receive full points for this question".format(train_loss, loss_threshold)) else: print("Your training timed out, therefore no final loss to report and test.")