def check_rl(tracker): import models, backend num_trials = 6 trials_satisfied = 0 trials_satisfied_required = 3 for trial_number in range(num_trials): model = models.DeepQModel() assert model.get_data_and_monitor == backend.get_data_and_monitor_rl, "DeepQModel.get_data_and_monitor is not set correctly" assert model.learning_rate > 0, "DeepQModel.learning_rate is not set correctly" model.train() stats = backend.get_stats(model) if stats['mean_reward'] >= stats['reward_threshold']: trials_satisfied += 1 if trials_satisfied >= trials_satisfied_required: tracker.add_points(1) return else: trials_left = num_trials - (trial_number + 1) if trials_satisfied + trials_left < trials_satisfied_required: break print( "To receive credit for this question, your agent must receive a mean reward of at least {} on {} out of {} trials" .format(stats['reward_threshold'], trials_satisfied_required, num_trials))
def check_odd_regression(tracker): loss_threshold = 0.02 asymmetry_threshold = 1e-8 import models, backend model = models.OddRegressionModel() assert model.get_data_and_monitor == backend.get_data_and_monitor_regression, "OddRegressionModel.get_data_and_monitor is not set correctly" assert model.learning_rate > 0, "OddRegressionModel.learning_rate is not set correctly" x_vals = np.linspace(-2 * np.pi, 2 * np.pi, num=16)[:, np.newaxis] y_vals = model.run(x_vals) # debug use # import matplotlib.pyplot as plt # plt.plot(x_vals, y_vals) # plt.show() asymmetry_vals = np.abs(y_vals + y_vals[::-1]) max_asymmetry = np.max(asymmetry_vals) max_asymmetry_x = float(x_vals[np.argmax(asymmetry_vals)]) if max_asymmetry > asymmetry_threshold: print("You do not appear to be modelling an odd function.") print( "Prior to training, you have abs(f(x) + f(-x)) = {} for x = {:.3f}." .format(max_asymmetry, max_asymmetry_x)) print("An odd function has abs(f(x) + f(-x)) = 0 for all x") return f_0 = float(model.run(np.array([[0.]]))) if np.abs(f_0) > asymmetry_threshold: print("Your OddRegressionModel does not satisfy f(0) = 0") return model.train() stats = backend.get_stats(model) full_points = True if stats['loss'] > loss_threshold: full_points = False print( "Your final loss ({:f}) must be no more than {:.4f} to receive points for this question" .format(stats['loss'], loss_threshold)) if stats['max_asymmetry'] > asymmetry_threshold: full_points = False print("You do not appear to be modelling an odd function.") print( "After training, you have abs(f(x) + f(-x)) = {} for x = {:.3f}.". format(stats['max_asymmetry'], stats['max_asymmetry_x'])) print("An odd function has abs(f(x) + f(-x)) = 0 for all x") if full_points: tracker.add_points(1)
def check_lang_id(tracker): import models, backend model = models.LanguageIDModel() assert model.get_data_and_monitor == backend.get_data_and_monitor_lang_id, "LanguageIDModel.get_data_and_monitor is not set correctly" assert model.learning_rate > 0, "LanguageIDModel.learning_rate is not set correctly" model.train() stats = backend.get_stats(model) accuracy_threshold = 0.81 if stats['dev_accuracy'] >= accuracy_threshold: tracker.add_points(2) else: print( "Your final validation accuracy ({:%}) must be at least {:.0%} to receive points for this question" .format(stats['dev_accuracy'], accuracy_threshold))
def check_digit_classification(tracker): import models, backend model = models.DigitClassificationModel() assert model.get_data_and_monitor == backend.get_data_and_monitor_digit_classification, "DigitClassificationModel.get_data_and_monitor is not set correctly" assert model.learning_rate > 0, "DigitClassificationModel.learning_rate is not set correctly" model.train() stats = backend.get_stats(model) accuracy_threshold = 0.97 if stats['dev_accuracy'] >= accuracy_threshold: tracker.add_points(1) else: print( "Your final validation accuracy ({:%}) must be at least {:.0%} to receive points for this question" .format(stats['dev_accuracy'], accuracy_threshold))
def check_regression(tracker): import models, backend model = models.RegressionModel() assert model.get_data_and_monitor == backend.get_data_and_monitor_regression, "RegressionModel.get_data_and_monitor is not set correctly" assert model.learning_rate > 0, "RegressionModel.learning_rate is not set correctly" model.train() stats = backend.get_stats(model) loss_threshold = 0.02 if stats['loss'] <= loss_threshold: tracker.add_points(2) else: print( "Your final loss ({:f}) must be no more than {:.4f} to receive points for this question" .format(stats['loss'], loss_threshold))
def check_perceptron(tracker): import perceptron, backend print("Sanity checking perceptron...") np_random = np.random.RandomState(0) with no_graphics(): # Check that the perceptron weights are initialized to a zero vector of `dimensions` entries. for _ in range(10): dimensions = np_random.randint(1, 10) p = perceptron.Perceptron(dimensions) p_weights = p.get_weights() assert p_weights is not None,\ "Perceptron.get_weights() should return weights, not None" p_shape = np.asarray(p_weights).shape assert p_shape == (dimensions,),\ "Perceptron weights had shape {}, expected {}".format(p_shape, (dimensions,)) assert np.count_nonzero(p.get_weights( )) == 0, "Perceptron weights should be initialized to zero." # Check that the untrained perceptron predicts 1 on any point for _ in range(10): dimensions = np_random.randint(1, 10) p = perceptron.Perceptron(dimensions) point = np_random.uniform(-10, 10, dimensions) pred = p.predict(point) assert pred == 1, "Untrained perceptron should predict 1 instead of {} for {}.".format( pred, pred) # Check that a correctly classified point does not update the perceptron. for _ in range(10): dimensions = np_random.randint(1, 10) p = perceptron.Perceptron(dimensions) point = np_random.uniform(-10, 10, dimensions) old_weights = p.get_weights().copy() # All points should be classified as 1 at this point updated = p.update(point, 1) assert updated is not None, "Perceptron.update should return True or False, not None" assert not updated, "Updating with a correctly classified point ({}) should return `False`.".format( point) new_weights = p.get_weights() assert np.allclose(new_weights, old_weights),\ "Updating with a correctly classified point ({}) should not change weights from {} to {}".format(point, new_weights, old_weights) # Check that the perceptron weight updates are correct xs = np.array([[0., -1., 1., 1.8], [-1., 0., -2., 2.6]]).T ys = np.array([-1, -1, -1, -1], dtype=int) expected_weights = np.array([[0., 1., 1., -0.8], [1., 1., 1., -1.6]]).T expected_returns = [True, True, False, True] p = perceptron.Perceptron(2) for i in range(xs.shape[0]): old_weights = p.get_weights().copy() res = p.update(xs[i], ys[i]) assert res == expected_returns[i],\ """Perceptron.update returned {}. Expected: {} Old weights: {} x: {} y: {}""".format(res, expected_returns[i], old_weights, xs[i], ys[i]) assert np.allclose(p.get_weights(), expected_weights[i]),\ """Perceptron weights are {}. Expected: {} Old weights: {} x: {} y: {}""".format(p.get_weights(), expected_weights[i], old_weights, xs[i], ys[i]) print("Sanity checking complete. Now training perceptron") p = perceptron.Perceptron(3) p.train() backend.maybe_sleep_and_close(1) stats = backend.get_stats(p) if stats is None: print("Your perceptron never trained for a full epoch!") return if stats["accuracy"] < 1.0: print( "The weights learned by your perceptron correctly classified {:.2%} of training examples" .format(stats['accuracy'])) print( "To receive points for this question, your perceptron must converge to 100% accuracy" ) else: tracker.add_points(3)