def check_rl(tracker):
import models, backend
num_trials = 6
trials_satisfied = 0
trials_satisfied_required = 3
for trial_number in range(num_trials):
model = models.DeepQModel()
assert model.get_data_and_monitor == backend.get_data_and_monitor_rl, "DeepQModel.get_data_and_monitor is not set correctly"
assert model.learning_rate > 0, "DeepQModel.learning_rate is not set correctly"
model.train()
stats = backend.get_stats(model)
if stats['mean_reward'] >= stats['reward_threshold']:
trials_satisfied += 1
if trials_satisfied >= trials_satisfied_required:
tracker.add_points(1)
return
else:
trials_left = num_trials - (trial_number + 1)
if trials_satisfied + trials_left < trials_satisfied_required:
break
print(
"To receive credit for this question, your agent must receive a mean reward of at least {} on {} out of {} trials"
.format(stats['reward_threshold'], trials_satisfied_required,
num_trials))
def check_odd_regression(tracker):
loss_threshold = 0.02
asymmetry_threshold = 1e-8
import models, backend
model = models.OddRegressionModel()
assert model.get_data_and_monitor == backend.get_data_and_monitor_regression, "OddRegressionModel.get_data_and_monitor is not set correctly"
assert model.learning_rate > 0, "OddRegressionModel.learning_rate is not set correctly"
x_vals = np.linspace(-2 * np.pi, 2 * np.pi, num=16)[:, np.newaxis]
y_vals = model.run(x_vals)
# debug use
# import matplotlib.pyplot as plt
# plt.plot(x_vals, y_vals)
# plt.show()
asymmetry_vals = np.abs(y_vals + y_vals[::-1])
max_asymmetry = np.max(asymmetry_vals)
max_asymmetry_x = float(x_vals[np.argmax(asymmetry_vals)])
if max_asymmetry > asymmetry_threshold:
print("You do not appear to be modelling an odd function.")
print(
"Prior to training, you have abs(f(x) + f(-x)) = {} for x = {:.3f}."
.format(max_asymmetry, max_asymmetry_x))
print("An odd function has abs(f(x) + f(-x)) = 0 for all x")
return
f_0 = float(model.run(np.array([[0.]])))
if np.abs(f_0) > asymmetry_threshold:
print("Your OddRegressionModel does not satisfy f(0) = 0")
return
model.train()
stats = backend.get_stats(model)
full_points = True
if stats['loss'] > loss_threshold:
full_points = False
print(
"Your final loss ({:f}) must be no more than {:.4f} to receive points for this question"
.format(stats['loss'], loss_threshold))
if stats['max_asymmetry'] > asymmetry_threshold:
full_points = False
print("You do not appear to be modelling an odd function.")
print(
"After training, you have abs(f(x) + f(-x)) = {} for x = {:.3f}.".
format(stats['max_asymmetry'], stats['max_asymmetry_x']))
print("An odd function has abs(f(x) + f(-x)) = 0 for all x")
if full_points:
tracker.add_points(1)
def check_lang_id(tracker):
import models, backend
model = models.LanguageIDModel()
assert model.get_data_and_monitor == backend.get_data_and_monitor_lang_id, "LanguageIDModel.get_data_and_monitor is not set correctly"
assert model.learning_rate > 0, "LanguageIDModel.learning_rate is not set correctly"
model.train()
stats = backend.get_stats(model)
accuracy_threshold = 0.81
if stats['dev_accuracy'] >= accuracy_threshold:
tracker.add_points(2)
else:
print(
"Your final validation accuracy ({:%}) must be at least {:.0%} to receive points for this question"
.format(stats['dev_accuracy'], accuracy_threshold))
def check_digit_classification(tracker):
import models, backend
model = models.DigitClassificationModel()
assert model.get_data_and_monitor == backend.get_data_and_monitor_digit_classification, "DigitClassificationModel.get_data_and_monitor is not set correctly"
assert model.learning_rate > 0, "DigitClassificationModel.learning_rate is not set correctly"
model.train()
stats = backend.get_stats(model)
accuracy_threshold = 0.97
if stats['dev_accuracy'] >= accuracy_threshold:
tracker.add_points(1)
else:
print(
"Your final validation accuracy ({:%}) must be at least {:.0%} to receive points for this question"
.format(stats['dev_accuracy'], accuracy_threshold))
def check_regression(tracker):
import models, backend
model = models.RegressionModel()
assert model.get_data_and_monitor == backend.get_data_and_monitor_regression, "RegressionModel.get_data_and_monitor is not set correctly"
assert model.learning_rate > 0, "RegressionModel.learning_rate is not set correctly"
model.train()
stats = backend.get_stats(model)
loss_threshold = 0.02
if stats['loss'] <= loss_threshold:
tracker.add_points(2)
else:
print(
"Your final loss ({:f}) must be no more than {:.4f} to receive points for this question"
.format(stats['loss'], loss_threshold))
def check_perceptron(tracker):
import perceptron, backend
print("Sanity checking perceptron...")
np_random = np.random.RandomState(0)
with no_graphics():
# Check that the perceptron weights are initialized to a zero vector of `dimensions` entries.
for _ in range(10):
dimensions = np_random.randint(1, 10)
p = perceptron.Perceptron(dimensions)
p_weights = p.get_weights()
assert p_weights is not None,\
"Perceptron.get_weights() should return weights, not None"
p_shape = np.asarray(p_weights).shape
assert p_shape == (dimensions,),\
"Perceptron weights had shape {}, expected {}".format(p_shape, (dimensions,))
assert np.count_nonzero(p.get_weights(
)) == 0, "Perceptron weights should be initialized to zero."
# Check that the untrained perceptron predicts 1 on any point
for _ in range(10):
dimensions = np_random.randint(1, 10)
p = perceptron.Perceptron(dimensions)
point = np_random.uniform(-10, 10, dimensions)
pred = p.predict(point)
assert pred == 1, "Untrained perceptron should predict 1 instead of {} for {}.".format(
pred, pred)
# Check that a correctly classified point does not update the perceptron.
for _ in range(10):
dimensions = np_random.randint(1, 10)
p = perceptron.Perceptron(dimensions)
point = np_random.uniform(-10, 10, dimensions)
old_weights = p.get_weights().copy()
# All points should be classified as 1 at this point
updated = p.update(point, 1)
assert updated is not None, "Perceptron.update should return True or False, not None"
assert not updated, "Updating with a correctly classified point ({}) should return `False`.".format(
point)
new_weights = p.get_weights()
assert np.allclose(new_weights, old_weights),\
"Updating with a correctly classified point ({}) should not change weights from {} to {}".format(point, new_weights, old_weights)
# Check that the perceptron weight updates are correct
xs = np.array([[0., -1., 1., 1.8], [-1., 0., -2., 2.6]]).T
ys = np.array([-1, -1, -1, -1], dtype=int)
expected_weights = np.array([[0., 1., 1., -0.8], [1., 1., 1., -1.6]]).T
expected_returns = [True, True, False, True]
p = perceptron.Perceptron(2)
for i in range(xs.shape[0]):
old_weights = p.get_weights().copy()
res = p.update(xs[i], ys[i])
assert res == expected_returns[i],\
"""Perceptron.update returned {}. Expected: {}
Old weights: {}
x: {}
y: {}""".format(res, expected_returns[i], old_weights, xs[i], ys[i])
assert np.allclose(p.get_weights(), expected_weights[i]),\
"""Perceptron weights are {}. Expected: {}
Old weights: {}
x: {}
y: {}""".format(p.get_weights(), expected_weights[i], old_weights, xs[i], ys[i])
print("Sanity checking complete. Now training perceptron")
p = perceptron.Perceptron(3)
p.train()
backend.maybe_sleep_and_close(1)
stats = backend.get_stats(p)
if stats is None:
print("Your perceptron never trained for a full epoch!")
return
if stats["accuracy"] < 1.0:
print(
"The weights learned by your perceptron correctly classified {:.2%} of training examples"
.format(stats['accuracy']))
print(
"To receive points for this question, your perceptron must converge to 100% accuracy"
)
else:
tracker.add_points(3)
