def check_logreg(tracker):
import models
print("Sanity checking logistic regression...")
np_random = np.random.RandomState(0)
pts = 0
# Check that run returns a node, and that the score in the node is correct
for dimensions in range(1, 11):
p = models.LogisticRegressionModel(dimensions)
p_weights = np.array(p.get_weights())
if (len(p_weights) == dimensions):
pts = pts + 0.1
else:
print('Weight List Initialized Does Not Have Correct Dimensions')
point = np_random.uniform(-10, 10, (1, dimensions))
score = p.DotProduct(p_weights.flatten(), point.flatten())
expected_score = float(np.dot(point.flatten(), p_weights.flatten()))
if np.isclose(score, expected_score):
pts = pts + 0.1
else:
print(
"The score computed by LogisticRegressionModel.run() ({:.4f}) does not match the expected score ({:.4f})"
.format(score, expected_score))
tracker.add_points(pts) # Partial credit for passing sanity checks
print("Sanity checking complete. Now training perceptron")
model = models.LogisticRegressionModel(3)
dataset = backend.LogisticRegressionDataset(model)
model.train(dataset)
backend.maybe_sleep_and_close(2)
assert dataset.epoch != 0, "LogisticRegressionModel code never iterated over the training data"
accuracy = np.mean(
np.where(
np.dot(dataset.x,
np.array(model.get_weights()).flatten()) >= 0.0, 1.0, -1.0)
== dataset.y.T)
if accuracy < 1.0:
print(
"The weights learned by your perceptron correctly classified {:.2%} of training examples"
.format(accuracy))
print(
"To receive full points for this question, your perceptron must converge to 100% accuracy"
)
return
tracker.add_points(4)
def check_perceptron(tracker):
import perceptron, backend
print("Sanity checking perceptron...")
np_random = np.random.RandomState(0)
with no_graphics():
# Check that the perceptron weights are initialized to a zero vector of `dimensions` entries.
for _ in range(10):
dimensions = np_random.randint(1, 10)
p = perceptron.Perceptron(dimensions)
p_weights = p.get_weights()
assert p_weights is not None,\
"Perceptron.get_weights() should return weights, not None"
p_shape = np.asarray(p_weights).shape
assert p_shape == (dimensions,),\
"Perceptron weights had shape {}, expected {}".format(p_shape, (dimensions,))
assert np.count_nonzero(p.get_weights(
)) == 0, "Perceptron weights should be initialized to zero."
# Check that the untrained perceptron predicts 1 on any point
for _ in range(10):
dimensions = np_random.randint(1, 10)
p = perceptron.Perceptron(dimensions)
point = np_random.uniform(-10, 10, dimensions)
pred = p.predict(point)
assert pred == 1, "Untrained perceptron should predict 1 instead of {} for {}.".format(
pred, pred)
# Check that a correctly classified point does not update the perceptron.
for _ in range(10):
dimensions = np_random.randint(1, 10)
p = perceptron.Perceptron(dimensions)
point = np_random.uniform(-10, 10, dimensions)
old_weights = p.get_weights().copy()
# All points should be classified as 1 at this point
updated = p.update(point, 1)
assert updated is not None, "Perceptron.update should return True or False, not None"
assert not updated, "Updating with a correctly classified point ({}) should return `False`.".format(
point)
new_weights = p.get_weights()
assert np.allclose(new_weights, old_weights),\
"Updating with a correctly classified point ({}) should not change weights from {} to {}".format(point, new_weights, old_weights)
# Check that the perceptron weight updates are correct
xs = np.array([[0., -1., 1., 1.8], [-1., 0., -2., 2.6]]).T
ys = np.array([-1, -1, -1, -1], dtype=int)
expected_weights = np.array([[0., 1., 1., -0.8], [1., 1., 1., -1.6]]).T
expected_returns = [True, True, False, True]
p = perceptron.Perceptron(2)
for i in range(xs.shape[0]):
old_weights = p.get_weights().copy()
res = p.update(xs[i], ys[i])
assert res == expected_returns[i],\
"""Perceptron.update returned {}. Expected: {}
Old weights: {}
x: {}
y: {}""".format(res, expected_returns[i], old_weights, xs[i], ys[i])
assert np.allclose(p.get_weights(), expected_weights[i]),\
"""Perceptron weights are {}. Expected: {}
Old weights: {}
x: {}
y: {}""".format(p.get_weights(), expected_weights[i], old_weights, xs[i], ys[i])
print("Sanity checking complete. Now training perceptron")
p = perceptron.Perceptron(3)
p.train()
backend.maybe_sleep_and_close(1)
stats = backend.get_stats(p)
if stats is None:
print("Your perceptron never trained for a full epoch!")
return
if stats["accuracy"] < 1.0:
print(
"The weights learned by your perceptron correctly classified {:.2%} of training examples"
.format(stats['accuracy']))
print(
"To receive points for this question, your perceptron must converge to 100% accuracy"
)
else:
tracker.add_points(3)
def check_polyregression(tracker):
import models
model = models.PolyRegressionModel(2)
dataset = backend.PolyRegressionDataset(model, 2)
detected_parameters = None
for batch_size in (1, 1, 1):
inp_x = nn.Constant(dataset.x[:batch_size])
inp_y = nn.Constant(dataset.y[:batch_size])
#print(nn.as_scalar(inp_x))
output_node = model.run(inp_x)
verify_node(output_node, 'node', (batch_size, 1),
"PolyRegressionModel.run()")
trace = trace_node(output_node)
if detected_parameters is None:
detected_parameters = [
node for node in trace if isinstance(node, nn.Parameter)
]
for node in trace:
assert not isinstance(
node, nn.Parameter
) or node in detected_parameters, (
"Calling PolyRegressionModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected"
)
for batch_size in (1, 1, 1):
inp_x = nn.Constant(dataset.x[:batch_size])
inp_y = nn.Constant(dataset.y[:batch_size])
loss_node = model.get_loss(inp_x, inp_y)
verify_node(loss_node, 'loss', None, "PolyRegressionModel.get_loss()")
trace = trace_node(loss_node)
#assert inp_x in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)"
assert inp_y in trace, "Node returned fromPolyRegressionModel.get_loss() does not depend on the provided labels (y)"
for node in trace:
assert not isinstance(
node, nn.Parameter
) or node in detected_parameters, (
"PolyRegressionModel.get_loss() should not use additional parameters not used by RegressionModel.run()"
)
tracker.add_points(2) # Partial credit for passing sanity checks
model.train(dataset)
backend.maybe_sleep_and_close(1)
train_loss = model.get_loss(nn.Constant(dataset.x), nn.Constant(dataset.y))
verify_node(train_loss, 'loss', None, "PolyRegressionModel.get_loss()")
train_loss = nn.as_scalar(train_loss)
# Re-compute the loss ourselves: otherwise get_loss() could be hard-coded
# to always return zero
train_predicted = model.run(nn.Constant(dataset.x))
verify_node(train_predicted, 'node', (dataset.x.shape[0], 1),
"PolyRegressionModel.run()")
sanity_loss = 0.5 * np.mean((train_predicted.data - dataset.y)**2)
assert np.isclose(train_loss, sanity_loss), (
"PolyRegressionModel.get_loss() returned a loss of {:.4f}, "
"but the autograder computed a loss of {:.4f} "
"based on the output of PolyRegressionModel.run()".format(
train_loss, sanity_loss))
loss_threshold = 0.1
if train_loss <= loss_threshold:
print("Your final loss is: {:f}".format(train_loss))
tracker.add_points(4)
else:
print(
"Your final loss ({:f}) must be no more than {:.4f} to receive full points for this question"
.format(train_loss, loss_threshold))
def check_perceptron(tracker):
import models
print("Sanity checking perceptron...")
np_random = np.random.RandomState(0)
# Check that the perceptron weights are initialized to a vector with `dimensions` entries.
for dimensions in range(1, 10):
p = models.PerceptronModel(dimensions)
p_weights = p.get_weights()
verify_node(p_weights, 'parameter', (1, dimensions),
"PerceptronModel.get_weights()")
# Check that run returns a node, and that the score in the node is correct
for dimensions in range(1, 10):
p = models.PerceptronModel(dimensions)
p_weights = p.get_weights()
verify_node(p_weights, 'parameter', (1, dimensions),
"PerceptronModel.get_weights()")
point = np_random.uniform(-10, 10, (1, dimensions))
score = p.run(nn.Constant(point))
verify_node(score, 'node', (1, 1), "PerceptronModel.run()")
calculated_score = nn.as_scalar(score)
expected_score = float(
np.dot(point.flatten(), p_weights.data.flatten()))
assert np.isclose(calculated_score, expected_score), (
"The score computed by PerceptronModel.run() ({:.4f}) does not match the expected score ({:.4f})"
.format(calculated_score, expected_score))
# Check that get_prediction returns the correct values, including the
# case when a point lies exactly on the decision boundary
for dimensions in range(1, 10):
p = models.PerceptronModel(dimensions)
random_point = np_random.uniform(-10, 10, (1, dimensions))
for point in (random_point, np.zeros_like(random_point)):
prediction = p.get_prediction(nn.Constant(point))
assert prediction == 1 or prediction == -1, (
"PerceptronModel.get_prediction() should return 1 or -1, not {}"
.format(prediction))
expected_prediction = np.asscalar(
np.where(np.dot(point,
p.get_weights().data.T) >= 0, 1, -1))
assert prediction == expected_prediction, (
"PerceptronModel.get_prediction() returned {}; expected {}".
format(prediction, expected_prediction))
tracker.add_points(2) # Partial credit for passing sanity checks
print("Sanity checking perceptron weight updates...")
# Test weight updates. This involves constructing a dataset that
# requires 0 or 1 updates before convergence, and testing that weight
# values change as expected. Note that (multiplier < -1 or multiplier > 1)
# must be true for the testing code to be correct.
dimensions = 2
for multiplier in (-5, -2, 2, 5):
p = models.PerceptronModel(dimensions)
orig_weights = p.get_weights().data.reshape((1, dimensions)).copy()
if np.abs(orig_weights).sum() == 0.0:
# This autograder test doesn't work when weights are exactly zero
continue
point = multiplier * orig_weights
sanity_dataset = backend.Dataset(x=np.tile(point, (500, 1)),
y=np.ones((500, 1)) * -1.0)
p.train(sanity_dataset)
new_weights = p.get_weights().data.reshape((1, dimensions))
if multiplier < 0:
expected_weights = orig_weights
else:
expected_weights = orig_weights - point
if not np.all(new_weights == expected_weights):
print()
print("Initial perceptron weights were: [{:.4f}, {:.4f}]".format(
orig_weights[0, 0], orig_weights[0, 1]))
print("All data points in the dataset were identical and had:")
print(" x = [{:.4f}, {:.4f}]".format(point[0, 0], point[0, 1]))
print(" y = -1")
print("Your trained weights were: [{:.4f}, {:.4f}]".format(
new_weights[0, 0], new_weights[0, 1]))
print("Expected weights after training: [{:.4f}, {:.4f}]".format(
expected_weights[0, 0], expected_weights[0, 1]))
print()
assert False, "Weight update sanity check failed"
print("Sanity checking complete. Now training perceptron")
model = models.PerceptronModel(3)
dataset = backend.PerceptronDataset(model)
model.train(dataset)
backend.maybe_sleep_and_close(1)
assert dataset.epoch != 0, "Perceptron code never iterated over the training data"
accuracy = np.mean(
np.where(
np.dot(dataset.x,
model.get_weights().data.T) >= 0.0, 1.0, -1.0) == dataset.y)
if accuracy < 1.0:
print(
"The weights learned by your perceptron correctly classified {:.2%} of training examples"
.format(accuracy))
print(
"To receive full points for this question, your perceptron must converge to 100% accuracy"
)
return
tracker.add_points(4)
def check_regression(tracker):
import models
model = models.RegressionModel()
dataset = backend.RegressionDataset(model)
detected_parameters = None
for batch_size in (1, 2, 4):
inp_x = nn.Constant(dataset.x[:batch_size])
inp_y = nn.Constant(dataset.y[:batch_size])
output_node = model.run(inp_x)
verify_node(output_node, 'node', (batch_size, 1), "RegressionModel.run()")
trace = trace_node(output_node)
assert inp_x in trace, "Node returned from RegressionModel.run() does not depend on the provided input (x)"
if detected_parameters is None:
detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)]
for node in trace:
assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
"Calling RegressionModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected")
for batch_size in (1, 2, 4):
inp_x = nn.Constant(dataset.x[:batch_size])
inp_y = nn.Constant(dataset.y[:batch_size])
loss_node = model.get_loss(inp_x, inp_y)
verify_node(loss_node, 'loss', None, "RegressionModel.get_loss()")
trace = trace_node(loss_node)
assert inp_x in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)"
assert inp_y in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)"
for node in trace:
assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
"RegressionModel.get_loss() should not use additional parameters not used by RegressionModel.run()")
tracker.add_points(2) # Partial credit for passing sanity checks
time_out = False
if platform == "linux" or platform == "linux2" or platform == "darwin":
# https://stackoverflow.com/questions/492519/timeout-on-a-function-call
# linux
# OS X
# print("Using MacOS")
signal.signal(signal.SIGALRM, handler)
signal.alarm(240)
try:
model.train(dataset)
except Exception as exc:
print(exc)
time_out = True
signal.alarm(0)
elif platform == "win32":
# Windows...
print("Using Windows no automatic timeout included")
model.train(dataset)
if time_out == False:
model.train(dataset)
backend.maybe_sleep_and_close(1)
train_loss = model.get_loss(nn.Constant(dataset.x), nn.Constant(dataset.y))
verify_node(train_loss, 'loss', None, "RegressionModel.get_loss()")
train_loss = nn.as_scalar(train_loss)
# Re-compute the loss ourselves: otherwise get_loss() could be hard-coded
# to always return zero
train_predicted = model.run(nn.Constant(dataset.x))
verify_node(train_predicted, 'node', (dataset.x.shape[0], 1), "RegressionModel.run()")
sanity_loss = 0.5 * np.mean((train_predicted.data - dataset.y)**2)
assert np.isclose(train_loss, sanity_loss), (
"RegressionModel.get_loss() returned a loss of {:.4f}, "
"but the autograder computed a loss of {:.4f} "
"based on the output of RegressionModel.run()".format(
train_loss, sanity_loss))
loss_threshold = 0.02
if train_loss <= loss_threshold:
print("Your final loss is: {:f}".format(train_loss))
tracker.add_points(4)
else:
print("Your final loss ({:f}) must be no more than {:.4f} to receive full points for this question".format(train_loss, loss_threshold))
else:
print("Your training timed out, therefore no final loss to report and test.")
