def check_digit_classification(tracker): import models model = models.DigitClassificationModel() dataset = backend.DigitClassificationDataset(model) detected_parameters = None for batch_size in (1, 2, 4): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) output_node = model.run(inp_x) verify_node(output_node, 'node', (batch_size, 10), "DigitClassificationModel.run()") trace = trace_node(output_node) assert inp_x in trace, "Node returned from DigitClassificationModel.run() does not depend on the provided input (x)" if detected_parameters is None: detected_parameters = [ node for node in trace if isinstance(node, nn.Parameter) ] for node in trace: assert not isinstance( node, nn.Parameter ) or node in detected_parameters, ( "Calling DigitClassificationModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected" ) for batch_size in (1, 2, 4): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) loss_node = model.get_loss(inp_x, inp_y) verify_node(loss_node, 'loss', None, "DigitClassificationModel.get_loss()") trace = trace_node(loss_node) assert inp_x in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided input (x)" assert inp_y in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided labels (y)" for node in trace: assert not isinstance( node, nn.Parameter ) or node in detected_parameters, ( "DigitClassificationModel.get_loss() should not use additional parameters not used by DigitClassificationModel.run()" ) tracker.add_points(2) # Partial credit for passing sanity checks model.train(dataset) test_logits = model.run(nn.Constant(dataset.test_images)).data test_predicted = np.argmax(test_logits, axis=1) test_accuracy = np.mean(test_predicted == dataset.test_labels) accuracy_threshold = 0.97 if test_accuracy >= accuracy_threshold: print("Your final test set accuracy is: {:%}".format(test_accuracy)) tracker.add_points(4) else: print( "Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question" .format(test_accuracy, accuracy_threshold))
def check_digit_classification(tracker): import models, backend model = models.DigitClassificationModel() assert model.get_data_and_monitor == backend.get_data_and_monitor_digit_classification, "DigitClassificationModel.get_data_and_monitor is not set correctly" assert model.learning_rate > 0, "DigitClassificationModel.learning_rate is not set correctly" model.train() stats = backend.get_stats(model) accuracy_threshold = 0.97 if stats['dev_accuracy'] >= accuracy_threshold: tracker.add_points(1) else: print( "Your final validation accuracy ({:%}) must be at least {:.0%} to receive points for this question" .format(stats['dev_accuracy'], accuracy_threshold))
def main(): import models model = models.PerceptronModel(3) dataset = PerceptronDataset(model) model.train(dataset) model = models.RegressionModel() dataset = RegressionDataset(model) model.train(dataset) model = models.DigitClassificationModel() dataset = DigitClassificationDataset(model) model.train(dataset) model = models.LanguageIDModel() dataset = LanguageIDDataset(model) model.train(dataset)
def check_digit_classification(tracker): import models model = models.DigitClassificationModel() dataset = backend.DigitClassificationDataset(model) detected_parameters = None for batch_size in (1, 2, 4): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) output_node = model.run(inp_x) verify_node(output_node, 'node', (batch_size, 10), "DigitClassificationModel.run()") trace = trace_node(output_node) assert inp_x in trace, "Node returned from DigitClassificationModel.run() does not depend on the provided input (x)" if detected_parameters is None: detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)] for node in trace: assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( "Calling DigitClassificationModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected") for batch_size in (1, 2, 4): inp_x = nn.Constant(dataset.x[:batch_size]) inp_y = nn.Constant(dataset.y[:batch_size]) loss_node = model.get_loss(inp_x, inp_y) verify_node(loss_node, 'loss', None, "DigitClassificationModel.get_loss()") trace = trace_node(loss_node) assert inp_x in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided input (x)" assert inp_y in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided labels (y)" for node in trace: assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( "DigitClassificationModel.get_loss() should not use additional parameters not used by DigitClassificationModel.run()") tracker.add_points(0.5) # Partial credit for passing sanity checks time_out = False if platform == "linux" or platform == "linux2" or platform == "darwin": # linux # OS X # print("Using MacOS") signal.signal(signal.SIGALRM, handler) signal.alarm(600) try: model.train(dataset) except Exception as exc: print(exc) time_out = True signal.alarm(0) elif platform == "win32": # Windows... print("Using Windows no automatic timeout included") model.train(dataset) if time_out == False: test_logits = model.run(nn.Constant(dataset.test_images)).data test_predicted = np.argmax(test_logits, axis=1) test_accuracy = np.mean(test_predicted == dataset.test_labels) accuracy_threshold = 0.97 if test_accuracy >= accuracy_threshold: print("Your final test set accuracy is: {:%}".format(test_accuracy)) tracker.add_points(1.5) else: print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold)) else: print("Your training timed out, therefore no test set accuracy is reported and tested.")