def test_category_cross_entropy(self): layer = tl.CategoryCrossEntropy() targets = np.array([0, 1]) # Near-perfect prediction (for both items in batch). model_outputs = np.array([[9., 2., 0., -2.], [2., 9., 0., -2.]]) loss = layer([model_outputs, targets]) self.assertAlmostEqual(loss, .001, places=3) # More right than wrong (for both items in batch). model_outputs = np.array([[2.2, 2., 0., -2.], [2., 2.2, 0., -2.]]) loss = layer([model_outputs, targets]) self.assertAlmostEqual(loss, .665, places=3) # First item near perfect, second item more right than wrong. model_outputs = np.array([[9., 2., 0., -2.], [2., 2.2, 0., -2.]]) loss = layer([model_outputs, targets]) self.assertAlmostEqual(loss, .333, places=3)
def test_category_cross_entropy_with_label_smoothing(self): epsilon = 0.01 layer = tl.CategoryCrossEntropy(label_smoothing=epsilon) targets = np.array([0, 1]) # Near-perfect prediction (for both items in batch). model_outputs = np.array([[9., 2., 0., -2.], [2., 9., 0., -2.]]) loss = layer([model_outputs, targets]) self.assertAlmostEqual(loss, .069, places=3) # More right than wrong (for both items in batch). model_outputs = np.array([[2.2, 2., 0., -2.], [2., 2.2, 0., -2.]]) loss = layer([model_outputs, targets]) self.assertAlmostEqual(loss, .682, places=3) # First item near perfect, second item more right than wrong. model_outputs = np.array([[9., 2., 0., -2.], [2., 2.2, 0., -2.]]) loss = layer([model_outputs, targets]) self.assertAlmostEqual(loss, .375, places=3)
inputs_test = trax.data.Serial( trax.data.Shuffle(), trax.data.BucketByLength(boundaries=[4, 8, 16, 32, 89], batch_sizes=[64, 64, 32, 32, 16 ]))(dt.get_input_sequence_and_gt( train_data, len_input, batch_size)) recommender = RecommenderTransformer(n_classes_in=len(classes), embedding_size=embedding_size, n_out_classes=len(classes), dropout_rate=dropout_rate) n_train_b = math.floor(float(len(train_data)) / float(batch_size)) train_task = ts.training.TrainTask( labeled_data=inputs_train, loss_layer=tl.CategoryCrossEntropy(), optimizer=trax.optimizers.Adam(learning_rate), n_steps_per_checkpoint= 400, #This will print the results at every 200 training steps. ) n_eval_b = math.floor(float(len(val_data)) / float(batch_size)) # Evaluaton task. eval_task = ts.training.EvalTask( labeled_data=inputs_test, metrics=[tl.CategoryCrossEntropy(), tl.CategoryAccuracy()], n_eval_batches=n_eval_b) training_loop = ts.training.Loop(recommender, train_task, eval_tasks=[eval_task],