def test_sequence_accuracy_last_position_zero_weight(self): layer = tl.SequenceAccuracy() targets = np.array([[0, 1, 0, 0], [1, 0, 1, 0]]) weights = np.array([[1., 1., 1., 0.], [1., 1., 1., 0.]]) # Model gets both sequences right; output in final position would give # wrong category but is ignored. model_outputs = np.array([[[.9, .1], [.2, .8], [.7, .3], [.35, .65]], [[.3, .7], [.8, .2], [.1, .9], [.35, .65]]]) accuracy = layer([model_outputs, targets, weights]) self.assertEqual(accuracy, 1.) # Model gets the first element of the first sequence barely wrong. model_outputs = np.array([[[.45, .55], [.2, .8], [.7, .3], [.6, .4]], [[.3, .7], [.8, .2], [.1, .9], [.6, .4]]]) accuracy = layer([model_outputs, targets, weights]) self.assertEqual(accuracy, .5) # Model gets second-to-last element of each sequence barely wrong. model_outputs = np.array([[[.9, .1], [.2, .8], [.48, .52], [.6, .4]], [[.3, .7], [.8, .2], [.51, .49], [.6, .4]]]) accuracy = layer([model_outputs, targets, weights]) self.assertEqual(accuracy, 0.)
def test_names(self): layer = tl.L2Loss() self.assertEqual('L2Loss_in3', str(layer)) layer = tl.Accuracy() self.assertEqual('Accuracy_in3', str(layer)) layer = tl.SequenceAccuracy() self.assertEqual('SequenceAccuracy_in3', str(layer)) layer = tl.CrossEntropyLoss() self.assertEqual('CrossEntropyLoss_in3', str(layer)) layer = tl.CrossEntropySum() self.assertEqual('CrossEntropySum_in3', str(layer))
def test_names(self): layer = tl.L2Loss() self.assertEqual('L2Loss_in3', str(layer)) layer = tl.BinaryClassifier() self.assertEqual('BinaryClassifier', str(layer)) layer = tl.MulticlassClassifier() self.assertEqual('MulticlassClassifier', str(layer)) layer = tl.Accuracy() self.assertEqual('Accuracy_in3', str(layer)) layer = tl.SequenceAccuracy() self.assertEqual('SequenceAccuracy_in3', str(layer)) layer = tl.BinaryCrossEntropyLoss() self.assertEqual('BinaryCrossEntropyLoss_in3', str(layer)) layer = tl.CrossEntropyLoss() self.assertEqual('CrossEntropyLoss_in3', str(layer)) layer = tl.BinaryCrossEntropySum() self.assertEqual('BinaryCrossEntropySum_in3', str(layer)) layer = tl.CrossEntropySum() self.assertEqual('CrossEntropySum_in3', str(layer))
def test_sequence_accuracy_weights_all_ones(self): layer = tl.SequenceAccuracy() targets = np.array([[0, 1, 0, 1], [1, 0, 1, 1]]) weights = np.ones_like(targets) # Model gets both sequences right; for each position in each sequence, the # category (integer ID) selected by argmax matches the target category. model_outputs = np.array([[[.9, .1], [.2, .8], [.7, .3], [.4, .6]], [[.3, .7], [.8, .2], [.1, .9], [.4, .6]]]) accuracy = layer([model_outputs, targets, weights]) self.assertEqual(accuracy, 1.) # Model gets the first element of the first sequence barely wrong. model_outputs = np.array([[[.45, .55], [.2, .8], [.7, .3], [.4, .6]], [[.3, .7], [.8, .2], [.1, .9], [.4, .6]]]) accuracy = layer([model_outputs, targets, weights]) self.assertEqual(accuracy, .5) # Model gets the last element of each sequence barely wrong. model_outputs = np.array([[[.9, .1], [.2, .8], [.7, .3], [.55, .45]], [[.3, .7], [.8, .2], [.1, .9], [.52, .48]]]) accuracy = layer([model_outputs, targets, weights]) self.assertEqual(accuracy, 0.)
OptState = collections.namedtuple( '_OptState', [ 'weights', # Model weights. 'slots', # Per-parameter optimizer state, e.g. gradient moments. 'opt_params', # Optimizer (hyper)parameters, e.g. learning rate, momentum. ]) _DEFAULT_METRICS = { 'loss': tl.Serial(tl.LogSoftmax(), tl.CrossEntropyLoss()), 'accuracy': tl.Accuracy(), 'sequence_accuracy': tl.SequenceAccuracy(), 'neg_log_perplexity': tl.Serial(tl.LogSoftmax(), tl.CrossEntropyLoss(), tl.Negate()), 'weights_per_batch_per_core': tl.Serial(tl.Drop(), tl.Drop(), tl.Sum()), } class Trainer: """Trax trainer. A trainer allows to make training steps, train for full epochs, save the training state and access evaluation data. """ def __init__(self, model,
'history', # trax.history.History. 'model_state', # Auxilliary state of the model. ]) OptState = collections.namedtuple( '_OptState', [ 'weights', # Model weights. 'slots', # Per-parameter optimizer state, e.g. gradient moments. 'opt_params', # Optimizer (hyper)parameters, e.g. learning rate, momentum. ]) _DEFAULT_METRICS = { 'loss': tl.CrossEntropyLoss(), 'accuracy': tl.Accuracy(), 'sequence_accuracy': tl.SequenceAccuracy(), 'neg_log_perplexity': tl.Serial(tl.CrossEntropyLoss(), tl.Negate()), 'weights_per_batch_per_core': tl.SumOfWeights(), } class Trainer(object): """Trax trainer. A trainer allows to make training steps, train for full epochs, save the training state and access evaluation data. """ def __init__(self, model, loss_fn, optimizer,
Latent_METRICS = { 'next_state_loss': tl.Serial(tl.Select([0, 1, 9]), tl.WeightedCategoryCrossEntropy()), # DropLast()), 'recon_state_loss': tl.Serial(tl.Select([2, 3, 10]), tl.WeightedCategoryCrossEntropy()), 'recon_action_loss': tl.Serial(tl.Select([4, 5, 11]), tl.WeightedCategoryCrossEntropy()), 'next_state_accuracy': tl.Serial(tl.Select([0, 1, 9]), tl.Accuracy()), # DropLast()), 'recon_state_accuracy': tl.Serial(tl.Select([2, 3, 10]), tl.Accuracy()), 'recon_action_accuracy': tl.Serial(tl.Select([4, 5, 11]), tl.Accuracy()), 'next_state_sequence_accuracy': tl.Serial(tl.Select([0, 1, 9]), tl.SequenceAccuracy()), # DropLast()), 'recon_state_sequence_accuracy': tl.Serial(tl.Select([2, 3, 10]), tl.SequenceAccuracy()), 'recon_action_sequence_accuracy': tl.Serial(tl.Select([4, 5, 11]), tl.SequenceAccuracy()), # 'neg_log_perplexity': Serial(WeightedCategoryCrossEntropy(), # Negate()), # 'weights_per_batch_per_core': Serial(tl.Drop(), Drop(), Sum()), } @gin.configurable def latent_fn(): return Latent_METRICS