def test_smoothed_cross_entropy_loss(): alpha = 0.5 vocab_target_size = 4 loss = sockeye.loss.get_loss( sockeye.model.ModelConfig(loss=C.SMOOTHED_CROSS_ENTROPY, vocab_target_size=vocab_target_size, smoothed_cross_entropy_alpha=alpha)) assert isinstance(loss, sockeye.loss.SmoothedCrossEntropyLoss) logits = mx.sym.Variable("logits") labels = mx.sym.Variable("labels") sym = mx.sym.Group(loss.get_loss(logits, labels)) assert sym.list_arguments() == ['labels', 'logits'] assert sym.list_outputs() == [ C.SMOOTHED_CROSS_ENTROPY + "_output", C.SOFTMAX_NAME + "_output" ] logits_np = mx.nd.array([[1, 2, 3, 4], [4, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]) labels_np = mx.nd.array([1, 0, 2, 3]) # C.PAD_ID == 0 expected_softmax = np.asarray( [[0.0320586, 0.08714432, 0.23688284, 0.64391428], [0.71123451, 0.09625512, 0.09625512, 0.09625512], [0.25, 0.25, 0.25, 0.25], [0.25, 0.25, 0.25, 0.25]]) expected_cross_entropy = np.asarray( [2.10685635, 0., 1.38629436, 1.38629436]) expected_grads = np.asarray( [[-0.13460806, -0.41285568, 0.07021617, 0.4772476], [0., 0., 0., 0.], [0.08333333, 0.08333333, -0.25, 0.08333333], [0.08333333, 0.08333333, 0.08333333, -0.25]]) _, out_shapes, _ = (sym.infer_shape(logits=logits_np.shape, labels=labels_np.shape)) assert len(out_shapes) == 2 assert out_shapes[0] == (4, ) assert out_shapes[1] == logits_np.shape executor = sym.simple_bind(ctx=mx.cpu(), logits=logits_np.shape, labels=labels_np.shape) executor.arg_dict["logits"][:] = logits_np executor.arg_dict["labels"][:] = labels_np outputs = executor.forward(is_train=True) smoothed_cross_entropy = outputs[0].asnumpy() softmax = outputs[1].asnumpy() assert np.isclose(softmax, expected_softmax).all() assert np.isclose(smoothed_cross_entropy, expected_cross_entropy).all() executor.backward() grads = executor.grad_dict["logits"].asnumpy() assert np.isclose(grads, expected_grads).all() label_grad_sum = executor.grad_dict["labels"].asnumpy().sum() assert label_grad_sum == 0
def sym_gen(seq_lens): """ Returns a (grouped) loss symbol given source & target input lengths. Also returns data and label names for the BucketingModule. """ source_seq_len, target_seq_len = seq_lens source_encoded = self.encoder.encode(source, source_length, seq_len=source_seq_len) source_lexicon = self.lexicon.lookup(source) if self.lexicon else None logits = self.decoder.decode(source_encoded, source_seq_len, source_length, target, target_seq_len, source_lexicon) outputs = loss.get_loss(logits, labels) return mx.sym.Group(outputs), data_names, label_names
def test_smoothed_cross_entropy_loss(): config = sockeye.loss.LossConfig(name=C.CROSS_ENTROPY, vocab_size=4, normalization_type=C.LOSS_NORM_BATCH, label_smoothing=0.5) loss = sockeye.loss.get_loss(config) assert isinstance(loss, sockeye.loss.CrossEntropyLoss) logits = mx.sym.Variable("logits") labels = mx.sym.Variable("labels") sym = mx.sym.Group(loss.get_loss(logits, labels)) assert sym.list_arguments() == ['logits', 'labels'] assert sym.list_outputs() == [C.SOFTMAX_NAME + "_output"] logits_np = mx.nd.array([[1, 2, 3, 4], [4, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]) labels_np = mx.nd.array([1, 0, 2, 3]) # C.PAD_ID == 0 expected_softmax = np.asarray([[0.0320586, 0.08714432, 0.23688284, 0.64391428], [0.71123451, 0.09625512, 0.09625512, 0.09625512], [0.25, 0.25, 0.25, 0.25], [0.25, 0.25, 0.25, 0.25]]) expected_grads = np.asarray([[-0.13460806, -0.41285568, 0.07021617, 0.4772476], [0., 0., 0., 0.], [0.08333333, 0.08333333, -0.25, 0.08333333], [0.08333333, 0.08333333, 0.08333333, -0.25]]) _, out_shapes, _ = (sym.infer_shape(logits=logits_np.shape, labels=labels_np.shape)) assert out_shapes[0] == logits_np.shape executor = sym.simple_bind(ctx=mx.cpu(), logits=logits_np.shape, labels=labels_np.shape) executor.arg_dict["logits"][:] = logits_np executor.arg_dict["labels"][:] = labels_np outputs = executor.forward(is_train=True) softmax = outputs[0].asnumpy() assert np.isclose(softmax, expected_softmax).all() executor.backward() grads = executor.grad_dict["logits"].asnumpy() assert np.isclose(grads, expected_grads).all() label_grad_sum = executor.grad_dict["labels"].asnumpy().sum() assert label_grad_sum == 0
def test_smoothed_cross_entropy_loss(): config = sockeye.loss.LossConfig(name=C.CROSS_ENTROPY, vocab_size=4, normalization_type=C.LOSS_NORM_BATCH, label_smoothing=0.5) loss = sockeye.loss.get_loss(config) assert isinstance(loss, sockeye.loss.CrossEntropyLoss) logits = mx.sym.Variable("logits") labels = mx.sym.Variable("labels") sym = mx.sym.Group(loss.get_loss(logits, labels)) assert sym.list_arguments() == ['logits', 'labels'] assert sym.list_outputs() == [C.SOFTMAX_NAME + "_output"] logits_np = mx.nd.array([[1, 2, 3, 4], [4, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]) labels_np = mx.nd.array([1, 0, 2, 3]) # C.PAD_ID == 0 expected_softmax = np.asarray([[0.0320586, 0.08714432, 0.23688284, 0.64391428], [0.71123451, 0.09625512, 0.09625512, 0.09625512], [0.25, 0.25, 0.25, 0.25], [0.25, 0.25, 0.25, 0.25]]) expected_grads = np.asarray([[-0.13460806, -0.41285568, 0.07021617, 0.4772476], [0., 0., 0., 0.], [0.08333333, 0.08333333, -0.25, 0.08333333], [0.08333333, 0.08333333, 0.08333333, -0.25]]) _, out_shapes, _ = (sym.infer_shape(logits=logits_np.shape, labels=labels_np.shape)) assert out_shapes[0] == logits_np.shape executor = sym.simple_bind(ctx=mx.cpu(), logits=logits_np.shape, labels=labels_np.shape) executor.arg_dict["logits"][:] = logits_np executor.arg_dict["labels"][:] = labels_np outputs = executor.forward(is_train=True) softmax = outputs[0].asnumpy() assert np.isclose(softmax, expected_softmax).all() executor.backward() grads = executor.grad_dict["logits"].asnumpy() assert np.isclose(grads, expected_grads).all() label_grad_sum = executor.grad_dict["labels"].asnumpy().sum() assert label_grad_sum == 0
def test_cross_entropy_loss(): loss = sockeye.loss.get_loss( sockeye.model.ModelConfig(loss=C.CROSS_ENTROPY)) assert isinstance(loss, sockeye.loss.CrossEntropyLoss) logits = mx.sym.Variable("logits") labels = mx.sym.Variable("labels") sym = mx.sym.Group(loss.get_loss(logits, labels)) assert sym.list_arguments() == ['logits', 'labels'] assert sym.list_outputs() == [C.SOFTMAX_NAME + "_output"] logits_np = mx.nd.array([[1, 2, 3, 4], [4, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]) labels_np = mx.nd.array([1, 0, 2, 3]) # C.PAD_ID == 0 expected_softmax = np.asarray( [[0.0320586, 0.08714432, 0.23688284, 0.64391428], [0.71123451, 0.09625512, 0.09625512, 0.09625512], [0.25, 0.25, 0.25, 0.25], [0.25, 0.25, 0.25, 0.25]]) expected_grads = np.asarray( [[0.0320586, -0.91285568, 0.23688284, 0.64391428], [0., 0., 0., 0.], [0.25, 0.25, -0.75, 0.25], [0.25, 0.25, 0.25, -0.75]]) _, out_shapes, _ = (sym.infer_shape(logits=logits_np.shape, labels=labels_np.shape)) assert out_shapes[0] == logits_np.shape executor = sym.simple_bind(ctx=mx.cpu(), logits=logits_np.shape, labels=labels_np.shape) executor.arg_dict["logits"][:] = logits_np executor.arg_dict["labels"][:] = labels_np softmax = executor.forward(is_train=True)[0].asnumpy() assert np.isclose(softmax, expected_softmax).all() executor.backward() grads = executor.grad_dict["logits"].asnumpy() assert np.isclose(grads, expected_grads).all() label_grad_sum = executor.grad_dict["labels"].asnumpy().sum() assert label_grad_sum == 0