def test_ctc_pseudo_cost_skip_softmax_stability(): LENGTH = 500 BATCHES = 40 CLASSES = 10 N_LABELS = 45 y_hat = T.tensor3('features') input_mask = T.matrix('features_mask') y_hat_mask = input_mask y = T.lmatrix('phonemes') y_mask = T.matrix('phonemes_mask') pseudo_cost = ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask, skip_softmax=True) Y_hat = np.asarray(np.random.normal(0, 1, (LENGTH, BATCHES, CLASSES + 1)), dtype=floatX) Y = np.zeros((N_LABELS, BATCHES), dtype='int64') Y[25:, :] = 1 Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX) Y_hat_mask[-5:] = 0 # default blank symbol is the highest class index (3 in this case) Y_mask = np.asarray(np.ones_like(Y), dtype=floatX) Y_mask[30:] = 0 pseudo_grad = T.grad(pseudo_cost.sum(), y_hat) test_grad = pseudo_grad.eval({y_hat: Y_hat, y: Y, y_hat_mask: Y_hat_mask, y_mask: Y_mask}) y_hat_softmax = T.exp(y_hat) / T.exp(y_hat).sum(2)[:, :, None] pseudo_cost2 = ctc_cost.pseudo_cost_old(y, y_hat_softmax, y_mask, y_hat_mask, skip_softmax=False) pseudo_grad2 = T.grad(pseudo_cost2.sum(), y_hat) test_grad2 = pseudo_grad2.eval({y_hat: Y_hat, y: Y, y_hat_mask: Y_hat_mask, y_mask: Y_mask}) testing.assert_almost_equal(test_grad, test_grad2, decimal=4)
def test_ctc_pseudo_cost(): LENGTH = 500 BATCHES = 40 CLASSES = 2 N_LABELS = 45 y_hat = T.tensor3('features') input_mask = T.matrix('features_mask') y_hat_mask = input_mask y = T.lmatrix('phonemes') y_mask = T.matrix('phonemes_mask') pseudo_cost = ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask) Y_hat = np.zeros((LENGTH, BATCHES, CLASSES + 1), dtype=floatX) Y_hat[:, :, 0] = .75 Y_hat[:, :, 1] = .2 Y_hat[:, :, 2] = .05 Y_hat[3, 0, 0] = .3 Y_hat[3, 0, 1] = .4 Y_hat[3, 0, 2] = .3 Y = np.zeros((N_LABELS, BATCHES), dtype='int64') Y[25:, :] = 1 Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX) Y_hat_mask[-5:] = 0 # default blank symbol is the highest class index (3 in this case) Y_mask = np.asarray(np.ones_like(Y), dtype=floatX) Y_mask[30:] = 0 cost = pseudo_cost.eval({y_hat: Y_hat, y: Y, y_hat_mask: Y_hat_mask, y_mask: Y_mask}) pseudo_grad = T.grad(ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask).sum(), y_hat) #test_grad2 = pseudo_grad.eval({y_hat: Y_hat, y: Y, # y_hat_mask: Y_hat_mask, y_mask: Y_mask}) # TODO: write some more meaningful asserts here assert cost.sum() > 0
def test_ctc_pseudo_cost_skip_softmax_stability(): LENGTH = 500 BATCHES = 40 CLASSES = 10 N_LABELS = 45 y_hat = T.tensor3('features') input_mask = T.matrix('features_mask') y_hat_mask = input_mask y = T.lmatrix('phonemes') y_mask = T.matrix('phonemes_mask') pseudo_cost = ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask, skip_softmax=True) Y_hat = np.asarray(np.random.normal(0, 1, (LENGTH, BATCHES, CLASSES + 1)), dtype=floatX) Y = np.zeros((N_LABELS, BATCHES), dtype='int64') Y[25:, :] = 1 Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX) Y_hat_mask[-5:] = 0 # default blank symbol is the highest class index (3 in this case) Y_mask = np.asarray(np.ones_like(Y), dtype=floatX) Y_mask[30:] = 0 pseudo_grad = T.grad(pseudo_cost.sum(), y_hat) test_grad = pseudo_grad.eval({ y_hat: Y_hat, y: Y, y_hat_mask: Y_hat_mask, y_mask: Y_mask }) y_hat_softmax = T.exp(y_hat) / T.exp(y_hat).sum(2)[:, :, None] pseudo_cost2 = ctc_cost.pseudo_cost_old(y, y_hat_softmax, y_mask, y_hat_mask, skip_softmax=False) pseudo_grad2 = T.grad(pseudo_cost2.sum(), y_hat) test_grad2 = pseudo_grad2.eval({ y_hat: Y_hat, y: Y, y_hat_mask: Y_hat_mask, y_mask: Y_mask }) testing.assert_almost_equal(test_grad, test_grad2, decimal=4)
def test_ctc_pseudo_cost(): LENGTH = 500 BATCHES = 40 CLASSES = 2 N_LABELS = 45 y_hat = T.tensor3('features') input_mask = T.matrix('features_mask') y_hat_mask = input_mask y = T.lmatrix('phonemes') y_mask = T.matrix('phonemes_mask') pseudo_cost = ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask) Y_hat = np.zeros((LENGTH, BATCHES, CLASSES + 1), dtype=floatX) Y_hat[:, :, 0] = .75 Y_hat[:, :, 1] = .2 Y_hat[:, :, 2] = .05 Y_hat[3, 0, 0] = .3 Y_hat[3, 0, 1] = .4 Y_hat[3, 0, 2] = .3 Y = np.zeros((N_LABELS, BATCHES), dtype='int64') Y[25:, :] = 1 Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX) Y_hat_mask[-5:] = 0 # default blank symbol is the highest class index (3 in this case) Y_mask = np.asarray(np.ones_like(Y), dtype=floatX) Y_mask[30:] = 0 cost = pseudo_cost.eval({ y_hat: Y_hat, y: Y, y_hat_mask: Y_hat_mask, y_mask: Y_mask }) pseudo_grad = T.grad( ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask).sum(), y_hat) #test_grad2 = pseudo_grad.eval({y_hat: Y_hat, y: Y, # y_hat_mask: Y_hat_mask, y_mask: Y_mask}) # TODO: write some more meaningful asserts here assert cost.sum() > 0