def test_ctc_pseudo_cost_skip_softmax_stability():
LENGTH = 500
BATCHES = 40
CLASSES = 10
N_LABELS = 45
y_hat = T.tensor3('features')
input_mask = T.matrix('features_mask')
y_hat_mask = input_mask
y = T.lmatrix('phonemes')
y_mask = T.matrix('phonemes_mask')
pseudo_cost = ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask,
skip_softmax=True)
Y_hat = np.asarray(np.random.normal(0, 1, (LENGTH, BATCHES, CLASSES + 1)),
dtype=floatX)
Y = np.zeros((N_LABELS, BATCHES), dtype='int64')
Y[25:, :] = 1
Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX)
Y_hat_mask[-5:] = 0
# default blank symbol is the highest class index (3 in this case)
Y_mask = np.asarray(np.ones_like(Y), dtype=floatX)
Y_mask[30:] = 0
pseudo_grad = T.grad(pseudo_cost.sum(), y_hat)
test_grad = pseudo_grad.eval({y_hat: Y_hat, y: Y,
y_hat_mask: Y_hat_mask, y_mask: Y_mask})
y_hat_softmax = T.exp(y_hat) / T.exp(y_hat).sum(2)[:, :, None]
pseudo_cost2 = ctc_cost.pseudo_cost_old(y, y_hat_softmax, y_mask, y_hat_mask,
skip_softmax=False)
pseudo_grad2 = T.grad(pseudo_cost2.sum(), y_hat)
test_grad2 = pseudo_grad2.eval({y_hat: Y_hat, y: Y,
y_hat_mask: Y_hat_mask, y_mask: Y_mask})
testing.assert_almost_equal(test_grad, test_grad2, decimal=4)
def test_ctc_pseudo_cost():
LENGTH = 500
BATCHES = 40
CLASSES = 2
N_LABELS = 45
y_hat = T.tensor3('features')
input_mask = T.matrix('features_mask')
y_hat_mask = input_mask
y = T.lmatrix('phonemes')
y_mask = T.matrix('phonemes_mask')
pseudo_cost = ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask)
Y_hat = np.zeros((LENGTH, BATCHES, CLASSES + 1), dtype=floatX)
Y_hat[:, :, 0] = .75
Y_hat[:, :, 1] = .2
Y_hat[:, :, 2] = .05
Y_hat[3, 0, 0] = .3
Y_hat[3, 0, 1] = .4
Y_hat[3, 0, 2] = .3
Y = np.zeros((N_LABELS, BATCHES), dtype='int64')
Y[25:, :] = 1
Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX)
Y_hat_mask[-5:] = 0
# default blank symbol is the highest class index (3 in this case)
Y_mask = np.asarray(np.ones_like(Y), dtype=floatX)
Y_mask[30:] = 0
cost = pseudo_cost.eval({y_hat: Y_hat, y: Y,
y_hat_mask: Y_hat_mask, y_mask: Y_mask})
pseudo_grad = T.grad(ctc_cost.pseudo_cost_old(y, y_hat,
y_mask, y_hat_mask).sum(),
y_hat)
#test_grad2 = pseudo_grad.eval({y_hat: Y_hat, y: Y,
# y_hat_mask: Y_hat_mask, y_mask: Y_mask})
# TODO: write some more meaningful asserts here
assert cost.sum() > 0
def test_ctc_pseudo_cost_skip_softmax_stability():
LENGTH = 500
BATCHES = 40
CLASSES = 10
N_LABELS = 45
y_hat = T.tensor3('features')
input_mask = T.matrix('features_mask')
y_hat_mask = input_mask
y = T.lmatrix('phonemes')
y_mask = T.matrix('phonemes_mask')
pseudo_cost = ctc_cost.pseudo_cost_old(y,
y_hat,
y_mask,
y_hat_mask,
skip_softmax=True)
Y_hat = np.asarray(np.random.normal(0, 1, (LENGTH, BATCHES, CLASSES + 1)),
dtype=floatX)
Y = np.zeros((N_LABELS, BATCHES), dtype='int64')
Y[25:, :] = 1
Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX)
Y_hat_mask[-5:] = 0
# default blank symbol is the highest class index (3 in this case)
Y_mask = np.asarray(np.ones_like(Y), dtype=floatX)
Y_mask[30:] = 0
pseudo_grad = T.grad(pseudo_cost.sum(), y_hat)
test_grad = pseudo_grad.eval({
y_hat: Y_hat,
y: Y,
y_hat_mask: Y_hat_mask,
y_mask: Y_mask
})
y_hat_softmax = T.exp(y_hat) / T.exp(y_hat).sum(2)[:, :, None]
pseudo_cost2 = ctc_cost.pseudo_cost_old(y,
y_hat_softmax,
y_mask,
y_hat_mask,
skip_softmax=False)
pseudo_grad2 = T.grad(pseudo_cost2.sum(), y_hat)
test_grad2 = pseudo_grad2.eval({
y_hat: Y_hat,
y: Y,
y_hat_mask: Y_hat_mask,
y_mask: Y_mask
})
testing.assert_almost_equal(test_grad, test_grad2, decimal=4)
def test_ctc_pseudo_cost():
LENGTH = 500
BATCHES = 40
CLASSES = 2
N_LABELS = 45
y_hat = T.tensor3('features')
input_mask = T.matrix('features_mask')
y_hat_mask = input_mask
y = T.lmatrix('phonemes')
y_mask = T.matrix('phonemes_mask')
pseudo_cost = ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask)
Y_hat = np.zeros((LENGTH, BATCHES, CLASSES + 1), dtype=floatX)
Y_hat[:, :, 0] = .75
Y_hat[:, :, 1] = .2
Y_hat[:, :, 2] = .05
Y_hat[3, 0, 0] = .3
Y_hat[3, 0, 1] = .4
Y_hat[3, 0, 2] = .3
Y = np.zeros((N_LABELS, BATCHES), dtype='int64')
Y[25:, :] = 1
Y_hat_mask = np.ones((LENGTH, BATCHES), dtype=floatX)
Y_hat_mask[-5:] = 0
# default blank symbol is the highest class index (3 in this case)
Y_mask = np.asarray(np.ones_like(Y), dtype=floatX)
Y_mask[30:] = 0
cost = pseudo_cost.eval({
y_hat: Y_hat,
y: Y,
y_hat_mask: Y_hat_mask,
y_mask: Y_mask
})
pseudo_grad = T.grad(
ctc_cost.pseudo_cost_old(y, y_hat, y_mask, y_hat_mask).sum(), y_hat)
#test_grad2 = pseudo_grad.eval({y_hat: Y_hat, y: Y,
# y_hat_mask: Y_hat_mask, y_mask: Y_mask})
# TODO: write some more meaningful asserts here
assert cost.sum() > 0
