def evaluation(tfdata_dev, model):
list_loss = []
list_acc = []
start_time = time()
num_processed = 0
progress = 0
total_cer_dist = 0
total_cer_len = 0
for batch in tfdata_dev:
x, y, aligns = batch
logits = model(x, training=False)
loss = model.align_loss(logits, y, args.dim_output, confidence=0.9)
acc = model.align_accuracy(logits, y)
list_loss.append(loss)
list_acc.append(acc)
preds = model.get_predicts(logits)
batch_cer_dist, batch_cer_len = batch_cer(preds.numpy(), y)
total_cer_dist += batch_cer_dist
total_cer_len += batch_cer_len
num_processed += len(x)
progress = num_processed / args.data.dev_size
cer = total_cer_dist / total_cer_len
print(
'dev loss: {:.3f}\t dev FER: {:.3f}\t dev PER: {:.3f}\t {:.2f}min {} / {}'
.format(np.mean(list_loss), 1 - np.mean(list_acc), cer,
(time() - start_time) / 60, num_processed, args.data.dev_size))
return 1 - np.mean(list_acc)
def evaluate(feature, dataset, dev_size, model):
num_processed = 0
total_cer_dist = 0
total_cer_len = 0
total_res_len = 0
for batch in feature:
uttids, x = batch
# import pdb; pdb.set_trace()
# preds = forward(x, model)
logits = model(x)
len_logits = get_logits_len(logits)
preds = ctc_decode(logits, len_logits)
# logits_shrunk = ctc_shrink(logits)
# preds = tf.argmax(logits_shrunk, -1)
trans = dataset.get_attrs('trans', uttids.numpy())
batch_cer_dist, batch_cer_len, batch_res_len = batch_cer(
preds.numpy(), trans)
total_cer_dist += batch_cer_dist
total_cer_len += batch_cer_len
total_res_len += batch_res_len
num_processed += len(x)
sys.stdout.write('\rinfering {} / {} ...'.format(
num_processed, dev_size))
sys.stdout.flush()
cer = total_cer_dist / total_cer_len
print('dev PER: {:.3f}\t{} / {}'.format(cer, num_processed, dev_size))
return cer
def evaluate(feature, dataset, dev_size, assigner, model):
num_processed = 0
total_cer_dist = 0
total_cer_len = 0
total_res_len = 0
for batch in feature:
uttids, x = batch
# trans = dataset.get_attrs('trans', uttids.numpy(), args.max_label_len)
trans = dataset.get_attrs('trans', uttids.numpy())
hidden, _, alpha = assigner(x)
musk = tf.cast(tf.reduce_sum(tf.abs(x), -1) > 0, tf.float32)
alpha *= musk
l = CIF(hidden, alpha, threshold=args.model.attention.threshold)
logits = model(l)
preds = get_predicts(logits)
# len_logits = get_x_len(x)
# preds = ctc_decode(logits1, len_logits)
batch_cer_dist, batch_cer_len, batch_res_len = batch_cer(preds.numpy(), trans)
total_cer_dist += batch_cer_dist
total_cer_len += batch_cer_len
total_res_len += batch_res_len
num_processed += len(x)
cer = total_cer_dist/total_cer_len
over_fire_rate = total_res_len/total_cer_len
print('dev PER: {:.3f}\tover_fire_rate: {:.2f}\t {} / {}'.format(cer, over_fire_rate, num_processed, dev_size))
return cer
def evaluate(feature, dataset, dev_size, encoder, decoder):
num_processed = 0
total_cer_dist = 0
total_cer_len = 0
total_res_len = 0
for batch in feature:
uttids, x = batch
# preds = forward(x, model)
encoded = encoder(x)
logits = decoder(encoded)
len_logits = get_logits_len(logits)
preds = ctc_decode(logits, len_logits)
trans = dataset.get_attrs('trans', uttids.numpy())
batch_cer_dist, batch_cer_len, batch_res_len = batch_cer(
preds.numpy(), trans)
total_cer_dist += batch_cer_dist
total_cer_len += batch_cer_len
total_res_len += batch_res_len
num_processed += len(x)
print('infering {} / {} ...'.format(num_processed, dev_size))
cer = total_cer_dist / total_cer_len
print('dev PER: {:.3f}\t{} / {}'.format(cer, num_processed, dev_size))
return cer
def evaluate(feature, dataset, dev_size, model):
num_processed = 0
total_cer_dist = 0
total_cer_len = 0
total_res_len = 0
for batch in feature:
uttids, x = batch
preds = forward(x, model, args.token2idx['<sos>'],
args.token2idx['<eos>'], args.max_label_len)
trans = dataset.get_attrs('trans', uttids.numpy())
batch_cer_dist, batch_cer_len, batch_res_len = batch_cer(
preds.numpy(), trans)
total_cer_dist += batch_cer_dist
total_cer_len += batch_cer_len
total_res_len += batch_res_len
num_processed += len(x)
print('infering {} / {} ...'.format(num_processed, dev_size))
cer = total_cer_dist / total_cer_len
print('dev PER: {:.3f}\t{} / {}'.format(cer, num_processed, dev_size))
return cer