예제 #1
0
def eval(epoch,
         config,
         model,
         validating_data,
         logger,
         dev_dataset,
         visualizer=None):
    model.eval()
    total_loss = 0
    total_dist = 0
    total_word = 0
    batch_steps = len(validating_data)
    for step, (inputs, targets, origin_length, inputs_length,
               targets_length) in enumerate(validating_data):

        if config.training.num_gpu > 0:
            inputs, inputs_length = inputs.cuda(), inputs_length.cuda()
            targets, targets_length = targets.cuda(), targets_length.cuda()

        max_inputs_length = origin_length.max().item()
        max_targets_length = targets_length.max().item()
        inputs = inputs[:, :max_inputs_length, :]
        targets = targets[:, :max_targets_length]

        preds = model.recognize(inputs, inputs_length)
        transcripts = [
            targets.cpu().numpy()[i][:targets_length[i].item()]
            for i in range(targets.size(0))
        ]

        print(''.join(
            [dev_dataset.index2word.get(index) for index in preds[0]]))
        print(''.join(
            [dev_dataset.index2word.get(index) for index in transcripts[0]]))
        dist, num_words = computer_cer(preds, transcripts)
        total_dist += dist
        total_word += num_words

        cer = total_dist / total_word * 100
        if step % config.training.show_interval == 0:
            process = step / batch_steps * 100
            logger.info('-Validation-Epoch:%d(%.5f%%), CER: %.5f %%' %
                        (epoch, process, cer))

    val_loss = total_loss / batch_steps
    logger.info(
        '-Validation-Epoch:%4d, AverageLoss:%.5f, AverageCER: %.5f %%' %
        (epoch, val_loss, cer))

    if visualizer is not None:
        visualizer.add_scalar('cer', cer, epoch)

    return cer
예제 #2
0
def test(config, model, test_dataset, validate_data, logger):
    model.eval()
    total_dist = 0
    total_word = 0
    batch_steps = len(validate_data)
    for step, (inputs, targets, origin_length, inputs_length,
               targets_length) in enumerate(validate_data):
        if config.training.num_gpu > 0:
            inputs, inputs_length = inputs.cuda(), inputs_length.cuda()
            targets, targets_length = targets.cuda(), targets_length.cuda()

        max_inputs_length = inputs_length.max().item()
        max_targets_length = targets_length.max().item()
        # inputs = inputs[:, :max_inputs_length, :]
        targets = targets[:, :max_targets_length]
        preds = model.recognize(inputs, inputs_length)
        transcripts = [
            targets.cpu().numpy()[i][:targets_length[i].item()]
            for i in range(targets.size(0))
        ]
        print(''.join(
            [test_dataset.index2word.get(index) for index in preds[0]]))
        print(''.join(
            [test_dataset.index2word.get(index) for index in transcripts[0]]))

        dist, num_words = computer_cer(preds, transcripts)
        total_dist += dist
        total_word += num_words

        cer = total_dist / total_word * 100
        if step % config.training.show_interval == 0:
            process = step / batch_steps * 100
            logger.info('-Validation-Epoch:%d(%.5f%%), CER: %.5f %%' %
                        (1, process, cer))

    logger.info('-Validation-Epoch:%4d, AverageCER: %.5f %%' % (1, cer))
    return cer
예제 #3
0
                        if l != previous:
                            new_labels.append(l)
                            previous = l
                    # 删除blank
                    new_labels = [l for l in new_labels if l != blank]
                    return new_labels

                pred_outs = [remove_blank(pred) for pred in preds]
                targets_y = [remove_blank(label) for label in targets_y]
                print(''.join(
                    dev_dataset.index2word.get(index)
                    for index in pred_outs[0]))
                print(''.join(
                    dev_dataset.index2word.get(index)
                    for index in targets_y[0]))
                diff, total = computer_cer(pred_outs, targets_y)
                # print(diff)
                # print(total)
                total_diff += diff
                total_nums += total
            wer = total_diff / total_nums * 100
            print('ctc model wer : {}%'.format(wer))
            if wer < old_wer:
                old_wer = wer
                print('complete trained model save!')
                save_path = os.path.join(home_dir, 'ctc_model')
                if not os.path.exists(save_path):
                    os.makedirs(save_path)
                torch.save(
                    model.state_dict(),
                    'ctc_model/{}_{:.4f}_enecoder_model'.format(
예제 #4
0
def eval_cer(config, model, validating_data):
    rle_decoder = GreedyDecoder(int2base, blank_index=0)
    base_decoder = GreedyDecoder(int2base, blank_index=0)
    model.eval()
    total_loss = 0
    total_dist = 0
    total_rle_dist = 0
    total_word = 0
    total_rle_word = 0
    total_match, total_insert, total_delete, total_mismatch, total_length = 0, 0, 0, 0, 0
    batch_steps = len(validating_data)
    for step, (regions, inputs, inputs_length, targets, targets_length,
               rle_bases, rle_bases_length, rles,
               rles_length) in enumerate(validating_data):

        if config.training.num_gpu > 0:
            inputs, inputs_length = inputs.cuda(), inputs_length.cuda()
            targets, targets_length = targets.cuda(), targets_length.cuda()
            rle_bases, rle_bases_length, rles, rles_length = rle_bases.cuda(
            ), rle_bases_length.cuda(), rles.cuda(), rles_length.cuda()

        max_inputs_length = inputs_length.max().item()
        max_targets_length = targets_length.max().item()
        max_rle_bases_length = rle_bases_length.max().item()
        max_rles_length = rles_length.max().item()
        inputs = inputs[:, :max_inputs_length, :]  # [N, max_inputs_length, c]
        targets = targets[:, :max_targets_length]  # [N, max_targets_length]
        rle_bases = rle_bases[:, :max_rle_bases_length]
        rles = rles[:, :max_rles_length]

        base_logits, rle_logits = model.recognize(inputs,
                                                  inputs_length)  # [B,L,o]
        pred_base_strings, offset = base_decoder.decode(
            base_logits, inputs_length)
        pred_rle_strings, offset = rle_decoder.decode(rle_logits,
                                                      inputs_length)
        pred_base_strings = [v.upper() for v in pred_base_strings]
        pred_rle_strings = [v.upper() for v in pred_rle_strings]
        # print(pred_base_strings, pred_rle_strings)

        # print('preds')
        # print(preds)

        base_targets = [
            rle_bases.cpu().numpy()[i][:rle_bases_length[i].item()].tolist()
            for i in range(rle_bases.size(0))
        ]
        base_transcripts = []
        for i in range(rle_bases.size(0)):
            rle_target_seq = ""
            for v in base_targets[i]:
                rle_target_seq += int2base[v]
            base_transcripts.append(''.join(rle_target_seq))

        rle_targets = [
            targets.cpu().numpy()[i][:targets_length[i].item()].tolist()
            for i in range(rles.size(0))
        ]
        rle_transcripts = []
        for i in range(targets.size(0)):
            rle_target_seq = ""
            for v in rle_targets[i]:
                rle_target_seq += int2base[v]
            rle_transcripts.append(''.join(rle_target_seq))

        #         # print('transcripts')
        #         # print(transcripts)
        #
        dist, num_words = computer_cer(pred_base_strings, base_transcripts)
        total_dist += dist
        total_word += num_words
        # print('base:',pred_base_strings, transcripts)

        rle_dist, rle_num_words = computer_cer(pred_rle_strings,
                                               rle_transcripts)
        total_rle_dist += rle_dist
        total_rle_word += rle_num_words
        # print('rle:',pred_rle_strings, rle_transcripts)
        # print()

        match, insert, delete, mismatch, batch_length, insert_set = calculate_identity(
            pred_rle_strings, rle_transcripts)
        for idx in insert_set:
            print(regions[idx])
            print("pred :", pred_rle_strings[idx])
            print("label:", rle_transcripts[idx])
            print(inputs.cpu()[idx][:inputs_length[idx]][:, :7])
            print('-' * 10)

        total_match += match
        total_insert += insert
        total_delete += delete
        total_mismatch += mismatch
        total_length += batch_length
        #
        cer = total_dist / total_word * 100
        rle_cer = total_rle_dist / total_rle_word * 100
        if step % config.training.show_interval == 0:
            process = step / batch_steps * 100
            print('-Validation:(%.5f%%), CER: %.5f %%, Rle CER: %.5f %%' %
                  (process, cer, rle_cer))

    val_loss = total_loss / (step + 1)
    print(
        '-Validation:, AverageLoss:%.5f, AverageCER: %.5f %%, AverageRleCER: %.5f %%, Mismatch: %.5f %%, Insertion: %.5f %%, Deletion: %.5f %%'
        %
        (val_loss, cer, rle_cer, total_mismatch / total_length * 100,
         total_insert / total_length * 100, total_delete / total_length * 100))