コード例 #1
0
def get_prediction(model, sentence, save_dir, mark='Eval', verbose=False):
    """Evaluate the model on `steps` batches."""
    # set model to evaluation mode
    model.eval()

    idx2lbl = load_obj(save_dir + "idx2lbl.json")
    idx2cls = load_obj(save_dir + "idx2cls.json")

    enc = sentence.to(device)
    enc_self_attn_mask = get_attn_pad_mask(enc, enc)
    enc_self_attn_mask.to(device)

    # get results from model
    logits_tgt, logits_clsf = model(enc, enc_self_attn_mask)

    # get sentence length
    pad_num = enc.data.eq(0).sum(axis=1)

    score_cls, cls_idx = torch.max(logits_clsf, dim=-1)
    pred_cls = cls_idx[0].data.tolist()

    # get valid slot for a specific intent
    idx_mask = load_mask(save_dir)

    masked_logits_tgt = softmax_mask(logits_tgt, cls_idx, idx_mask)
    score_tgt, tgt_idx = torch.max(masked_logits_tgt, dim=-1)

    pred_tags = tgt_idx[0, 0:-pad_num].data.tolist()

    pred_lbls = []
    for idx in pred_tags:
        pred_lbls.append(idx2lbl[str(idx)])
    pred_cls = idx2cls[str(pred_cls)]

    return pred_cls, pred_lbls
コード例 #2
0
    pred_cls, pred_lbls = test(model,
                               test_data,
                               args.save_dir,
                               mark='Test',
                               verbose=True)
    # Format and print response sentence
    # pred_tags[:] = [x for x in pred_tags if x != 'PAD']
    # print('{0}\n{1}'.format(input_sentence, ' '.join(pred_lbls)).strip(), flush=True)
    # print('intent:', pred_cls)
    pretty_print(tokens, pred_lbls, pred_cls)
    # exit()

    import onnx
    model = model
    enc = test_data.to(device)
    enc_self_attn_mask = get_attn_pad_mask(enc, enc)
    enc_self_attn_mask.to(device)
    x = (enc, enc_self_attn_mask)
    # get results from model
    logits_tgt, logits_clsf = model(enc, enc_self_attn_mask)
    # logits_tgt, logits_clsf = model(enc,enc_self_attn_mask)
    torch.onnx.export(
        model,  # model being run
        x,  # model input (or a tuple for multiple inputs)
        args.onnx_dir +
        "transformer_mix.onnx",  # where to save the model (can be a file or file-like object)
        export_params=
        True,  # store the trained parameter weights inside the model file
        opset_version=10,  # the ONNX version to export the model to
        do_constant_folding=
        True,  # whether to execute constant folding for optimization
コード例 #3
0
def evaluate_f1_no_mask(model, dl_test, save_dir, criterion_clsf = nn.CrossEntropyLoss().to(device), criterion_tgt = nn.CrossEntropyLoss(ignore_index=PAD).to(device), verbose = False):
    loss_test = 0
    pred_tags = []
    true_tags = []

    pred_clss = []
    true_clss = []
    criterion_clsf = criterion_clsf
    criterion_tgt = criterion_tgt
    idx2lbl = load_obj(save_dir+'idx2lbl.json')
    for enc, tgt, cls in dl_test[:]:
        model.eval()
        with torch.no_grad():
            enc = enc.to(device)
            tgt = tgt.to(device)
            cls = cls.to(device)
            enc_self_attn_mask = get_attn_pad_mask(enc, enc)
            enc_self_attn_mask.to(device)

            logits_tgt, logits_clsf = model(enc,enc_self_attn_mask)
            loss_tgt = criterion_tgt(logits_tgt.transpose(1, 2), tgt) # for masked LM
            loss_tgt = (loss_tgt.float()).mean()
            loss_clsf = criterion_clsf(logits_clsf, cls)# for sentence classification
            loss = loss_clsf + loss_tgt
            # loss = loss_clsf
            loss_test+=loss

        pad_mask = enc.data.eq(0).sum(axis = 1)

        score_tgt, tgt_idx = torch.max(logits_tgt,dim = -1)
        score_cls, cls_idx = torch.max(logits_clsf, dim = -1)

        for pre, true, pad_num in zip(tgt_idx, tgt, pad_mask):
            pred_tags += pre[0:-pad_num].data.tolist()
            true_tags += true[0:-pad_num].data.tolist()

        # print(cls_idx.size())
        pred_clss += cls_idx.tolist()
        true_clss += cls.tolist()
        # print(len(pred_tags), len(true_tags))
        # print(pred_tags)
        # print(true_tags)
        # print(len(pred_clss), len(true_clss))
        # print(pred_clss)

        # print(true_clss)
        assert len(pred_tags) == len(true_tags)
        assert len(pred_clss) == len(true_clss)
    # print(pred_clss[-20:])
    # print(true_clss[-20:])
    # print(pred_tags[-20:])
    # print(true_tags[-20:])

    # print(enc[-20:])

    f1_tgt = f1_score(pred_tags, true_tags, average='micro')
    f1_cls = f1_score(pred_clss, true_clss, average='micro')

    # logging loss, f1 and report

    metrics = {}
    true_lbls = []
    pred_lbls = []

    for t,p in zip(true_tags,pred_tags):
        true_lbls.append(idx2lbl[str(t)])
        pred_lbls.append(idx2lbl[str(p)])

    f1_tgt_merged = f1_score_merged(true_lbls, pred_lbls)

    if verbose:
        report = classification_report(true_lbls, pred_lbls)
        print("============no_mask_slot================")
        print(report, flush=True)

    return loss_test/len(dl_test), f1_cls*100, f1_tgt*100, f1_tgt_merged
コード例 #4
0
def generate_report_txt(
        model,
        dl_test,
        save_dir,
        criterion_clsf=nn.CrossEntropyLoss().to(device),
        criterion_tgt=nn.CrossEntropyLoss(ignore_index=PAD).to(device),
        verbose=False):
    loss_test = 0
    pred_tags = []
    true_tags = []

    pred_clss = []
    true_clss = []
    criterion_clsf = criterion_clsf
    criterion_tgt = criterion_tgt

    idx2lbl = load_obj(save_dir + 'idx2lbl.json')
    idx2cls = load_obj(save_dir + "idx2cls.json")

    sents = load_obj(save_dir + "TestDataSentence.txt")
    for enc, tgt, cls in dl_test[:]:
        model.eval()
        with torch.no_grad():
            enc = enc.to(device)
            tgt = tgt.to(device)
            cls = cls.to(device)
            enc_self_attn_mask = get_attn_pad_mask(enc, enc)
            enc_self_attn_mask.to(device)

            logits_tgt, logits_clsf = model(enc, enc_self_attn_mask)
            loss_tgt = criterion_tgt(logits_tgt.transpose(1, 2),
                                     tgt)  # for masked LM
            loss_tgt = (loss_tgt.float()).mean()
            loss_clsf = criterion_clsf(logits_clsf,
                                       cls)  # for sentence classification
            loss = loss_clsf + loss_tgt
            # loss = loss_clsf
            loss_test += loss

        pad_mask = enc.data.eq(0).sum(axis=1)

        score_cls, cls_idx = torch.max(logits_clsf, dim=-1)
        # get valid slot for a specific intent
        idx_mask = load_mask(save_dir)
        masked_logits_tgt = softmax_mask(logits_tgt, cls_idx, idx_mask)
        score_tgt, tgt_idx = torch.max(masked_logits_tgt, dim=-1)

        for pre, true, pad_num in zip(tgt_idx, tgt, pad_mask):
            pred_tags.append(pre[0:-pad_num].data.tolist())
            true_tags.append(true[0:-pad_num].data.tolist())

        pred_clss += cls_idx.tolist()
        true_clss += cls.tolist()

    print("Prediction completed", flush=True)

    lines_correct = []
    lines_intent_error = []
    lines_slot_error = []
    for idx in range(len(true_clss)):
        tokens = sents[idx].split(' ')
        true_lbls = []
        pred_lbls = []
        true_tags_idx = true_tags[idx]
        pred_tags_idx = pred_tags[idx]
        for t, p in zip(true_tags_idx, pred_tags_idx):
            true_lbls.append(idx2lbl[str(t)])
            pred_lbls.append(idx2lbl[str(p)])

        true_entities = get_entities(true_lbls)
        pred_entities = get_entities(pred_lbls)

        slots_true = []
        slots_pred = []

        for chunk_true, chunk_pred, cls_true, cls_pred in zip(
                true_entities, pred_entities, true_clss, pred_clss):
            tag, start, end = chunk_true[0], chunk_true[1], chunk_true[2]
            tok = ''.join(tokens[start:end + 1])
            slot_true = '<{0}>: {1}'.format(tag, tok)
            slots_true.append(slot_true)

            tag, start, end = chunk_pred[0], chunk_pred[1], chunk_pred[2]
            tok = ''.join(tokens[start:end + 1])
            slot_pred = '<{0}>: {1}'.format(tag, tok)
            slots_pred.append(slot_pred)

        intent_true = idx2cls[str(true_clss[idx])]
        intent_pred = idx2cls[str(pred_clss[idx])]

        line = "Sentence:{0:}\nExpect: \t{1}\t{2}\nPredict:\t{3}\t{4}\n".format(
            sents[idx], intent_true, slots_true, intent_pred, slots_pred)
        if intent_true != intent_pred:
            lines_intent_error.append(line)
        elif slots_true != slots_pred:
            lines_slot_error.append(line)
        else:
            lines_correct.append(line)

    correct_num = len(lines_correct)
    intent_w_num = len(lines_intent_error)
    slot_w_num = len(lines_slot_error)
    total_line = len(lines_intent_error) + len(lines_correct) + len(
        lines_slot_error)

    score1 = 'total line = {0}; Exact match = {1}, with intent fail = {2}, with slot fail = {3};'.format(
        total_line, correct_num, intent_w_num, slot_w_num)
    score2 = 'Accuracy = {0:.4f}'.format(correct_num / total_line)
    scores = [score1, score2]

    # saveing report
    print("Saving reports...", flush=True)
    report_dir = os.path.join(save_dir, 'reports', '')
    create_dir(report_dir)

    remove_old_file(report_dir + 'reports_correct.txt')
    remove_old_file(report_dir + 'reports_intent_error.txt')
    remove_old_file(report_dir + 'reports_slot_error.txt')
    remove_old_file(report_dir + 'scores.txt')

    with open(report_dir + 'reports_correct.txt', 'w', encoding='utf-8') as f:
        for line in lines_correct:
            f.write("{0}".format(line + '\n'))
    with open(report_dir + 'reports_intent_error.txt', 'w',
              encoding='utf-8') as f:
        for line in lines_intent_error:
            f.write("{0}".format(line + '\n'))
    with open(report_dir + 'reports_slot_error.txt', 'w',
              encoding='utf-8') as f:
        for line in lines_slot_error:
            f.write("{0}".format(line + '\n'))
    with open(report_dir + 'scores.txt', 'w', encoding='utf-8') as f:
        for line in scores:
            f.write("{0}".format(line + '\n'))