Esempio n. 1
0
def train(model, num_epochs=10, lr=0.0003, print_every=100):
    """Train a model on IWSLT"""

    if params['USE_CUDA']:
        model.cuda()

    # optionally add label smoothing; see the Annotated Transformer
    criterion = nn.NLLLoss(reduction="sum", ignore_index=PAD_INDEX)
    optim = torch.optim.Adam(model.parameters(), lr=lr)

    dev_perplexities = []

    for epoch in range(num_epochs):

        print("Epoch", epoch)
        model.train()
        train_perplexity = run_epoch(
            (rebatch(PAD_INDEX, b) for b in train_iter),
            model,
            SimpleLossCompute(model.generator, criterion, optim),
            print_every=print_every)

        model.eval()
        with torch.no_grad():
            print_examples((rebatch(PAD_INDEX, x) for x in valid_iter),
                           model,
                           n=3,
                           src_vocab=SRC.vocab,
                           trg_vocab=TRG.vocab)

            dev_perplexity = run_epoch(
                (rebatch(PAD_INDEX, b) for b in valid_iter), model,
                SimpleLossCompute(model.generator, criterion, None))
            print("Validation perplexity: %f" % dev_perplexity)
            dev_perplexities.append(dev_perplexity)

    return dev_perplexities
Esempio n. 2
0
def run_lrp(data_iter, vocab, model_file):
    net = BiLSTM(model_file=model_file)
    method = 'all'
    linear = 'abs'
    if linear == 'zero':
        alpha = 2
    else:
        alpha = 1

    print("LRP - ", method)
    print("Linear - ", linear)
    print("alpha - ", alpha)
    num_del = 1

    while num_del <= 5:
        iterator = (rebatch(PAD_INDEX, b) for b in data_iter)
        deletion_iter(num_del, method, linear, alpha, iterator, net)
        num_del += 1
Esempio n. 3
0
def run_lrp(data_iter, vocab, model_file):
	net = BiLSTM(model_file=model_file)
	vocab_size = len(vocab)
	method = 'abs'
	linear = 'epsilon'
	if linear == 'zero':
		alpha = 2
	else:
		alpha = 1

	print("LRP - ", method)
	print("Linear - ", linear)
	print("alpha - ", alpha)
	num_replace = 1
	iters = 8
	result_true, result_false = {}, {}

	while num_replace <= 5:
		result_true[num_replace] = []
		result_false[num_replace] = []

		for it in range(iters):
			iterator = (rebatch(PAD_INDEX, b) for b in data_iter)
			count1, count2, correct_total, incorrect_total = replace_iter(num_replace, method, linear, alpha, iterator, net, vocab_size)
			result_true[num_replace].append(count1)
			result_false[num_replace].append(count2)

		num_replace += 1


	print("Initially correct: ", result_true)
	print("Initially false: ", result_false)

	count_true, count_false = 0, 0
	for k in result_true:
		count_true += sum(result_true[k]) / len(result_true[k])
		count_false += sum(result_false[k]) / len(result_false[k])

		print("Number of replacement: ", k)
		print("Accuracy (initially correct): {0:0.2f}".format(count_true/correct_total*100))
		print("Accuracy (initially false): {0:0.2f}".format(count_false/incorrect_total*100))

		count_true, count_false = 0, 0
def run_test(model, test_data, test_iter):
    preds = []

    for k, batch in enumerate(test_iter):
        model.eval()
        with torch.no_grad():
            batch = rebatch(PAD_INDEX, batch)
            pred, attn = greedy_decode(model,
                                       batch.src,
                                       batch.src_mask,
                                       batch.src_lengths,
                                       max_len=25,
                                       sos_index=TRG.vocab.stoi[SOS_TOKEN],
                                       eos_index=TRG.vocab.stoi[EOS_TOKEN])

            preds.append(pred)

    hypotheses = [lookup_words(pred, TRG.vocab) for pred in preds]
    hypotheses = [" ".join(h) for h in hypotheses]
    references = [" ".join(data.trg) for data in test_data]
    bleu = sacrebleu.raw_corpus_bleu(hypotheses, [references], .01).score
    print("BLEU score: ", bleu)

    return bleu
def run_lrp(data_iter, vocab, model_file):
    net = BiLSTM(model_file=model_file)
    method = 'all'
    # print("LRP - ", method)
    num_del = 1
    iters = 8
    result_true, result_false = {}, {}

    while num_del <= 5:
        result_true[num_del] = []
        result_false[num_del] = []

        for it in range(iters):
            iterator = (rebatch(PAD_INDEX, b) for b in data_iter)
            count1, count2, correct_total, incorrect_total = deletion_iter(
                num_del, method, iterator, net)
            result_true[num_del].append(count1)
            result_false[num_del].append(count2)

        num_del += 1

    print("Initially correct: ", result_true)
    print("Initially false: ", result_false)

    count_true, count_false = 0, 0
    for k in result_true:
        count_true += sum(result_true[k]) / len(result_true[k])
        count_false += sum(result_false[k]) / len(result_false[k])

        print("Number of deletion: ", k)
        print("Accuracy (initially correct): {0:0.2f}".format(
            count_true / correct_total * 100))
        print("Accuracy (initially false): {0:0.2f}".format(
            count_false / incorrect_total * 100))

        count_true, count_false = 0, 0
Esempio n. 6
0
            f.write("%s\n" % i)
    with open("result/valid_loss.txt", "w") as f:
        for i in valid_losses:
            f.write("%s\n" % i)
    with open("result/valid_prec.txt", "w") as f:
        for i in valid_prec_score:
            f.write("%s\n" % i)
    with open("result/valid_recall.txt", "w") as f:
        for i in valid_recall_score:
            f.write("%s\n" % i)
    with open("result/valid_f1.txt", "w") as f:
        for i in valid_f1_score:
            f.write("%s\n" % i)

    print("Finish training ... ")
    print_examples((rebatch(x, SRC_PAD_INDEX, TRG_PAD_INDEX, TRG_UNK_INDEX,
                            len(TRG.vocab), SRC.vocab) for x in test_iter),
                   model,
                   num=params['TEST_EXAMPLE_TO_PRINT'],
                   src_vocab=SRC.vocab,
                   trg_vocab=TRG.vocab)

    ########################
    print("Start testing ... ")
    test_f1_score, test_prec_score, test_recall_score = run_test(
        (rebatch(x, SRC_PAD_INDEX, TRG_PAD_INDEX, TRG_UNK_INDEX, len(
            TRG.vocab), SRC.vocab) for x in test_iter),
        model,
        trg_vocab=TRG.vocab)

    print("test precision score: ", test_prec_score)
    print("test recall score: ", test_recall_score)
Esempio n. 7
0
def german_english_translation():
    print("german_english_translation task")
    SRC, TGT, train, val, test = data_help.data_load()

    # GPUs to use
    devices = [0, 1]
    pad_idx = TGT.vocab.stoi["<blank>"]
    print("Size:", len(SRC.vocab), len(TGT.vocab))
    model = model_help.make_model(len(SRC.vocab), len(TGT.vocab),
                                  N=6).to(args.device)
    print("+===============+")
    criterion = model_help.LabelSmoothing(size=len(TGT.vocab),
                                          padding_idx=pad_idx,
                                          smoothing=0.1).to(args.device)
    BATCH_SIZE = 12000
    train_iter = data_help.MyIterator(train,
                                      batch_size=BATCH_SIZE,
                                      device=devices[0],
                                      repeat=False,
                                      sort_key=lambda x:
                                      (len(x.src), len(x.trg)),
                                      batch_size_fn=batch_size_fn,
                                      train=True)
    valid_iter = data_help.MyIterator(val,
                                      batch_size=BATCH_SIZE,
                                      device=devices[0],
                                      repeat=False,
                                      sort_key=lambda x:
                                      (len(x.src), len(x.trg)),
                                      batch_size_fn=batch_size_fn,
                                      train=False)
    model_par = nn.DataParallel(model, device_ids=devices)

    model_opt = model_help.NoamOpt(
        model.src_embed[0].d_model, 1, 2000,
        torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),
                         eps=1e-9))
    for epoch in range(10):
        model_par.train()
        run_epoch((data_help.rebatch(pad_idx, b) for b in train_iter),
                  model_par,
                  model_help.MultiGPULossCompute(model.generator,
                                                 criterion,
                                                 devices=devices,
                                                 opt=model_opt))
        model_par.eval()
        loss = run_epoch((data_help.rebatch(pad_idx, b) for b in valid_iter),
                         model_par,
                         model_help.MultiGPULossCompute(model.generator,
                                                        criterion,
                                                        devices=devices,
                                                        opt=None))
        print(loss)
    """Once trained we can decode the model to produce a set of translations. 
    Here we simply translate the first sentence in the validation set. This dataset 
    is pretty small so the translations with greedy search are reasonably accurate."""
    for i, batch in enumerate(valid_iter):
        src = batch.src.transpose(0, 1)[:1]
        src_mask = (src != SRC.vocab.stoi["<blank>"]).unsqueeze(-2)
        out = greedy_decode(model,
                            src,
                            src_mask,
                            max_len=60,
                            start_symbol=TGT.vocab.stoi["<s>"])
        print("Translation:", end="\t")
        for i in range(1, out.size(1)):
            sym = TGT.vocab.itos[out[0, i]]
            if sym == "</s>": break
            print(sym, end=" ")
        print()
        print("Target:", end="\t")
        for i in range(1, batch.trg.size(0)):
            sym = TGT.vocab.itos[batch.trg.data[i, 0]]
            if sym == "</s>": break
            print(sym, end=" ")
        print()
        break
Esempio n. 8
0
	model = make_model(SRC.vocab, output_class=3, embed_size=params['EMBEDDING_DIM'], 
		hidden_size=params['HIDDEN_SIZE'], num_layers=params['NUM_LAYERS'], 
		dropout=params['DROPOUT_PROB'])

	model.apply(init_weight)
	#############################

	print("Start training ... ")
	# print(SRC.vocab.itos[1], SRC.vocab.itos[2], SRC.vocab.itos[3], SRC.vocab.itos[4])

	train_losses, valid_losses, train_ac, valid_ac = train(model, num_epochs=params['NUM_EPOCHS'], learning_rate=params['LEARNING_RATE'])

	print("Start test ... ")

	print_examples((rebatch(PAD_INDEX, b) for b in test_iter), model, n=5, src_vocab=SRC.vocab)
	# sos_idx=SOS_INDEX, eos_idx=EOS_INDEX, 
	print()

	test_accuracy = run_test((rebatch(PAD_INDEX, b) for b in test_iter), model)

	print('Test accuracy: ', test_accuracy)
		







Esempio n. 9
0
    # train_iter = data.Iterator(train_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE)

    valid_iter = data.Iterator(valid_data,
                               batch_size=1,
                               train=False,
                               sort=False,
                               repeat=False,
                               device=DEVICE)

    test_iter = data.Iterator(test_data,
                              batch_size=1,
                              train=False,
                              sort=False,
                              repeat=False,
                              device=DEVICE)

    # print_data_info(train_data, valid_data, test_data, SRC, LABEL)

    #############################

    run_lrp((rebatch(PAD_INDEX, b) for b in test_iter),
            vocab=SRC.vocab,
            model_file='sa_model4.pt')

    # true: pos, pred: pos, words: ['a', 'masterpiece', 'four', 'years', 'in', 'the', 'making', '.'], relevance: [ 0.13147668  1.88773544  0.72980414  0.71849921 -0.02894038  0.04202898 0.21119616 -0.01914013]
    # true: neg, pred: neg, words: , relevance:
    # true: neu, pred: neu, words: ['effective', 'but', 'too', '-', 'tepid', 'biopic'], relevance: [-0.39086432  0.0311396   0.18368873  0.16158153  1.10337837 -0.17609372]

    # true: pos, pred: neg, words: , relevance:
    # true: neg, pred: pos, words: ['it', 'took', '19', 'predecessors', 'to', 'get', 'this', '?'], relevance: [ 0.30267079 -0.08001834  0.43165412  0.63442843  0.20579954  0.08087923  0.31217654 -0.55773837]
Esempio n. 10
0
	# print(LABEL.vocab.freqs.most_common(10))

	#############################
	# define iterator
	train_iter = data.BucketIterator(train_data, batch_size=params['BATCH_SIZE'],
		device=DEVICE, sort_within_batch=True, 
		sort_key=lambda x: len(x.text), 
		train=True, repeat=False)

	# train_iter = data.Iterator(train_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE)

	valid_iter = data.Iterator(valid_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE)

	test_iter = data.Iterator(test_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE)

	print_data_info(train_data, valid_data, test_data, SRC, LABEL)

	#############################
	

	run_lrp((rebatch(PAD_INDEX, b) for b in test_iter), vocab=SRC.vocab, model_file='sa_model4.pt')