def train(model, num_epochs=10, lr=0.0003, print_every=100):
"""Train a model on IWSLT"""
if params['USE_CUDA']:
model.cuda()
# optionally add label smoothing; see the Annotated Transformer
criterion = nn.NLLLoss(reduction="sum", ignore_index=PAD_INDEX)
optim = torch.optim.Adam(model.parameters(), lr=lr)
dev_perplexities = []
for epoch in range(num_epochs):
print("Epoch", epoch)
model.train()
train_perplexity = run_epoch(
(rebatch(PAD_INDEX, b) for b in train_iter),
model,
SimpleLossCompute(model.generator, criterion, optim),
print_every=print_every)
model.eval()
with torch.no_grad():
print_examples((rebatch(PAD_INDEX, x) for x in valid_iter),
model,
n=3,
src_vocab=SRC.vocab,
trg_vocab=TRG.vocab)
dev_perplexity = run_epoch(
(rebatch(PAD_INDEX, b) for b in valid_iter), model,
SimpleLossCompute(model.generator, criterion, None))
print("Validation perplexity: %f" % dev_perplexity)
dev_perplexities.append(dev_perplexity)
return dev_perplexities
def run_lrp(data_iter, vocab, model_file):
net = BiLSTM(model_file=model_file)
method = 'all'
linear = 'abs'
if linear == 'zero':
alpha = 2
else:
alpha = 1
print("LRP - ", method)
print("Linear - ", linear)
print("alpha - ", alpha)
num_del = 1
while num_del <= 5:
iterator = (rebatch(PAD_INDEX, b) for b in data_iter)
deletion_iter(num_del, method, linear, alpha, iterator, net)
num_del += 1
def run_lrp(data_iter, vocab, model_file):
net = BiLSTM(model_file=model_file)
vocab_size = len(vocab)
method = 'abs'
linear = 'epsilon'
if linear == 'zero':
alpha = 2
else:
alpha = 1
print("LRP - ", method)
print("Linear - ", linear)
print("alpha - ", alpha)
num_replace = 1
iters = 8
result_true, result_false = {}, {}
while num_replace <= 5:
result_true[num_replace] = []
result_false[num_replace] = []
for it in range(iters):
iterator = (rebatch(PAD_INDEX, b) for b in data_iter)
count1, count2, correct_total, incorrect_total = replace_iter(num_replace, method, linear, alpha, iterator, net, vocab_size)
result_true[num_replace].append(count1)
result_false[num_replace].append(count2)
num_replace += 1
print("Initially correct: ", result_true)
print("Initially false: ", result_false)
count_true, count_false = 0, 0
for k in result_true:
count_true += sum(result_true[k]) / len(result_true[k])
count_false += sum(result_false[k]) / len(result_false[k])
print("Number of replacement: ", k)
print("Accuracy (initially correct): {0:0.2f}".format(count_true/correct_total*100))
print("Accuracy (initially false): {0:0.2f}".format(count_false/incorrect_total*100))
count_true, count_false = 0, 0
def run_test(model, test_data, test_iter):
preds = []
for k, batch in enumerate(test_iter):
model.eval()
with torch.no_grad():
batch = rebatch(PAD_INDEX, batch)
pred, attn = greedy_decode(model,
batch.src,
batch.src_mask,
batch.src_lengths,
max_len=25,
sos_index=TRG.vocab.stoi[SOS_TOKEN],
eos_index=TRG.vocab.stoi[EOS_TOKEN])
preds.append(pred)
hypotheses = [lookup_words(pred, TRG.vocab) for pred in preds]
hypotheses = [" ".join(h) for h in hypotheses]
references = [" ".join(data.trg) for data in test_data]
bleu = sacrebleu.raw_corpus_bleu(hypotheses, [references], .01).score
print("BLEU score: ", bleu)
return bleu
def run_lrp(data_iter, vocab, model_file):
net = BiLSTM(model_file=model_file)
method = 'all'
# print("LRP - ", method)
num_del = 1
iters = 8
result_true, result_false = {}, {}
while num_del <= 5:
result_true[num_del] = []
result_false[num_del] = []
for it in range(iters):
iterator = (rebatch(PAD_INDEX, b) for b in data_iter)
count1, count2, correct_total, incorrect_total = deletion_iter(
num_del, method, iterator, net)
result_true[num_del].append(count1)
result_false[num_del].append(count2)
num_del += 1
print("Initially correct: ", result_true)
print("Initially false: ", result_false)
count_true, count_false = 0, 0
for k in result_true:
count_true += sum(result_true[k]) / len(result_true[k])
count_false += sum(result_false[k]) / len(result_false[k])
print("Number of deletion: ", k)
print("Accuracy (initially correct): {0:0.2f}".format(
count_true / correct_total * 100))
print("Accuracy (initially false): {0:0.2f}".format(
count_false / incorrect_total * 100))
count_true, count_false = 0, 0
f.write("%s\n" % i)
with open("result/valid_loss.txt", "w") as f:
for i in valid_losses:
f.write("%s\n" % i)
with open("result/valid_prec.txt", "w") as f:
for i in valid_prec_score:
f.write("%s\n" % i)
with open("result/valid_recall.txt", "w") as f:
for i in valid_recall_score:
f.write("%s\n" % i)
with open("result/valid_f1.txt", "w") as f:
for i in valid_f1_score:
f.write("%s\n" % i)
print("Finish training ... ")
print_examples((rebatch(x, SRC_PAD_INDEX, TRG_PAD_INDEX, TRG_UNK_INDEX,
len(TRG.vocab), SRC.vocab) for x in test_iter),
model,
num=params['TEST_EXAMPLE_TO_PRINT'],
src_vocab=SRC.vocab,
trg_vocab=TRG.vocab)
########################
print("Start testing ... ")
test_f1_score, test_prec_score, test_recall_score = run_test(
(rebatch(x, SRC_PAD_INDEX, TRG_PAD_INDEX, TRG_UNK_INDEX, len(
TRG.vocab), SRC.vocab) for x in test_iter),
model,
trg_vocab=TRG.vocab)
print("test precision score: ", test_prec_score)
print("test recall score: ", test_recall_score)
def german_english_translation():
print("german_english_translation task")
SRC, TGT, train, val, test = data_help.data_load()
# GPUs to use
devices = [0, 1]
pad_idx = TGT.vocab.stoi["<blank>"]
print("Size:", len(SRC.vocab), len(TGT.vocab))
model = model_help.make_model(len(SRC.vocab), len(TGT.vocab),
N=6).to(args.device)
print("+===============+")
criterion = model_help.LabelSmoothing(size=len(TGT.vocab),
padding_idx=pad_idx,
smoothing=0.1).to(args.device)
BATCH_SIZE = 12000
train_iter = data_help.MyIterator(train,
batch_size=BATCH_SIZE,
device=devices[0],
repeat=False,
sort_key=lambda x:
(len(x.src), len(x.trg)),
batch_size_fn=batch_size_fn,
train=True)
valid_iter = data_help.MyIterator(val,
batch_size=BATCH_SIZE,
device=devices[0],
repeat=False,
sort_key=lambda x:
(len(x.src), len(x.trg)),
batch_size_fn=batch_size_fn,
train=False)
model_par = nn.DataParallel(model, device_ids=devices)
model_opt = model_help.NoamOpt(
model.src_embed[0].d_model, 1, 2000,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),
eps=1e-9))
for epoch in range(10):
model_par.train()
run_epoch((data_help.rebatch(pad_idx, b) for b in train_iter),
model_par,
model_help.MultiGPULossCompute(model.generator,
criterion,
devices=devices,
opt=model_opt))
model_par.eval()
loss = run_epoch((data_help.rebatch(pad_idx, b) for b in valid_iter),
model_par,
model_help.MultiGPULossCompute(model.generator,
criterion,
devices=devices,
opt=None))
print(loss)
"""Once trained we can decode the model to produce a set of translations.
Here we simply translate the first sentence in the validation set. This dataset
is pretty small so the translations with greedy search are reasonably accurate."""
for i, batch in enumerate(valid_iter):
src = batch.src.transpose(0, 1)[:1]
src_mask = (src != SRC.vocab.stoi["<blank>"]).unsqueeze(-2)
out = greedy_decode(model,
src,
src_mask,
max_len=60,
start_symbol=TGT.vocab.stoi["<s>"])
print("Translation:", end="\t")
for i in range(1, out.size(1)):
sym = TGT.vocab.itos[out[0, i]]
if sym == "</s>": break
print(sym, end=" ")
print()
print("Target:", end="\t")
for i in range(1, batch.trg.size(0)):
sym = TGT.vocab.itos[batch.trg.data[i, 0]]
if sym == "</s>": break
print(sym, end=" ")
print()
break
model = make_model(SRC.vocab, output_class=3, embed_size=params['EMBEDDING_DIM'],
hidden_size=params['HIDDEN_SIZE'], num_layers=params['NUM_LAYERS'],
dropout=params['DROPOUT_PROB'])
model.apply(init_weight)
#############################
print("Start training ... ")
# print(SRC.vocab.itos[1], SRC.vocab.itos[2], SRC.vocab.itos[3], SRC.vocab.itos[4])
train_losses, valid_losses, train_ac, valid_ac = train(model, num_epochs=params['NUM_EPOCHS'], learning_rate=params['LEARNING_RATE'])
print("Start test ... ")
print_examples((rebatch(PAD_INDEX, b) for b in test_iter), model, n=5, src_vocab=SRC.vocab)
# sos_idx=SOS_INDEX, eos_idx=EOS_INDEX,
print()
test_accuracy = run_test((rebatch(PAD_INDEX, b) for b in test_iter), model)
print('Test accuracy: ', test_accuracy)
# train_iter = data.Iterator(train_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE)
valid_iter = data.Iterator(valid_data,
batch_size=1,
train=False,
sort=False,
repeat=False,
device=DEVICE)
test_iter = data.Iterator(test_data,
batch_size=1,
train=False,
sort=False,
repeat=False,
device=DEVICE)
# print_data_info(train_data, valid_data, test_data, SRC, LABEL)
#############################
run_lrp((rebatch(PAD_INDEX, b) for b in test_iter),
vocab=SRC.vocab,
model_file='sa_model4.pt')
# true: pos, pred: pos, words: ['a', 'masterpiece', 'four', 'years', 'in', 'the', 'making', '.'], relevance: [ 0.13147668 1.88773544 0.72980414 0.71849921 -0.02894038 0.04202898 0.21119616 -0.01914013]
# true: neg, pred: neg, words: , relevance:
# true: neu, pred: neu, words: ['effective', 'but', 'too', '-', 'tepid', 'biopic'], relevance: [-0.39086432 0.0311396 0.18368873 0.16158153 1.10337837 -0.17609372]
# true: pos, pred: neg, words: , relevance:
# true: neg, pred: pos, words: ['it', 'took', '19', 'predecessors', 'to', 'get', 'this', '?'], relevance: [ 0.30267079 -0.08001834 0.43165412 0.63442843 0.20579954 0.08087923 0.31217654 -0.55773837]
# print(LABEL.vocab.freqs.most_common(10)) ############################# # define iterator train_iter = data.BucketIterator(train_data, batch_size=params['BATCH_SIZE'], device=DEVICE, sort_within_batch=True, sort_key=lambda x: len(x.text), train=True, repeat=False) # train_iter = data.Iterator(train_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE) valid_iter = data.Iterator(valid_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE) test_iter = data.Iterator(test_data, batch_size=1, train=False, sort=False, repeat=False, device=DEVICE) print_data_info(train_data, valid_data, test_data, SRC, LABEL) ############################# run_lrp((rebatch(PAD_INDEX, b) for b in test_iter), vocab=SRC.vocab, model_file='sa_model4.pt')