bleu_per_sentence[dutch] = [bleu, eng, output]
for n in range(1, N + 1):
total_clipped_counts[n] += ngrams_clipped_counts[n]
total_counts[n] += ngrams_counts[n]
bar.update(i)
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(bleu_per_sentence)
print("bleu on corpus:",
computeBlue(total_clipped_counts, total_counts, bp, N))
if __name__ == "__main__":
input_lang = Lang(nld_data)
output_lang = Lang(eng_data)
hidden_size = 256
encoder1 = EncoderRNN(input_lang.n_words, hidden_size)
encoder1.load_state_dict(
torch.load('models_project6/encoder.pt',
map_location=lambda storage, loc: storage))
attn_decoder1 = AttnDecoderRNN(hidden_size,
output_lang.n_words,
1,
dropout_p=0.1)
attn_decoder1.load_state_dict(
torch.load('models_project6/decoder.pt',
map_location=lambda storage, loc: storage))
readTrainData("data/dutch-sentences.txt")
# evaluateAndShowAttention("zij vertrekken morgenochtend uit japan")
loss_all += print_loss_avg
plot_loss_avg = plot_loss_total / plot_every
plot_loss_total = 0
hidden_size = 2046
encoder1 = EncoderRNN(2046, hidden_size)
attn_decoder1 = DecoderRNN(hidden_size, caption_list.n_words)
if use_cuda:
encoder1 = encoder1.cuda()
attn_decoder1 = attn_decoder1.cuda()
encoder1.load_state_dict(torch.load('encoder.pt'))
attn_decoder1.load_state_dict(torch.load('decoder.pt'))
def evaluate(encoder, decoder, vid_ID,max_length=MAX_LENGTH):
input_variable, empty = variableFromId(vid_ID)
input_length = len(input_variable)
encoder_hidden = encoder.initHidden()
# encoder_outputs = Variable(torch.zeros(max_length, encoder.hidden_size))
# encoder_outputs = encoder_outputs.cuda() if use_cuda else encoder_outputs
for ei in range(input_length):
encoder_output, encoder_hidden = encoder(input_variable[ei], encoder_hidden)
# encoder_outputs[ei] = encoder_outputs[ei] + encoder_output[0][0]
decoder_input = Variable(torch.LongTensor([[SOS_token]])) # SOS
encoder_optimizer_sd = checkpoint['en_opt']
decoder_optimizer_sd = checkpoint['de_opt']
embedding_sd = checkpoint['embedding']
voc.__dict__ = checkpoint['voc_dict']
print('Building encoder and decoder ...')
# 初始化word embedding
embedding = nn.Embedding(voc.num_words, hidden_size)
if model_checkpoint:
embedding.load_state_dict(embedding_sd)
# 初始化encoder和decoder模型
encoder = EncoderRNN(hidden_size, embedding, encoder_n_layers, dropout)
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, voc.num_words, decoder_n_layers, dropout)
if model_checkpoint:
encoder.load_state_dict(encoder_sd)
decoder.load_state_dict(decoder_sd)
# 使用合适的设备
encoder = encoder.to(device)
decoder = decoder.to(device)
print('Models built and ready to go!')
class GreedySearchDecoder(nn.Module):
def __init__(self, encoder, decoder):
super(GreedySearchDecoder, self).__init__()
self.encoder = encoder
self.decoder = decoder
def forward(self, input_seq, input_length, max_length):
# Encoder的Forward计算
gen_first_word_other_aux = []
gen_first_word_other_word = []
while direcs_to_process:
if not os.path.exists(directory + "_" + str(counter)):
direcs_to_process = 0
else:
directory_now = directory + "_" + str(counter)
counter += 1
dec_list = sorted(os.listdir(directory_now))
dec = sorted(dec_list[:int(len(dec_list)/2)], key=lambda x:float(".".join(x.split(".")[2:4])))[0]
print("This directory:", dec)
enc = dec.replace("decoder", "encoder")
encoder1.load_state_dict(torch.load(directory_now + "/" + enc))
decoder1.load_state_dict(torch.load(directory_now + "/" + dec))
evaluateRandomly(encoder1, decoder1)
right = 0
rightpos = 0
total = 0
test = open(testFile, "r")
test_set = test.readlines()
for batch in test_batches[200:400]:
pred_words, att = evaluate(encoder1, decoder1, batch)
#print(pred_words)
for index in range(batch_size):
