def evaluate(lang, embedding, encoder, decoder, dev_pairs, max_length, infer_batch_size): target_words = [] decoded_words = [] for i in range(0, len(dev_pairs), infer_batch_size): # print("batch: ", int(i / infer_batch_size)) dev_batch = dev_pairs[i:i + infer_batch_size] # 排序并padding dev_batch = sorted(dev_batch, key=lambda x: len(x[0]), reverse=True) target_words.extend(map(lambda x: x[1], dev_batch)) dev_batch = list(map(lambda x: indexesFromPair(lang, x), dev_batch)) enc_lens = list(map(lambda x: len(x[0]), dev_batch)) # print("enc_lens: ", enc_lens) enc_max_len = max(enc_lens) enc = [] for t in dev_batch: enc.append(t[0] + (enc_max_len - len(t[0])) * [0]) enc = torch.tensor(enc, dtype=torch.long, device=device_cuda) decoded_words.extend( inference(lang, enc, enc_lens, embedding, encoder, decoder, max_length)) target_sentences = list(map(lambda x: ''.join(x), target_words)) output_sentences = list(map(lambda x: ''.join(x), decoded_words)) with open('bleu/gold', 'w', encoding='utf-8') as gw: gw.write('\n'.join(target_sentences)) gw.close() with open('bleu/predict', 'w', encoding='utf-8') as pr: pr.write('\n'.join(output_sentences)) pr.close() bleu_score = bleu(target_sentences, output_sentences) return bleu_score
def compute_scores(self, matchs): bleu_matched = [] nb_matched_all = [] for match in matchs: if len(match) > 3: test_src, test_tgt, direction, nb_matched, matched, matched_rendered = match bleu_matched.append( utils.bleu([test_tgt.source], [matched_rendered])) nb_matched_all.append(nb_matched) total = len(matchs) multi_matched = sum(1 if len(e) > 3 and e[3] > 1 else 0 for e in matchs) _matched = [len(e) > 3 for e in matchs] return { 'bleu_total': sum(bleu_matched) / total, 'bleu_matched': sum(bleu_matched) / len(bleu_matched), 'nb_entry_matched': sum(_matched), 'avg_nb_entry_matched': sum(_matched) / total, 'nb_entry_multi_matched': multi_matched, 'avg_nb_entry_multi_matched': multi_matched / total, 'nb_database_match': sum(nb_matched_all), 'avg-matched_nb_database_match': sum(nb_matched_all) / sum(_matched), 'nb_no_matched': sum(1 if not e else 0 for e in _matched), 'avg_no_matched': sum(1 if not e else 0 for e in _matched) / total, 'nb_bleu_1': sum(1 if d == 1.0 else 0 for d in bleu_matched), 'avg-matched_nb_bleu_1': sum(1 if d == 1.0 else 0 for d in bleu_matched) / sum(_matched), 'nb_bleu_between_1_0.7': sum(1 if d < 1.0 and d > 0.7 else 0 for d in bleu_matched), 'avg-matched_nb_bleu_between_1_0.7': sum(1 if d < 1.0 and d > 0.7 else 0 for d in bleu_matched) / sum(_matched), 'nb_bleu_below_0.7': sum(1 if d < 0.7 else 0 for d in bleu_matched), 'avg-matched_nb_bleu_below_0.7': sum(1 if d < 0.7 else 0 for d in bleu_matched) / sum(_matched), 'total': total, }
load_checkpoint(torch.load("my_checkpoint.pth.tar"), model, optimizer) # sentence = "ein pferd geht unter einer brücke neben einem boot." # # translated_sentence = translate_sentence( # model, sentence, german, english, device, max_length=50 # ) sentence1 = [ 'ein', 'pferd', 'geht', 'unter', 'einer', 'brücke', 'neben', 'einem', 'boot', '.' ] translated_sentence = translate_sentence(model, sentence1, german, english, device, max_length=50) # exit() # print(f"Translated1 example sentence: \n {sentence}") # print(f"Translated1 example sentence: \n {translated_sentence}") # exit() train(model, device, load_model, save_model, german, english, train_data, valid_data, test_data, batch_size) # running on entire test data takes a while score = bleu(train_data[1:100], model, german, english, device) print(f"Final Train Bleu score {score * 100:.2f}") score = bleu(test_data[1:100], model, german, english, device) print(f"Final Test Bleu score {score * 100:.2f}")
def train(model, device, load_model, save_model, german_vocab, english_vocab, train_data, valid_data, test_data, batch_size): optimizer = optim.Adam(model.parameters(), lr=learning_rate) if load_model: load_checkpoint(torch.load("my_checkpoint.pth.tar"), model, optimizer) sentence = "ein pferd geht unter einer brücke neben einem boot." # sentence = 'a little girl climbing into a wooden playhouse.' # sentence = "man stuffed smiling lion" #6 1 4 7 3 2 5 0 # sentence = ['ein', 'pferd', 'geht', 'unter', 'einer', 'brücke', 'neben', 'einem', 'boot', '.'] #sentence = ['The', 'study’s', 'questions', 'are', 'carefully', 'worded', 'and', 'chosen', '.'] # sentence = 'The study questions are carefully worded and chosen.' # sentence = ['a', 'little', 'girl', 'climbing', 'into', 'a', 'wooden', 'playhouse', '.'] scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=10, verbose=True) pad_idx = english_vocab.stoi["<pad>"] criterion = nn.CrossEntropyLoss(ignore_index=pad_idx) # train_iterator, valid_iterator, test_iterator = Batcher(train_data, valid_data, test_data) train_iterator, valid_iterator, test_iterator = BucketIterator.splits( (train_data, valid_data, test_data), batch_size=batch_size, sort_within_batch=True, sort_key=lambda x: len(x.src), device=device, ) step = 0 for epoch in range(num_epochs): print(f"[Epoch {epoch} / {num_epochs}]") if save_model: checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } save_checkpoint(checkpoint) model.eval() # sentence = "Das wird sehr seltsam" # sentence = "Frankreich wird wohl Deutschland angreifen" translated_sentence = translate_sentence(model, sentence, german_vocab, english_vocab, device, max_length=50) print(f"Translated example sentence: \n {sentence}") print(f"Translated example sentence: \n {translated_sentence}") # exit() # running on entire test data takes a while print("here1") score = bleu(train_data[1:10], model, german_vocab, english_vocab, device) print(f"Train Bleu score {score * 100:.2f}") print("here2") score = bleu(test_data[1:50], model, german_vocab, english_vocab, device) print(f"Test Bleu score {score * 100:.2f}") model.train() losses = [] for batch_idx, batch in enumerate(train_iterator): # Get input and targets and get to cuda # print(batch_idx) inp_data = batch.src inp_data = inp_data.to(device) target = batch.trg target = target.to(device) # inp_data = batch[0].to(device) # target = batch[1].to(device) # Forward prop # print(target) # printSentences(inp_data, german_vocab) # printSentences2(target, english_vocab, inp_data, german_vocab) trg = target[:-1, :] # print(trg.shape) output = model(inp_data, trg) # Output is of shape (trg_len, batch_size, output_dim) but Cross Entropy Loss # doesn't take input in that form. For example if we have MNIST we want to have # output to be: (N, 10) and targets just (N). Here we can view it in a similar # way that we have output_words * batch_size that we want to send in into # our cost function, so we need to do some reshapin. # Let's also remove the start token while we're at it output = output.reshape(-1, output.shape[2]) target = target[1:].reshape(-1) optimizer.zero_grad() loss = criterion(output, target) losses.append(loss.item()) # Back prop loss.backward() # Clip to avoid exploding gradient issues, makes sure grads are # within a healthy range torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1) # Gradient descent step optimizer.step() # plot to tensorboard # writer.add_scalar("Training loss", loss, global_step=step) step += 1 mean_loss = sum(losses) / len(losses) scheduler.step(mean_loss)
output = output[1:].reshape(-1, output.shape[2]) target = target[1:].reshape(-1) optimizer.zero_grad() # GPUtil.showUtilization() loss = criterion(output, target) del output, inp_data, target torch.cuda.empty_cache() # for i in range(100000000): # continue # Back prop loss.mean().backward() # Clip to avoid exploding gradient issues, makes sure grads are # within a healthy range torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1) # Gradient descent step optimizer.step() # Plot to tensorboard writer.add_scalar("Training loss", loss, global_step=step) step += 1 del loss torch.cuda.empty_cache() # for i in range(100000000): # continue # running on entire test data takes a while score = bleu(val[1:100], model, english, english, device) print("Bleu score {}".format(score * 100))
# print(spe_dec.decode(src)) # print(" ", trg) # print(spe_dec.decode(trg)) src_vocab_size = len(spe_dec) trg_vocab_size = len(spe_dec) print("src vocabulary size: ", src_vocab_size) print("trg vocabulary size: ", trg_vocab_size) embedding_size = 256 src_pad_idx = spe_dec.pad_id() print("pad_index = ", src_pad_idx) print("===============================after loading") model = Transformer(device, embedding_size, src_vocab_size, trg_vocab_size, src_pad_idx).to(device) load_model = True save_model = True learning_rate = 3e-4 optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) if load_model: load_checkpoint(torch.load("my_checkpoint.pth.tar"), model, optimizer) # with torch.no_grad(): score1 = bleu(train_data[1:50], model, spe_dec, spe_dec, device) score2 = bleu(test_data[1:50], model, spe_dec, spe_dec, device) print(f"Train Bleu score1 {score1 * 100:.2f}") print(f"Test Bleu score2 {score2 * 100:.2f}")
# output to be: (N, 10) and targets just (N). Here we can view it in a similar # way that we have output_words * batch_size that we want to send in into # our cost function, so we need to do some reshapin. While we're at it # Let's also remove the start token while we're at it # (N, 10) and targets would be (N) output = output[1:].reshape( -1, output.shape[2] ) # output dimension which would be the size of the vocabulary and just gonna put everyting else together target = target[1:].reshape(-1) optimizer.zero_grad() loss = criterion(output, target) # Back prop loss.backward() # To avoid exploding problems deprecated errors happen #clipping_value = 1 # arbitrary value of your choosing #torch.nn.utils.clip_grad_norm(model.parameters(), clipping_value) clipping_value = 1 # arbitrary value of your choosing torch.nn.utils.clip_grad_norm_(model.parameters(), clipping_value) # Gradient descent step optimizer.step() writer.add_scalar('Training loss', loss, global_step=step) step += 1 score = bleu(test_data, model, german, english, device) print(f'Bleu score {score*100:.2f}')
if valid_loss < best_valid_loss: best_valid_loss = valid_loss torch.save(seq2seq.state_dict(), 'tut2-model.pt') print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s') print(f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}') print(f'\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}') seq2seq.load_state_dict(torch.load('tut2-model.pt')) test_loss = evaluate(seq2seq, test_iterator, criterion) print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |') print("Bleu Score is :") print(bleu(test_data, seq2seq, config.source, config.target, config.device)) example_idx = 0 example = train_data.examples[example_idx] print('source sentence: ', ' '.join(example.src)) print('target sentence: ', ' '.join(example.trg)) src_tensor = config.source.process([example.src]).to(config.device) trg_tensor = config.target.process([example.trg]).to(config.device) seq2seq.eval() with torch.no_grad(): outputs = seq2seq(src_tensor, trg_tensor, teacher_forcing_ratio=0) output_idx = outputs[1:].squeeze(1).argmax(1) result = ' '.join([config.target.vocab.itos[idx] for idx in output_idx])
def train(model, device, load_model, save_model, german_vocab, english_vocab, train_data, valid_data, test_data, batch_size, LOAD_NEW_METHOD): optimizer = optim.Adam(model.parameters(), lr=learning_rate) if load_model: load_checkpoint(torch.load("my_checkpoint.pth.tar"), model, optimizer) # sentence = "ein pferd geht unter einer brücke neben einem boot." # sentence = 'a little girl climbing into a wooden playhouse.' # sentence = "is man lion a stuffed A at smiling." # sentence = ['ein', 'pferd', 'geht', 'unter', 'einer', 'brücke', 'neben', 'einem', 'boot', '.'] #sentence = ['The', 'study’s', 'questions', 'are', 'carefully', 'worded', 'and', 'chosen', '.'] # sentence = 'The study questions are carefully worded and chosen.' # sentence = ['a', 'little', 'girl', 'climbing', 'into', 'a', 'wooden', 'playhouse', '.'] scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=10, verbose=True) pad_idx = english_vocab.stoi["<pad>"] criterion = nn.CrossEntropyLoss(ignore_index=pad_idx) if LOAD_NEW_METHOD: train_iterator, valid_iterator, test_iterator = Batcher( train_data, valid_data, test_data) else: train_iterator, valid_iterator, test_iterator = BucketIterator.splits( (train_data, valid_data, test_data), batch_size=batch_size, # sort_within_batch=True, # shuffle=True, sort_key=lambda x: len(x.src), device=device, ) step = 0 for epoch in range(num_epochs): print(f"[Epoch {epoch} / {num_epochs}]") if save_model: checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } save_checkpoint(checkpoint) model.eval() # sentence = "Das wird sehr seltsam" # sentence = "Frankreich wird wohl Deutschland angreifen" src_sents = [ "There was no possibility of taking a walk that day.", "His own words is a pledge of this.", "His own words are a pledge of this.", "A man are playing the banjo at a concert.", "No; you is less than a servant, for you do nothing for your keep." ] # for src, trg in train_data: # # translation = translate_sentence_lstm(model, src, german_vocab, english_vocab, device) # # translated_sentence = translate_sentence( # model, # src, german_vocab, english_vocab, device, max_length=50, inputIsTensor=True # ) # src_eng = [german_vocab.itos[idx] for idx in src] # print(f"Translated example sentence: \n {src_eng}") # print(f"Translated example sentence: \n {translated_sentence}") # running on entire test data takes a while print("here1") score = bleu(train_data[1:10], model, german_vocab, english_vocab, device, LOAD_NEW_METHOD) print(f"Train Bleu score {score * 100:.2f}") # print("here2") score = bleu(test_data[1:10], model, german_vocab, english_vocab, device, LOAD_NEW_METHOD) print(f"Test Bleu score {score * 100:.2f}") model.train() losses = [] t1 = time.time() for batch_idx, batch in enumerate(train_iterator): # Get input and targets and get to cuda if LOAD_NEW_METHOD: inp_data = batch[0].to(device) target = batch[1].to(device) else: inp_data = batch.src.to(device) target = batch.trg.to(device) # continue # Forward prop # print(target) # printSentences(inp_data, german_vocab) # printSentences(target, english_vocab) trg = target[:-1, :] # print(trg.shape) output = model(inp_data, trg) # Output is of shape (trg_len, batch_size, output_dim) but Cross Entropy Loss # doesn't take input in that form. For example if we have MNIST we want to have # output to be: (N, 10) and targets just (N). Here we can view it in a similar # way that we have output_words * batch_size that we want to send in into # our cost function, so we need to do some reshapin. # Let's also remove the start token while we're at it output = output.reshape(-1, output.shape[2]) target = target[1:].reshape(-1) optimizer.zero_grad() loss = criterion(output, target) losses.append(loss.item()) # Back prop loss.backward() # Clip to avoid exploding gradient issues, makes sure grads are # within a healthy range torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1) # Gradient descent step optimizer.step() # plot to tensorboard # writer.add_scalar("Training loss", loss, global_step=step) step += 1 if batch_idx % 100 == 0: print("batch " + str(batch_idx)) t2 = time.time() print("epoch time = ", t2 - t1) mean_loss = sum(losses) / len(losses) scheduler.step(mean_loss)
print("src vocabulary size: ", src_vocab_size) print("trg vocabulary size: ", trg_vocab_size) embedding_size = 512 src_pad_idx = english_vocab.stoi["<pad>"] print(src_pad_idx) print(english_vocab.itos[src_pad_idx]) print("===============================after loading ") model = Transformer(device, embedding_size, src_vocab_size, trg_vocab_size, src_pad_idx).to(device) load_model = True save_model = True learning_rate = 3e-4 optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) if load_model: load_checkpoint_zaid("my_checkpoint.pth.tar", model, optimizer) print("here1") score = bleu(train_data[1:10], model, german_vocab, english_vocab, device) print(f"Train Bleu score {score * 100:.2f}") print("here2") score = bleu(test_data[1:50], model, german_vocab, english_vocab, device) print(f"Test Bleu score {score * 100:.2f}") if save_model: save_checkpoint_zaid("my_checkpointx.pth.tar", model, optimizer)
# Back prop loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1) optimizer.step() training_loss.append(loss.item()) avg_train_loss = np.average(training_loss) avg_trainloss_plot.append(avg_train_loss) model.eval() BleuScore = bleu(test_data[:3000], model, source_lang, target_lang, device,max_length=50,generate_outputs=False) Bleu_scores_plot.append(BleuScore) #print(f"Bleu score {score*100:.2f}") print(f'\n[epoch:{epoch}] -->>'+f'training loss is: {avg_train_loss:.5f}' + f' Bleu Score is: {BleuScore*100:.5f}') #reset losses for next epoch training_loss = [] #Earlystop check if Max_BleuScore == None: Max_BleuScore = BleuScore #print("First min loss",min_loss) print("saving first model") torch.save(model,checkpoint_path) elif BleuScore > Max_BleuScore: Max_BleuScore = BleuScore print("New max bleu score found , saving model")
def oracle(matched_many, target): return sorted(range(len(matched_many)), key=lambda i: -utils.bleu([matched_many[i]], [target]))[0]
def main(): file = { "model_group": "/seq_len_exp", "model_name": "/generation_6", "model_version": 0, "project_file": "/home/mattd/PycharmProjects/reddit/generation" } file["dataset_path"] = "{}/data/".format(file["project_file"]) file["model_filename"] = '{}{}s{}_{}'.format(file["project_file"], file["model_group"], file["model_name"], file["model_version"]) file["output_file"] = '{}{}_outputs{}_{}'.format(file["project_file"], file["model_group"], file["model_name"], file["model_version"]) #check_files(file) use_old_model = file["model_version"] != 0 params = {} if use_old_model: file["old_model_filename"] = '{}{}s{}_{}'.format( file["project_file"], file["model_group"], file["model_name"], file["model_version"] - 1) params, old_files = load_params(file["old_model_filename"]) use_old_model = old_files != {} if not use_old_model: params = { "attention": True, "batch_size": 325, "hidden_size": 256, "embedding_dim": 300, "pretrained_embeddings": True, "max_grad_norm": 5, "max_len": 30, "min_count": 2, "weight_decay": 0.00001, "learning_rate": 0.005, } params["num_training_examples"] = 78260 params["num_val_examples"] = -1 params["nb_epochs"] = 40 if params["pretrained_embeddings"]: file["pretrained_embeddings_file"] = \ "/embeddings/embeddings_min{}_max{}.npy".format( params["min_count"], params["max_len"]) string = "" for k, v in file.items(): string += "{}: {}\n".format(k, v) for k, v in params.items(): string += "{}: {}\n".format(k, v) print(string) output = string + '\n' # eng_fr_filename = '/mnt/data1/datasets/yelp/merged/train' dataset_train_filename = "{}train.csv".format(file["dataset_path"]) dataset_val_filename = "{}validation.csv".format(file["dataset_path"]) dataset_train = SentenceDataset(dataset_train_filename, params["max_len"], params["min_count"]) dataset_val = SentenceDataset(dataset_val_filename, params["max_len"], params["min_count"], dataset_train.vocab) string = 'Vocab size {}\n'.format(len(dataset_train.vocab)) string += 'Train {} '.format(len(dataset_train)) if params["num_training_examples"] != -1: dataset_train.prune_examples(params["num_training_examples"]) string += '-> {}'.format(len(dataset_train)) string += '\nVal: {}'.format(len(dataset_val)) if params["num_val_examples"] != -1: dataset_val.prune_examples(params["num_val_examples"]) string += '-> {}'.format(len(dataset_val)) print(string) output += string + '\n' if params["pretrained_embeddings"]: embeddings_dir = '{}{}'.format(file["project_file"], file["pretrained_embeddings_file"]) pretrained_embeddings = cuda(get_pretrained_embeddings(embeddings_dir)) params["embedding_dim"] = pretrained_embeddings.shape[1] else: pretrained_embeddings = None data_loader_train = torch.utils.data.DataLoader(dataset_train, params["batch_size"], shuffle=True) data_loader_val = torch.utils.data.DataLoader(dataset_val, params["batch_size"], shuffle=False) vocab_size = len(dataset_train.vocab) padding_idx = dataset_train.vocab[SentenceDataset.PAD_TOKEN] init_idx = dataset_train.vocab[SentenceDataset.INIT_TOKEN] if params["attention"] is True: model = Seq2SeqModelAttention(params["hidden_size"], padding_idx, init_idx, params["max_len"], vocab_size, params["embedding_dim"], pretrained_embeddings) else: model = Seq2SeqModel(params["hidden_size"], padding_idx, init_idx, params["max_len"], vocab_size, params["embedding_dim"], pretrained_embeddings) model = cuda(model) parameters = list(model.parameters()) optimizer = torch.optim.Adam(parameters, amsgrad=True, weight_decay=params["weight_decay"], lr=params["learning_rate"]) criterion = torch.nn.CrossEntropyLoss() if use_old_model: model, optimizer = load_checkpoint(file["old_model_filename"], model, optimizer) lowest_loss = 100 train_loss = [] val_loss = [] best_model = model best_optimizer = optimizer average_epoch_loss = 0 metrics = { "token_accuracy": [], "sentence_accuracy": [], "perplexity": [], "bleu": { 'bleu_1': [], 'bleu_2': [], 'bleu_3': [], 'bleu_4': [] } } outfile = open(file["output_file"], 'w') outfile.write(output) outfile.close() phases = [ 'train', 'val', ] data_loaders = [ data_loader_train, data_loader_val, ] intervals = 2 highest_acc = 0 for epoch in range(0, params["nb_epochs"]): start = time.clock() string = 'Epoch: {}\n'.format(epoch) print(string, end='') output = output + '\n' + string #if epoch == 6: # model.unfreeze_embeddings() # parameters = list(model.parameters()) # optimizer = torch.optim.Adam( # parameters, amsgrad=True, weight_decay=weight_decay) for phase, data_loader in zip(phases, data_loaders): if phase == 'train': model.train() string = 'Train: \n' else: model.eval() string = 'Validation \n' print(string, end='') output = output + '\n' + string epoch_bleu = { 'bleu_1': [], 'bleu_2': [], 'bleu_3': [], 'bleu_4': [] } epoch_loss = [] epoch_sentenence_accuracy = [] epoch_token_accuracy = [] j = 1 for i, (sentence_1, sentence_2) in tqdm(enumerate(data_loader)): optimizer.zero_grad() sentence_1 = variable(sentence_1) sentence_2 = variable(sentence_2) outputs = model(sentence_1, sentence_2) targets = sentence_2.view(-1) outputs = outputs.view(targets.size(0), -1) loss = criterion(outputs, targets) epoch_loss.append(float(loss)) average_epoch_loss = np.mean(epoch_loss) if phase == 'train': loss.backward() torch.nn.utils.clip_grad_norm_(parameters, params["max_grad_norm"]) optimizer.step() if (len(data_loader) / intervals) * j <= i + 1: string = ('Example {:03d} | {} loss: {:.3f}'.format( i, phase, average_epoch_loss)) print(string, end='\n') output = output + string + '\n' j += 1 else: # get result metrics predicted = torch.argmax( outputs.view(-1, params["max_len"], vocab_size), -1) batch_sentence_accuracy, batch_token_accuracy = \ encoder_accuracy( targets.view(-1, params["max_len"]), predicted) epoch_sentenence_accuracy.append(batch_sentence_accuracy) epoch_token_accuracy.append(batch_token_accuracy) bleu_1 = bleu(targets.view(-1, params["max_len"]), predicted, 1) bleu_2 = bleu(targets.view(-1, params["max_len"]), predicted, 2) bleu_3 = bleu(targets.view(-1, params["max_len"]), predicted, 3) bleu_4 = bleu(targets.view(-1, params["max_len"]), predicted, 4) epoch_bleu["bleu_1"].append(bleu_1) epoch_bleu["bleu_2"].append(bleu_2) epoch_bleu["bleu_3"].append(bleu_3) epoch_bleu["bleu_4"].append(bleu_4) if phase == 'val': time_taken = time.clock() - start val_loss.append(average_epoch_loss) string = ' {} loss: {:.3f} | time: {:.3f}'.format( phase, average_epoch_loss, time_taken) string += ' | lowest loss: {:.3f} highest accuracy: ' \ '{:.3f}'.format(lowest_loss, highest_acc) print(string, end='\n') output = output + '\n' + string + '\n' average_epoch_sentenence_accuracy = np.mean( epoch_sentenence_accuracy) average_epoch_token_accuracy = np.mean(epoch_token_accuracy) perplexity = np.exp(average_epoch_loss) average_epoch_bleu = { 'bleu_1': np.mean(epoch_bleu['bleu_1']), 'bleu_2': np.mean(epoch_bleu['bleu_2']), 'bleu_3': np.mean(epoch_bleu['bleu_3']), 'bleu_4': np.mean(epoch_bleu['bleu_4']) } metrics["token_accuracy"].append(average_epoch_token_accuracy) metrics["sentence_accuracy"].append( average_epoch_sentenence_accuracy) metrics["perplexity"].append(perplexity) metrics["bleu"]["bleu_1"].append(average_epoch_bleu['bleu_1']) metrics["bleu"]["bleu_2"].append(average_epoch_bleu['bleu_2']) metrics["bleu"]["bleu_3"].append(average_epoch_bleu['bleu_3']) metrics["bleu"]["bleu_4"].append(average_epoch_bleu['bleu_4']) if average_epoch_loss < lowest_loss: best_model = model best_optimizer = optimizer best_epoch = epoch lowest_loss = average_epoch_loss save_checkpoint(best_epoch, best_model, best_optimizer, epoch, model, optimizer, train_loss, val_loss, metrics, params, file) if average_epoch_token_accuracy > highest_acc: highest_acc = average_epoch_token_accuracy string = "Token_accuracy: {:.3f}\nSentence_accuracy: {:.3f}\n".format( average_epoch_token_accuracy, average_epoch_sentenence_accuracy) string += "Perplexity: {:.3f}\n".format(perplexity) string += "Bleu: {}\n".format(average_epoch_bleu) print(string, end='\n') output = output + string + '\n' random_idx = np.random.randint(len(dataset_val)) sentence_1, sentence_2 = dataset_val[random_idx] sentence_1_var = variable(sentence_1) sentence_2_var = variable(sentence_2) outputs_var = model(sentence_1_var.unsqueeze(0)) # unsqueeze # to get the batch dimension outputs = argmax(outputs_var).squeeze(0).data.cpu().numpy() string = '> {}\n'.format( get_sentence_from_indices(sentence_1, dataset_val.vocab, SentenceDataset.EOS_TOKEN)) string = string + u'< {}\n'.format( get_sentence_from_indices(outputs, dataset_val.vocab, SentenceDataset.EOS_TOKEN)) string = string + u'= {}\n'.format( get_sentence_from_indices(sentence_2, dataset_val.vocab, SentenceDataset.EOS_TOKEN)) print(string, end='\n\n') output = output + string + '\n' + '\n' else: train_loss.append(average_epoch_loss) outfile = open(file["output_file"], 'w') outfile.write(output) outfile.close()