def runTest(args, n_layers, hidden_size, reverse, modelFile, beam_size, batch_size, input, corpus): data, length = loadPrepareData(args) voc = data.voc print('load data...') user_length, item_length = length #, user_length2, item_length2 = length # train_batches = batchify(data.train, data.user_text, user_length, data.item_text, item_length, batch_size) # val_batches = batchify(data.dev, data.user_text, user_length, data.item_text, item_length, batch_size) test_batches = batchify(data.test, data.user_text, user_length, data.item_text, item_length, batch_size) print('Building encoder and decoder ...') embedding = nn.Embedding(data.voc.n_words, hidden_size) encoderU = EncoderRNNlinear(data.voc.n_words, hidden_size, embedding, data.dmax, n_layers) encoderB = EncoderRNNlinear(data.voc.n_words, hidden_size, embedding, data.dmax, n_layers) attn_model = 'dot' decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, data.voc.n_words, n_layers) # load model checkpoint = torch.load(modelFile) encoderU.load_state_dict(checkpoint['enU']) encoderB.load_state_dict(checkpoint['enB']) decoder.load_state_dict(checkpoint['de']) # train mode set to false, effect only on dropout, batchNorm encoderU.train(False) encoderB.train(False) decoder.train(False) if USE_CUDA: encoderU = encoderU.cuda() encoderB = encoderB.cuda() decoder = decoder.cuda() if not args.sample: # evaluate on test # for test_batch in tqdm(test_batches): for test_i, test_batch in enumerate(test_batches): if test_i > 1: break input_index, input_variable, lengths, target_variable, mask, max_target_len = test_batch user_input_variable, business_input_variable = input_variable user_lengths, business_lengths = lengths # evaluate on train evaluateRandomly(encoderU, encoderB, decoder, voc, \ input_index, user_input_variable, business_input_variable, \ user_lengths, business_lengths, \ target_variable, mask, max_target_len, reverse, beam_size) else: # evaluate using sample sample(encoderU, encoderB, decoder, voc, test_batches, reverse)
def trainIters(corpus, reverse, n_iteration, learning_rate, batch_size, n_layers, hidden_size, print_every, save_every, loadFilename=None, attn_model='dot', decoder_learning_ratio=5.0): voc, pairs = loadPrepareData(corpus) # training data corpus_name = os.path.split(corpus)[-1].split('.')[0] training_batches = None try: training_batches = torch.load(os.path.join(save_dir, 'training_data', corpus_name, '{}_{}_{}.tar'.format(n_iteration, \ filename(reverse, 'training_batches'), \ batch_size))) except BaseException: #OWEN: was FileNotFoundError print('Training pairs not found, generating ...') training_batches = [ batch2TrainData(voc, [random.choice(pairs) for _ in range(batch_size)], reverse) for _ in range(n_iteration) ] torch.save(training_batches, os.path.join(save_dir, 'training_data', corpus_name, '{}_{}_{}.tar'.format(n_iteration, \ filename(reverse, 'training_batches'), \ batch_size))) # model checkpoint = None print('Building encoder and decoder ...') embedding = nn.Embedding(voc.n_words, hidden_size) encoder = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers) attn_model = 'dot' decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, voc.n_words, n_layers) if loadFilename: checkpoint = torch.load(loadFilename) encoder.load_state_dict(checkpoint['en']) decoder.load_state_dict(checkpoint['de']) # use cuda if USE_CUDA: encoder = encoder.cuda() decoder = decoder.cuda() # optimizer print('Building optimizers ...') encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate) decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate * decoder_learning_ratio) if loadFilename: encoder_optimizer.load_state_dict(checkpoint['en_opt']) decoder_optimizer.load_state_dict(checkpoint['de_opt']) # initialize print('Initializing ...') start_iteration = 1 perplexity = [] print_loss = 0 if loadFilename: start_iteration = checkpoint['iteration'] + 1 perplexity = checkpoint['plt'] for iteration in tqdm(range(start_iteration, n_iteration + 1)): training_batch = training_batches[iteration - 1] input_variable, lengths, target_variable, mask, max_target_len = training_batch loss = train(input_variable, lengths, target_variable, mask, max_target_len, encoder, decoder, embedding, encoder_optimizer, decoder_optimizer, batch_size) print_loss += loss perplexity.append(loss) if iteration % print_every == 0: print_loss_avg = math.exp(print_loss / print_every) perplexity.append(print_loss_avg) # show perplexity (lots of numbers!): #print(perplexity, iteration) # plotPerplexity(perplexity, iteration) print('%d %d%% %.4f' % (iteration, iteration / n_iteration * 100, print_loss_avg)) print_loss = 0 if (iteration % save_every == 0): directory = os.path.join( save_dir, 'model', corpus_name, '{}-{}_{}'.format(n_layers, n_layers, hidden_size)) if not os.path.exists(directory): os.makedirs(directory) torch.save( { 'iteration': iteration, 'en': encoder.state_dict(), 'de': decoder.state_dict(), 'en_opt': encoder_optimizer.state_dict(), 'de_opt': decoder_optimizer.state_dict(), 'loss': loss, 'plt': perplexity }, os.path.join( directory, '{}_{}.tar'.format(iteration, filename(reverse, 'backup_bidir_model'))))
def trainIters(corpus, pre_modelFile, reverse, n_iteration, learning_rate, batch_size, n_layers, hidden_size, print_every, save_every, loadFilename=None, attn_model='dot', decoder_learning_ratio=5.0): voc, pairs = loadPrepareData(corpus) # training data corpus_name = os.path.split(corpus)[-1].split('.')[0] training_batches = None try: training_batches = torch.load(os.path.join(save_dir, 'training_data', corpus_name, '{}_{}_{}.tar'.format(n_iteration, \ filename(reverse, 'training_batches'), \ batch_size))) except FileNotFoundError: print('Training pairs not found, generating ...') training_batches = [ batch2TrainData(voc, [random.choice(pairs) for _ in range(batch_size)], reverse) for _ in range(n_iteration) ] torch.save(training_batches, os.path.join(save_dir, 'training_data', corpus_name, '{}_{}_{}.tar'.format(n_iteration, \ filename(reverse, 'training_batches'), \ batch_size))) # model checkpoint = None #print('Building pretrained word2vector model...') embedding = nn.Embedding( 300, hidden_size) #The dimension of google's model is 300 #----------------------------------------------------------------- #my code ''' EMBEDDING_DIM = 300 #Should be the same as hidden_size! if EMBEDDING_DIM != hidden_size: sys.exit("EMBEDDING_DIM do not equal to hidden_size. Please correct it.") CONTEXT_SIZE = 2 pre_checkpoint = torch.load(pre_modelFile) pretrained_model = NGramLanguageModeler(voc.n_words, EMBEDDING_DIM, CONTEXT_SIZE) pretrained_model.load_state_dict(pre_checkpoint['w2v']) pretrained_model.train(False) embedding = pretrained_model ''' if USE_CUDA: embedding = embedding.cuda() #----------------------------------------------------------------- #replace embedding by pretrained_model print('Building encoder and decoder ...') encoder = EncoderRNN(300, hidden_size, embedding, n_layers) attn_model = 'dot' decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, voc.n_words, n_layers) if loadFilename: checkpoint = torch.load(loadFilename) encoder.load_state_dict(checkpoint['en']) decoder.load_state_dict(checkpoint['de']) # use cuda if USE_CUDA: encoder = encoder.cuda() decoder = decoder.cuda() # optimizer print('Building optimizers ...') encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate) decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate * decoder_learning_ratio) if loadFilename: encoder_optimizer.load_state_dict(checkpoint['en_opt']) decoder_optimizer.load_state_dict(checkpoint['de_opt']) # Load Google's pre-trained Word2Vec model. print('Loading w2v_model ...') logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) w2v_model = gensim.models.KeyedVectors.load_word2vec_format(pre_modelFile, binary=True) print("Loading complete!") # initialize print('Initializing ...') start_iteration = 1 perplexity = [] print_loss = 0 if loadFilename: start_iteration = checkpoint['iteration'] + 1 perplexity = checkpoint['plt'] for iteration in tqdm(range(start_iteration, n_iteration + 1)): training_batch = training_batches[iteration - 1] input_variable, lengths, target_variable, mask, max_target_len = training_batch loss = train(input_variable, lengths, target_variable, mask, max_target_len, encoder, decoder, embedding, encoder_optimizer, decoder_optimizer, batch_size, w2v_model, voc) print_loss += loss perplexity.append(loss) if iteration % print_every == 0: print_loss_avg = math.exp(print_loss / print_every) # perplexity.append(print_loss_avg) # plotPerplexity(perplexity, iteration) print('%d %d%% %.4f' % (iteration, iteration / n_iteration * 100, print_loss_avg)) print_loss = 0 if (iteration % save_every == 0): directory = os.path.join( save_dir, 'model', corpus_name, '{}-{}_{}'.format(n_layers, n_layers, hidden_size)) if not os.path.exists(directory): os.makedirs(directory) torch.save( { 'iteration': iteration, 'en': encoder.state_dict(), 'de': decoder.state_dict(), 'en_opt': encoder_optimizer.state_dict(), 'de_opt': decoder_optimizer.state_dict(), 'loss': loss, 'plt': perplexity }, os.path.join( directory, '{}_{}.tar'.format(iteration, filename(reverse, 'backup_bidir_model'))))
decoder = LuongAttnDecoderRNN(attn_model, hidden_size, len(dictionary), n_layers, dropout=dropout) # Initialize optimizers and criterion encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate) decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate * decoder_learning_ratio) criterion = nn.CrossEntropyLoss() # Move models to GPU if USE_CUDA: encoder.cuda() decoder.cuda() # ============= # Begin! # ============= print_loss_total = 0.0 start = time.time() while epoch < n_epochs: epoch += 1 # Get training data for this cycle input_batches, input_lengths, target_batches, target_lengths = random_batch( raw_data, batch_size, dictionary) # Run the train function
def main(): epoch = 1000 batch_size = 64 hidden_dim = 300 use_cuda = True encoder = Encoder(num_words, hidden_dim) if args.attn: attn_model = 'dot' decoder = LuongAttnDecoderRNN(attn_model, hidden_dim, num_words) else: decoder = DecoderRhyme(hidden_dim, num_words, num_target_lengths, num_rhymes) if args.train: weight = torch.ones(num_words) weight[word2idx_mapping[PAD_TOKEN]] = 0 if use_cuda: encoder = encoder.cuda() decoder = decoder.cuda() weight = weight.cuda() encoder_optimizer = Adam(encoder.parameters(), lr=0.001) decoder_optimizer = Adam(decoder.parameters(), lr=0.001) criterion = nn.CrossEntropyLoss(weight=weight) np.random.seed(1124) order = np.arange(len(train_data)) best_loss = 1e10 best_epoch = 0 for e in range(epoch): #if e - best_epoch > 20: break np.random.shuffle(order) shuffled_train_data = train_data[order] shuffled_x_lengths = input_lengths[order] shuffled_y_lengths = target_lengths[order] shuffled_y_rhyme = target_rhymes[order] train_loss = 0 valid_loss = 0 for b in tqdm(range(int(len(order) // batch_size))): #print(b, '\r', end='') batch_x = torch.LongTensor( shuffled_train_data[b * batch_size:(b + 1) * batch_size][:, 0].tolist()).t() batch_y = torch.LongTensor( shuffled_train_data[b * batch_size:(b + 1) * batch_size][:, 1].tolist()).t() batch_x_lengths = shuffled_x_lengths[b * batch_size:(b + 1) * batch_size] batch_y_lengths = shuffled_y_lengths[b * batch_size:(b + 1) * batch_size] batch_y_rhyme = shuffled_y_rhyme[b * batch_size:(b + 1) * batch_size] if use_cuda: batch_x, batch_y = batch_x.cuda(), batch_y.cuda() train_loss += train(batch_x, batch_y, batch_y_lengths, max(batch_y_lengths), batch_y_rhyme, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, use_cuda, False) train_loss /= b ''' for b in range(len(valid_data) // batch_size): batch_x = torch.LongTensor(valid_data[b*batch_size: (b+1)*batch_size][:, 0].tolist()).t() batch_y = torch.LongTensor(valid_data[b*batch_size: (b+1)*batch_size][:, 1].tolist()).t() if use_cuda: batch_x, batch_y = batch_x.cuda(), batch_y.cuda() valid_loss += train(batch_x, batch_y, max_seqlen, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, use_cuda, True) valid_loss /= b ''' print( "epoch {}, train_loss {:.4f}, valid_loss {:.4f}, best_epoch {}, best_loss {:.4f}" .format(e, train_loss, valid_loss, best_epoch, best_loss)) ''' if valid_loss < best_loss: best_loss = valid_loss best_epoch = e torch.save(encoder.state_dict(), args.encoder_path + '.best') torch.save(decoder.state_dict(), args.decoder_path + '.best') ''' torch.save(encoder.state_dict(), args.encoder_path) torch.save(decoder.state_dict(), args.decoder_path) print(encoder) print(decoder) print("==============") else: encoder.load_state_dict(torch.load( args.encoder_path)) #, map_location=torch.device('cpu'))) decoder.load_state_dict(torch.load( args.decoder_path)) #, map_location=torch.device('cpu'))) print(encoder) print(decoder) predict(encoder, decoder)
def runTest(n_layers, hidden_size, reverse, modelFile, beam_size, input, corpus): voc, pairs, valid_pairs, test_pairs = loadPrepareData(corpus) print('Building encoder and decoder ...') # aspect with open(os.path.join(save_dir, '15_aspect.pkl'), 'rb') as fp: aspect_ids = pickle.load(fp) aspect_num = 15 # 15 | 20 main aspects and each of them has 100 words aspect_ids = Variable( torch.LongTensor(aspect_ids), requires_grad=False ) # convert list into torch Variable, used to index word embedding # attribute embeddings attr_size = 64 # attr_num = 2 with open(os.path.join(save_dir, 'user_item.pkl'), 'rb') as fp: user_dict, item_dict = pickle.load(fp) num_user = len(user_dict) num_item = len(item_dict) attr_embeddings = [] attr_embeddings.append(nn.Embedding(num_user, attr_size)) attr_embeddings.append(nn.Embedding(num_item, attr_size)) aspect_embeddings = [] aspect_embeddings.append(nn.Embedding(num_user, aspect_num)) aspect_embeddings.append(nn.Embedding(num_item, aspect_num)) if USE_CUDA: for attr_embedding in attr_embeddings: attr_embedding = attr_embedding.cuda() for aspect_embedding in aspect_embeddings: aspect_embedding = aspect_embedding.cuda() aspect_ids = aspect_ids.cuda() encoder1 = AttributeEncoder(attr_size, attr_num, hidden_size, attr_embeddings, n_layers) encoder2 = AttributeEncoder(aspect_num, attr_num, hidden_size, aspect_embeddings, n_layers) embedding = nn.Embedding(voc.n_words, hidden_size) encoder3 = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers) attn_model = 'dot' decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, attr_size, voc.n_words, aspect_ids, n_layers) checkpoint = torch.load(modelFile) encoder1.load_state_dict(checkpoint['en1']) encoder2.load_state_dict(checkpoint['en2']) encoder3.load_state_dict(checkpoint['en3']) decoder.load_state_dict(checkpoint['de']) # use cuda if USE_CUDA: encoder1 = encoder1.cuda() encoder2 = encoder2.cuda() encoder3 = encoder3.cuda() decoder = decoder.cuda() # train mode set to false, effect only on dropout, batchNorm encoder1.train(False) encoder2.train(False) encoder3.train(False) decoder.train(False) #evaluateRandomly(encoder1, encoder2, encoder3, decoder, voc, pairs, reverse, beam_size, 100) evaluateRandomly(encoder1, encoder2, encoder3, decoder, voc, test_pairs, reverse, beam_size, len(test_pairs))
def load_network_stageI(self): from model import STAGE1_G, STAGE1_D from model import EncoderRNN, LuongAttnDecoderRNN from model import STAGE1_ImageEncoder, EncodingDiscriminator netG = STAGE1_G() netG.apply(weights_init) #print(netG) netD = STAGE1_D() netD.apply(weights_init) #print(netD) emb_dim = 300 encoder = EncoderRNN(emb_dim, self.txt_emb, 1, dropout=0.0) attn_model = 'general' decoder = LuongAttnDecoderRNN(attn_model, emb_dim, len(self.txt_dico.id2word), self.txt_emb, n_layers=1, dropout=0.0) image_encoder = STAGE1_ImageEncoder() image_encoder.apply(weights_init) e_disc = EncodingDiscriminator(emb_dim) if cfg.NET_G != '': state_dict = \ torch.load(cfg.NET_G, map_location=lambda storage, loc: storage) netG.load_state_dict(state_dict) print('Load from: ', cfg.NET_G) if cfg.NET_D != '': state_dict = \ torch.load(cfg.NET_D, map_location=lambda storage, loc: storage) netD.load_state_dict(state_dict) print('Load from: ', cfg.NET_D) if cfg.ENCODER != '': state_dict = \ torch.load(cfg.ENCODER, map_location=lambda storage, loc: storage) encoder.load_state_dict(state_dict) print('Load from: ', cfg.ENCODER) if cfg.DECODER != '': state_dict = \ torch.load(cfg.DECODER, map_location=lambda storage, loc: storage) decoder.load_state_dict(state_dict) print('Load from: ', cfg.DECODER) if cfg.IMAGE_ENCODER != '': state_dict = \ torch.load(cfg.IMAGE_ENCODER, map_location=lambda storage, loc: storage) image_encoder.load_state_dict(state_dict) print('Load from: ', cfg.IMAGE_ENCODER) # load classification model and freeze weights #clf_model = models.alexnet(pretrained=True) clf_model = models.vgg16(pretrained=True) for param in clf_model.parameters(): param.requires_grad = False if cfg.CUDA: netG.cuda() netD.cuda() encoder.cuda() decoder.cuda() image_encoder.cuda() e_disc.cuda() clf_model.cuda() # ## finetune model for a bit??? # output_size = 512 * 2 * 2 # num_classes = 200 # clf_model.classifier = nn.Sequential( # nn.Linear(output_size, 1024, bias=True), # nn.LeakyReLU(0.2), # nn.Dropout(0.5), # nn.Linear(1024, num_classes, bias=True) # ) # clf_optim = optim.SGD(clf_model.parameters(), lr=1e-2, momentum=0.9) return netG, netD, encoder, decoder, image_encoder, e_disc, clf_model
def trainIters(corpus, reverse, n_epoch, learning_rate, batch_size, n_layers, hidden_size, print_every, loadFilename=None, attn_model='dot', decoder_learning_ratio=1.0): print( "corpus: {}, reverse={}, n_epoch={}, learning_rate={}, batch_size={}, n_layers={}, hidden_size={}, decoder_learning_ratio={}" .format(corpus, reverse, n_epoch, learning_rate, batch_size, n_layers, hidden_size, decoder_learning_ratio)) voc, pairs, valid_pairs, test_pairs = loadPrepareData(corpus) print('load data...') path = "data/expansion" # training data corpus_name = corpus training_batches = None try: training_batches = torch.load( os.path.join( save_dir, path, '{}_{}.tar'.format(filename(reverse, 'training_batches'), batch_size))) except FileNotFoundError: print('Training pairs not found, generating ...') training_batches = batchify(pairs, batch_size, voc, reverse) print('Complete building training pairs ...') torch.save( training_batches, os.path.join( save_dir, path, '{}_{}.tar'.format(filename(reverse, 'training_batches'), batch_size))) # validation/test data eval_batch_size = 10 try: val_batches = torch.load( os.path.join( save_dir, path, '{}_{}.tar'.format(filename(reverse, 'val_batches'), eval_batch_size))) except FileNotFoundError: print('Validation pairs not found, generating ...') val_batches = batchify(valid_pairs, eval_batch_size, voc, reverse, evaluation=True) print('Complete building validation pairs ...') torch.save( val_batches, os.path.join( save_dir, path, '{}_{}.tar'.format(filename(reverse, 'val_batches'), eval_batch_size))) try: test_batches = torch.load( os.path.join( save_dir, path, '{}_{}.tar'.format(filename(reverse, 'test_batches'), eval_batch_size))) except FileNotFoundError: print('Test pairs not found, generating ...') test_batches = batchify(test_pairs, eval_batch_size, voc, reverse, evaluation=True) print('Complete building test pairs ...') torch.save( test_batches, os.path.join( save_dir, path, '{}_{}.tar'.format(filename(reverse, 'test_batches'), eval_batch_size))) # model checkpoint = None print('Building encoder and decoder ...') # aspect with open(os.path.join(save_dir, '15_aspect.pkl'), 'rb') as fp: aspect_ids = pickle.load(fp) aspect_num = 15 # 15 | 20 main aspects and each of them has 100 words aspect_ids = Variable( torch.LongTensor(aspect_ids), requires_grad=False ) # convert list into torch Variable, used to index word embedding # attribute embeddings attr_size = 64 # attr_num = 2 print( "corpus: {}, reverse={}, n_words={}, n_epoch={}, learning_rate={}, batch_size={}, n_layers={}, hidden_size={}, decoder_learning_ratio={}, attr_size={}, aspect_num={}" .format(corpus, reverse, voc.n_words, n_epoch, learning_rate, batch_size, n_layers, hidden_size, decoder_learning_ratio, attr_size, aspect_num)) with open(os.path.join(save_dir, 'user_item.pkl'), 'rb') as fp: user_dict, item_dict = pickle.load(fp) num_user = len(user_dict) num_item = len(item_dict) attr_embeddings = [] attr_embeddings.append(nn.Embedding(num_user, attr_size)) attr_embeddings.append(nn.Embedding(num_item, attr_size)) aspect_embeddings = [] aspect_embeddings.append(nn.Embedding(num_user, aspect_num)) aspect_embeddings.append(nn.Embedding(num_item, aspect_num)) if USE_CUDA: for attr_embedding in attr_embeddings: attr_embedding = attr_embedding.cuda() for aspect_embedding in aspect_embeddings: aspect_embedding = aspect_embedding.cuda() aspect_ids = aspect_ids.cuda() encoder1 = AttributeEncoder(attr_size, attr_num, hidden_size, attr_embeddings, n_layers) encoder2 = AttributeEncoder(aspect_num, attr_num, hidden_size, aspect_embeddings, n_layers) embedding = nn.Embedding(voc.n_words, hidden_size) encoder3 = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers) attn_model = 'dot' decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, attr_size, voc.n_words, aspect_ids, n_layers) if loadFilename: checkpoint = torch.load(loadFilename) encoder1.load_state_dict(checkpoint['en1']) encoder2.load_state_dict(checkpoint['en2']) encoder3.load_state_dict(checkpoint['en3']) decoder.load_state_dict(checkpoint['de']) # use cuda if USE_CUDA: encoder1 = encoder1.cuda() encoder2 = encoder2.cuda() encoder3 = encoder3.cuda() decoder = decoder.cuda() # optimizer print('Building optimizers ...') encoder1_optimizer = optim.Adam(encoder1.parameters(), lr=learning_rate) encoder2_optimizer = optim.Adam(encoder2.parameters(), lr=learning_rate) encoder3_optimizer = optim.Adam(encoder3.parameters(), lr=learning_rate) decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate * decoder_learning_ratio) if loadFilename: encoder1_optimizer.load_state_dict(checkpoint['en1_opt']) encoder2_optimizer.load_state_dict(checkpoint['en2_opt']) encoder3_optimizer.load_state_dict(checkpoint['en3_opt']) decoder_optimizer.load_state_dict(checkpoint['de_opt']) # initialize print('Initializing ...') start_epoch = 0 perplexity = [] best_val_loss = None print_loss = 0 if loadFilename: start_epoch = checkpoint['epoch'] + 1 perplexity = checkpoint['plt'] for epoch in range(start_epoch, n_epoch): epoch_start_time = time.time() # train epoch encoder1.train() encoder2.train() encoder3.train() decoder.train() print_loss = 0 start_time = time.time() for batch, training_batch in enumerate(training_batches): attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = training_batch loss = train(attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len, encoder1, encoder2, encoder3, decoder, embedding, encoder1_optimizer, encoder2_optimizer, encoder3_optimizer, decoder_optimizer, batch_size) print_loss += loss perplexity.append(loss) #print("batch {} loss={}".format(batch, loss)) if batch % print_every == 0 and batch > 0: cur_loss = print_loss / print_every elapsed = time.time() - start_time print( '| epoch {:3d} | {:5d}/{:5d} batches | lr {:05.5f} | ms/batch {:5.2f} | ' 'loss {:5.2f} | ppl {:8.2f}'.format( epoch, batch, len(training_batches), learning_rate, elapsed * 1000 / print_every, cur_loss, math.exp(cur_loss))) print_loss = 0 start_time = time.time() # evaluate val_loss = 0 for val_batch in val_batches: attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = val_batch loss = evaluate(attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len, encoder1, encoder2, encoder3, decoder, embedding, encoder1_optimizer, encoder2_optimizer, encoder3_optimizer, decoder_optimizer, batch_size) val_loss += loss val_loss /= len(val_batches) print('-' * 89) print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' 'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time), val_loss, math.exp(val_loss))) print('-' * 89) # Save the model if the validation loss is the best we've seen so far. if not best_val_loss or val_loss < best_val_loss: directory = os.path.join(save_dir, 'model', '{}_{}'.format(n_layers, hidden_size)) if not os.path.exists(directory): os.makedirs(directory) torch.save( { 'epoch': epoch, 'en1': encoder1.state_dict(), 'en2': encoder2.state_dict(), 'en3': encoder3.state_dict(), 'de': decoder.state_dict(), 'en1_opt': encoder1_optimizer.state_dict(), 'en2_opt': encoder2_optimizer.state_dict(), 'en3_opt': encoder3_optimizer.state_dict(), 'de_opt': decoder_optimizer.state_dict(), 'loss': loss, 'plt': perplexity }, os.path.join( directory, '{}_{}.tar'.format( epoch, filename(reverse, 'lexicon_title_expansion_model')))) best_val_loss = val_loss # Run on test data. test_loss = 0 for test_batch in test_batches: attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = test_batch loss = evaluate(attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len, encoder1, encoder2, encoder3, decoder, embedding, encoder1_optimizer, encoder2_optimizer, encoder3_optimizer, decoder_optimizer, batch_size) test_loss += loss test_loss /= len(test_batches) print('-' * 89) print('| test loss {:5.2f} | test ppl {:8.2f}'.format( test_loss, math.exp(test_loss))) print('-' * 89) if val_loss > best_val_loss: break