def main(): """ Training and validation. """ global best_bleu4, epochs_since_improvement, checkpoint, start_epoch, fine_tune_encoder, data_name, word_map, rev_word_map # Read word map word_map_file = os.path.join(data_folder, 'WORDMAP_' + data_name + '.json') with open(word_map_file, 'r') as j: word_map = json.load(j) rev_word_map = {v: k for k, v in word_map.items()} # Initialize / load checkpoint if checkpoint is None: decoder = DecoderWithAttention(attention_dim=attention_dim, embed_dim=emb_dim, decoder_dim=decoder_dim, vocab_size=len(word_map), dropout=dropout) pretrained_embs, pretrained_embs_dim = load_embeddings( '/home/Iwamura/datasets/datasets/GloVe/glove.6B.300d.txt', word_map) assert pretrained_embs_dim == decoder.embed_dim decoder.load_pretrained_embeddings(pretrained_embs) decoder.fine_tune_embeddings(True) decoder_optimizer = torch.optim.Adam(params=filter( lambda p: p.requires_grad, decoder.parameters()), lr=decoder_lr) encoder = Encoder() encoder_opt = Encoder() encoder.fine_tune(fine_tune_encoder) encoder_opt.fine_tune(fine_tune_encoder_opt) encoder_optimizer = torch.optim.Adam( params=filter(lambda p: p.requires_grad, encoder.parameters()), lr=encoder_lr) if fine_tune_encoder else None encoder_optimizer_opt = torch.optim.Adam( params=filter(lambda p: p.requires_grad, encoder_opt.parameters()), lr=encoder_opt_lr) if fine_tune_encoder_opt else None else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] best_bleu4 = checkpoint['bleu-4'] decoder = checkpoint['decoder'] decoder_optimizer = checkpoint['decoder_optimizer'] encoder_opt = checkpoint['encoder_opt'] encoder_optimizer_opt = checkpoint['encoder_optimizer_opt'] # if fine_tune_encoder is True and encoder_optimizer is None and encoder_optimizer_opt is None if fine_tune_encoder_opt is True and encoder_optimizer_opt is None: encoder_opt.fine_tune(fine_tune_encoder_opt) encoder_optimizer_opt = torch.optim.Adam(params=filter( lambda p: p.requires_grad, encoder_opt.parameters()), lr=encoder_opt_lr) # Move to GPU, if available decoder = decoder.to(device) encoder_opt = encoder_opt.to(device) # Loss function criterion = nn.CrossEntropyLoss().to(device) # Custom dataloaders normalize_opt = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader(CaptionDataset( data_folder, data_name, 'TRAIN', transform=transforms.Compose([normalize_opt])), batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(CaptionDataset( data_folder, data_name, 'VAL', transform=transforms.Compose([normalize_opt])), batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True) # Epochs for epoch in range(start_epoch, epochs): # Decay learning rate if there is no improvement for 8 consecutive epochs, and terminate training after 20 if epochs_since_improvement == 10: break if epoch > 0 and epoch % 4 == 0: adjust_learning_rate(decoder_optimizer, 0.8) if fine_tune_encoder_opt: adjust_learning_rate(encoder_optimizer_opt, 0.8) # One epoch's training train(train_loader=train_loader, encoder_opt=encoder_opt, decoder=decoder, criterion=criterion, encoder_optimizer_opt=encoder_optimizer_opt, decoder_optimizer=decoder_optimizer, epoch=epoch) # One epoch's validation recent_bleu4 = validate(val_loader=val_loader, encoder_opt=encoder_opt, decoder=decoder, criterion=criterion) # Check if there was an improvement is_best = recent_bleu4 > best_bleu4 best_bleu4 = max(recent_bleu4, best_bleu4) if not is_best: epochs_since_improvement += 1 print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement, )) else: epochs_since_improvement = 0 # Save checkpoint save_checkpoint(data_name, epoch, epochs_since_improvement, encoder_opt, decoder, encoder_optimizer_opt, decoder_optimizer, recent_bleu4, is_best)
def fit(t_params, checkpoint=None, m_params=None, logger=None): # info data_name = t_params['data_name'] imgs_path = t_params['imgs_path'] df_path = t_params['df_path'] vocab = t_params['vocab'] start_epoch = 0 epochs_since_improvement = 0 best_bleu4 = 0 epochs = t_params['epochs'] batch_size = t_params['batch_size'] workers = t_params['workers'] encoder_lr = t_params['encoder_lr'] decoder_lr = t_params['decoder_lr'] fine_tune_encoder = t_params['fine_tune_encoder'] # pretrained word embeddings pretrained_embeddings = t_params['pretrained_embeddings'] if pretrained_embeddings: fine_tune_embeddings = t_params['fine_tune_embeddings'] embeddings_matrix = m_params['embeddings_matrix'] # init / load checkpoint if checkpoint is None: # getting hyperparameters attention_dim = m_params['attention_dim'] embed_dim = m_params['embed_dim'] decoder_dim = m_params['decoder_dim'] encoder_dim = m_params['encoder_dim'] dropout = m_params['dropout'] decoder = DecoderWithAttention(attention_dim=attention_dim, embed_dim=embed_dim, decoder_dim=decoder_dim, encoder_dim=encoder_dim, vocab_size=len(vocab), dropout=dropout) if pretrained_embeddings: decoder.load_pretrained_embeddings( torch.tensor(embeddings_matrix, dtype=torch.float32)) decoder.fine_tune_embeddings(fine_tune=fine_tune_embeddings) decoder_optimizer = torch.optim.RMSprop(params=filter( lambda p: p.requires_grad, decoder.parameters()), lr=decoder_lr) encoder = Encoder() encoder.fine_tune(fine_tune_encoder) encoder_optimizer = torch.optim.RMSprop( params=filter(lambda p: p.requires_grad, encoder.parameters()), lr=encoder_lr) if fine_tune_encoder else None # load checkpoint else: checkpoint = torch.load(checkpoint) print('Loaded Checkpoint!!') start_epoch = checkpoint['epoch'] + 1 print(f"Starting Epoch: {start_epoch}") epochs_since_improvement = checkpoint['epochs_since_imrovment'] best_bleu4 = checkpoint['bleu-4'] decoder = checkpoint['decoder'] decoder_optimizer = checkpoint['deocder_optimizer'] encoder = checkpoint['encoder'] encoder_optimizer = checkpoint['encoder_optimizer'] if fine_tune_encoder is True and encoder_optimizer is None: encoder.fine_tune(fine_tune_encoder) encoder_optimizer = torch.optim.RMSprop(params=filter( lambda p: p.requires_grad, encoder.parameters()), lr=encoder_lr) # Schedulers decoder_scheduler = ReduceLROnPlateau(decoder_optimizer, patience=2, verbose=True) if fine_tune_encoder: encoder_scheduler = ReduceLROnPlateau(encoder_optimizer, patience=2, verbose=True) # move to gpu, if available decoder = decoder.to(device) encoder = encoder.to(device) # loss function criterion = nn.CrossEntropyLoss().to(device) # dataloaders transform = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) print('Loading Data') train_loader, val_loader = get_loaders(batch_size, imgs_path, df_path, transform, vocab, False, workers) print('_' * 50) print('-' * 20, 'Fitting', '-' * 20) for epoch in range(start_epoch, epochs): # if epochs_since_improvement > 0 and epochs_since_improvement % 2 == 0: # adjust_learning_rate(decoder_optimizer, 0.8) # if fine_tune_encoder: # adjust_learning_rate(encoder_optimizer, 0.8) print('_' * 50) print('-' * 20, 'Training', '-' * 20) # one epoch of training epoch_time = AverageMeter() start_time = time.time() train(train_loader=train_loader, encoder=encoder, decoder=decoder, criterion=criterion, encoder_optimizer=encoder_optimizer, decoder_optimizer=decoder_optimizer, epoch=epoch, logger=logger) epoch_time.update(time.time() - start_time) print(f"Epoch train time {epoch_time.val:.3f} (epoch_time.avg:.3f)") # one epoch of validation epoch_time = AverageMeter() start_time = time.time() print('-' * 20, 'Validation', '-' * 20) b1, b2, b3, recent_bleu4 = validate(val_loader=val_loader, encoder=encoder, decoder=decoder, criterion=criterion, vocab=vocab, epoch=epoch, logger=logger) epoch_time.update(time.time() - start_time) # tensorboard logger.add_scalar(f'b-1/valid', b1, epoch) logger.add_scalar(f'b-2/valid', b2, epoch) logger.add_scalar(f'b-3/valid', b3, epoch) logger.add_scalar(f'b-4/valid', recent_bleu4, epoch) # logger.add_scalar(f'Meteor/valid', m, epoch) print( f"Epoch validation time {epoch_time.val:.3f} (epoch_time.avg:.3f)") # check for improvement is_best = recent_bleu4 > best_bleu4 best_bleu4 = max(recent_bleu4, best_bleu4) if not is_best: epochs_since_improvement += 1 print( f'\nEpochs since last improvement: {epochs_since_improvement,}' ) else: # reset epochs_since_improvement = 0 # stop training if no improvement for 5 epochs if epochs_since_improvement == 5: print('No improvement for 5 consecutive epochs, terminating...') break # learning rate schedular decoder_scheduler.step(recent_bleu4) if fine_tune_encoder: encoder_scheduler.step(recent_bleu4) save_checkpoint(data_name, epoch, epochs_since_improvement, encoder, decoder, encoder_optimizer, decoder_optimizer, recent_bleu4, is_best)
def main(): """ Training and validation. """ global best_bleu4, epochs_since_improvement, checkpoint, start_epoch, fine_tune_encoder, data_name, word_map, glove_path, emb_dim, rev_word_map # Read word map word_map_file = os.path.join(data_folder, 'WORDMAP_' + data_name + '.json') with open(word_map_file, 'r') as j: word_map = json.load(j) rev_word_map = {v: k for k, v in word_map.items()} #get glove vectors = bcolz.open(f'{glove_path}/6B.300.dat')[:] words = pickle.load(open(f'{glove_path}/6B.300_words.pkl', 'rb')) word2idx = pickle.load(open(f'{glove_path}/6B.300_idx.pkl', 'rb')) glove = {w: vectors[word2idx[w]] for w in words} matrix_len = len(word_map) weights_matrix = np.zeros((matrix_len, emb_dim)) words_found = 0 for i, word in enumerate(word_map.keys()): try: weights_matrix[i] = glove[word] words_found += 1 except KeyError: weights_matrix[i] = np.random.normal(scale=0.6, size=(emb_dim, )) # weights_matrix = np.float64(weights_matrix) # weights_matrix = torch.from_numpy(weights_matrix) # pretrained_embedding = weights_matrix.to(dtype=torch.float) # print(pretrained_embedding.dtype) # if device.type == 'cpu' : # pretrained_embedding = torch.FloatTensor(weights_matrix) # else: # pretrained_embedding = torch.cuda.FloatTensor(weights_matrix) pretrained_embedding = torch.FloatTensor(weights_matrix) # Initialize / load checkpoint if checkpoint is None: decoder = DecoderWithAttention(attention_dim=attention_dim, embed_dim=emb_dim, decoder_dim=decoder_dim, vocab_size=len(word_map), dropout=dropout) decoder.load_pretrained_embeddings( pretrained_embedding ) # pretrained_embeddings should be of dimensions (len(word_map), emb_dim) decoder.fine_tune_embeddings(True) # or False decoder_optimizer = torch.optim.Adam(params=filter( lambda p: p.requires_grad, decoder.parameters()), lr=decoder_lr) encoder = Encoder() encoder.fine_tune(fine_tune_encoder) encoder_optimizer = torch.optim.Adam( params=filter(lambda p: p.requires_grad, encoder.parameters()), lr=encoder_lr) if fine_tune_encoder else None else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] best_bleu4 = checkpoint['bleu-4'] decoder = checkpoint['decoder'] decoder_optimizer = checkpoint['decoder_optimizer'] encoder = checkpoint['encoder'] encoder_optimizer = checkpoint['encoder_optimizer'] if fine_tune_encoder is True and encoder_optimizer is None: encoder.fine_tune(fine_tune_encoder) encoder_optimizer = torch.optim.Adam(params=filter( lambda p: p.requires_grad, encoder.parameters()), lr=encoder_lr) # Move to GPU, if available decoder = decoder.to(device) encoder = encoder.to(device) # Loss function criterion = nn.CrossEntropyLoss().to(device) # Custom dataloaders normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader(CaptionDataset( data_folder, data_name, 'TRAIN', transform=transforms.Compose([normalize])), batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(CaptionDataset( data_folder, data_name, 'VAL', transform=transforms.Compose([normalize])), batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True) # Epochs for epoch in range(start_epoch, epochs): # Decay learning rate if there is no improvement for 8 consecutive epochs, and terminate training after 20 if epochs_since_improvement == 20: break if epochs_since_improvement > 0 and epochs_since_improvement % 8 == 0: adjust_learning_rate(decoder_optimizer, 0.8) if fine_tune_encoder: adjust_learning_rate(encoder_optimizer, 0.8) # One epoch's training # train(train_loader=train_loader, # encoder=encoder, # decoder=decoder, # criterion=criterion, # encoder_optimizer=encoder_optimizer, # decoder_optimizer=decoder_optimizer, # epoch=epoch) # One epoch's validation recent_bleu4 = validate(val_loader=val_loader, encoder=encoder, decoder=decoder, criterion=criterion) # Check if there was an improvement is_best = recent_bleu4 > best_bleu4 best_bleu4 = max(recent_bleu4, best_bleu4) if not is_best: epochs_since_improvement += 1 print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement, )) else: epochs_since_improvement = 0 # Save checkpoint save_checkpoint(data_name, epoch, epochs_since_improvement, encoder, decoder, encoder_optimizer, decoder_optimizer, recent_bleu4, is_best)
def main(): """ Training and validation. """ global best_bleu4, epochs_since_improvement, checkpoint, start_epoch, fine_tune_encoder, data_name, word_map # Read word map word_map_file = os.path.join(data_folder, 'WORDMAP_' + data_name + '.json') with open(word_map_file, 'r') as j: word_map = json.load(j) # Initialize / load checkpoint if checkpoint is None: emb_dim=100 #remove if not usiong pretrained model decoder = DecoderWithAttention(attention_dim=attention_dim, embed_dim=emb_dim, decoder_dim=decoder_dim, vocab_size=len(word_map), dropout=dropout) pretrained_embeddings = decoder.create_pretrained_embedding_matrix(word_map) decoder.load_pretrained_embeddings( pretrained_embeddings) # pretrained_embeddings should be of dimensions (len(word_map), emb_dim) decoder.fine_tune_embeddings(True) decoder_optimizer = torch.optim.Adam(params=filter(lambda p: p.requires_grad, decoder.parameters()), lr=decoder_lr) encoder = Encoder() encoder.fine_tune(fine_tune_encoder) encoder_optimizer = torch.optim.Adam(params=filter(lambda p: p.requires_grad, encoder.parameters()), lr=encoder_lr) if fine_tune_encoder else None else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] best_bleu4 = checkpoint['bleu-4'] decoder = checkpoint['decoder'] decoder_optimizer = checkpoint['decoder_optimizer'] encoder = checkpoint['encoder'] encoder_optimizer = checkpoint['encoder_optimizer'] if fine_tune_encoder is True and encoder_optimizer is None: encoder.fine_tune(fine_tune_encoder) encoder_optimizer = torch.optim.Adam(params=filter(lambda p: p.requires_grad, encoder.parameters()), lr=encoder_lr) # Move to GPU, if available decoder = decoder.to(device) encoder = encoder.to(device) # Loss function criterion = nn.CrossEntropyLoss().to(device) # Custom dataloaders normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader( CaptionDataset(data_folder, data_name, 'TRAIN', transform=transforms.Compose([normalize])), batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True) val_loader = torch.utils.data.DataLoader( CaptionDataset(data_folder, data_name, 'VAL', transform=transforms.Compose([normalize])), batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True) # Epochs for epoch in range(start_epoch, epochs): # Decay learning rate if there is no improvement for 8 consecutive epochs, and terminate training after 20 if epochs_since_improvement == 20: break if epochs_since_improvement > 0 and epochs_since_improvement % 8 == 0: adjust_learning_rate(decoder_optimizer, 0.8) if fine_tune_encoder: adjust_learning_rate(encoder_optimizer, 0.8) # One epoch's training train(train_loader=train_loader, encoder=encoder, decoder=decoder, criterion=criterion, encoder_optimizer=encoder_optimizer, decoder_optimizer=decoder_optimizer, epoch=epoch) # One epoch's validation recent_bleu4, val_loss_avg, val_accu_avg = validate(val_loader=val_loader, encoder=encoder, decoder=decoder, criterion=criterion) #write to tensorboard writer.add_scalar('validation_loss', val_loss_avg, epoch) writer.add_scalar('validation_accuracy', val_accu_avg, epoch) writer.add_scalar('validation_bleu4', recent_bleu4, epoch) # Check if there was an improvement is_best = recent_bleu4 > best_bleu4 best_bleu4 = max(recent_bleu4, best_bleu4) if not is_best: epochs_since_improvement += 1 print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement,)) else: epochs_since_improvement = 0 # Save checkpoint print("Saving model to file",ckpt_name.format(epoch, bleu=recent_bleu4, loss=val_loss_avg, acc=val_accu_avg)) save_checkpoint(ckpt_name.format(epoch, bleu=recent_bleu4, loss=val_loss_avg, acc=val_accu_avg), epoch, epochs_since_improvement, encoder, decoder, encoder_optimizer, decoder_optimizer, recent_bleu4, is_best) #close tensorboard writer writer.close()