def test(args): vocab = Vocabulary() vocab.load_vocab(os.path.join(args['data_dir'], 'vocabulary.json')) args['voca_size'] = vocab.get_vocab_size() test_data = get_dataloader( os.path.join(args['data_dir'], 'encoded_test_dialogue_pair.json'), os.path.join(args['data_dir'], 'vocabulary.json'), 1) test_sent_pair_list = [] model = Seq2Seq(args).eval() if torch.cuda.is_available(): model = model.cuda() path = Checkpoint.get_latest_checkpoint(args['exp_dir']) model.load_state_dict(torch.load(os.path.join(path, 'model.pt'))) for batch_idx, (sour, sour_len, targ, targ_len) in enumerate(test_data): if torch.cuda.is_available(): sour = sour.cuda() targ = targ.cuda() enco_hidd_state = model.encoder.encoder_forward(sour, sour_len) out_prob = model.decoder.decoder_forward(targ, targ_len, enco_hidd_state, 0) sent_list = [(out_prob.topk(1)[1].view(-1).tolist(), 0)] test_sent_pair_list += process_sent_list(vocab, sour, targ, sent_list) # logger.info('batch_idx:{} \nsent:{}'.format(batch_idx,test_sent_pair_list)) save_test_sent(args['exp_data'], 'generated_test_sent.txt', test_sent_pair_list)
def main(): train_loader, test_loader = get_mnist_data('../%s' % opt.dataset, opt.batch_size) model = CapsuleNetwork(opt) if opt.cuda == True: model = model.cuda() if opt.is_train == True: if opt.resume == True: latest_checkpoint_path = Checkpoint.get_latest_checkpoint( opt.save_folder) resume_checkpoint = Checkpoint.load(latest_checkpoint_path) model = resume_checkpoint.model optimizer = resume_checkpoint.optimizer start_epoch = resume_checkpoint.epoch + 1 else: start_epoch = 0 optimizer = Adam(model.parameters()) for epoch in range(start_epoch, opt.n_epochs): train(epoch, model, train_loader, test_loader, optimizer) Checkpoint(model=model, optimizer=optimizer, epoch=epoch).save(opt.save_folder) else: run_test(model, test_loader)
def train(self, model, data, num_epochs=5, resume=False, dev_data=None, optimizer=None, teacher_forcing_ratio=0): """ Run training for a given model. Args: model (seq2seq.models): model to run training on, if `resume=True`, it would be overwritten by the model loaded from the latest checkpoint. data (seq2seq.dataset.dataset.Dataset): dataset object to train on num_epochs (int, optional): number of epochs to run (default 5) resume(bool, optional): resume training with the latest checkpoint, (default False) dev_data (seq2seq.dataset.dataset.Dataset, optional): dev Dataset (default None) optimizer (seq2seq.optim.Optimizer, optional): optimizer for training (default: Optimizer(pytorch.optim.Adam, max_grad_norm=5)) teacher_forcing_ratio (float, optional): teaching forcing ratio (default 0) Returns: model (seq2seq.models): trained model. """ # If training is set to resume if resume: latest_checkpoint_path = Checkpoint.get_latest_checkpoint( self.expt_dir) resume_checkpoint = Checkpoint.load(latest_checkpoint_path) model = resume_checkpoint.model self.optimizer = resume_checkpoint.optimizer # A walk around to set optimizing parameters properly resume_optim = self.optimizer.optimizer defaults = resume_optim.param_groups[0] defaults.pop('params', None) defaults.pop('initial_lr', None) self.optimizer.optimizer = resume_optim.__class__( model.parameters(), **defaults) start_epoch = resume_checkpoint.epoch step = resume_checkpoint.step else: start_epoch = 1 step = 0 if optimizer is None: optimizer = Optimizer(optim.Adam(model.parameters()), max_grad_norm=5) self.optimizer = optimizer self.logger.info("Optimizer: %s, Scheduler: %s" % (self.optimizer.optimizer, self.optimizer.scheduler)) self._train_epochs(data, model, num_epochs, start_epoch, step, dev_data=dev_data, teacher_forcing_ratio=teacher_forcing_ratio) return model
def load_model(): checkpoint_path = "" if not FLAGS.load_checkpoint is None: checkpoint_path = os.path.join(FLAGS.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, FLAGS.load_checkpoint) else: checkpoint_path = Checkpoint.get_latest_checkpoint(FLAGS.expt_dir) logging.info("loading checkpoint from {}".format(checkpoint_path)) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model # these are vocab classes with members stoi and itos input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab classifier = (seq2seq, input_vocab, output_vocab) return classifier
def train(self,encoder, decoder, n_epochs, train_data, dev_data, resume, optimizer, log_file): """ ------------------------------------------------------------------------ Args: encoder: Self explanatory. decoder: Self explanatory. n_epoch (int): Number of epochs to train the model. train_data (Composition): Self explanatory. dev_data (Composition): Self explanatory. resume (bool): If true, load last checkpoint. ------------------------------------------------------------------------ """ if resume: latest_checkpoint_path = Checkpoint.get_latest_checkpoint(self.exp_dir) resume_checkpoint = Checkpoint.load(latest_checkpoint_path) encoder = resume_checkpoint.encoder decoder = resume_checkpoint.decoder start_epoch = resume_checkpoint.epoch step = resume_checkpoint.step self.scheduler = resume_checkpoint.scheduler self.optimizer = resume_checkpoint.optimizer self.samp_rate = resume_checkpoint.samp_rate self.KL_rate = resume_checkpoint.KL_rate self.free_bits = resume_checkpoint.free_bits self.vocab_size = decoder.vocab_size else: self.optimizer = optimizer if optimizer is None: params = list(encoder.parameters()) + list(decoder.parameters()) self.optimizer = Adam(params, lr=1e-3) self.scheduler = LambdaLR(self.optimizer,decay) self.vocab_size = decoder.vocab_size start_epoch = 1 step = 0 self.train_epochs(encoder, decoder, start_epoch, step, train_data, dev_data, start_epoch + n_epochs, log_file) return encoder,decoder
def run_test(model, test_loader): latest_checkpoint_path = Checkpoint.get_latest_checkpoint(opt.save_folder) resume_checkpoint = Checkpoint.load(latest_checkpoint_path) model = resume_checkpoint.model optimizer = resume_checkpoint.optimizer model.eval() test_loss = 0 num_error = 0 num_data = 0 for batch_id, (data, target) in enumerate(test_loader): data = Variable(data) if opt.cuda == True: data = data.cuda() output, mask, recon = model(data) out_mag = torch.sqrt((output**2).sum(2)) out_mag = F.softmax(out_mag, dim=1) max_val, max_idx = out_mag.max(dim=1) for idx in range(data.size(0)): print "(batch_index, sample_index, estimated, target) : ", batch_id, idx, max_idx[ idx].data.cpu().numpy(), target[idx] if max_idx[idx].data.cpu().numpy() != target[idx]: num_error = num_error + 1 num_data = num_data + 1 if opt.vis == True: idx = random.randint(0, data.size(0) - 1) show_recon = recon[idx].data.cpu().numpy().reshape(28, 28) show_data = data[idx].data.cpu().numpy().reshape(28, 28) cv2.namedWindow("recon", cv2.WINDOW_NORMAL) cv2.imshow("recon", np.concatenate((show_data, show_recon), axis=1)) cv2.waitKey(1) print 'test error : ', float(num_error) / float(num_data)
def main(): torch.manual_seed(233) logging.basicConfig(level=logging.INFO, format='%(asctime)s [INFO] %(message)s') parser = argparse.ArgumentParser() parser.add_argument('--data_path', type=str,dest='data_path', default='/home/ml/ydong26/data/EditNTS_data/editnet_data/%s/'%dataset, help='Path to train vocab_data') parser.add_argument('--store_dir', action='store', dest='store_dir', default='/home/ml/ydong26/tmp_store/editNTS_%s'%dataset, help='Path to exp storage directory.') parser.add_argument('--vocab_path', type=str, dest='vocab_path', default='../vocab_data/', help='Path contains vocab, embedding, postag_set') parser.add_argument('--load_model', type=str, dest='load_model', default=None, help='Path for loading pre-trained model for further training') parser.add_argument('--vocab_size', dest='vocab_size', default=30000, type=int) parser.add_argument('--batch_size', dest='batch_size', default=32, type=int) parser.add_argument('--max_seq_len', dest='max_seq_len', default=100) parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--hidden', type=int, default=200) parser.add_argument('--lr', type=float, default=1e-4) parser.add_argument('--device', type=int, default=1, help='select GPU') #train_file = '/media/vocab_data/yue/TS/editnet_data/%s/train.df.filtered.pos'%dataset # test='/media/vocab_data/yue/TS/editnet_data/%s/test.df.pos' % args.dataset args = parser.parse_args() torch.cuda.set_device(args.device) # load vocab-related files and init vocab print('*'*10) vocab = data.Vocab() vocab.add_vocab_from_file(args.vocab_path+'vocab.txt', args.vocab_size) vocab.add_embedding(gloveFile=args.vocab_path+'glove.6B.100d.txt') pos_vocab = data.POSvocab(args.vocab_path) #load pos-tags embeddings print('*' * 10) print(args) print("generating config") hyperparams=collections.namedtuple( 'hps', #hyper=parameters ['vocab_size', 'embedding_dim', 'word_hidden_units', 'sent_hidden_units', 'pretrained_embedding', 'word2id', 'id2word', 'pos_vocab_size', 'pos_embedding_dim'] ) hps = hyperparams( vocab_size=vocab.count, embedding_dim=100, word_hidden_units=args.hidden, sent_hidden_units=args.hidden, pretrained_embedding=vocab.embedding, word2id=vocab.w2i, id2word=vocab.i2w, pos_vocab_size=pos_vocab.count, pos_embedding_dim=30 ) print('init editNTS model') edit_net = EditNTS(hps, n_layers=1) edit_net.cuda() if args.load_model is not None: ckpt_path = Checkpoint.get_latest_checkpoint(args.load_model) print("load edit_net for further training from %s", ckpt_path) ckpt = Checkpoint.load(ckpt_path) edit_net = ckpt.model edit_net.cuda() edit_net.train() training(edit_net, args.epochs, args, vocab)
def main(args): configure(os.path.join(args['exp_dir'], 'log_dir')) transform = transforms.Compose([ transforms.RandomCrop(args['crop_size']), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) data_loader = get_loader({ 'data_dir': args['data_dir'], 'exp_dir': args['exp_dir'], 'raw_data_dir': args['raw_data_dir'], 'batch_size': args['batch_size'], 'transform': transform, 'num_workers': args['num_workers'], 'shuffle': args['shuffle'], 'mode': 'train' }) # valid_data_loader=get_loader({'data_dir' : args['data_dir'], # 'raw_data_dir' : args['raw_data_dir'], # 'batch_size' : int(args['batch_size']/4), # 'transform' : transform, # 'num_workers' : args['num_workers'], # 'shuffle' : args['shuffle'], # 'mode':'validate'}) args['vocab_size'] = len(Vocabulary.load_vocab(args['exp_dir'])) encoder = EncoderCNN(args).train() decoder = DecoderRNN(args).train() if args['pretrained']: checkpoint_path = Checkpoint.get_latest_checkpoint(args['exp_dir']) checkpoint = Checkpoint.load(checkpoint_path) encoder.load_state_dict(checkpoint.encoder) decoder.load_state_dict(checkpoint.decoder) step = checkpoint.step epoch = checkpoint.epoch omit = True else: step = 0 epoch = 0 omit = False encoder.to(device) decoder.to(device) criterion = nn.CrossEntropyLoss() params = list(decoder.parameters()) + list( encoder.linear.parameters()) + list(encoder.bn.parameters()) # params=list(decoder.parameters()) + list(encoder.parameters()) optimizer = torch.optim.Adam(params, lr=args['lr']) scheduler = StepLR(optimizer, step_size=40, gamma=0.1) # optimizer=YFOptimizer(params) total_step = len(data_loader) min_valid_loss = float('inf') for epoch in range(epoch, args['num_epochs']): scheduler.step() for idx, (images, captions, leng) in enumerate(data_loader): if omit: if idx < (step - total_step * epoch): logger.info( 'idx:{},step:{}, epoch:{}, total_step:{}, diss:{}'. format(idx, step, epoch, total_step, step - total_step * epoch)) continue else: omit = False images = images.to(device) captions = captions.to(device) targets = pack_padded_sequence(captions, leng, batch_first=True)[0] features = encoder(images) outputs = decoder(features, captions, leng) loss = criterion(outputs, targets) decoder.zero_grad() encoder.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(decoder.parameters(), 5) optimizer.step() log_value('loss', loss.item(), step) step += 1 if step % args['log_step'] == 0: logger.info( 'Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Perplexity: {:5.4f}' .format(epoch, args['num_epochs'], idx, total_step, loss.item(), np.exp(loss.item()))) if step % args['valid_step'] == 0: # valid_loss=validate(encoder.eval(),decoder,criterion,valid_data_loader) # if valid_loss<min_valid_loss: # min_valid_loss=valid_loss Checkpoint(encoder, decoder, optimizer, epoch, step).save(args['exp_dir'])
#Prepare trainer. Ash_Ketchum = VAE_Trainer.Trainer(exp_dir=exp_dir, score_type=score_type, batch_size=128, random_seed=42, print_every=100, checkpoint_every=10000, samp_rate=None, KL_rate=0.99991, free_bits=125) #Check to see if we've already started the experiment resume = False if resume: latest_checkpoint_path = Checkpoint.get_latest_checkpoint("experiment_1") print("Resuming training...") Ash_Ketchum.train(None, None, n_epochs, train_comp, val_comp, True, None, log_file) else: vocab_size = 275 encoder_hidden_size = 256 decoder_hidden_size = 512 latent_size = 128 seq_size = 52 num_layers = 2 encoder = Encoder(vocab_size, encoder_hidden_size, latent_size, seq_size, num_layers).to(device) decoder = Decoder(latent_size, decoder_hidden_size, vocab_size, num_layers, seq_size).to(device)