def save_checkpoint(model: Vicl, model_optimizer: LocalSgd, moptim_scheduler: ExponentialLR, task: int, epoch: int, models_dir: str): halo = Halo( text=f'Saving checkpoint (epoch: {epoch + 1})', spinner='dots').start() checkpoint = { 'model': model.state(), 'model_optimizer': model_optimizer.state_dict(), 'moptim_scheduler': moptim_scheduler.state_dict(), 'epoch': epoch, } try: save_name = f'vicl-task-{task}-cp.pt' save_path = os.path.join(models_dir, save_name) torch.save(checkpoint, save_path) except Exception as e: halo.fail(f'Couldn\'t save checkpoint (error: {e})') else: halo.succeed(f'Successfully saved checkpoint (epoch: {epoch + 1})')
base_optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.decay_lr) optimizer = LARS(optimizer=base_optimizer, eps=1e-8, trust_coef=0.001) scheduler = ExponentialLR(optimizer, gamma=args.decay_lr) # Main training loop best_loss = np.inf for epoch in range(args.epochs): v_loss = execute_graph(model, loaders, optimizer, scheduler, epoch) if v_loss < best_loss: best_loss = v_loss print('Writing model checkpoint') state = { 'epoch': epoch, 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'base_optimizer': base_optimizer.state_dict(), 'scheduler': scheduler.state_dict(), 'val_loss': v_loss } torch.save( state, args.output_dir + "/" + str(validation_fold) + "/best.ckpt") print("Only training once. Done.") exit()
def main(rank, args): # Distributed setup if args.distributed: setup_distributed(rank, args.world_size) not_main_rank = args.distributed and rank != 0 logging.info("Start time: %s", datetime.now()) # Explicitly set seed to make sure models created in separate processes # start from same random weights and biases torch.manual_seed(args.seed) # Empty CUDA cache torch.cuda.empty_cache() # Change backend for flac files torchaudio.set_audio_backend("soundfile") # Transforms melkwargs = { "n_fft": args.win_length, "n_mels": args.n_bins, "hop_length": args.hop_length, } sample_rate_original = 16000 if args.type == "mfcc": transforms = torch.nn.Sequential( torchaudio.transforms.MFCC( sample_rate=sample_rate_original, n_mfcc=args.n_bins, melkwargs=melkwargs, ), ) num_features = args.n_bins elif args.type == "waveform": transforms = torch.nn.Sequential(UnsqueezeFirst()) num_features = 1 else: raise ValueError("Model type not supported") if args.normalize: transforms = torch.nn.Sequential(transforms, Normalize()) augmentations = torch.nn.Sequential() if args.freq_mask: augmentations = torch.nn.Sequential( augmentations, torchaudio.transforms.FrequencyMasking( freq_mask_param=args.freq_mask), ) if args.time_mask: augmentations = torch.nn.Sequential( augmentations, torchaudio.transforms.TimeMasking(time_mask_param=args.time_mask), ) # Text preprocessing char_blank = "*" char_space = " " char_apostrophe = "'" labels = char_blank + char_space + char_apostrophe + string.ascii_lowercase language_model = LanguageModel(labels, char_blank, char_space) # Dataset training, validation = split_process_librispeech( [args.dataset_train, args.dataset_valid], [transforms, transforms], language_model, root=args.dataset_root, folder_in_archive=args.dataset_folder_in_archive, ) # Decoder if args.decoder == "greedy": decoder = GreedyDecoder() else: raise ValueError("Selected decoder not supported") # Model model = Wav2Letter( num_classes=language_model.length, input_type=args.type, num_features=num_features, ) if args.jit: model = torch.jit.script(model) if args.distributed: n = torch.cuda.device_count() // args.world_size devices = list(range(rank * n, (rank + 1) * n)) model = model.to(devices[0]) model = torch.nn.parallel.DistributedDataParallel(model, device_ids=devices) else: devices = ["cuda" if torch.cuda.is_available() else "cpu"] model = model.to(devices[0], non_blocking=True) model = torch.nn.DataParallel(model) n = count_parameters(model) logging.info("Number of parameters: %s", n) # Optimizer if args.optimizer == "adadelta": optimizer = Adadelta( model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay, eps=args.eps, rho=args.rho, ) elif args.optimizer == "sgd": optimizer = SGD( model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay, ) elif args.optimizer == "adam": optimizer = Adam( model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay, ) elif args.optimizer == "adamw": optimizer = AdamW( model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay, ) else: raise ValueError("Selected optimizer not supported") if args.scheduler == "exponential": scheduler = ExponentialLR(optimizer, gamma=args.gamma) elif args.scheduler == "reduceonplateau": scheduler = ReduceLROnPlateau(optimizer, patience=10, threshold=1e-3) else: raise ValueError("Selected scheduler not supported") criterion = torch.nn.CTCLoss(blank=language_model.mapping[char_blank], zero_infinity=False) # Data Loader collate_fn_train = collate_factory(model_length_function, augmentations) collate_fn_valid = collate_factory(model_length_function) loader_training_params = { "num_workers": args.workers, "pin_memory": True, "shuffle": True, "drop_last": True, } loader_validation_params = loader_training_params.copy() loader_validation_params["shuffle"] = False loader_training = DataLoader( training, batch_size=args.batch_size, collate_fn=collate_fn_train, **loader_training_params, ) loader_validation = DataLoader( validation, batch_size=args.batch_size, collate_fn=collate_fn_valid, **loader_validation_params, ) # Setup checkpoint best_loss = 1.0 load_checkpoint = args.checkpoint and os.path.isfile(args.checkpoint) if args.distributed: torch.distributed.barrier() if load_checkpoint: logging.info("Checkpoint: loading %s", args.checkpoint) checkpoint = torch.load(args.checkpoint) args.start_epoch = checkpoint["epoch"] best_loss = checkpoint["best_loss"] model.load_state_dict(checkpoint["state_dict"]) optimizer.load_state_dict(checkpoint["optimizer"]) scheduler.load_state_dict(checkpoint["scheduler"]) logging.info("Checkpoint: loaded '%s' at epoch %s", args.checkpoint, checkpoint["epoch"]) else: logging.info("Checkpoint: not found") save_checkpoint( { "epoch": args.start_epoch, "state_dict": model.state_dict(), "best_loss": best_loss, "optimizer": optimizer.state_dict(), "scheduler": scheduler.state_dict(), }, False, args.checkpoint, not_main_rank, ) if args.distributed: torch.distributed.barrier() torch.autograd.set_detect_anomaly(False) for epoch in range(args.start_epoch, args.epochs): logging.info("Epoch: %s", epoch) train_one_epoch( model, criterion, optimizer, scheduler, loader_training, decoder, language_model, devices[0], epoch, args.clip_grad, not_main_rank, not args.reduce_lr_valid, ) loss = evaluate( model, criterion, loader_validation, decoder, language_model, devices[0], epoch, not_main_rank, ) if args.reduce_lr_valid and isinstance(scheduler, ReduceLROnPlateau): scheduler.step(loss) is_best = loss < best_loss best_loss = min(loss, best_loss) save_checkpoint( { "epoch": epoch + 1, "state_dict": model.state_dict(), "best_loss": best_loss, "optimizer": optimizer.state_dict(), "scheduler": scheduler.state_dict(), }, is_best, args.checkpoint, not_main_rank, ) logging.info("End time: %s", datetime.now()) if args.distributed: torch.distributed.destroy_process_group()
best_loss = checkpoint['val_loss'] epoch = checkpoint['epoch'] print('Loading model: {}. Resuming from epoch: {}'.format(args.load_model, epoch)) else: print('Model: {} not found'.format(args.load_model)) for epoch in range(args.epochs): v_loss = execute_graph(model, loader, optimizer, schedular, epoch, use_cuda) if v_loss < best_loss: best_loss = v_loss print('Writing model checkpoint') state = { 'epoch': epoch, 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'schedular': schedular.state_dict(), 'val_loss': v_loss } t = time.localtime() timestamp = time.strftime('%b-%d-%Y_%H%M', t) file_name = 'models/{}_{}_{}_{:04.4f}.pt'.format(timestamp, args.uid, epoch, v_loss) torch.save(state, file_name) # TensorboardX logger logger.close() # save model / restart training
def main() -> None: """Entrypoint. """ config: Any = importlib.import_module(args.config) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_data = tx.data.MonoTextData(config.train_data_hparams, device=device) val_data = tx.data.MonoTextData(config.val_data_hparams, device=device) test_data = tx.data.MonoTextData(config.test_data_hparams, device=device) iterator = tx.data.DataIterator({ "train": train_data, "valid": val_data, "test": test_data }) opt_vars = { 'learning_rate': config.lr_decay_hparams["init_lr"], 'best_valid_nll': 1e100, 'steps_not_improved': 0, 'kl_weight': config.kl_anneal_hparams["start"] } decay_cnt = 0 max_decay = config.lr_decay_hparams["max_decay"] decay_factor = config.lr_decay_hparams["decay_factor"] decay_ts = config.lr_decay_hparams["threshold"] save_dir = f"./models/{config.dataset}" if not os.path.exists(save_dir): os.makedirs(save_dir) suffix = f"{config.dataset}_{config.decoder_type}Decoder.ckpt" save_path = os.path.join(save_dir, suffix) # KL term annealing rate anneal_r = 1.0 / (config.kl_anneal_hparams["warm_up"] * (len(train_data) / config.batch_size)) vocab = train_data.vocab model = VAE(train_data.vocab.size, config) model.to(device) start_tokens = torch.full((config.batch_size, ), vocab.bos_token_id, dtype=torch.long).to(device) end_token = vocab.eos_token_id optimizer = tx.core.get_optimizer(params=model.parameters(), hparams=config.opt_hparams) scheduler = ExponentialLR(optimizer, decay_factor) def _run_epoch(epoch: int, mode: str, display: int = 10) \ -> Tuple[Tensor, float]: iterator.switch_to_dataset(mode) if mode == 'train': model.train() opt_vars["kl_weight"] = min(1.0, opt_vars["kl_weight"] + anneal_r) kl_weight = opt_vars["kl_weight"] else: model.eval() kl_weight = 1.0 step = 0 start_time = time.time() num_words = 0 nll_total = 0. avg_rec = tx.utils.AverageRecorder() for batch in iterator: ret = model(batch, kl_weight, start_tokens, end_token) if mode == "train": opt_vars["kl_weight"] = min(1.0, opt_vars["kl_weight"] + anneal_r) kl_weight = opt_vars["kl_weight"] ret["nll"].backward() optimizer.step() optimizer.zero_grad() batch_size = len(ret["lengths"]) num_words += torch.sum(ret["lengths"]).item() nll_total += ret["nll"].item() * batch_size avg_rec.add([ ret["nll"].item(), ret["kl_loss"].item(), ret["rc_loss"].item() ], batch_size) if step % display == 0 and mode == 'train': nll = avg_rec.avg(0) klw = opt_vars["kl_weight"] KL = avg_rec.avg(1) rc = avg_rec.avg(2) log_ppl = nll_total / num_words ppl = math.exp(log_ppl) time_cost = time.time() - start_time print( f"{mode}: epoch {epoch}, step {step}, nll {nll:.4f}, " f"klw {klw:.4f}, KL {KL:.4f}, rc {rc:.4f}, " f"log_ppl {log_ppl:.4f}, ppl {ppl:.4f}, " f"time_cost {time_cost:.1f}", flush=True) step += 1 nll = avg_rec.avg(0) KL = avg_rec.avg(1) rc = avg_rec.avg(2) log_ppl = nll_total / num_words ppl = math.exp(log_ppl) print(f"\n{mode}: epoch {epoch}, nll {nll:.4f}, KL {KL:.4f}, " f"rc {rc:.4f}, log_ppl {log_ppl:.4f}, ppl {ppl:.4f}") return nll, ppl # type: ignore @torch.no_grad() def _generate(start_tokens: torch.LongTensor, end_token: int, filename: Optional[str] = None): ckpt = torch.load(args.model) model.load_state_dict(ckpt['model']) model.eval() batch_size = train_data.batch_size dst = MultivariateNormalDiag(loc=torch.zeros(batch_size, config.latent_dims), scale_diag=torch.ones( batch_size, config.latent_dims)) latent_z = dst.rsample().to(device) helper = model.decoder.create_helper(decoding_strategy='infer_sample', start_tokens=start_tokens, end_token=end_token) outputs = model.decode(helper=helper, latent_z=latent_z, max_decoding_length=100) sample_tokens = vocab.map_ids_to_tokens_py(outputs.sample_id.cpu()) if filename is None: fh = sys.stdout else: fh = open(filename, 'w', encoding='utf-8') for sent in sample_tokens: sent = tx.utils.compat_as_text(list(sent)) end_id = len(sent) if vocab.eos_token in sent: end_id = sent.index(vocab.eos_token) fh.write(' '.join(sent[:end_id + 1]) + '\n') print('Output done') fh.close() if args.mode == "predict": _generate(start_tokens, end_token, args.out) return # Counts trainable parameters total_parameters = sum(param.numel() for param in model.parameters()) print(f"{total_parameters} total parameters") best_nll = best_ppl = 0. for epoch in range(config.num_epochs): _, _ = _run_epoch(epoch, 'train', display=200) val_nll, _ = _run_epoch(epoch, 'valid') test_nll, test_ppl = _run_epoch(epoch, 'test') if val_nll < opt_vars['best_valid_nll']: opt_vars['best_valid_nll'] = val_nll opt_vars['steps_not_improved'] = 0 best_nll = test_nll best_ppl = test_ppl states = { "model": model.state_dict(), "optimizer": optimizer.state_dict(), "scheduler": scheduler.state_dict() } torch.save(states, save_path) else: opt_vars['steps_not_improved'] += 1 if opt_vars['steps_not_improved'] == decay_ts: old_lr = opt_vars['learning_rate'] opt_vars['learning_rate'] *= decay_factor opt_vars['steps_not_improved'] = 0 new_lr = opt_vars['learning_rate'] ckpt = torch.load(save_path) model.load_state_dict(ckpt['model']) optimizer.load_state_dict(ckpt['optimizer']) scheduler.load_state_dict(ckpt['scheduler']) scheduler.step() print(f"-----\nchange lr, old lr: {old_lr}, " f"new lr: {new_lr}\n-----") decay_cnt += 1 if decay_cnt == max_decay: break print(f"\nbest testing nll: {best_nll:.4f}," f"best testing ppl {best_ppl:.4f}\n")
class Parser(object): NAME = None MODEL = None def __init__(self, args, model, transform): self.args = args self.model = model self.transform = transform def train(self, train, dev, test, buckets=32, batch_size=5000, update_steps=1, clip=5.0, epochs=5000, patience=100, **kwargs): args = self.args.update(locals()) init_logger(logger, verbose=args.verbose) self.transform.train() batch_size = batch_size // update_steps if dist.is_initialized(): batch_size = batch_size // dist.get_world_size() logger.info("Loading the data") train = Dataset(self.transform, args.train, **args).build(batch_size, buckets, True, dist.is_initialized()) dev = Dataset(self.transform, args.dev).build(batch_size, buckets) test = Dataset(self.transform, args.test).build(batch_size, buckets) logger.info( f"\n{'train:':6} {train}\n{'dev:':6} {dev}\n{'test:':6} {test}\n") if args.encoder == 'lstm': self.optimizer = Adam(self.model.parameters(), args.lr, (args.mu, args.nu), args.eps, args.weight_decay) self.scheduler = ExponentialLR(self.optimizer, args.decay**(1 / args.decay_steps)) else: from transformers import AdamW, get_linear_schedule_with_warmup steps = len(train.loader) * epochs // args.update_steps self.optimizer = AdamW([{ 'params': p, 'lr': args.lr * (1 if n.startswith('encoder') else args.lr_rate) } for n, p in self.model.named_parameters()], args.lr) self.scheduler = get_linear_schedule_with_warmup( self.optimizer, int(steps * args.warmup), steps) if dist.is_initialized(): self.model = DDP(self.model, device_ids=[args.local_rank], find_unused_parameters=True) self.epoch, self.best_e, self.patience, self.best_metric, self.elapsed = 1, 1, patience, Metric( ), timedelta() if self.args.checkpoint: self.optimizer.load_state_dict( self.checkpoint_state_dict.pop('optimizer_state_dict')) self.scheduler.load_state_dict( self.checkpoint_state_dict.pop('scheduler_state_dict')) set_rng_state(self.checkpoint_state_dict.pop('rng_state')) for k, v in self.checkpoint_state_dict.items(): setattr(self, k, v) train.loader.batch_sampler.epoch = self.epoch for epoch in range(self.epoch, args.epochs + 1): start = datetime.now() logger.info(f"Epoch {epoch} / {args.epochs}:") self._train(train.loader) loss, dev_metric = self._evaluate(dev.loader) logger.info(f"{'dev:':5} loss: {loss:.4f} - {dev_metric}") loss, test_metric = self._evaluate(test.loader) logger.info(f"{'test:':5} loss: {loss:.4f} - {test_metric}") t = datetime.now() - start self.epoch += 1 self.patience -= 1 self.elapsed += t if dev_metric > self.best_metric: self.best_e, self.patience, self.best_metric = epoch, patience, dev_metric if is_master(): self.save_checkpoint(args.path) logger.info(f"{t}s elapsed (saved)\n") else: logger.info(f"{t}s elapsed\n") if self.patience < 1: break parser = self.load(**args) loss, metric = parser._evaluate(test.loader) parser.save(args.path) logger.info(f"Epoch {self.best_e} saved") logger.info(f"{'dev:':5} {self.best_metric}") logger.info(f"{'test:':5} {metric}") logger.info(f"{self.elapsed}s elapsed, {self.elapsed / epoch}s/epoch") def evaluate(self, data, buckets=8, batch_size=5000, **kwargs): args = self.args.update(locals()) init_logger(logger, verbose=args.verbose) self.transform.train() logger.info("Loading the data") dataset = Dataset(self.transform, data) dataset.build(batch_size, buckets) logger.info(f"\n{dataset}") logger.info("Evaluating the dataset") start = datetime.now() loss, metric = self._evaluate(dataset.loader) elapsed = datetime.now() - start logger.info(f"loss: {loss:.4f} - {metric}") logger.info( f"{elapsed}s elapsed, {len(dataset)/elapsed.total_seconds():.2f} Sents/s" ) return loss, metric def predict(self, data, pred=None, lang=None, buckets=8, batch_size=5000, prob=False, **kwargs): args = self.args.update(locals()) init_logger(logger, verbose=args.verbose) self.transform.eval() if args.prob: self.transform.append(Field('probs')) logger.info("Loading the data") dataset = Dataset(self.transform, data, lang=lang) dataset.build(batch_size, buckets) logger.info(f"\n{dataset}") logger.info("Making predictions on the dataset") start = datetime.now() preds = self._predict(dataset.loader) elapsed = datetime.now() - start for name, value in preds.items(): setattr(dataset, name, value) if pred is not None and is_master(): logger.info(f"Saving predicted results to {pred}") self.transform.save(pred, dataset.sentences) logger.info( f"{elapsed}s elapsed, {len(dataset) / elapsed.total_seconds():.2f} Sents/s" ) return dataset def _train(self, loader): raise NotImplementedError @torch.no_grad() def _evaluate(self, loader): raise NotImplementedError @torch.no_grad() def _predict(self, loader): raise NotImplementedError @classmethod def build(cls, path, **kwargs): raise NotImplementedError @classmethod def load(cls, path, reload=False, src='github', checkpoint=False, **kwargs): r""" Loads a parser with data fields and pretrained model parameters. Args: path (str): - a string with the shortcut name of a pretrained model defined in ``supar.MODEL`` to load from cache or download, e.g., ``'biaffine-dep-en'``. - a local path to a pretrained model, e.g., ``./<path>/model``. reload (bool): Whether to discard the existing cache and force a fresh download. Default: ``False``. src (str): Specifies where to download the model. ``'github'``: github release page. ``'hlt'``: hlt homepage, only accessible from 9:00 to 18:00 (UTC+8). Default: ``'github'``. checkpoint (bool): If ``True``, loads all checkpoint states to restore the training process. Default: ``False``. kwargs (dict): A dict holding unconsumed arguments for updating training configs and initializing the model. Examples: >>> from supar import Parser >>> parser = Parser.load('biaffine-dep-en') >>> parser = Parser.load('./ptb.biaffine.dep.lstm.char') """ args = Config(**locals()) args.device = 'cuda' if torch.cuda.is_available() else 'cpu' state = torch.load(path if os.path.exists(path) else download( supar.MODEL[src].get(path, path), reload=reload)) cls = supar.PARSER[state['name']] if cls.NAME is None else cls args = state['args'].update(args) model = cls.MODEL(**args) model.load_pretrained(state['pretrained']) model.load_state_dict(state['state_dict'], False) model.to(args.device) transform = state['transform'] parser = cls(args, model, transform) parser.checkpoint_state_dict = state[ 'checkpoint_state_dict'] if args.checkpoint else None return parser def save(self, path): model = self.model if hasattr(model, 'module'): model = self.model.module args = model.args state_dict = {k: v.cpu() for k, v in model.state_dict().items()} pretrained = state_dict.pop('pretrained.weight', None) state = { 'name': self.NAME, 'args': args, 'state_dict': state_dict, 'pretrained': pretrained, 'transform': self.transform } torch.save(state, path, pickle_module=dill) def save_checkpoint(self, path): model = self.model if hasattr(model, 'module'): model = self.model.module args = model.args checkpoint_state_dict = { k: getattr(self, k) for k in ['epoch', 'best_e', 'patience', 'best_metric', 'elapsed'] } checkpoint_state_dict.update({ 'optimizer_state_dict': self.optimizer.state_dict(), 'scheduler_state_dict': self.scheduler.state_dict(), 'rng_state': get_rng_state() }) state_dict = {k: v.cpu() for k, v in model.state_dict().items()} pretrained = state_dict.pop('pretrained.weight', None) state = { 'name': self.NAME, 'args': args, 'state_dict': state_dict, 'pretrained': pretrained, 'checkpoint_state_dict': checkpoint_state_dict, 'transform': self.transform } torch.save(state, path, pickle_module=dill)
def main(): """Entrypoint. """ config: Any = importlib.import_module(args.config) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # train_data = tx.data.MonoTextData(config.train_data_hparams, device=device) # val_data = tx.data.MonoTextData(config.val_data_hparams, device=device) # test_data = tx.data.MonoTextData(config.test_data_hparams, device=device) train_data = tx.data.MonoTextData(config.train_data_hparams, device=torch.device("cpu")) val_data = tx.data.MonoTextData(config.val_data_hparams, device=torch.device("cpu")) test_data = tx.data.MonoTextData(config.test_data_hparams, device=torch.device("cpu")) iterator = tx.data.DataIterator({ "train": train_data, "valid": val_data, "test": test_data }) opt_vars = { 'learning_rate': config.lr_decay_hparams["init_lr"], 'best_valid_nll': 1e100, 'steps_not_improved': 0, 'kl_weight': config.kl_anneal_hparams["start"] } decay_cnt = 0 max_decay = config.lr_decay_hparams["max_decay"] decay_factor = config.lr_decay_hparams["decay_factor"] decay_ts = config.lr_decay_hparams["threshold"] if 'pid' in args.model_name: save_dir = args.model_name + '_' + str(config.dataset) + '_KL' + str( args.exp_kl) elif 'cost' in args.model_name: save_dir = args.model_name + '_' + str(config.dataset) + '_step' + str( args.anneal_steps) elif 'cyclical' in args.model_name: save_dir = args.model_name + '_' + str(config.dataset) + '_cyc_' + str( args.cycle) if not os.path.exists(save_dir): os.makedirs(save_dir) suffix = f"{config.dataset}_{config.decoder_type}Decoder.ckpt" save_path = os.path.join(save_dir, suffix) # KL term annealing rate warm_up=10 ## replace it with sigmoid function anneal_r = 1.0 / (config.kl_anneal_hparams["warm_up"] * (len(train_data) / config.batch_size)) vocab = train_data.vocab model = VAE(train_data.vocab.size, config) model.to(device) start_tokens = torch.full((config.batch_size, ), vocab.bos_token_id, dtype=torch.long).to(device) end_token = vocab.eos_token_id optimizer = tx.core.get_optimizer(params=model.parameters(), hparams=config.opt_hparams) scheduler = ExponentialLR(optimizer, decay_factor) ## max iteration max_iter = config.num_epochs * len(train_data) / config.batch_size max_iter = min(max_iter, args.max_steps) print('max steps:', max_iter) pbar = tqdm(total=int(max_iter)) if args.mode == "train": outFile = os.path.join(save_dir, 'train.log') fw_log = open(outFile, "w") global_steps = {} global_steps['step'] = 0 pid = PIDControl() opt_vars["kl_weight"] = 0.0 Kp = args.Kp Ki = args.Ki exp_kl = args.exp_kl ## train model def _run_epoch(epoch: int, mode: str, display: int = 10) \ -> Tuple[Tensor, float]: iterator.switch_to_dataset(mode) if mode == 'train': model.train() kl_weight = opt_vars["kl_weight"] else: model.eval() kl_weight = 1.0 # kl_weight = opt_vars["kl_weight"] start_time = time.time() num_words = 0 nll_total = 0. avg_rec = tx.utils.AverageRecorder() for batch in iterator: ## run model to get loss function if global_steps['step'] >= args.max_steps: break ret = model(batch, kl_weight, start_tokens, end_token) if mode == "train": pbar.update(1) global_steps['step'] += 1 kl_loss = ret['kl_loss'].item() rec_loss = ret['rc_loss'].item() total_loss = ret["nll"].item() if 'cost' in args.model_name: kl_weight = _cost_annealing(global_steps['step'], 1.0, args.anneal_steps) elif 'pid' in args.model_name: kl_weight = pid.pid(exp_kl, kl_loss, Kp, Ki) elif 'cyclical' in args.model_name: kl_weight = _cyclical_annealing(global_steps['step'], max_iter / args.cycle) opt_vars["kl_weight"] = kl_weight ## total loss ret["nll"].backward() optimizer.step() optimizer.zero_grad() fw_log.write('epoch:{0} global_step:{1} total_loss:{2:.3f} kl_loss:{3:.3f} rec_loss:{4:.3f} kl_weight:{5:.4f}\n'\ .format(epoch, global_steps['step'], total_loss, kl_loss, rec_loss, kl_weight)) fw_log.flush() batch_size = len(ret["lengths"]) num_words += torch.sum(ret["lengths"]).item() nll_total += ret["nll"].item() * batch_size avg_rec.add([ ret["nll"].item(), ret["kl_loss"].item(), ret["rc_loss"].item() ], batch_size) if global_steps['step'] % display == 1 and mode == 'train': nll = avg_rec.avg(0) klw = opt_vars["kl_weight"] KL = avg_rec.avg(1) rc = avg_rec.avg(2) writer.add_scalar(f'Loss/Rec_loss_{args.model_name}', rc, global_steps['step']) writer.add_scalar(f'Loss/KL_diverg_{args.model_name}', KL, global_steps['step']) writer.add_scalar(f'Loss/KL_weight_{args.model_name}', klw, global_steps['step']) nll = avg_rec.avg(0) KL = avg_rec.avg(1) rc = avg_rec.avg(2) if num_words > 0: log_ppl = nll_total / num_words ppl = math.exp(log_ppl) else: log_ppl = 100 ppl = math.exp(log_ppl) nll = 1000 KL = args.exp_kl print(f"\n{mode}: epoch {epoch}, nll {nll:.4f}, KL {KL:.4f}, " f"rc {rc:.4f}, log_ppl {log_ppl:.4f}, ppl {ppl:.4f}") return nll, ppl # type: ignore args.model = save_path @torch.no_grad() def _generate(start_tokens: torch.LongTensor, end_token: int, filename: Optional[str] = None): ckpt = torch.load(args.model) model.load_state_dict(ckpt['model']) model.eval() batch_size = train_data.batch_size dst = MultivariateNormalDiag(loc=torch.zeros(batch_size, config.latent_dims), scale_diag=torch.ones( batch_size, config.latent_dims)) # latent_z = dst.rsample().to(device) latent_z = torch.FloatTensor(batch_size, config.latent_dims).uniform_(-1, 1).to(device) # latent_z = torch.randn(batch_size, config.latent_dims).to(device) helper = model.decoder.create_helper(decoding_strategy='infer_sample', start_tokens=start_tokens, end_token=end_token) outputs = model.decode(helper=helper, latent_z=latent_z, max_decoding_length=100) if config.decoder_type == "transformer": outputs = outputs[0] sample_tokens = vocab.map_ids_to_tokens_py(outputs.sample_id.cpu()) if filename is None: fh = sys.stdout else: fh = open(filename, 'a', encoding='utf-8') for sent in sample_tokens: sent = tx.utils.compat_as_text(list(sent)) end_id = len(sent) if vocab.eos_token in sent: end_id = sent.index(vocab.eos_token) fh.write(' '.join(sent[:end_id + 1]) + '\n') print('Output done') fh.close() if args.mode == "predict": out_path = os.path.join(save_dir, 'results.txt') for _ in range(10): _generate(start_tokens, end_token, out_path) return # Counts trainable parameters total_parameters = sum(param.numel() for param in model.parameters()) print(f"{total_parameters} total parameters") best_nll = best_ppl = 0. ## start running model for epoch in range(config.num_epochs): _, _ = _run_epoch(epoch, 'train', display=200) val_nll, _ = _run_epoch(epoch, 'valid') test_nll, test_ppl = _run_epoch(epoch, 'test') if val_nll < opt_vars['best_valid_nll']: opt_vars['best_valid_nll'] = val_nll opt_vars['steps_not_improved'] = 0 best_nll = test_nll best_ppl = test_ppl states = { "model": model.state_dict(), "optimizer": optimizer.state_dict(), "scheduler": scheduler.state_dict() } torch.save(states, save_path) else: opt_vars['steps_not_improved'] += 1 if opt_vars['steps_not_improved'] == decay_ts: old_lr = opt_vars['learning_rate'] opt_vars['learning_rate'] *= decay_factor opt_vars['steps_not_improved'] = 0 new_lr = opt_vars['learning_rate'] ckpt = torch.load(save_path) model.load_state_dict(ckpt['model']) optimizer.load_state_dict(ckpt['optimizer']) scheduler.load_state_dict(ckpt['scheduler']) scheduler.step() print(f"-----\nchange lr, old lr: {old_lr}, " f"new lr: {new_lr}\n-----") decay_cnt += 1 if decay_cnt == max_decay: break if global_steps['step'] >= args.max_steps: break print(f"\nbest testing nll: {best_nll:.4f}," f"best testing ppl {best_ppl:.4f}\n") if args.mode == "train": fw_log.write(f"\nbest testing nll: {best_nll:.4f}," f"best testing ppl {best_ppl:.4f}\n") fw_log.close()
def main_worker(train_loader, val_loader, ntokens, args, device): global best_ppl model_kwargs = { 'dropout': args.dropout, 'tie_weights': not args.not_tied, 'norm': args.norm_mode, 'alpha_fwd': args.afwd, 'alpha_bkw': args.abkw, 'batch_size': args.batch_size, # Deprecated 'ecm': args.ecm, 'cell_norm': args.cell_norm, } # create model print("=> creating model: '{}'".format(args.ru_type)) model = models.RNNModel(args.ru_type, ntokens, args.emsize, args.nhid, args.nlayers, **model_kwargs).to(device) print(model) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().to(device) optimizer = torch.optim.SGD(model.parameters(), args.lr, weight_decay=args.weight_decay) scheduler = ExponentialLR(optimizer, gamma=1 / args.lr_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_ppl = checkpoint['best_ppl'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) scheduler.load_state_dict(checkpoint['scheduler']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = False if args.seed else True if args.evaluate: validate(val_loader, model, criterion, device, args, ntokens) return for epoch in range(args.start_epoch, args.epochs): if epoch: scheduler.step() # train for one epoch train(train_loader, model, criterion, optimizer, epoch, device, args, ntokens) # evaluate on validation set ppl = validate(val_loader, model, criterion, device, args, ntokens) # remember best ppl and save checkpoint is_best = ppl < best_ppl best_ppl = min(ppl, best_ppl) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_ppl': best_ppl, 'optimizer': optimizer.state_dict(), 'scheduler': scheduler.state_dict(), }, is_best, args)
def run(model, train_loader, eval_loader_all, opt, train_loader_all, train_loader_for_regularization=None, train_loader_except_regularization=None, eval_loader_for_regularization=None, eval_loader_except_regularization=None): """Contains the main training loop and test logic. Also, handles saving/loading of checkpoints """ def _eval_extra(): eval_extra(model, epoch, log_file, train_loader_all, train_loader_for_regularization, train_loader_except_regularization, eval_loader_for_regularization, eval_loader_except_regularization, opt) if not os.path.exists(opt.checkpoint_path): os.makedirs(opt.checkpoint_path) # Preliminary setup of optimizer, lr scheduler, logs optim = create_optim(opt, model) lr_scheduler = ExponentialLR(optim, gamma=opt.lr_gamma) best_eval_score = 0 ans_cossim = pickle.load(open(f'{opt.data_dir}/ans_cossim.pkl', 'rb')) log_file = open(opt.checkpoint_path + '/log.txt', 'a') print(json.dumps(vars(opt), indent=4, sort_keys=True), file=log_file) log_file.flush() ## If load_checkpoint_path flag is specified, then we need to load model from that state if opt.load_checkpoint_path is not None and len( opt.load_checkpoint_path) > 0: ckpt = torch.load(os.path.join(opt.load_checkpoint_path)) if 'epoch' in ckpt: states_ = ckpt['model_state_dict'] else: states_ = ckpt model.load_state_dict(states_) ## Handle test logic if opt.test: print("Evaluating ...") if 'epoch' in ckpt: epoch = ckpt['epoch'] else: epoch = 'unknown' evaluate_and_log("Eval on Test", model, eval_loader_all, opt, epoch, log_file, save_sensitivities=True, save_prediction_scores=True, prefix='test') return if opt.test_on_train: print("Evaluating ...") if 'epoch' in ckpt: epoch = ckpt['epoch'] else: epoch = 'unknown' evaluate_and_log("Eval on Train", model, train_loader_all, opt, epoch, log_file, save_sensitivities=False, save_prediction_scores=True, prefix='train') return ## The main training loop for epoch in range(opt.max_epochs): print(f"Training epoch {epoch}...") iter_num = 0 if opt.var_random_subset_ratio is not None: ## Gather a new training subset every epoch. The subset is selected randomly. train_dset = train_loader.dataset subset_sampler = RandomSubsetSampler( torch.LongTensor(range(0, len(train_dset))), int(len(train_dset) * opt.var_random_subset_ratio)) train_loader = DataLoader(train_dset, opt.batch_size, shuffle=False, num_workers=opt.num_workers, sampler=subset_sampler) for objs, qns, answers, hint_scores, _, question_ids, image_ids, hint_flags in iter( train_loader): if opt.change_scores_every_epoch: ## Assign random scores every epoch, if the flag says to do so. hint_scores = torch.rand(hint_scores.shape).cuda() objs = objs.cuda().float().requires_grad_() # B x num_objs x emb qns = qns.cuda().long() # B x len answers = answers.cuda() # B x num classes hint_scores = hint_scores.cuda().float() # B x num_objs x 1 words, logits, attended_objs, ans_idxs = model( qns, objs) # pred: B x num classes loss = compute_loss(opt, train_loader, epoch, iter_num, objs, answers, logits, ans_idxs, hint_flags, hint_scores, ans_cossim) loss.backward() nn.utils.clip_grad_norm(model.parameters(), opt.grad_clip) optim.step() optim.zero_grad() log_file.flush() iter_num += 1 print("##\n") lr_scheduler.step() print(f"lr {lr_scheduler.get_lr()}") if epoch in opt.log_epochs: eval_score = evaluate_and_log("Eval", model, eval_loader_all, opt, epoch, log_file, save_sensitivities=True, save_prediction_scores=True) else: eval_score = evaluate_and_log("Eval", model, eval_loader_all, opt, epoch, log_file, save_sensitivities=False, save_prediction_scores=True) log_file.flush() # Save model state = { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optim_state_dict': optim.state_dict(), 'lr_state_dict': lr_scheduler.state_dict() } if epoch in opt.log_epochs: torch.save( state, os.path.join(opt.checkpoint_path, f'model-epoch-{epoch}.pth')) if eval_score > best_eval_score: torch.save(state, os.path.join(opt.checkpoint_path, 'model-best.pth')) best_eval_score = eval_score if epoch in opt.log_epochs: _eval_extra() print("Evaluating extra stuff on the best model ...") best_model_path = os.path.join(opt.checkpoint_path, 'model-best.pth') model.load_state_dict(torch.load(best_model_path)['model_state_dict']) _eval_extra()
best_g_loss = np.inf # Main training loop for epoch in range(1, args.epochs): g_v_loss, _ = execute_graph(E, G, D, EG_optim, D_optim, EG_scheduler, D_scheduler, loader, epoch, use_tb) if g_v_loss < best_g_loss: best_g_loss = g_v_loss print('Writing model checkpoint') state = { 'epoch': epoch, 'E': E.state_dict(), 'G': G.state_dict(), 'D': D.state_dict(), 'EG_optim': EG_optim.state_dict(), 'D_optim': D_optim.state_dict(), 'EG_scheduler': EG_scheduler.state_dict(), 'D_scheduler': D_scheduler.state_dict(), 'val_loss': g_v_loss } t = time.localtime() timestamp = time.strftime('%b-%d-%Y_%H%M', t) file_name = 'models/{}_{}_{}_{}_{:04.4f}.pt'.format( timestamp, args.uid, epoch, latent_size, g_v_loss) torch.save(state, file_name) # TensorboardX logger logger.close()