def build_scheduler(optimizer, config, optimizer_config, epochs, steps_per_epoch): if config.type == 'cosine': scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, epochs * steps_per_epoch) elif config.type == 'onecycle': scheduler = OneCycleScheduler(optimizer, min_lr=optimizer_config.lr / 100, beta_range=[0.95, 0.85], max_steps=epochs * steps_per_epoch, annealing='linear', peak_pos=0.45, end_pos=0.9) elif config.type == 'coswarm': scheduler = CosineWithWarmup(optimizer, warmup_steps=steps_per_epoch, max_steps=epochs * steps_per_epoch) elif config.type == 'step': scheduler = torch.optim.lr_scheduler.StepLR( optimizer, step_size=(epochs * steps_per_epoch) // 3, gamma=0.1) else: raise AssertionError('invalid scheduler {}'.format(config.type)) return scheduler
def train_fold(fold, lr): train_indices, eval_indices = indices_for_fold(fold, len(train_data)) train_dataset = TrainEvalDataset(train_data.iloc[train_indices], transform=train_transform) train_data_loader = torch.utils.data.DataLoader( train_dataset, batch_size=config.batch_size, drop_last=True, shuffle=True, num_workers=args.workers) if config.mixup is not None: train_data_loader = MixupDataLoader(train_data_loader, config.mixup) eval_dataset = TrainEvalDataset(train_data.iloc[eval_indices], transform=eval_transform) eval_data_loader = torch.utils.data.DataLoader( eval_dataset, batch_size=config.batch_size, num_workers=args.workers) model = Model(config.model, NUM_CLASSES) model = model.to(DEVICE) optimizer = build_optimizer(config.opt.type, model.parameters(), lr, config.opt.beta, weight_decay=config.opt.weight_decay) if config.sched.type == 'onecycle': scheduler = lr_scheduler_wrapper.StepWrapper( OneCycleScheduler(optimizer, lr=(lr / 20, lr), beta_range=config.sched.onecycle.beta, max_steps=len(train_data_loader) * config.epochs, annealing=config.sched.onecycle.anneal)) elif config.sched.type == 'cyclic': # TODO: add cyclic min/max momentum to config scheduler = lr_scheduler_wrapper.StepWrapper( CyclicLR(optimizer, 0., lr, step_size_up=len(train_data_loader), step_size_down=len(train_data_loader), mode='triangular2', cycle_momentum=True, base_momentum=config.sched.cyclic.beta[1], max_momentum=config.sched.cyclic.beta[0])) elif config.sched.type == 'cawr': scheduler = lr_scheduler_wrapper.StepWrapper( torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer, T_0=len(train_data_loader), T_mult=2)) elif config.sched.type == 'plateau': scheduler = lr_scheduler_wrapper.ScoreWrapper( torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=0, verbose=True)) else: raise AssertionError('invalid sched {}'.format(config.sched.type)) best_score = 0 for epoch in range(config.epochs): train_epoch(model=model, optimizer=optimizer, scheduler=scheduler, data_loader=train_data_loader, fold=fold, epoch=epoch) gc.collect() score = eval_epoch(model=model, data_loader=eval_data_loader, fold=fold, epoch=epoch) gc.collect() scheduler.step_epoch() scheduler.step_score(score) if score > best_score: best_score = score torch.save( model.state_dict(), os.path.join(args.experiment_path, 'model_{}.pth'.format(fold)))
def train_fold(fold, train_eval_data, unsup_data): train_indices, eval_indices = indices_for_fold(fold, train_eval_data) train_dataset = TrainEvalDataset(train_eval_data.iloc[train_indices], transform=train_transform) train_data_loader = torch.utils.data.DataLoader( train_dataset, batch_size=config.batch_size, drop_last=True, shuffle=True, num_workers=args.workers, worker_init_fn=worker_init_fn) eval_dataset = TrainEvalDataset(train_eval_data.iloc[eval_indices], transform=eval_transform) eval_data_loader = torch.utils.data.DataLoader( eval_dataset, batch_size=config.batch_size, num_workers=args.workers, worker_init_fn=worker_init_fn) unsup_data = TestDataset(unsup_data, transform=unsup_transform) unsup_data_loader = torch.utils.data.DataLoader( unsup_data, batch_size=config.batch_size // 2, num_workers=args.workers, worker_init_fn=worker_init_fn) model = Model(config.model, NUM_CLASSES) model = model.to(DEVICE) if args.restore_path is not None: model.load_state_dict( torch.load( os.path.join(args.restore_path, 'model_{}.pth'.format(fold)))) optimizer = build_optimizer(config.opt, model.parameters()) if config.sched.type == 'onecycle': scheduler = lr_scheduler_wrapper.StepWrapper( OneCycleScheduler(optimizer, lr=(config.opt.lr / 20, config.opt.lr), beta_range=config.sched.onecycle.beta, max_steps=len(train_data_loader) * config.epochs, annealing=config.sched.onecycle.anneal, peak_pos=config.sched.onecycle.peak_pos, end_pos=config.sched.onecycle.end_pos)) elif config.sched.type == 'step': scheduler = lr_scheduler_wrapper.EpochWrapper( torch.optim.lr_scheduler.StepLR( optimizer, step_size=config.sched.step.step_size, gamma=config.sched.step.decay)) elif config.sched.type == 'cyclic': step_size_up = len( train_data_loader) * config.sched.cyclic.step_size_up step_size_down = len( train_data_loader) * config.sched.cyclic.step_size_down scheduler = lr_scheduler_wrapper.StepWrapper( torch.optim.lr_scheduler.CyclicLR( optimizer, 0., config.opt.lr, step_size_up=step_size_up, step_size_down=step_size_down, mode='triangular2', gamma=config.sched.cyclic.decay**( 1 / (step_size_up + step_size_down)), cycle_momentum=True, base_momentum=0.85, max_momentum=0.95)) elif config.sched.type == 'cawr': scheduler = lr_scheduler_wrapper.StepWrapper( torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer, T_0=len(train_data_loader), T_mult=2)) elif config.sched.type == 'plateau': scheduler = lr_scheduler_wrapper.ScoreWrapper( torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, mode='max', factor=config.sched.plateau.decay, patience=config.sched.plateau.patience, verbose=True)) else: raise AssertionError('invalid sched {}'.format(config.sched.type)) best_score = 0 for epoch in range(1, config.epochs + 1): optimizer.train() train_epoch(model=model, optimizer=optimizer, scheduler=scheduler, data_loader=train_data_loader, unsup_data_loader=unsup_data_loader, fold=fold, epoch=epoch) gc.collect() optimizer.eval() metric = eval_epoch(model=model, data_loader=eval_data_loader, fold=fold, epoch=epoch) gc.collect() score = metric['accuracy@1'] scheduler.step_epoch() scheduler.step_score(score) if score > best_score: best_score = score torch.save( model.state_dict(), os.path.join(args.experiment_path, 'model_{}.pth'.format(fold)))
def train_fold(fold, train_eval_data, args, config): train_indices, eval_indices = indices_for_fold(fold, len(train_eval_data), seed=config.seed) train_dataset = Dataset([train_eval_data[i] for i in train_indices]) eval_dataset = Dataset([train_eval_data[i] for i in eval_indices]) train_data_loader = DataLoader(train_dataset, batch_size=config.batch_size, drop_last=True, shuffle=True, num_workers=args['workers'], worker_init_fn=worker_init_fn) eval_data_loader = DataLoader(eval_dataset, batch_size=config.batch_size, num_workers=args['workers'], worker_init_fn=worker_init_fn) model = Model(config.model) model = model.to(DEVICE) optimizer = build_optimizer(config.opt.type, model.parameters(), config.opt.lr, config.opt.beta, weight_decay=config.opt.weight_decay) if config.sched.type == 'onecycle': scheduler = lr_scheduler_wrapper.StepWrapper( OneCycleScheduler(optimizer, lr=(lr / 20, lr), beta_range=config.sched.onecycle.beta, max_steps=len(train_data_loader) * config.epochs, annealing=config.sched.onecycle.anneal)) elif config.sched.type == 'cyclic': scheduler = lr_scheduler_wrapper.StepWrapper( torch.optim.lr_scheduler.CyclicLR( optimizer, 0., lr, step_size_up=len(train_data_loader), step_size_down=len(train_data_loader), mode='triangular2', cycle_momentum=True, base_momentum=config.sched.cyclic.beta[1], max_momentum=config.sched.cyclic.beta[0])) elif config.sched.type == 'cawr': scheduler = lr_scheduler_wrapper.StepWrapper( torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer, T_0=len(train_data_loader), T_mult=2)) elif config.sched.type == 'plateau': scheduler = lr_scheduler_wrapper.ScoreWrapper( torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=2, verbose=True)) else: raise AssertionError('invalid sched {}'.format(config.sched.type)) best_score = 0 for epoch in range(config.epochs): train_epoch(model=model, optimizer=optimizer, scheduler=scheduler, data_loader=train_data_loader, fold=fold, epoch=epoch, args=args, config=config) gc.collect() score = eval_epoch(model=model, data_loader=eval_data_loader, fold=fold, epoch=epoch, args=args, config=config) gc.collect() scheduler.step_epoch() scheduler.step_score(score) if score > best_score: best_score = score torch.save( model.state_dict(), os.path.join(args['experiment_path'], 'model_{}.pth'.format(fold)))
def train_fold(fold, train_eval_data): model = Model(config.model, NUM_CLASSES) model = model.to(DEVICE) if args.restore_path is not None: model.load_state_dict( torch.load( os.path.join(args.restore_path, 'model_{}.pth'.format(fold)))) optimizer = build_optimizer(config.opt, model.parameters()) if config.sched.type == 'onecycle': scheduler = lr_scheduler_wrapper.EpochWrapper( OneCycleScheduler(optimizer, lr=(config.opt.lr / 20, config.opt.lr), beta_range=config.sched.onecycle.beta, max_steps=config.epochs, annealing=config.sched.onecycle.anneal, peak_pos=config.sched.onecycle.peak_pos, end_pos=config.sched.onecycle.end_pos)) elif config.sched.type == 'cyclic': step_size_up = len( train_data_loader) * config.sched.cyclic.step_size_up step_size_down = len( train_data_loader) * config.sched.cyclic.step_size_down scheduler = lr_scheduler_wrapper.StepWrapper( torch.optim.lr_scheduler.CyclicLR( optimizer, 0., config.opt.lr, step_size_up=step_size_up, step_size_down=step_size_down, mode='exp_range', gamma=config.sched.cyclic.decay**( 1 / (step_size_up + step_size_down)), cycle_momentum=True, base_momentum=0.75, max_momentum=0.95)) elif config.sched.type == 'cawr': scheduler = lr_scheduler_wrapper.StepWrapper( torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer, T_0=len(train_data_loader), T_mult=2)) elif config.sched.type == 'plateau': scheduler = lr_scheduler_wrapper.ScoreWrapper( torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, mode='max', factor=config.sched.plateau.decay, patience=config.sched.plateau.patience, verbose=True)) else: raise AssertionError('invalid sched {}'.format(config.sched.type)) best_score = 0 for epoch in range(1, config.epochs + 1): train_indices, eval_indices = indices_for_fold(fold, train_eval_data) eval_pl = pd.read_csv( './tf_log/cells/tmp-512-progres-crop-norm-la/eval_{}.csv'.format( fold)) eval_pl['root'] = os.path.join(args.dataset_path, 'train') test_pl = pd.read_csv( './tf_log/cells/tmp-512-progres-crop-norm-la/test.csv') test_pl['root'] = os.path.join(args.dataset_path, 'test') pl = pd.concat([eval_pl, test_pl]) pl_size = len(pl) pl = pl.sample(frac=np.linspace(1., 0., config.epochs)[epoch - 1].item()) print('frac: {:.4f}, lr: {:.8f}'.format( len(pl) / pl_size, scheduler.get_lr())) train_dataset = TrainEvalDataset(pd.concat( [train_eval_data.iloc[train_indices], pl]), transform=train_transform) train_data_loader = torch.utils.data.DataLoader( train_dataset, batch_size=config.batch_size, drop_last=True, shuffle=True, num_workers=args.workers, worker_init_fn=worker_init_fn) eval_dataset = TrainEvalDataset(train_eval_data.iloc[eval_indices], transform=eval_transform) eval_data_loader = torch.utils.data.DataLoader( eval_dataset, batch_size=config.batch_size, num_workers=args.workers, worker_init_fn=worker_init_fn) train_epoch(model=model, optimizer=optimizer, scheduler=scheduler, data_loader=train_data_loader, fold=fold, epoch=epoch) gc.collect() metric = eval_epoch(model=model, data_loader=eval_data_loader, fold=fold, epoch=epoch) gc.collect() score = metric['accuracy@1'] scheduler.step_epoch() scheduler.step_score(score) if score > best_score: best_score = score torch.save( model.state_dict(), os.path.join(args.experiment_path, 'model_{}.pth'.format(fold)))