def experiment(args): table = ExperimentTable(args.dir, args.name) exp = CachedExperiment(table, run) # load data data_loaders = loaders(args.dataset)( dir=args.dir, use_validation=not args.no_validation, val_ratio=args.val_ratio, batch_size=args.batch_size, ) train_loader = data_loaders['train'] valid_loader = data_loaders['valid'] # parse model num_classes = len(np.unique(train_loader.dataset.targets)) model_cfg = getattr(models, args.model) model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs) if args.cuda: model.cuda() # parse optimizer optimizer_cls = getattr(torch.optim, args.optimizer) if optimizer_cls == torch.optim.SGD: optimizer = torch.optim.SGD(model.parameters(), lr=args.lr_init, weight_decay=args.l2, momentum=args.momentum) else: #optimizer_cls == torch.optim.Adam: optimizer = torch.optim.Adam(model.parameters(), lr=args.lr_init, weight_decay=args.l2, betas=(args.beta_1, args.beta_2)) # parse criterion criterion = getattr(torch.nn, args.criterion)() exp.run( model=model, name=args.name, optimizer=optimizer, criterion=criterion, train_loader=train_loader, valid_loader=valid_loader, lr_init=args.lr_init, lr_final=args.lr_final, epochs=args.epochs, using_cuda=args.cuda, verbose=args.verbose, save_graph=args.save_graph, call=str(args), ) return exp
def experiment(args): experiment = ExperimentDirectory(args.dir, args.name) table_name = 'heatmap_loss_{}'.format(args.table_seed) experiment.add_table(table_name) model_cfg = getattr(models, args.model) criterion = getattr(torch.nn, args.criterion)() # load data data_loaders = loaders(args.dataset)( dir=experiment.path, use_validation=not args.no_validation, val_ratio=args.val_ratio, batch_size=args.batch_size, test_transforms=model_cfg.transform_test, train_transforms=model_cfg.transform_train, ) train_loader = data_loaders['train'] valid_loader = data_loaders['valid'] test_loader = data_loaders['test'] num_classes = len(np.unique(train_loader.dataset.targets)) if args.on_test: target_loader = test_loader else: target_loader = valid_loader num_models = range(args.max_num_models) if args.rank == -1: ranks = range(args.min_rank, args.max_rank + 1, args.step_rank) else: ranks = range(args.rank, args.rank + 1) # load posteriors posterior_names = [] for posterior_name in tqdm(os.listdir(experiment.posteriors_path)): if posterior_name.endswith('.pt'): posterior_names.append(posterior_name[:-3]) np.random.seed(args.table_seed) posterior_names = list(np.random.choice(posterior_names, size=len(posterior_names), replace=False)) print('Posterior order:', posterior_names) posteriors = [] for posterior_name in posterior_names: model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs) posterior = SWAGPosterior(model, rank=args.full_rank) posterior.load_state_dict(experiment.cached_state_dict(posterior_name, folder='posteriors')[0]) posteriors.append(posterior) for rank in tqdm(ranks): _, cache_row = experiment.cached_table_row({'rank': rank}, table_name=table_name) if cache_row: loss_valids = [] accu_valids = [] ensembler = Ensembler(target_loader) for n in tqdm(num_models): # n^th global model posterior = posteriors[n] if args.cuda: posterior.cuda() # add local models for k in tqdm(list(range(args.local_samples))): posterior.sample(rank=rank) posterior.renormalize(train_loader) ensembler.add_model(posterior) loss_valids.append(ensembler.evaluate(criterion).item()) accu_valids.append(ensembler.evaluate(accuracy)) cache_row({ 'losses_valid': loss_valids, 'accues_valid': accu_valids })
def experiment(args): experiment = ExperimentDirectory(args.dir, args.name) # load data data_loaders = loaders(args.dataset)( dir=experiment.path, use_validation=not args.no_validation, val_ratio=args.val_ratio, batch_size=args.batch_size, ) train_loader = data_loaders['train'] valid_loader = data_loaders['valid'] num_classes = len(np.unique(train_loader.dataset.targets)) # parse optimizer and criterion optimizer_cls = getattr(torch.optim, args.optimizer) criterion = getattr(torch.nn, args.criterion)() model_cfg = getattr(models, args.model) # parse optimizer and criterion optimizer_cls = getattr(torch.optim, args.optimizer) criterion = getattr(torch.nn, args.criterion)() for model_num in tqdm(list(range(1, args.num_models + 1))): model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs) if args.cuda: model.cuda() if optimizer_cls == torch.optim.SGD: optimizer = torch.optim.SGD(model.parameters(), lr=args.lr_init, weight_decay=args.l2, momentum=args.momentum) else: optimizer = torch.optim.Adam(model.parameters(), lr=args.lr_init, weight_decay=args.l2, betas=(args.beta_1, args.beta_2)) model_name = 'model_{}'.format(model_num) model_state_dict, cache_model = experiment.cached_state_dict(model_name) optim_name = 'optim_{}'.format(model_num) optim_state_dict, cache_optim = experiment.cached_state_dict(optim_name, folder='optims') if cache_model or cache_optim: seed(model_num) model, optimizer, tracker, res_train, res_valid = train_model( model=model, optimizer=optimizer, criterion=criterion, train_loader=train_loader, valid_loader=valid_loader, lr_init=args.lr_init, lr_final=args.lr_final, epochs=args.training_epochs, using_cuda=args.cuda, verbose=args.verbose ) cache_model(model.state_dict()) cache_optim(optimizer.state_dict()) else: model.load_state_dict(model_state_dict) optimizer.load_state_dict(optim_state_dict) posterior = SWAGPosterior(model, rank=args.rank) if args.cuda: posterior.cuda() posterior_name = 'posterior_{}'.format(model_num) posterior_state_dict, cache_posterior = experiment.cached_state_dict(posterior_name, folder='posteriors') if cache_posterior: sample_freq = int(len(train_loader.dataset) / (train_loader.batch_size * args.sample_rate)) sampler = SWAGSampler(posterior, optimizer, sample_freq=sample_freq) posterior, tracker = train_swag_from_pretrained( posterior=posterior, sampler=sampler, criterion=criterion, train_loader=train_loader, valid_loader=valid_loader, swag_epochs=args.swag_epochs, using_cuda=args.cuda, verbose=args.verbose, ) cache_posterior(posterior.state_dict())
def experiment(args): experiment = ExperimentDirectory(args.dir, args.name) table_name = 'heatmap_loss_{}'.format(args.table_seed) experiment.add_table(table_name) model_cfg = getattr(models, args.model) criterion = getattr(torch.nn, args.criterion)() # load data data_loaders = loaders(args.dataset)( dir=experiment.path, use_validation=not args.no_validation, val_ratio=args.val_ratio, batch_size=args.batch_size, test_transforms=model_cfg.transform_test, train_transforms=model_cfg.transform_train, ) train_loader = data_loaders['train'] valid_loader = data_loaders['valid'] test_loader = data_loaders['test'] #num_classes = len(np.unique(train_loader.dataset.targets)) if args.on_test: target_loader = test_loader else: target_loader = valid_loader num_models = range(args.max_num_models) if args.rank == -1: ranks = range(args.min_rank, args.max_rank + 1, args.step_rank) else: ranks = range(args.rank, args.rank + 1) # load predictions table predictions = experiment.tables['predictions'].read() posterior_names = list(predictions.loc[predictions['type'] == 'SWA', 'model'].unique()) np.random.seed(args.table_seed) posterior_names = list(np.random.choice(posterior_names, size=len(posterior_names), replace=False)) print('Model names:', posterior_names) for rank in tqdm(ranks): print('RANK:', rank) _, cache_row = experiment.cached_table_row({'rank': rank}, table_name=table_name) if cache_row: loss_valids = [] accu_valids = [] ensembler = Ensembler(target_loader) for n in tqdm(num_models): # n^th global model posterior_name = posterior_names[n] # add local models for k in tqdm(list(range(1, args.local_samples + 1))): prediction_file = predictions.loc[ (predictions['model'] == posterior_name) & (predictions['rank'] == rank) & (predictions['sample'] == k), 'path' ].iloc[0] print('predicting with: ', prediction_file) prediction = torch.load(prediction_file) ensembler.add_predictions(prediction) loss_valids.append(ensembler.evaluate(criterion).item()) accu_valids.append(ensembler.evaluate(accuracy)) print(loss_valids) cache_row({ 'losses_valid': loss_valids, 'accues_valid': accu_valids })
def experiment(args): experiment = ExperimentDirectory(args.dir, args.name) experiment.add_folder('predictions') experiment.add_table('predictions') # check args if not (args.train or args.test or args.validation): print('Missing --train, --test, or --validation flag/flags.') return model_cfg = getattr(models, args.model) criterion = getattr(torch.nn, args.criterion)() # load data data_loaders = loaders(args.dataset)( dir=experiment.path, use_validation=True, val_ratio=args.val_ratio, batch_size=args.batch_size, test_transforms=model_cfg.transform_test, train_transforms=model_cfg.transform_train, ) train_loader = data_loaders['train'] valid_loader = data_loaders['valid'] test_loader = data_loaders['test'] if args.train: loader = train_loader if args.validation: loader = valid_loader if args.test: loader = test_loader num_classes = len(np.unique(train_loader.dataset.targets)) if args.rank == -1: ranks = list(range(args.min_rank, args.max_rank + 1, args.step_rank)) else: ranks = list(range(args.rank, args.rank + 1)) # SGD print('Predicting with SGD solutions') for model_name in tqdm(os.listdir(experiment.models_path)): if not model_name.endswith('.pt'): continue model_name = model_name[:-3] _, cache_row = experiment.cached_table_row( { 'model': model_name, 'type': 'SGD', 'rank': None, 'sample': None }, table_name='predictions') if cache_row: path = os.path.join(experiment.predictions_path, model_name + '_SGD.pt') model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs) model.load_state_dict( experiment.cached_state_dict(model_name, folder='models')[0]) if args.cuda: model.cuda() ensembler = Ensembler(loader) bn_update(train_loader, model) ensembler.add_model(model) torch.save(ensembler.predictions, path) cache_row({'path': path}) # SWA print('Predicting with SWA solutions') for posterior_name in tqdm(os.listdir(experiment.posteriors_path)): if not posterior_name.endswith('.pt'): continue posterior_name = posterior_name[:-3] _, cache_row = experiment.cached_table_row( { 'model': posterior_name, 'type': 'SWA', 'rank': None, 'sample': None }, table_name='predictions') if cache_row: path = os.path.join(experiment.predictions_path, posterior_name + '_SWA.pt') model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs) posterior = SWAGPosterior(model, rank=args.full_rank) posterior.load_state_dict( experiment.cached_state_dict(posterior_name, folder='posteriors')[0]) if args.cuda: posterior.cuda() ensembler = Ensembler(loader) posterior.expected() posterior.renormalize(train_loader) ensembler.add_model(posterior) torch.save(ensembler.predictions, path) cache_row({'path': path}) # SWAG print('Predicting with SWAG solutions') for posterior_name in tqdm(os.listdir(experiment.posteriors_path)): if not posterior_name.endswith('.pt'): continue posterior_name = posterior_name[:-3] model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs) posterior = SWAGPosterior(model, rank=args.full_rank) posterior.load_state_dict( experiment.cached_state_dict(posterior_name, folder='posteriors')[0]) if args.cuda: posterior.cuda() for rank in ranks: for k in range(1, args.local_samples + 1): _, cache_row = experiment.cached_table_row( { 'model': posterior_name, 'type': 'SWAG', 'rank': rank, 'sample': k }, table_name='predictions') if cache_row: path = os.path.join( experiment.predictions_path, posterior_name + '_SWAG_rank_{}_sample_{}.pt'.format(rank, k)) ensembler = Ensembler(loader) posterior.sample(rank=rank) posterior.renormalize(train_loader) ensembler.add_model(posterior) torch.save(ensembler.predictions, path) cache_row({'path': path})