def experiment(args):
table = ExperimentTable(args.dir, args.name)
exp = CachedExperiment(table, run)
# load data
data_loaders = loaders(args.dataset)(
dir=args.dir,
use_validation=not args.no_validation,
val_ratio=args.val_ratio,
batch_size=args.batch_size,
)
train_loader = data_loaders['train']
valid_loader = data_loaders['valid']
# parse model
num_classes = len(np.unique(train_loader.dataset.targets))
model_cfg = getattr(models, args.model)
model = model_cfg.model(*model_cfg.args,
num_classes=num_classes,
**model_cfg.kwargs)
if args.cuda:
model.cuda()
# parse optimizer
optimizer_cls = getattr(torch.optim, args.optimizer)
if optimizer_cls == torch.optim.SGD:
optimizer = torch.optim.SGD(model.parameters(),
lr=args.lr_init,
weight_decay=args.l2,
momentum=args.momentum)
else: #optimizer_cls == torch.optim.Adam:
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr_init,
weight_decay=args.l2,
betas=(args.beta_1, args.beta_2))
# parse criterion
criterion = getattr(torch.nn, args.criterion)()
exp.run(
model=model,
name=args.name,
optimizer=optimizer,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
lr_init=args.lr_init,
lr_final=args.lr_final,
epochs=args.epochs,
using_cuda=args.cuda,
verbose=args.verbose,
save_graph=args.save_graph,
call=str(args),
)
return exp
def experiment(args):
experiment = ExperimentDirectory(args.dir, args.name)
table_name = 'heatmap_loss_{}'.format(args.table_seed)
experiment.add_table(table_name)
model_cfg = getattr(models, args.model)
criterion = getattr(torch.nn, args.criterion)()
# load data
data_loaders = loaders(args.dataset)(
dir=experiment.path,
use_validation=not args.no_validation,
val_ratio=args.val_ratio,
batch_size=args.batch_size,
test_transforms=model_cfg.transform_test,
train_transforms=model_cfg.transform_train,
)
train_loader = data_loaders['train']
valid_loader = data_loaders['valid']
test_loader = data_loaders['test']
num_classes = len(np.unique(train_loader.dataset.targets))
if args.on_test:
target_loader = test_loader
else:
target_loader = valid_loader
num_models = range(args.max_num_models)
if args.rank == -1:
ranks = range(args.min_rank, args.max_rank + 1, args.step_rank)
else:
ranks = range(args.rank, args.rank + 1)
# load posteriors
posterior_names = []
for posterior_name in tqdm(os.listdir(experiment.posteriors_path)):
if posterior_name.endswith('.pt'):
posterior_names.append(posterior_name[:-3])
np.random.seed(args.table_seed)
posterior_names = list(np.random.choice(posterior_names, size=len(posterior_names), replace=False))
print('Posterior order:', posterior_names)
posteriors = []
for posterior_name in posterior_names:
model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs)
posterior = SWAGPosterior(model, rank=args.full_rank)
posterior.load_state_dict(experiment.cached_state_dict(posterior_name, folder='posteriors')[0])
posteriors.append(posterior)
for rank in tqdm(ranks):
_, cache_row = experiment.cached_table_row({'rank': rank}, table_name=table_name)
if cache_row:
loss_valids = []
accu_valids = []
ensembler = Ensembler(target_loader)
for n in tqdm(num_models):
# n^th global model
posterior = posteriors[n]
if args.cuda:
posterior.cuda()
# add local models
for k in tqdm(list(range(args.local_samples))):
posterior.sample(rank=rank)
posterior.renormalize(train_loader)
ensembler.add_model(posterior)
loss_valids.append(ensembler.evaluate(criterion).item())
accu_valids.append(ensembler.evaluate(accuracy))
cache_row({
'losses_valid': loss_valids,
'accues_valid': accu_valids
})
def experiment(args):
experiment = ExperimentDirectory(args.dir, args.name)
# load data
data_loaders = loaders(args.dataset)(
dir=experiment.path,
use_validation=not args.no_validation,
val_ratio=args.val_ratio,
batch_size=args.batch_size,
)
train_loader = data_loaders['train']
valid_loader = data_loaders['valid']
num_classes = len(np.unique(train_loader.dataset.targets))
# parse optimizer and criterion
optimizer_cls = getattr(torch.optim, args.optimizer)
criterion = getattr(torch.nn, args.criterion)()
model_cfg = getattr(models, args.model)
# parse optimizer and criterion
optimizer_cls = getattr(torch.optim, args.optimizer)
criterion = getattr(torch.nn, args.criterion)()
for model_num in tqdm(list(range(1, args.num_models + 1))):
model = model_cfg.model(*model_cfg.args, num_classes=num_classes, **model_cfg.kwargs)
if args.cuda:
model.cuda()
if optimizer_cls == torch.optim.SGD:
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr_init, weight_decay=args.l2,
momentum=args.momentum)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr_init, weight_decay=args.l2,
betas=(args.beta_1, args.beta_2))
model_name = 'model_{}'.format(model_num)
model_state_dict, cache_model = experiment.cached_state_dict(model_name)
optim_name = 'optim_{}'.format(model_num)
optim_state_dict, cache_optim = experiment.cached_state_dict(optim_name, folder='optims')
if cache_model or cache_optim:
seed(model_num)
model, optimizer, tracker, res_train, res_valid = train_model(
model=model,
optimizer=optimizer,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
lr_init=args.lr_init,
lr_final=args.lr_final,
epochs=args.training_epochs,
using_cuda=args.cuda,
verbose=args.verbose
)
cache_model(model.state_dict())
cache_optim(optimizer.state_dict())
else:
model.load_state_dict(model_state_dict)
optimizer.load_state_dict(optim_state_dict)
posterior = SWAGPosterior(model, rank=args.rank)
if args.cuda:
posterior.cuda()
posterior_name = 'posterior_{}'.format(model_num)
posterior_state_dict, cache_posterior = experiment.cached_state_dict(posterior_name, folder='posteriors')
if cache_posterior:
sample_freq = int(len(train_loader.dataset) / (train_loader.batch_size * args.sample_rate))
sampler = SWAGSampler(posterior, optimizer, sample_freq=sample_freq)
posterior, tracker = train_swag_from_pretrained(
posterior=posterior,
sampler=sampler,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
swag_epochs=args.swag_epochs,
using_cuda=args.cuda,
verbose=args.verbose,
)
cache_posterior(posterior.state_dict())
def experiment(args):
experiment = ExperimentDirectory(args.dir, args.name)
table_name = 'heatmap_loss_{}'.format(args.table_seed)
experiment.add_table(table_name)
model_cfg = getattr(models, args.model)
criterion = getattr(torch.nn, args.criterion)()
# load data
data_loaders = loaders(args.dataset)(
dir=experiment.path,
use_validation=not args.no_validation,
val_ratio=args.val_ratio,
batch_size=args.batch_size,
test_transforms=model_cfg.transform_test,
train_transforms=model_cfg.transform_train,
)
train_loader = data_loaders['train']
valid_loader = data_loaders['valid']
test_loader = data_loaders['test']
#num_classes = len(np.unique(train_loader.dataset.targets))
if args.on_test:
target_loader = test_loader
else:
target_loader = valid_loader
num_models = range(args.max_num_models)
if args.rank == -1:
ranks = range(args.min_rank, args.max_rank + 1, args.step_rank)
else:
ranks = range(args.rank, args.rank + 1)
# load predictions table
predictions = experiment.tables['predictions'].read()
posterior_names = list(predictions.loc[predictions['type'] == 'SWA', 'model'].unique())
np.random.seed(args.table_seed)
posterior_names = list(np.random.choice(posterior_names, size=len(posterior_names), replace=False))
print('Model names:', posterior_names)
for rank in tqdm(ranks):
print('RANK:', rank)
_, cache_row = experiment.cached_table_row({'rank': rank}, table_name=table_name)
if cache_row:
loss_valids = []
accu_valids = []
ensembler = Ensembler(target_loader)
for n in tqdm(num_models):
# n^th global model
posterior_name = posterior_names[n]
# add local models
for k in tqdm(list(range(1, args.local_samples + 1))):
prediction_file = predictions.loc[
(predictions['model'] == posterior_name) & (predictions['rank'] == rank) & (predictions['sample'] == k), 'path'
].iloc[0]
print('predicting with: ', prediction_file)
prediction = torch.load(prediction_file)
ensembler.add_predictions(prediction)
loss_valids.append(ensembler.evaluate(criterion).item())
accu_valids.append(ensembler.evaluate(accuracy))
print(loss_valids)
cache_row({
'losses_valid': loss_valids,
'accues_valid': accu_valids
})
def experiment(args):
experiment = ExperimentDirectory(args.dir, args.name)
experiment.add_folder('predictions')
experiment.add_table('predictions')
# check args
if not (args.train or args.test or args.validation):
print('Missing --train, --test, or --validation flag/flags.')
return
model_cfg = getattr(models, args.model)
criterion = getattr(torch.nn, args.criterion)()
# load data
data_loaders = loaders(args.dataset)(
dir=experiment.path,
use_validation=True,
val_ratio=args.val_ratio,
batch_size=args.batch_size,
test_transforms=model_cfg.transform_test,
train_transforms=model_cfg.transform_train,
)
train_loader = data_loaders['train']
valid_loader = data_loaders['valid']
test_loader = data_loaders['test']
if args.train:
loader = train_loader
if args.validation:
loader = valid_loader
if args.test:
loader = test_loader
num_classes = len(np.unique(train_loader.dataset.targets))
if args.rank == -1:
ranks = list(range(args.min_rank, args.max_rank + 1, args.step_rank))
else:
ranks = list(range(args.rank, args.rank + 1))
# SGD
print('Predicting with SGD solutions')
for model_name in tqdm(os.listdir(experiment.models_path)):
if not model_name.endswith('.pt'):
continue
model_name = model_name[:-3]
_, cache_row = experiment.cached_table_row(
{
'model': model_name,
'type': 'SGD',
'rank': None,
'sample': None
},
table_name='predictions')
if cache_row:
path = os.path.join(experiment.predictions_path,
model_name + '_SGD.pt')
model = model_cfg.model(*model_cfg.args,
num_classes=num_classes,
**model_cfg.kwargs)
model.load_state_dict(
experiment.cached_state_dict(model_name, folder='models')[0])
if args.cuda:
model.cuda()
ensembler = Ensembler(loader)
bn_update(train_loader, model)
ensembler.add_model(model)
torch.save(ensembler.predictions, path)
cache_row({'path': path})
# SWA
print('Predicting with SWA solutions')
for posterior_name in tqdm(os.listdir(experiment.posteriors_path)):
if not posterior_name.endswith('.pt'):
continue
posterior_name = posterior_name[:-3]
_, cache_row = experiment.cached_table_row(
{
'model': posterior_name,
'type': 'SWA',
'rank': None,
'sample': None
},
table_name='predictions')
if cache_row:
path = os.path.join(experiment.predictions_path,
posterior_name + '_SWA.pt')
model = model_cfg.model(*model_cfg.args,
num_classes=num_classes,
**model_cfg.kwargs)
posterior = SWAGPosterior(model, rank=args.full_rank)
posterior.load_state_dict(
experiment.cached_state_dict(posterior_name,
folder='posteriors')[0])
if args.cuda:
posterior.cuda()
ensembler = Ensembler(loader)
posterior.expected()
posterior.renormalize(train_loader)
ensembler.add_model(posterior)
torch.save(ensembler.predictions, path)
cache_row({'path': path})
# SWAG
print('Predicting with SWAG solutions')
for posterior_name in tqdm(os.listdir(experiment.posteriors_path)):
if not posterior_name.endswith('.pt'):
continue
posterior_name = posterior_name[:-3]
model = model_cfg.model(*model_cfg.args,
num_classes=num_classes,
**model_cfg.kwargs)
posterior = SWAGPosterior(model, rank=args.full_rank)
posterior.load_state_dict(
experiment.cached_state_dict(posterior_name,
folder='posteriors')[0])
if args.cuda:
posterior.cuda()
for rank in ranks:
for k in range(1, args.local_samples + 1):
_, cache_row = experiment.cached_table_row(
{
'model': posterior_name,
'type': 'SWAG',
'rank': rank,
'sample': k
},
table_name='predictions')
if cache_row:
path = os.path.join(
experiment.predictions_path, posterior_name +
'_SWAG_rank_{}_sample_{}.pt'.format(rank, k))
ensembler = Ensembler(loader)
posterior.sample(rank=rank)
posterior.renormalize(train_loader)
ensembler.add_model(posterior)
torch.save(ensembler.predictions, path)
cache_row({'path': path})