def __init__(self, hyperparameters, model=None, train_loader=None, model_loss='multi_class_linear_output',
device=torch.device('cpu')):
'''
:param hyperparameters: Hyperparameters include {'lr', 'prior_std', 'num_samples'}
:param model: Pytorch model to run SGLD on.
:param train_loader: DataLoader for train data
:param model_loss: Loss function to use for the model. (e.g.: 'multi_class_linear_output')
:param device: Device on which model is present (e.g.: torch.device('cpu'))
'''
if hyperparameters == None:
# Initialise as some default values
hyperparameters = {'lr': 0.1, 'epochs':10, 'momentum': 0.9, 'weight_decay': 0.001}
super(SGD, self).__init__(hyperparameters, model, train_loader, device)
self.lr = hyperparameters['lr']
self.num_samples = 1
self.burn_in_epochs = hyperparameters['epochs']
self.momentum = hyperparameters['momentum']
self.model = model.to(device)
self.train_loader = train_loader
self.device = device
self.dataset_size = len(train_loader.dataset)
self.weight_decay = hyperparameters['weight_decay']
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum,
weight_decay=self.weight_decay)
self.loss_criterion = get_loss_criterion(loss=model_loss)
self.burnt_in = False
self.epochs_run = 0
self.lr_final = self.lr / 100.
self.optimizer_scheduler = CosineAnnealingLR(optimizer=self.optimizer, T_max=
self.burn_in_epochs + self.num_samples, eta_min=self.lr_final)
def __init__(self, hyperparameters, model=None, train_loader=None, model_loss='multi_class_linear_output',
device=torch.device('cpu')):
if hyperparameters == None:
# Initialise as some default values
hyperparameters = {'lr_0': 0.001000, 'prior_std': 10.1000, 'num_samples_per_cycle': 5, 'cycle_length': 20, 'burn_in_epochs': 5, 'num_cycles': 10, 'alpha': 1.,}
super(cSGHMC, self).__init__(hyperparameters, model, train_loader, device)
self.lr_0 = hyperparameters['lr_0']
self.prior_std = hyperparameters['prior_std']
self.num_samples_per_cycle = hyperparameters['num_samples_per_cycle']
self.cycle_length = hyperparameters['cycle_length']
self.alpha = hyperparameters['alpha']
self.burn_in_epochs = hyperparameters['burn_in_epochs']
self.num_cycles = hyperparameters['num_cycles']
self.batch_size = train_loader.batch_size
self.num_batch = len(train_loader.dataset) / self.batch_size + 1
self.num_batch = max(1, self.num_batch)
self.model_loss = model_loss
self.model = model
self.train_loader = train_loader
self.device = device
self.dataset_size = len(train_loader.dataset)
self.optimizer = optimSGHMC(params=self.model.parameters(), lr=self.lr_0, momentum=1 - self.alpha,
num_training_samples=self.dataset_size, weight_decay=1 / (self.prior_std ** 2))
self.loss_criterion = get_loss_criterion(loss=model_loss)
self.burnt_in = False
self.epochs_run = 0
self.total_epochs = self.cycle_length * self.num_cycles
self.dataloader_batch_size = self.train_loader.batch_size
self.total_iterations = self.total_epochs * self.num_batch
assert ((self.cycle_length - self.burn_in_epochs - self.num_samples_per_cycle) > 0)
def __init__(self,
hyperparameters,
model=None,
train_loader=None,
model_loss='multi_class_linear_output',
device=torch.device('cpu'),
**subspace_kwargs):
super(SWA, self).__init__(hyperparameters,
model=None,
train_loader=None,
device=torch.device('cpu'))
if hyperparameters == None:
# Initialise as some default values
hyperparameters = {
'swag_lr': 0.001,
'swag_wd': 0.001,
'lr_init': 0.001,
'num_samples': 20,
'momentum': 0.1,
'burn_in_epochs': 100,
'num_iterates': 50
}
self.hyperparameters = hyperparameters
self.swag_model = deepcopy(model)
self.model = model
self.num_parameters = sum(param.numel()
for param in self.swag_model.parameters())
self.weight_mean = torch.zeros(self.num_parameters)
self.sq_mean = torch.zeros(self.num_parameters)
self.num_models_collected = torch.zeros(1, dtype=torch.long)
self.var_clamp = 1e-30
self.device = device
self.train_loader = train_loader
self.loss_criterion = get_loss_criterion(loss=model_loss)
self.dataset_size = len(train_loader.dataset)
self.burnt_in = False
self.epochs_run = 0
self.burn_in_epochs = self.hyperparameters['burn_in_epochs']
self.num_iterates = self.hyperparameters['num_iterates']
self.momentum = self.hyperparameters['momentum']
self.lr_init = self.hyperparameters['lr_init']
self.swag_lr = self.hyperparameters['swag_lr']
self.swag_wd = self.hyperparameters['swag_wd']
self.optimizer = SGD(params=self.model.parameters(),
lr=self.lr_init,
momentum=self.momentum,
weight_decay=self.swag_wd)
if 'subspace_type' not in hyperparameters.keys():
self.subspace_type = 'pca'
else:
self.subspace_type = hyperparameters['subspace_type']
if subspace_kwargs is None:
subspace_kwargs = dict()
self.subspace = Subspace.create(self.subspace_type,
num_parameters=self.num_parameters,
**subspace_kwargs)
self.cov_factor = None
def __init__(self,
hyperparameters,
model=None,
train_loader=None,
model_loss='multi_class_linear_output',
device=torch.device('cpu')):
'''
:param hyperparameters: Hyperparameters include {'lr', 'prior_std', 'num_samples'}
:param model: Pytorch model to run SGLD on.
:param train_loader: DataLoader for train data
:param model_loss: Loss function to use for the model. (e.g.: 'multi_class_linear_output')
:param device: Device on which model is present (e.g.: torch.device('cpu'))
'''
if hyperparameters == None:
# Initialise as some default values
hyperparameters = {
'lr': 0.001,
'prior_std': 10,
'num_samples': 2,
'alpha': 0.1,
'burn_in_epochs': 10
}
super(SGHMC, self).__init__(hyperparameters, model, train_loader,
device)
self.lr = hyperparameters['lr']
self.prior_std = hyperparameters['prior_std']
self.num_samples = hyperparameters['num_samples']
self.alpha = hyperparameters['alpha']
self.burn_in_epochs = hyperparameters['burn_in_epochs']
self.model_loss = model_loss
self.model = model
self.train_loader = train_loader
self.device = device
self.dataset_size = len(train_loader.dataset)
self.optimizer = optimSGHMC(params=self.model.parameters(),
lr=self.lr,
momentum=1 - self.alpha,
num_training_samples=self.dataset_size,
weight_decay=1 / (self.prior_std**2))
self.loss_criterion = get_loss_criterion(loss=model_loss)
self.burnt_in = False
self.epochs_run = 0
self.lr_final = self.lr / 2
# self.optimizer_scheduler = CosineAnnealingLR(optimizer=self.optimizer, T_max=
# (self.burn_in_epochs + self.num_samples), eta_min=self.lr_final)
self.optimizer_scheduler = OneCycleLR(
optimizer=self.optimizer,
max_lr=self.lr * 5,
steps_per_epoch=len(self.train_loader),
epochs=self.burn_in_epochs + self.num_samples)
def __init__(self,
hyperparameters,
model=None,
train_loader=None,
device=torch.device('cpu'),
model_loss='multi_class_linear_output'):
"""
Inputs:
model: torch.nn.model (TODO Check this is flexible to other models)
hyperparameters: list of hyperparameters in order expected by inference engine e.g. [[0.0], [2., 4.]]
train_loader: torch.utils.data.DataLoader
device: default 'cpu'
"""
self.model = model
self.hyperparameters = hyperparameters
self.train_loader = train_loader
self.device = device
self.loss_criterion = get_loss_criterion(loss=model_loss)
def __init__(self, hyperparameters, model=None, train_loader=None, model_loss='multi_class_linear_output',
device=torch.device('cpu')):
'''
:param hyperparameters: Hyperparameters include {'lr', 'prior_std', 'num_samples'}
:param model: Pytorch model to run SGLD on.
:param train_loader: DataLoader for train data
:param model_loss: Loss function to use for the model. (e.g.: 'multi_class_linear_output')
:param device: Device on which model is present (e.g.: torch.device('cpu'))
'''
if hyperparameters == None:
# Initialise as some default values
hyperparameters = {'lr': 0.1, 'epochs':10, 'dropout': 0.2, 'lengthscale': 0.01, 'num_samples': 10, 'momentum': 0.9, 'weight_decay': 0}
super(MCdropout, self).__init__(hyperparameters, model, train_loader, device)
self.lr = hyperparameters['lr']
self.num_samples = hyperparameters['num_samples']
self.burn_in_epochs = hyperparameters['epochs']
self.dropout = hyperparameters['dropout']
self.momentum = hyperparameters['momentum']
self.model = change_to_dropout_model(model, self.dropout).to(device)
self.train_loader = train_loader
self.device = device
self.dataset_size = len(train_loader.dataset)
if hyperparameters['weight_decay'] != 0:
self.weight_decay = hyperparameters['weight_decay']
else:
self.weight_decay = hyperparameters['lengthscale'] ** 2 * (1 - self.dropout) / (2. * self.dataset_size)
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum,
weight_decay=self.weight_decay)
self.loss_criterion = get_loss_criterion(loss=model_loss)
self.burnt_in = False
self.epochs_run = 0
self.lr_final = self.lr / 100.
# self.optimizer_scheduler = CosineAnnealingLR(optimizer=self.optimizer, T_max=
# self.burn_in_epochs + self.num_samples, eta_min=self.lr_final)
self.optimizer_scheduler = OneCycleLR(optimizer=self.optimizer, max_lr=self.lr * 5,
steps_per_epoch=len(self.train_loader),
epochs=self.burn_in_epochs + self.num_samples)