def set_hyperparameters(params,
num_classes,
model,
checkpoint,
dontcare_val,
loss_fn,
optimizer,
class_weights=None,
inference: str = ''):
"""
Function to set hyperparameters based on values provided in yaml config file.
If none provided, default functions values may be used.
:param params: (dict) Parameters found in the yaml config file
:param num_classes: (int) number of classes for current task
:param model: initialized model
:param checkpoint: (dict) state dict as loaded by model_choice.py
:param dontcare_val: value in label to ignore during loss calculation
:param loss_fn: loss function
:param optimizer: optimizer function
:param class_weights: class weights for loss function
:param inference: (str) path to inference checkpoint (used in load_from_checkpoint())
:return: model, criterion, optimizer, lr_scheduler, num_gpus
"""
# set mandatory hyperparameters values with those in config file if they exist
lr = get_key_def('learning_rate', params['training'], None)
weight_decay = get_key_def('weight_decay', params['training'], None)
step_size = get_key_def('step_size', params['training'], None)
gamma = get_key_def('gamma', params['training'], None)
class_weights = torch.tensor(class_weights) if class_weights else None
# Loss function
criterion = MultiClassCriterion(loss_type=loss_fn,
ignore_index=dontcare_val,
weight=class_weights)
# Optimizer
opt_fn = optimizer
optimizer = create_optimizer(params=model.parameters(),
mode=opt_fn,
base_lr=lr,
weight_decay=weight_decay)
lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer,
step_size=step_size,
gamma=gamma)
if checkpoint:
tqdm.write(f'Loading checkpoint...')
model, optimizer = load_from_checkpoint(checkpoint,
model,
optimizer=optimizer,
inference=inference)
return model, criterion, optimizer, lr_scheduler
def set_hyperparameters(params, num_classes, model, checkpoint):
"""
Function to set hyperparameters based on values provided in yaml config file.
Will also set model to GPU, if available.
If none provided, default functions values may be used.
:param params: (dict) Parameters found in the yaml config file
:param num_classes: (int) number of classes for current task
:param model: Model loaded from model_choice.py
:param checkpoint: (dict) state dict as loaded by model_choice.py
:return: model, criterion, optimizer, lr_scheduler, num_gpus
"""
# set mandatory hyperparameters values with those in config file if they exist
lr = get_key_def('learning_rate', params['training'], None,
"missing mandatory learning rate parameter")
weight_decay = get_key_def('weight_decay', params['training'], None,
"missing mandatory weight decay parameter")
step_size = get_key_def('step_size', params['training'], None,
"missing mandatory step size parameter")
gamma = get_key_def('gamma', params['training'], None,
"missing mandatory gamma parameter")
# optional hyperparameters. Set to None if not in config file
class_weights = torch.tensor(
params['training']
['class_weights']) if params['training']['class_weights'] else None
if params['training']['class_weights']:
verify_weights(num_classes, class_weights)
ignore_index = get_key_def('ignore_index', params['training'], -1)
# Loss function
criterion = MultiClassCriterion(loss_type=params['training']['loss_fn'],
ignore_index=ignore_index,
weight=class_weights)
# Optimizer
opt_fn = params['training']['optimizer']
optimizer = create_optimizer(params=model.parameters(),
mode=opt_fn,
base_lr=lr,
weight_decay=weight_decay)
lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer,
step_size=step_size,
gamma=gamma)
if checkpoint:
tqdm.write(f'Loading checkpoint...')
model, optimizer = load_from_checkpoint(checkpoint,
model,
optimizer=optimizer)
return model, criterion, optimizer, lr_scheduler
def set_hyperparameters(params, model, checkpoint):
"""
Function to set hyperparameters based on values provided in yaml config file.
Will also set model to GPU, if available.
If none provided, default functions values may be used.
:param params: (dict) Parameters found in the yaml config file
:param model: Model loaded from model_choice.py
:param checkpoint: (dict) state dict as loaded by model_choice.py
:return: model, criterion, optimizer, lr_scheduler, num_gpus
"""
# set mandatory hyperparameters values with those in config file if they exist
lr = params['training']['learning_rate']
assert lr is not None and lr > 0, "missing mandatory learning rate parameter"
weight_decay = params['training']['weight_decay']
assert weight_decay is not None and weight_decay >= 0, "missing mandatory weight decay parameter"
step_size = params['training']['step_size']
assert step_size is not None and step_size > 0, "missing mandatory step size parameter"
gamma = params['training']['gamma']
assert gamma is not None and gamma >= 0, "missing mandatory gamma parameter"
# optional hyperparameters. Set to None if not in config file
class_weights = torch.tensor(
params['training']
['class_weights']) if params['training']['class_weights'] else None
if params['training']['class_weights']:
verify_weights(params['global']['num_classes'], class_weights)
ignore_index = -100
if params['training']['ignore_index'] is not None:
ignore_index = params['training']['ignore_index']
# Loss function
criterion = MultiClassCriterion(loss_type=params['training']['loss_fn'],
ignore_index=ignore_index,
weight=class_weights)
# Optimizer
opt_fn = params['training']['optimizer']
optimizer = create_optimizer(params=model.parameters(),
mode=opt_fn,
base_lr=lr,
weight_decay=weight_decay)
lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer,
step_size=step_size,
gamma=gamma)
if checkpoint:
model, optimizer = load_from_checkpoint(checkpoint,
model,
optimizer=optimizer)
return model, criterion, optimizer, lr_scheduler
def set_hyperparameters(params, model, state_dict_path):
"""
Function to set hyperparameters based on values provided in yaml config file.
Will also set model to GPU, if available.
If none provided, default functions values are used.
:param params: (dict) Parameters found in the yaml config file
:param model: Model loaded from model_choice.py
:param state_dict_path: (str) Full file path to the state dict
:return: model, criterion, optimizer, lr_scheduler, num_gpus
"""
# assign default values to hyperparameters
loss_signature = inspect.signature(nn.CrossEntropyLoss).parameters
optim_signature = inspect.signature(optim.Adam).parameters
lr_scheduler_signature = inspect.signature(
optim.lr_scheduler.StepLR).parameters
class_weights = loss_signature['weight'].default
ignore_index = loss_signature['ignore_index'].default
lr = optim_signature['lr'].default
weight_decay = optim_signature['weight_decay'].default
step_size = lr_scheduler_signature['step_size'].default
if not isinstance(step_size, int):
step_size = params['training']['num_epochs'] + 1
gamma = lr_scheduler_signature['gamma'].default
num_devices = 0
# replace default values by those in config file if they exist
if params['training']['class_weights']:
class_weights = torch.tensor(params['training']['class_weights'])
verify_weights(params['global']['num_classes'], class_weights)
if params['training']['ignore_index']:
ignore_index = params['training']['ignore_index']
if params['training']['learning_rate']:
lr = params['training']['learning_rate']
if params['training']['weight_decay']:
weight_decay = params['training']['weight_decay']
if params['training']['step_size']:
step_size = params['training']['step_size']
if params['training']['gamma']:
gamma = params['training']['gamma']
if params['global']['num_gpus']:
num_devices = params['global']['num_gpus']
# Loss function
criterion = MultiClassCriterion(loss_type=params['training']['loss_fn'],
ignore_index=ignore_index,
weight=class_weights)
# list of GPU devices that are available and unused. If no GPUs, returns empty list
lst_device_ids = get_device_ids(
num_devices) if torch.cuda.is_available() else []
num_devices = len(lst_device_ids) if lst_device_ids else 0
device = torch.device(f'cuda:{lst_device_ids[0]}' if torch.cuda.
is_available() and lst_device_ids else 'cpu')
if num_devices == 1:
print(f"Using Cuda device {lst_device_ids[0]}")
elif num_devices > 1:
print(f"Using data parallel on devices {str(lst_device_ids)[1:-1]}")
model = nn.DataParallel(model, device_ids=lst_device_ids
) # adds prefix 'module.' to state_dict keys
else:
warnings.warn(
f"No Cuda device available. This process will only run on CPU")
criterion = criterion.to(device)
model = model.to(device)
# Optimizer
opt_fn = params['training']['optimizer']
optimizer = create_optimizer(params=model.parameters(),
mode=opt_fn,
base_lr=lr,
weight_decay=weight_decay)
lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer,
step_size=step_size,
gamma=gamma)
if state_dict_path != '':
model, optimizer = load_from_checkpoint(state_dict_path,
model,
optimizer=optimizer)
return model, criterion, optimizer, lr_scheduler, device, num_devices