def set_hyperparameters(params, num_classes, model, checkpoint, dontcare_val, loss_fn, optimizer, class_weights=None, inference: str = ''): """ Function to set hyperparameters based on values provided in yaml config file. If none provided, default functions values may be used. :param params: (dict) Parameters found in the yaml config file :param num_classes: (int) number of classes for current task :param model: initialized model :param checkpoint: (dict) state dict as loaded by model_choice.py :param dontcare_val: value in label to ignore during loss calculation :param loss_fn: loss function :param optimizer: optimizer function :param class_weights: class weights for loss function :param inference: (str) path to inference checkpoint (used in load_from_checkpoint()) :return: model, criterion, optimizer, lr_scheduler, num_gpus """ # set mandatory hyperparameters values with those in config file if they exist lr = get_key_def('learning_rate', params['training'], None) weight_decay = get_key_def('weight_decay', params['training'], None) step_size = get_key_def('step_size', params['training'], None) gamma = get_key_def('gamma', params['training'], None) class_weights = torch.tensor(class_weights) if class_weights else None # Loss function criterion = MultiClassCriterion(loss_type=loss_fn, ignore_index=dontcare_val, weight=class_weights) # Optimizer opt_fn = optimizer optimizer = create_optimizer(params=model.parameters(), mode=opt_fn, base_lr=lr, weight_decay=weight_decay) lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer, step_size=step_size, gamma=gamma) if checkpoint: tqdm.write(f'Loading checkpoint...') model, optimizer = load_from_checkpoint(checkpoint, model, optimizer=optimizer, inference=inference) return model, criterion, optimizer, lr_scheduler
def set_hyperparameters(params, num_classes, model, checkpoint): """ Function to set hyperparameters based on values provided in yaml config file. Will also set model to GPU, if available. If none provided, default functions values may be used. :param params: (dict) Parameters found in the yaml config file :param num_classes: (int) number of classes for current task :param model: Model loaded from model_choice.py :param checkpoint: (dict) state dict as loaded by model_choice.py :return: model, criterion, optimizer, lr_scheduler, num_gpus """ # set mandatory hyperparameters values with those in config file if they exist lr = get_key_def('learning_rate', params['training'], None, "missing mandatory learning rate parameter") weight_decay = get_key_def('weight_decay', params['training'], None, "missing mandatory weight decay parameter") step_size = get_key_def('step_size', params['training'], None, "missing mandatory step size parameter") gamma = get_key_def('gamma', params['training'], None, "missing mandatory gamma parameter") # optional hyperparameters. Set to None if not in config file class_weights = torch.tensor( params['training'] ['class_weights']) if params['training']['class_weights'] else None if params['training']['class_weights']: verify_weights(num_classes, class_weights) ignore_index = get_key_def('ignore_index', params['training'], -1) # Loss function criterion = MultiClassCriterion(loss_type=params['training']['loss_fn'], ignore_index=ignore_index, weight=class_weights) # Optimizer opt_fn = params['training']['optimizer'] optimizer = create_optimizer(params=model.parameters(), mode=opt_fn, base_lr=lr, weight_decay=weight_decay) lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer, step_size=step_size, gamma=gamma) if checkpoint: tqdm.write(f'Loading checkpoint...') model, optimizer = load_from_checkpoint(checkpoint, model, optimizer=optimizer) return model, criterion, optimizer, lr_scheduler
def set_hyperparameters(params, model, checkpoint): """ Function to set hyperparameters based on values provided in yaml config file. Will also set model to GPU, if available. If none provided, default functions values may be used. :param params: (dict) Parameters found in the yaml config file :param model: Model loaded from model_choice.py :param checkpoint: (dict) state dict as loaded by model_choice.py :return: model, criterion, optimizer, lr_scheduler, num_gpus """ # set mandatory hyperparameters values with those in config file if they exist lr = params['training']['learning_rate'] assert lr is not None and lr > 0, "missing mandatory learning rate parameter" weight_decay = params['training']['weight_decay'] assert weight_decay is not None and weight_decay >= 0, "missing mandatory weight decay parameter" step_size = params['training']['step_size'] assert step_size is not None and step_size > 0, "missing mandatory step size parameter" gamma = params['training']['gamma'] assert gamma is not None and gamma >= 0, "missing mandatory gamma parameter" # optional hyperparameters. Set to None if not in config file class_weights = torch.tensor( params['training'] ['class_weights']) if params['training']['class_weights'] else None if params['training']['class_weights']: verify_weights(params['global']['num_classes'], class_weights) ignore_index = -100 if params['training']['ignore_index'] is not None: ignore_index = params['training']['ignore_index'] # Loss function criterion = MultiClassCriterion(loss_type=params['training']['loss_fn'], ignore_index=ignore_index, weight=class_weights) # Optimizer opt_fn = params['training']['optimizer'] optimizer = create_optimizer(params=model.parameters(), mode=opt_fn, base_lr=lr, weight_decay=weight_decay) lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer, step_size=step_size, gamma=gamma) if checkpoint: model, optimizer = load_from_checkpoint(checkpoint, model, optimizer=optimizer) return model, criterion, optimizer, lr_scheduler
def set_hyperparameters(params, model, state_dict_path): """ Function to set hyperparameters based on values provided in yaml config file. Will also set model to GPU, if available. If none provided, default functions values are used. :param params: (dict) Parameters found in the yaml config file :param model: Model loaded from model_choice.py :param state_dict_path: (str) Full file path to the state dict :return: model, criterion, optimizer, lr_scheduler, num_gpus """ # assign default values to hyperparameters loss_signature = inspect.signature(nn.CrossEntropyLoss).parameters optim_signature = inspect.signature(optim.Adam).parameters lr_scheduler_signature = inspect.signature( optim.lr_scheduler.StepLR).parameters class_weights = loss_signature['weight'].default ignore_index = loss_signature['ignore_index'].default lr = optim_signature['lr'].default weight_decay = optim_signature['weight_decay'].default step_size = lr_scheduler_signature['step_size'].default if not isinstance(step_size, int): step_size = params['training']['num_epochs'] + 1 gamma = lr_scheduler_signature['gamma'].default num_devices = 0 # replace default values by those in config file if they exist if params['training']['class_weights']: class_weights = torch.tensor(params['training']['class_weights']) verify_weights(params['global']['num_classes'], class_weights) if params['training']['ignore_index']: ignore_index = params['training']['ignore_index'] if params['training']['learning_rate']: lr = params['training']['learning_rate'] if params['training']['weight_decay']: weight_decay = params['training']['weight_decay'] if params['training']['step_size']: step_size = params['training']['step_size'] if params['training']['gamma']: gamma = params['training']['gamma'] if params['global']['num_gpus']: num_devices = params['global']['num_gpus'] # Loss function criterion = MultiClassCriterion(loss_type=params['training']['loss_fn'], ignore_index=ignore_index, weight=class_weights) # list of GPU devices that are available and unused. If no GPUs, returns empty list lst_device_ids = get_device_ids( num_devices) if torch.cuda.is_available() else [] num_devices = len(lst_device_ids) if lst_device_ids else 0 device = torch.device(f'cuda:{lst_device_ids[0]}' if torch.cuda. is_available() and lst_device_ids else 'cpu') if num_devices == 1: print(f"Using Cuda device {lst_device_ids[0]}") elif num_devices > 1: print(f"Using data parallel on devices {str(lst_device_ids)[1:-1]}") model = nn.DataParallel(model, device_ids=lst_device_ids ) # adds prefix 'module.' to state_dict keys else: warnings.warn( f"No Cuda device available. This process will only run on CPU") criterion = criterion.to(device) model = model.to(device) # Optimizer opt_fn = params['training']['optimizer'] optimizer = create_optimizer(params=model.parameters(), mode=opt_fn, base_lr=lr, weight_decay=weight_decay) lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer, step_size=step_size, gamma=gamma) if state_dict_path != '': model, optimizer = load_from_checkpoint(state_dict_path, model, optimizer=optimizer) return model, criterion, optimizer, lr_scheduler, device, num_devices