def test_validator_combine_objectives_no_problem(): v = Validator(model, dataloader, metrics, objectives) assert (v.combine_objectives(obj_results, alphas, max_normalization) == 0.505) assert (v.combine_objectives(obj_results, None, max_normalization) == 1.75) assert (v.combine_objectives(obj_results, alphas, None) == 0.51) assert (v.combine_objectives(obj_results) == sum(obj_results))
def test_validator_mock_opposite_model(): mock_dataset = MamoDataset(input_data, input_data.copy()) mock_dataloader = DataLoader(mock_dataset, batch_size=1, shuffle=False, drop_last=False) v_opposite = Validator(MockOpposite(), mock_dataloader, [RecallAtK(1)], [MSELoss()]) results = v_opposite.evaluate() assert isinstance(results, tuple) assert isinstance(results[0], list) assert (results[0][0] == 0) assert isinstance(results[1], list) assert (results[1][0] == 1) assert (v_opposite.combine_objectives(results[1]) == 1)
def test_validator_mock_shift_right_by_one_model(): mock_dataset = MamoDataset(input_data, np.roll(input_data.copy(), shift=1, axis=1)) mock_dataloader = DataLoader(mock_dataset, batch_size=1, shuffle=False, drop_last=False) v_shift_right = Validator(MockShiftRightByOne(), mock_dataloader, [RecallAtK(1)], [MSELoss()]) results = v_shift_right.evaluate() assert isinstance(results, tuple) assert isinstance(results[0], list) assert (results[0][0] == 1) assert isinstance(results[1], list) assert (results[1][0] == 0) assert (v_shift_right.combine_objectives(results[1]) == 0)
def test_validator_combine_objectives_bad_obj_results(): v = Validator(model, dataloader, metrics, objectives) with pytest.raises(TypeError, match='Argument: obj_results must be set.'): v.combine_objectives(None, alphas, max_normalization) with pytest.raises(TypeError, match='Argument:' + ' obj_results must be a list.'): v.combine_objectives('Results', alphas, max_normalization) with pytest.raises(TypeError, match='All elements of argument: obj_results' + ' must be of type int or float.'): v.combine_objectives([1, 2.5, 'number'], alphas, max_normalization)
def test_validator_mock_no_change_model(): mock_dataset = MamoDataset(input_data, input_data.copy()) mock_dataloader = DataLoader(mock_dataset, batch_size=1, shuffle=False, drop_last=False) v_no_change = Validator(MockNoChange(), mock_dataloader, [RecallAtK(1)], [MSELoss()]) results = v_no_change.evaluate() assert isinstance(results, tuple) assert isinstance(results[0], list) assert (results[0][0] == 0) assert isinstance(results[1], list) # Removing chosen elements -so: mse = np.mean(input_data) assert (round(results[1][0], 2) == round(mse, 2)) assert (round(v_no_change.combine_objectives(results[1]), 2) == round(mse, 2))
def test_validator_combine_objectives_bad_max_normalization(): v = Validator(model, dataloader, metrics, objectives) with pytest.raises(TypeError, match='Argument:' + ' max_normalization must be a list.'): v.combine_objectives(obj_results, alphas, 'max_normalization') with pytest.raises(TypeError, match='All elements of argument:' + ' max_normalization must be of type int or float.'): v.combine_objectives(obj_results, alphas, [1, 2.5, 'number']) with pytest.raises(ValueError, match='The length of max_normalization must' + ' be equal to that of obj_results'): v.combine_objectives(obj_results, alphas, [1, 2.5])
def test_validator_mock_all_zeros_model(): mock_dataset = MamoDataset(input_data, input_data.copy()) mock_dataloader = DataLoader(mock_dataset, batch_size=1, shuffle=False, drop_last=False) v_all_zeros = Validator(MockAllZeros(), mock_dataloader, [RecallAtK(1)], None) results = v_all_zeros.evaluate() assert isinstance(results, tuple) assert isinstance(results[0], list) assert (results[0][0] == 0) assert isinstance(results[1], list) assert (results[1] == []) v_all_zeros = Validator(MockAllZeros(), mock_dataloader, None, [MSELoss()]) results = v_all_zeros.evaluate() assert isinstance(results, tuple) assert isinstance(results[0], list) assert (results[0] == []) assert isinstance(results[1], list) mse = np.mean(input_data) assert (round(results[1][0], 2) == round(mse, 2)) assert (round(v_all_zeros.combine_objectives(results[1]), 2) == round(mse, 2))
class Trainer(): """The trainer class, the core of the framework, used for training models. All the needed objects for this class have to be given through the constructor. Additionally, the other parameters needed by this trainer have to be supplied in a YAML file named 'trainer_params.yaml' or a dictionary containing the parameters. For more details about the parameters supplied in this YAML file, please refer to 'Attributes from the YAML file' section below. Attributes: data_handler: A MamoDataHandler object which feeds the data set to the trainer. model: A torch.nn.Module object which is the model that is being trained. losses: A list of Loss objects which represent the losses/objectives that the model is trained on. validation_metrics: A list of MetricAtK objects which are used to evaluate the model while the training and validation process. save_to_path: A path to a directory where the trained models from the Pareto front will be saved during training. device: A variable indicating whether the model is trained on the gpu or on the cpu. _train_dataloader: A dataloader object used for feeding the data to the trainer. pareto_manager: A ParetoManager which is responsible for maintaining a pareto front of models and saving these models on permanent storage. validator: A Validator object which is used to evaluate the models on multiple objective and multiple losses. max_empirical_losses: A list of losses (float) which is the approximation of the maximum empirical losses the model will have. common_descent_vector: A MultiObjectiveCDV, is responsible for combining the multiple gradients from the multiple losses into a single gradient. optimizer: A pytorch optimizer which is used to train the model. Attributes from the YAML file: seed: An integer, used to initialize the numpy and pytorch random seeds, default = 42. normalize_gradients: A boolean value, indicating whether to normalize the gradients while training the model or not, default = True. learning_rate: A float value, the learning rate that is given to the pytorch optimizer, if the optimizer is not given in the constructor, default = 1e-3. batch_size_training: An integer value, representing the batch sizes in which the data is fed to the trainer, default = 500. shuffle_training: A boolean value, indicating if the training data should be shuffled, default = True. drop_last_batch_training: A boolean value, indicating to drop the last incomplete batch, if the training dataset size is not divisible by the batch size, default = True. batch_size_validation: An integer value, representing the batch sizes in which the data is fed to the validator, default = 500. shuffle_validation: A boolean value, indicating if the validation data should be shuffled, default = True. drop_last_batch_validation: A boolean value, indicating to drop the last incomplete batch, if the validation dataset size is not divisible by the batch size, default = False. number_of_epochs: An integer value, indicating for how many epochs should the model be trained, default = 50. frank_wolfe_max_iter: An integer value, indicating the maximum number of iterations to be used by the frank wolfe algorithm in the commonDescentVector object, default = 100. anneal: A boolean value, indicating if annealing should be used while training the model, default = True. beta_start: If the anneal is used, this will be the first value of the beta, default = 0. beta_cap: If the anneal is used, this will be the maximum value of the beta, default = 0.3. beta_step: If the anneal is used, this is the amount by which to increase the beta every batch, default = 0.3/10000. """ def __init__(self, data_handler, model, losses, validation_metrics, save_to_path, params='yaml_files/trainer_params.yaml', optimizer=None): """The constructor which initializes a trainer object. Arguments: data_handler: A MamoDataHandler object which feeds the data set to the trainer. model: A torch.nn.Module object which is the model that is being trained. losses: A list of Loss objects which represent the losses/objectives that the model is trained on. validation_metrics: A list of MetricAtK objects which are used to evaluate the model while the training and validation process. save_to_path: A path to a directory where the trained models from the Pareto front will be saved during training. params: Path to the yaml file with the trainger parameters, or a dictionary containing the parameters. optimizer: A pytorch optimizer which is used to train the model, if it is None, a default Adam optimizer is created. Raises: TypeError: If any of the arguments passed are not an instance of the expected class or are None, a TypeError will be raised. ValueError: If the directory which save_to_path references is not empty, a ValueError will be raised. """ logger.info('Trainer: Started with initializing trainer...') self._check_input_(data_handler, model, losses, validation_metrics, save_to_path, optimizer) self._read_params(params) self.data_handler = data_handler self.model = model self.losses = losses logger.info('Trainer: Losses: ') logger.info('Trainer: '.join( ['%s ' % loss.name for loss in self.losses])) self.validation_metrics = validation_metrics logger.info('Trainer: Validation metrics: ') logger.info('Trainer: '.join( ['%s ' % m.get_name() for m in self.validation_metrics])) self.save_to_path = save_to_path logger.info('Trainer: Saving models to: %s' % self.save_to_path) self.optimizer = optimizer # set cuda if available self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') logger.info('Trainer: Training on device: %s' % self.device) self._init_objects() logger.info('Trainer: Initialization done.') def _check_input_(self, data_handler, model, losses, validation_metrics, save_to_path, optimizer): """A helper function for the __init__ to check the input of the constructor. """ if not isinstance(data_handler, MamoDataHandler): raise TypeError( 'Please check you are using the right data handler object, or the right order of the attributes!' ) if not isinstance(model, nn.Module): raise TypeError( 'Please check you are using the right model object, or the right order of the attributes!' ) if not hasattr(model, 'initialize_model'): raise TypeError( 'Please check if your models has initialize_model() method defined!' ) # check if losses is None if losses is None: raise ValueError( 'The losses are None, please make sure to give valid losses!') if not all([isinstance(x, Loss) for x in losses]): raise TypeError( 'Please check you are using the right loss objects, or the right order of the attributes!' ) # check if there are at least two losses if len(losses) < 2: raise ValueError( 'Please check you have defined at least two losses,' + ' for training with one loss use the Single Objective Loss class!' ) # check if validation metrics is None if validation_metrics is None: raise ValueError( 'The validation_metrics are None, please make sure to give valid validation_metrics!' ) if not all([isinstance(x, MetricAtK) for x in validation_metrics]): raise TypeError( 'Please check you are using the right metric objects, or the right order of the attributes!' ) # check if length is at least 1 if len(validation_metrics) == 0: raise ValueError( 'Please check you have defined at least one validation metric!' ) if not os.path.exists(save_to_path): os.mkdir(save_to_path) # checking if the save_to_path directory is empty if os.listdir(save_to_path): raise ValueError( 'Please make sure that the directory where you want to save the models is empty!' ) # if the optimizer is not None, than has to be pytorch optimizer object if optimizer is not None: if not isinstance(optimizer, optim.Optimizer): raise TypeError( 'Please make sure that the optimizer is a pytorch Optimizer object!' ) def _read_params(self, params): """A helper function for the __init__ to read the configuration yaml file. """ logger.info('Trainer: Reading trainer parameters.') if type(params) is str: with open(params, 'r') as stream: params = yaml.safe_load(stream) self.seed = int(params.get('seed', 42)) logger.info('Trainer: Random seed: %d' % self.seed) self.normalize_gradients = bool(params.get('normalize_gradients', True)) logger.info('Trainer: Normalize gradients: %s' % self.normalize_gradients) self.learning_rate = float(params.get('learning_rate', 1e-3)) logger.info('Trainer: Learning rate: %f' % self.learning_rate) self.batch_size_training = int(params.get('batch_size_training', 500)) logger.info('Trainer: Batch size training: %d' % self.batch_size_training) self.shuffle_training = bool(params.get('shuffle_training', True)) logger.info('Trainer: Shuffle training: %d' % self.shuffle_training) self.drop_last_batch_training = bool( params.get('drop_last_batch_training', True)) logger.info('Trainer: Drop last batch training: %d' % self.drop_last_batch_training) self.batch_size_validation = int( params.get('batch_size_validation', 500)) logger.info('Trainer: Batch size validation: %d' % self.batch_size_validation) self.shuffle_validation = bool(params.get('shuffle_validation', True)) logger.info('Trainer: Shuffle validation: %d' % self.shuffle_validation) self.drop_last_batch_validation = bool( params.get('drop_last_batch_validation', False)) logger.info('Trainer: Drop last batch validation: %d' % self.drop_last_batch_validation) self.number_of_epochs = int(params.get('number_of_epochs', 50)) logger.info('Trainer: Number of epochs: %f' % self.number_of_epochs) self.frank_wolfe_max_iter = int(params.get('frank_wolfe_max_iter', 100)) logger.info('Trainer: Frank Wolfe max iterations: %d' % self.frank_wolfe_max_iter) self.anneal = bool(params.get('anneal', True)) logger.info('Trainer: Annealing: %s' % self.anneal) if self.anneal and ('beta_start' not in params or 'beta_cap' not in params or 'beta_step' not in params): raise ValueError( ('Please make sure that if anneal is set to True, ' 'the beta_start, beta_cap and beta_step are all ' 'present in the parameters yaml file!')) if self.anneal: self.beta_start = float(params.get('beta_start', 0)) logger.info('Trainer: Beta start: %f' % self.beta_start) self.beta_cap = float(params.get('beta_cap', 0.3)) logger.info('Trainer: Beta cap: %f' % self.beta_cap) self.beta_step = float(eval(params.get('beta_step', '0.3/10000'))) logger.info('Trainer: Beta step: %f' % self.beta_step) def _init_objects(self): """A helper function for the __init__ to initialize different objects. """ logger.info('Trainer: Initializing helper trainer objects.') np.random.seed(self.seed) torch.manual_seed(self.seed) self.model.initialize_model() self.model.to(self.device) self._train_dataloader = self.data_handler.get_train_dataloader( batch_size=self.batch_size_training, shuffle=self.shuffle_training, drop_last=self.drop_last_batch_training) self.pareto_manager = ParetoManager(PATH=self.save_to_path) val_dataloader = self.data_handler.get_validation_dataloader( batch_size=self.batch_size_validation, shuffle=self.shuffle_validation, drop_last=self.drop_last_batch_validation) self.validator = Validator(self.model, val_dataloader, self.validation_metrics, self.losses) self.max_empirical_losses = None if self.normalize_gradients: self.max_empirical_losses = self._compute_max_empirical_losses() logger.info('Trainer: Max empirical losses: %s' % self.max_empirical_losses) copsolver = None if len(self.losses) <= 2: copsolver = AnalyticalSolver() else: copsolver = FrankWolfeSolver(max_iter=self.frank_wolfe_max_iter) self.common_descent_vector = MultiObjectiveCDV( copsolver=copsolver, max_empirical_losses=self.max_empirical_losses, normalized=self.normalize_gradients) # create default optimizer if self.optimizer is None: self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) def _compute_max_empirical_losses(self): """A helper function for approximating the maximum empirical loss the model could have. It is called by _init_objects function. """ # approximate the max loss empirically max_losses = [0] * len(self.losses) cnt = 0 for batch in self._train_dataloader: # fetch data x = batch[0] x = Variable(x).to(self.device) y = batch[1] y = Variable(y).to(self.device) cnt += 1 # forward pass model_output = self.model(x) for i, loss in enumerate(self.losses): # if annealing is done, the KL divergence is ignored when computing # the max empirical loss, therefore the anneal is set to 0 if self.anneal: L = loss.compute_loss(y, model_output, anneal=0) else: L = loss.compute_loss(y, model_output) # compute the moving average term max_losses[i] = (cnt - 1) / cnt * \ max_losses[i] + 1 / cnt * L.item() return max_losses def _get_gradient_np(self): """A helper function for obtaining the gradients of the model in a numpy array. Before the first backward call, all grad attributes are set to None, and that is when the exception is thrown, and the parameters are returned. After the first backward pass, the gradient values are available and are returned by this function. """ gradient = [] try: for p in self.model.parameters(): gradient.append(p.grad.cpu().detach().numpy().ravel()) return np.concatenate(gradient) except Exception: size = 0 for p in self.model.parameters(): size += len(p.cpu().detach().numpy().ravel()) return np.zeros(shape=size) def train(self): """The main method of this class. By calling this method, the traning process starts. """ # model training logger.info('Trainer: Started training...') if self.anneal: beta = self.beta_start for epoch in range(self.number_of_epochs): # start time for current epoch start_time = time.time() # statistics training_loss = 0 average_alpha = [0] * len(self.losses) cnt = 0 # set model in train mode self.model.train() # do training for batch in self._train_dataloader: # create x x = batch[0] x = Variable(x).to(self.device) y = batch[1] y = Variable(y).to(self.device) # anneal beta if self.anneal: beta += self.beta_step beta = beta if beta < self.beta_cap else self.beta_cap # calculate the gradients gradients = [] for i, loss in enumerate(self.losses): # forward pass model_output = self.model(x) # calculate loss if self.anneal: L = loss.compute_loss(y, model_output, anneal=beta) else: L = loss.compute_loss(y, model_output) # zero gradient self.optimizer.zero_grad() # backward pass L.backward() # get gradient for correctness objective gradients.append(self._get_gradient_np()) # calculate the losses losses_computed = [] # forward pass model_output = self.model(x) for i, loss in enumerate(self.losses): if self.anneal: L = loss.compute_loss(y, model_output, anneal=beta) else: L = loss.compute_loss(y, model_output) losses_computed.append(L) # get the final loss to compute the common descent vector final_loss, alphas = self.common_descent_vector.get_descent_vector( losses_computed, gradients) # zero gradient self.optimizer.zero_grad() # backward pass final_loss.backward() # update parameters self.optimizer.step() # statistics.... cnt += 1 # moving average loss training_loss = (cnt - 1) / cnt * \ training_loss + 1 / cnt * final_loss.item() # moving average alpha for i, alpha in enumerate(alphas): average_alpha[i] = (cnt - 1) / cnt * \ average_alpha[i] + 1 / cnt * alpha # time in milliseconds for current batch batch_time = (time.time() - start_time) / cnt * 1000 # log progress if cnt % 10 == 0: average_alpha_string = ', '.join( ['%.4f'] * len(average_alpha)) % tuple(average_alpha) logger.info( 'Trainer: Batch: %d/%d, Batch time: %.2fms,' % (cnt, int( np.round(self.data_handler.get_traindata_len() / self.batch_size_training)), batch_time) + ' Training loss: %.3f, Alphas: [%s]' % (training_loss, average_alpha_string)) # do validation val_metrics, val_objectives = self.validator.evaluate( disable_anneal=self.anneal, verbose=False) val_loss = self.validator.combine_objectives( val_objectives, alphas=average_alpha, max_normalization=self.max_empirical_losses) # add the solution to the pareto manager self.pareto_manager.add_solution(val_metrics, self.model) # calculate epoch time epoch_time = time.time() - start_time val_metrics_string = ', '.join( ['%.4f'] * len(val_metrics)) % tuple(val_metrics) val_objectives_string = ', '.join( ['%.4f'] * len(val_objectives)) % tuple(val_objectives) logger.info( 'Trainer: Epoch: %d, Epoch time: %.2fs, Training loss: %.3f,' % (epoch + 1, epoch_time, training_loss) + ' Validation loss: %.3f, Validation metrics: [%s], Validation losses: [%s]' % (val_loss, val_metrics_string, val_objectives_string)) return val_loss