def train(dataset, args): n_weights = np.size(dataset.x, 1) model = LinearRegression(n_weights=n_weights) optimizer = Optimizer(dataset) losses = [] for _ in range(args.epochs): losses.append(mse(model, dataset)) optimizer.step(model, args.learning_rate) print ('Model:', model, '\n') print ('Loss:', mse(model, dataset)) sklearn_model = SklearnLinearRegression().fit(dataset.x, dataset.y) print ('Loss with sklearn:', mean_squared_error(dataset.y, sklearn_model.predict(dataset.x)), '\n') if args.plot: plot(model, n_weights, losses, dataset) return model
def display_model_metrics(model, dataset): print('--- Model\'s metrics ---') print(model) print('Mean square error:', mse(model, dataset)) print('Mean absolute error:', mae(model, dataset)) print('Mean percentage error: ', mpe(model, dataset)) print('Mean absolute percentage error: ', mape(model, dataset), '\n')
def add_loss_layer(self, layer_name, prediction_layer_id, ground_truth_layer_id, loss_type): """ Adds a layer corresponding to the loss function :param layer_name: The name of the layer. Type=string :param prediction_layer_id: The identifier for the prediction layer :param ground_truth_layer_id: The identifier for the ground truth layer :param loss_type: The loss function to use. Available options defined by LossTypes. :return: None """ layer_id = self._get_layer_id(layer_name) assert self._layer_verifier( layer_id), 'Invalid: This layer is already present.' assert not self._layer_verifier( prediction_layer_id), 'Invalid: Output layer id is invalid.' assert not self._layer_verifier( ground_truth_layer_id ), 'Invalid: Ground truth layer id is invalid.' output = self.layers[prediction_layer_id] ground_truth = self.layers[ground_truth_layer_id] if loss_type == LossTypes.mse: self.layers[layer_id] = mse(ground_truth, output) elif loss_type == LossTypes.cross_entropy: self.layers[layer_id] = cross_entropy(ground_truth, output) else: raise ValueError('The type of loss can only be one of ["mse"]')
def test_edge_cases_mse(self): """`loss.mse`: Edge Case Validator. Tests the behavior of `mse` with edge cases. Raises: Exception: If at least one `Exception` raised is not of the expected kind. """ with self.assertRaises(_InvalidObservationSetError): # Empty observation matrix. mse(_np.matrix([[]]), _random_matrix((self.n, 1))) with self.assertRaises(_InvalidObservationSetError): # Empty prediction matrix. mse(_random_matrix((self.n, 1)), _np.matrix([[]]))
def evaluate_batch(self, images: torch.tensor, labels: torch.tensor, print_labels=False) -> tuple: """ evaluations for batch """ self.eval() with torch.no_grad(): labels_pred = self.forward(images) if print_labels: for i, (prediction, target) in enumerate(zip(labels_pred, labels)): print( "target\t\t", np.around(target[self.considered_label_indices].cpu(), 3)) print("\033[1mprediction\t", np.around( prediction[self.considered_label_indices].cpu(), 3), end="\033[0m\n") if i >= 2: break print( "<target>\t", np.around( torch.mean(labels[:, self.considered_label_indices], dim=0).cpu(), 3)) print( "<target>\t", np.around( torch.std(labels[:, self.considered_label_indices], dim=0).cpu(), 3)) print("\033[1m<prediction>\t", np.around( torch.mean( labels_pred[:, self.considered_label_indices], dim=0).cpu(), 3), end="\033[0m\n") print("\033[1m<prediction>\t", np.around( torch.std(labels_pred[:, self.considered_label_indices], dim=0).cpu(), 3), end="\033[0m\n") loss_regression = torch.sqrt( mse(labels_pred[:, self.considered_label_indices], labels[:, self.considered_label_indices])).item() loss_variance = self.weight_loss_sample_variance * \ loss_sample_variance(labels_pred[:,self.considered_label_indices], threshold=self.sample_variance_threshold ).item() accs = measure_accuracy_classifier( labels_pred, labels, considered_groups=self.considered_groups.considered_groups) variance = get_sample_variance( labels_pred[:, self.considered_label_indices]).item() return loss_regression, loss_variance, accs, variance
def train_step(self, images: torch.tensor, labels: torch.tensor) -> float: self.train() labels_pred = self.forward(images, train=True) loss_regression = mse(labels_pred[:, self.considered_label_indices], labels[:, self.considered_label_indices]) loss_variance = self.weight_loss_sample_variance * \ loss_sample_variance(labels_pred[:,self.considered_label_indices], threshold=self.sample_variance_threshold) loss = loss_regression + loss_variance self.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.parameters(), max_norm=1) self.optimizer.step() self.iteration += 1 return loss.item()
def test_random_mse(self): """`loss.mse`: Randomized Validator. Tests the behavior of `mse` by feeding it randomly generated arguments. Raises: AssertionError: If `mse` needs debugging. """ for i in range(self.n_tests): Y = _random_matrix((self.n, 1), max_val=self.max_mean) """float: Random-valued observations.""" Y_hat = _random_matrix((self.n, 1), max_val=self.max_mean) """float: Random-valued predictions.""" delta_Y = abs(Y - Y_hat) """float: Distance between predictions and observations.""" squared_sum_delta_Y = _np.linalg.norm(delta_Y[1:, 0])**2 """float: Sum of the squares of all `delta_Y` values.""" # To ensure that the coercion does not result in the square of a # negative number, we can use the mean of the upper-bound # `squared_sum_delta_Y` as insurance that the computation will only # work with positive numbers. err = _uniform((squared_sum_delta_Y + 1.0) / self.n, (squared_sum_delta_Y + self.max_mean) / self.n) """float: MSE to coerce.""" # Coerce MSE by changing the first prediction to a strategic choice # and mathematically guaranteeing Y_hat[0, 0] = (_np.sqrt(self.n * err - squared_sum_delta_Y) - Y[0, 0]) * -1.0 result = mse(Y, Y_hat) """float: Test input.""" self.assertAlmostEqual(result, err)
def train_classifier_distributed(rank, world_size, optimizer=optimizer): # initialize distributed process group os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '12355' dist.init_process_group(rank=rank, world_size=world_size, backend="nccl") # prepare splitted dataloader make_data_loader = MakeDataLoader() sampler_train = DistributedSampler(make_data_loader.dataset_train, num_replicas=world_size, rank=rank, shuffle=True, drop_last=True) data_loader_train = make_data_loader.get_data_loader_train( batch_size=batch_size, num_workers=0, sampler=sampler_train, shuffle=False) data_loader_valid = make_data_loader.get_data_loader_valid( batch_size=batch_size, num_workers=0) # prepare model classifier = Classifier(seed=seed_parameter).to(rank) ddp_classifier = DDP(classifier, device_ids=list(range(world_size)), output_device=rank, find_unused_parameters=True) optimizer = optimizer(ddp_classifier.parameters(), lr=learning_rate_init) scheduler = MultiStepLR(optimizer, milestones=[292, 373], gamma=gamma) for epoch in range(epochs): data_loader_train.sampler.set_epoch(epoch) ddp_classifier.train() for image, label in data_loader_train: image = image.to(rank) label = label.to(rank) optimizer.zero_grad(set_to_none=True) prediction = ddp_classifier(image, train=True) loss_train = mse(prediction, label) loss_train.mean().backward() torch.nn.utils.clip_grad_norm_(ddp_classifier.parameters(), max_norm=1) optimizer.step() if rank == 0: accs_train = measure_accuracy_classifier(prediction, label) ddp_classifier.eval() loss_valid = 0 accs_valid = Counter({group: 0 for group in range(1, 12)}) with torch.no_grad(): for N_test, (image, label) in enumerate(data_loader_valid): image = image.to(rank) label = label.to(rank) prediction = ddp_classifier(image, train=False) loss_valid += mse(predction, label).item() accs_valid.update( measure_accuracy_classifier(prediction, label)) for group in accs.keys(): accs[group] /= N_test + 1 logs = { "loss_train": np.sqrt(loss_train.item()), "loss_valid": np.sqrt(loss_valid) } logs.update({ f"accuracy_Q{group}_train": acc for group, acc in accs_train.items() }) logs.update({ f"accuracy_Q{group}_valid": acc for group, acc in accs_valid.items() }) wandb.log(logs) # safe full model for later use if not epoch % 100: torch.save(ddp_classifier.model, folder_results + f"classifier_model_epoch{epoch}.pth") # end distributed process dist.destroy_process_group()
def test_invalid_args_mse(self): """`loss.mse`: Argument Validator. Tests the behavior of `mse` with invalid argument counts and values. Raises: Exception: If at least one `Exception` raised is not of the expected kind. """ with self.assertRaises(TypeError): # No arguments. mse() with self.assertRaises(TypeError): # Only one arguments. mse(123) with self.assertRaises(TypeError): # More than two arguments. mse(123, 123, 123, 1123) with self.assertRaises(_InvalidObservationSetError): # List of an empty list instead of observation matrix. mse([[]], _random_matrix((self.n, 1))) with self.assertRaises(_InvalidObservationSetError): # `None` instead of prediction matrix. mse(_random_matrix((self.n, 1)), [[]]) with self.assertRaises(_IncompatibleDataSetsError): # Incompatible observation and prediction matrices. mse(_random_matrix((self.n, 1)), _random_matrix((self.n + 1, 1)))