def train_autoencoder(self, epoch, performance_estimators=None): if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("train_loss")] performance_estimators += [FloatHelper("train_grad_norm")] print('\nTraining, epoch: %d' % epoch) self.net.train() supervised_grad_norm = 1. for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() unsupervised_loss_acc = 0 num_batches = 0 train_loader_subset = self.problem.train_loader_subset_range( 0, self.args.num_training) for batch_idx, (_, data_dict) in enumerate(train_loader_subset): inputs = data_dict["input"].to(self.device) num_batches += 1 inputs, targets = Variable(inputs), Variable(inputs, requires_grad=False) # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: self.net.train() self.optimizer_training.zero_grad() outputs = self.net(inputs) supervised_loss = self.criterion(outputs, targets) optimized_loss = supervised_loss optimized_loss.backward() self.optimizer_training.step() performance_estimators.set_metric_with_outputs( batch_idx, "train_loss", supervised_loss.item(), outputs, targets) supervised_grad_norm = grad_norm(self.net.parameters()) performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm) performance_estimators.set_metric_with_outputs( batch_idx, "optimized_loss", optimized_loss.item(), outputs, targets) progress_bar( batch_idx * self.mini_batch_size, self.max_training_examples, " ".join([ performance_estimator.progress_message() for performance_estimator in performance_estimators ])) if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break return performance_estimators
def test_semi_sup(self, epoch): print('\nTesting, epoch: %d' % epoch) performance_estimators = PerformanceList() performance_estimators += [LossHelper("test_supervised_loss")] performance_estimators += [LossHelper("test_reconstruction_loss")] performance_estimators += [AccuracyHelper("test_")] self.net.eval() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() validation_loader_subset = self.problem.validation_loader_range(0, self.args.num_validation) data_provider = MultiThreadedCpuGpuDataProvider(iterator=zip(validation_loader_subset), is_cuda=self.use_cuda, batch_names=["validation"], requires_grad={"validation": []}, volatile={"validation": ["input", "softmaxGenotype"]}) try: for batch_idx, (_, data_dict) in enumerate(data_provider): input_s = data_dict["validation"]["input"] target_s = data_dict["validation"]["softmaxGenotype"] # we need copies of the same tensors: input_u, target_u = Variable(input_s.data, volatile=True), Variable(input_s.data, volatile=True) output_s = self.net(input_s) output_u = self.net.autoencoder(input_u) output_s_p = self.get_p(output_s) _, target_index = torch.max(target_s, dim=1) supervised_loss = self.criterion_classifier(output_s, target_s) reconstruction_loss = self.criterion_autoencoder(output_u, target_u) performance_estimators.set_metric(batch_idx, "test_supervised_loss", supervised_loss.data[0]) performance_estimators.set_metric(batch_idx, "test_reconstruction_loss", reconstruction_loss.data[0]) performance_estimators.set_metric_with_outputs(batch_idx, "test_accuracy", supervised_loss.data[0], output_s_p, targets=target_index) progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples, performance_estimators.progress_message(["test_supervised_loss", "test_reconstruction_loss", "test_accuracy"])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break # print() finally: data_provider.close() test_metric = performance_estimators.get_metric(self.get_test_metric_name()) assert test_metric is not None, self.get_test_metric_name() + "must be found among estimated performance metrics" if not self.args.constant_learning_rates: self.scheduler_train.step(test_metric, epoch) return performance_estimators
def test_autoencoder(self, epoch, performance_estimators=None): print('\nTesting, epoch: %d' % epoch) if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("test_loss")] self.net.eval() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() for batch_idx, (_, data_dict) in enumerate( self.problem.validation_loader_range( 0, self.args.num_validation)): inputs = data_dict["input"] if self.use_cuda: inputs = inputs.cuda() inputs, targets = Variable(inputs, volatile=True), Variable(inputs, volatile=True) outputs = self.net(inputs) loss = self.criterion(outputs, targets) performance_estimators.set_metric_with_outputs( batch_idx, "test_loss", loss.data[0], outputs, targets) progress_bar( batch_idx * self.mini_batch_size, self.max_validation_examples, performance_estimators.progress_message(["test_loss"])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break # print() # Apply learning rate schedule: test_accuracy = performance_estimators.get_metric("test_loss") assert test_accuracy is not None, "test_loss must be found among estimated performance metrics" if not self.args.constant_learning_rates: self.scheduler_train.step(test_accuracy, epoch) return performance_estimators
def train_semisup(self, epoch): performance_estimators = PerformanceList() performance_estimators += [FloatHelper("optimized_loss")] performance_estimators += [FloatHelper("supervised_loss")] performance_estimators += [FloatHelper("reconstruction_loss")] performance_estimators += [AccuracyHelper("train_")] print('\nTraining, epoch: %d' % epoch) self.net.train() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() unsupervised_loss_acc = 0 num_batches = 0 train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training) unlabeled_loader = self.problem.unlabeled_loader() data_provider = MultiThreadedCpuGpuDataProvider(iterator=zip(train_loader_subset, unlabeled_loader),is_cuda=self.use_cuda, batch_names=["training", "unlabeled"], requires_grad={"training": ["input"], "unlabeled": ["input"]}, volatile={"training": ["metaData"], "unlabeled": []}, recode_functions={"softmaxGenotype": lambda x: recode_for_label_smoothing(x,self.epsilon)}) self.net.autoencoder.train() try: for batch_idx, (_, data_dict) in enumerate(data_provider): input_s = data_dict["training"]["input"] metadata = data_dict["training"]["metaData"] target_s = data_dict["training"]["softmaxGenotype"] input_u = data_dict["unlabeled"]["input"] num_batches += 1 # need a copy of input_u and input_s as output: target_u = Variable(input_u.data, requires_grad=False) target_output_s = Variable(input_s.data, requires_grad=False) # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: # Zero gradients: self.net.zero_grad() self.net.autoencoder.zero_grad() self.optimizer_training.zero_grad() output_s = self.net(input_s) output_u = self.net.autoencoder(input_u) input_output_s = self.net.autoencoder(input_s) output_s_p = self.get_p(output_s) _, target_index = torch.max(target_s, dim=1) supervised_loss = self.criterion_classifier(output_s, target_s) reconstruction_loss_unsup = self.criterion_autoencoder(output_u, target_u) reconstruction_loss_sup = self.criterion_autoencoder(input_output_s, target_output_s) reconstruction_loss = self.args.gamma * reconstruction_loss_unsup+reconstruction_loss_sup optimized_loss = supervised_loss + reconstruction_loss optimized_loss.backward() self.optimizer_training.step() performance_estimators.set_metric(batch_idx, "supervised_loss", supervised_loss.data[0]) performance_estimators.set_metric(batch_idx, "reconstruction_loss", reconstruction_loss.data[0]) performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0]) performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0], output_s_p, targets=target_index) progress_bar(batch_idx * self.mini_batch_size, self.max_training_examples, performance_estimators.progress_message(["supervised_loss", "reconstruction_loss", "train_accuracy"])) if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break finally: data_provider.close() return performance_estimators
def test_semisup_aae(self, epoch, performance_estimators=None): print('\nTesting, epoch: %d' % epoch) if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [FloatHelper("reconstruction_loss")] performance_estimators += [LossHelper("test_loss")] performance_estimators += [AccuracyHelper("test_")] performance_estimators += [FloatHelper("weight")] self.net.eval() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() validation_loader_subset = self.problem.validation_loader_range( 0, self.args.num_validation) data_provider = MultiThreadedCpuGpuDataProvider( iterator=zip(validation_loader_subset), is_cuda=self.use_cuda, batch_names=["validation"], requires_grad={"validation": []}, volatile={ "validation": ["input", "softmaxGenotype"], }, recode_functions={"input": self.normalize_inputs}) self.net.eval() try: for batch_idx, (_, data_dict) in enumerate(data_provider): input_s = data_dict["validation"]["input"] target_s = data_dict["validation"]["softmaxGenotype"] # Estimate the reconstruction loss on validation examples: reconstruction_loss = self.net.get_crossconstruction_loss( input_s, input_s, target_s) # now evaluate prediction of categories: categories_predicted, latent_code = self.net.encoder(input_s) # categories_predicted+=self.net.latent_to_categories(latent_code) categories_predicted_p = self.get_p(categories_predicted) categories_predicted_p[ categories_predicted_p != categories_predicted_p] = 0.0 _, target_index = torch.max(target_s, dim=1) categories_loss = self.net.semisup_loss_criterion( categories_predicted, target_s) weight = self.estimate_example_density_weight(latent_code) performance_estimators.set_metric(batch_idx, "reconstruction_loss", reconstruction_loss.data[0]) performance_estimators.set_metric(batch_idx, "weight", weight) performance_estimators.set_metric_with_outputs( batch_idx, "test_accuracy", reconstruction_loss.data[0], categories_predicted_p, target_index) performance_estimators.set_metric_with_outputs( batch_idx, "test_loss", categories_loss.data[0] * weight, categories_predicted_p, target_s) if not self.args.no_progress: progress_bar( batch_idx * self.mini_batch_size, self.max_validation_examples, performance_estimators.progress_message([ "test_loss", "test_accuracy", "reconstruction_loss" ])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break # print() finally: data_provider.close() # Apply learning rate schedules: test_metric = performance_estimators.get_metric( self.get_test_metric_name()) assert test_metric is not None, ( self.get_test_metric_name() + "must be found among estimated performance metrics") if not self.args.constant_learning_rates: for scheduler in self.schedulers: scheduler.step(test_metric, epoch) # Run the garbage collector to try to release memory we no longer need: import gc gc.collect() return performance_estimators