def test(self, epoch, performance_estimators=(LossHelper("test_loss"), AccuracyHelper("test_"))): print('\nTesting, epoch: %d' % epoch) self.net.eval() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)): if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = Variable(inputs, volatile=True), Variable(targets) outputs = self.net(inputs) loss = self.criterion(outputs, targets) for performance_estimator in performance_estimators: performance_estimator.observe_performance_metric(batch_idx, loss.data[0], outputs, targets) self.ureg.estimate_accuracy(inputs) progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples, " ".join([performance_estimator.progress_message() for performance_estimator in performance_estimators])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break print() # Apply learning rate schedule: test_accuracy = self.get_metric(performance_estimators, "test_accuracy") assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics" if not self.args.constant_learning_rates: self.scheduler_train.step(test_accuracy, epoch) self.scheduler_reg.step(test_accuracy, epoch) return performance_estimators
def test(self, performance_estimators=(LossHelper("test_loss"), AccuracyHelper("test_"))): self.model.eval() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() for batch_idx, (inputs, targets) in enumerate( self.problem.test_loader_range(0, self.num_validation)): if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True) outputs = self.model(inputs) loss = self.criterion(outputs, targets) for performance_estimator in performance_estimators: performance_estimator.observe_performance_metric( batch_idx, loss.data[0], outputs, targets) progress_bar( batch_idx * self.mini_batch_size, self.max_validation_examples, " ".join([ performance_estimator.progress_message() for performance_estimator in performance_estimators ])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break print() return performance_estimators
def test_acc(self, epoch, performance_estimators=None): print('\nTesting, epoch: %d' % epoch) if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")] self.net.eval() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() cm = ConfusionMeter(self.problem.num_classes(), normalized=False) for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)): if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() image1, image2 = half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda) encoded = self.image_encoder(image1) unsup_image = self.image_generator(encoded) if self.args.mode == "separate": inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True) outputs = self.net(inputs) elif self.args.mode == "average": inputs = (inputs + unsup_image.data) / 2 inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) outputs = self.net(inputs) elif self.args.mode=="uonly": inputs = unsup_image.data inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) outputs = self.net(inputs) loss = self.criterion(outputs, targets) # accumulate the confusion matrix: _, predicted = torch.max(outputs.data, 1) cm.add(predicted=predicted, target=targets.data) performance_estimators.set_metric_with_outputs(batch_idx, "test_loss", loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "test_accuracy", loss.data[0], outputs, targets) progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples, performance_estimators.progress_message(["test_loss", "test_accuracy"])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break # print() # Apply learning rate schedule: test_accuracy = performance_estimators.get_metric("test_accuracy") assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics" if not self.args.constant_learning_rates: self.scheduler_train.step(test_accuracy, epoch) self.confusion_matrix = cm.value().transpose() return performance_estimators
def test(self, epoch, performance_estimators=None): print('\nTesting, epoch: %d' % epoch) if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")] self.net.eval() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() cm = ConfusionMeter(self.problem.num_classes(), normalized=False) for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)): if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True) if not hasattr(self.net, 'is_dual'): outputs = self.net(inputs) else: outputs, _, _ =self.net(inputs,None) if self.args.mode=="capsules": one_hot_targets=Variable(self.problem.one_hot(targets.data),volatile=True) loss, capsule_loss, _ =self.net.loss(inputs, outputs, one_hot_targets) # ||vc|| also known as norm: v_c = torch.sqrt((outputs**2).sum(dim=2, keepdim=True)) outputs=v_c.view(v_c.size()[0],-1) # recover index of predicted class: #_, outputs =torch.max(v_c.view(10,-1),dim=0) else: loss = self.criterion(outputs, targets) # accumulate the confusion matrix: _, predicted = torch.max(outputs.data, 1) cm.add(predicted=predicted, target=targets.data) performance_estimators.set_metric_with_outputs(batch_idx, "test_loss", loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "test_accuracy", loss.data[0], outputs, targets) progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples, performance_estimators.progress_message(["test_loss", "test_accuracy"])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break # print() # Apply learning rate schedule: test_accuracy = performance_estimators.get_metric("test_accuracy") assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics" if not self.args.constant_learning_rates: self.scheduler_train.step(test_accuracy, epoch) self.confusion_matrix = cm.value().transpose() return performance_estimators
def test(self, epoch, performance_estimators=None): criterion = MSELoss() print('\nTesting, epoch: %d' % epoch) self.net.eval() self.image_generator.eval() self.image_encoder.eval() if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")] for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() # we used unsup set to train, use training to validate: for batch_idx, (inputs, _) in enumerate(self.problem.train_loader_subset(range(0, self.args.num_validation))): if self.use_cuda: inputs = inputs.cuda() image1, image2= half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda) # train the discriminator/generator pair on the first half of the image: encoded = self.image_encoder(image1) output = self.image_generator(encoded) if batch_idx == 0: self.save_images(epoch, image1, image2, generated_image2=output) full_image = Variable(inputs, requires_grad=False) loss = criterion(output, full_image) performance_estimators.set_metric(batch_idx, "test_loss", loss.data[0]) progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples, performance_estimators.progress_message(["test_loss"])) if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples: break # print() # Apply learning rate schedule: test_loss = performance_estimators.get_metric("test_loss") assert test_loss is not None, "test_loss must be found among estimated performance metrics" if not self.args.constant_learning_rates: self.scheduler_train.step(test_loss, epoch) return performance_estimators
def train_capsules(self, epoch, performance_estimators=None, train_supervised_model=True, ): if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("optimized_loss")] performance_estimators += [LossHelper("capsule_loss")] performance_estimators += [LossHelper("reconstruction_loss")] performance_estimators += [AccuracyHelper("train_")] performance_estimators += [FloatHelper("train_grad_norm")] #performance_estimators += [FloatHelper("reconstruct_grad_norm")] print('\nTraining, epoch: %d' % epoch) self.net.train() supervised_grad_norm = 1. for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() unsupervised_loss_acc = 0 num_batches = 0 train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training) for batch_idx, (inputs, targets) in enumerate(train_loader_subset): num_batches += 1 if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = Variable(inputs,requires_grad=True), Variable(targets, requires_grad=False) # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: self.net.train() self.net.zero_grad() self.optimizer_training.zero_grad() outputs = self.net(inputs) one_hot_targets = Variable(self.problem.one_hot(targets.data), requires_grad=False) if self.published_reconstruction_loss: (optimized_loss, capsule_loss, reconstruction_loss) = self.net.loss(inputs, outputs, one_hot_targets) optimized_loss.backward() self.optimizer_training.step() reconstruct_grad_norm=0 else: margin_loss = self.net.margin_loss(outputs, one_hot_targets) margin_loss = margin_loss.mean() margin_loss.backward(retain_graph=True) #reconstruct_grad_norm = grad_norm(inputs.grad) reconstruction_loss = self.net.focused_reconstruction_loss(inputs, inputs.grad, outputs, one_hot_targets) reconstruction_loss.backward() self.optimizer_training.step() optimized_loss=margin_loss+reconstruction_loss capsule_loss=margin_loss supervised_grad_norm = grad_norm(self.net.decoder.parameters()) performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm) #performance_estimators.set_metric(batch_idx, "reconstruct_grad_norm", reconstruct_grad_norm) performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0]) performance_estimators.set_metric(batch_idx,"reconstruction_loss", reconstruction_loss.data[0]) performance_estimators.set_metric(batch_idx,"capsule_loss", capsule_loss.data[0]) progress_bar(batch_idx * self.mini_batch_size, self.max_training_examples, performance_estimators.progress_message(["optimized_loss","capsule_loss","reconstruction_loss"])) if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break return performance_estimators
def train_with_fm_loss(self, epoch, gamma=1E-5, performance_estimators=None): if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("optimized_loss")] performance_estimators += [LossHelper("train_loss")] performance_estimators += [FloatHelper("fm_loss")] performance_estimators += [AccuracyHelper("train_")] performance_estimators += [FloatHelper("train_grad_norm")] print('\nTraining, epoch: %d' % epoch) self.net.train() supervised_grad_norm = 1. for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() unsupervised_loss_acc = 0 num_batches = 0 train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training) #sec_train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training) unsuploader_shuffled = self.problem.reg_loader_subset_range(0, self.args.num_shaving) unsupiter = itertools.cycle(unsuploader_shuffled) for batch_idx, ((inputs, targets), (uinputs, _)) in enumerate(zip(train_loader_subset , unsupiter)): num_batches += 1 if self.use_cuda: inputs = inputs.cuda() uinputs = uinputs.cuda() targets = targets.cuda() # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: self.net.train() self.net.zero_grad() self.optimizer_training.zero_grad() inputs, targets, uinputs = Variable(inputs), Variable(targets, requires_grad=False), Variable(uinputs, requires_grad=True) if self.use_cuda: inputs, targets, uinputs=inputs.cuda(),targets.cuda(), uinputs.cuda() outputs, outputu, fm_loss = self.net(inputs,uinputs) supervised_loss = self.criterion(outputs, targets) optimized_loss = supervised_loss+gamma*fm_loss optimized_loss.backward() self.optimizer_training.step() supervised_grad_norm = grad_norm(self.net.parameters()) performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm) performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0]) performance_estimators.set_metric(batch_idx, "fm_loss", fm_loss.data[0]) performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0], outputs, targets) # performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0], # outputs, targets) progress_bar(batch_idx * self.mini_batch_size, min(self.max_regularization_examples, self.max_training_examples), performance_estimators.progress_message(["optimized_loss","train_loss","train_accuracy"])) if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break return performance_estimators
def train(self, epoch, performance_estimators=None, train_supervised_model=True, ): if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("optimized_loss")] performance_estimators += [LossHelper("train_loss")] performance_estimators += [AccuracyHelper("train_")] performance_estimators += [FloatHelper("train_grad_norm")] print('\nTraining, epoch: %d' % epoch) self.net.train() supervised_grad_norm = 1. for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() unsupervised_loss_acc = 0 num_batches = 0 train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training) for batch_idx, (inputs, targets) in enumerate(train_loader_subset): num_batches += 1 if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: self.net.train() self.optimizer_training.zero_grad() outputs = self.net(inputs) if train_supervised_model and self.args.mode=="supervised": # if self.ureg._which_one_model is not None: # self.ureg.estimate_example_weights(inputs) supervised_loss = self.criterion(outputs, targets) optimized_loss = supervised_loss optimized_loss.backward() self.optimizer_training.step() performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0], outputs, targets) elif self.args.mode=="capsules": one_hot_targets= Variable(self.problem.one_hot(targets.data),requires_grad=False) (optimized_loss, capsule_loss, reconstruction_loss) = self.net.loss(inputs, outputs, one_hot_targets) optimized_loss.backward() self.optimizer_training.step() supervised_grad_norm = grad_norm(self.net.parameters()) performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm) performance_estimators.set_metric_with_outputs(batch_idx, "optimized_loss", optimized_loss.data[0], outputs, targets) progress_bar(batch_idx * self.mini_batch_size, self.max_training_examples, " ".join([performance_estimator.progress_message() for performance_estimator in performance_estimators])) if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break return performance_estimators
def train( self, epoch, performance_estimators=None, train_supervised_model=True, ): if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("optimized_loss")] performance_estimators += [LossHelper("train_loss")] performance_estimators += [AccuracyHelper("train_")] performance_estimators += [FloatHelper("train_grad_norm")] print('\nTraining, epoch: %d' % epoch) self.net.train() supervised_grad_norm = 1. for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() unsupervised_loss_acc = 0 num_batches = 0 unsup_examples = numpy.random.random_integers( 0, self.args.num_shaving - 1, int(self.args.unsup_proportion * self.args.num_training)) if self.args.label_strategy == "RANDOM_UNIFORM": made_up_label = lambda index: randint( 0, self.problem.num_classes() - 1) else: print("Unsupported --label-strategy: " + self.args.label_strategy + " only RANDOM_UNIFORM is supported with this mode.") exit(1) training_dataset = ConcatDataset(datasets=[ SubsetDataset(self.problem.train_set(), range(0, self.args.num_training)), SubsetDataset(self.problem.unsup_set(), unsup_examples, get_label=made_up_label) ]) length = len(training_dataset) train_loader_subset = torch.utils.data.DataLoader( training_dataset, batch_size=self.problem.mini_batch_size(), shuffle=True, num_workers=0) for batch_idx, (inputs, targets) in enumerate(train_loader_subset): num_batches += 1 if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: self.net.train() self.optimizer_training.zero_grad() outputs = self.net(inputs) if train_supervised_model: supervised_loss = self.criterion(outputs, targets) optimized_loss = supervised_loss optimized_loss.backward() self.optimizer_training.step() supervised_grad_norm = grad_norm(self.net.parameters()) performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm) performance_estimators.set_metric_with_outputs( batch_idx, "optimized_loss", optimized_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs( batch_idx, "train_accuracy", supervised_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs( batch_idx, "train_loss", supervised_loss.data[0], outputs, targets) progress_bar( batch_idx * self.mini_batch_size, length, performance_estimators.progress_message( ["train_loss", "train_accuracy"])) if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break return performance_estimators
def train_with_two_halves(self, epoch, optimizer_training, performance_estimators=None, train_supervised_model=True, ): if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("supervised_loss")] if self.args.mode == "separate": performance_estimators += [LossHelper("unsup_loss")] performance_estimators += [AccuracyHelper("train_")] performance_estimators += [FloatHelper("supervised_grad_norm")] if self.args.mode == "separate": performance_estimators += [FloatHelper("unsup_grad_norm")] print('\nTraining, epoch: %d' % epoch) self.net.train() supervised_grad_norm = 1. for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() unsupervised_loss_acc = 0 num_batches = 0 train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training) self.image_encoder.eval() self.image_generator.eval() self.net.train() for batch_idx, (inputs, targets) in enumerate(train_loader_subset): num_batches += 1 if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() self.net.zero_grad() optimizer_training.zero_grad() image1, image2 = half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda) encoded = self.image_encoder(image1) unsup_image = self.image_generator(encoded) if self.args.mode=="separate": # train the discriminator/generator pair on the first half of the image: inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) outputs = self.net(unsup_image.detach()) unsup_loss = self.criterion(outputs, targets) unsup_loss.backward() unsup_grad_norm = grad_norm(self.net.parameters()) optimizer_training.step() # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: self.net.zero_grad() optimizer_training.zero_grad() outputs = self.net(inputs) supervised_loss = self.criterion(outputs, targets) supervised_grad_norm = grad_norm(self.net.parameters()) supervised_loss.backward() unsup_grad_norm = grad_norm(self.net.parameters()) optimizer_training.step() elif self.args.mode=="average": inputs = (inputs + unsup_image.data) / 2 inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) outputs = self.net(inputs) supervised_loss = self.criterion(outputs, targets) supervised_grad_norm = grad_norm(self.net.parameters()) supervised_loss.backward() optimizer_training.step() elif self.args.mode=="uonly": inputs = unsup_image.data inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) outputs = self.net(inputs) supervised_loss = self.criterion(outputs, targets) supervised_grad_norm = grad_norm(self.net.parameters()) supervised_loss.backward() optimizer_training.step() performance_estimators.set_metric(batch_idx, "supervised_grad_norm", supervised_grad_norm) if self.args.mode == "separate": performance_estimators.set_metric(batch_idx, "unsup_grad_norm", unsup_grad_norm) performance_estimators.set_metric_with_outputs(batch_idx, "unsup_loss", unsup_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "supervised_loss", supervised_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0], outputs, targets) progress_bar(batch_idx * self.mini_batch_size, min(self.max_regularization_examples, self.max_training_examples), " ".join([performance_estimator.progress_message() for performance_estimator in performance_estimators])) if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break return performance_estimators
def train_linear_combination(self, epoch, performance_estimators=None, train_supervised_model=True, train_ureg=True, ): if performance_estimators is None: performance_estimators = PerformanceList() performance_estimators += [LossHelper("train_loss"), AccuracyHelper("train_")] performance_estimators += [LossHelper("reg_loss")] if train_ureg: performance_estimators += [LossHelper("ureg_loss"), FloatHelper("ureg_accuracy")] performance_estimators += [FloatHelper("train_grad_norm")] print('\nTraining, epoch: %d' % epoch) self.net.train() for performance_estimator in performance_estimators: performance_estimator.init_performance_metrics() num_batches = 0 train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training) unsuploader_shuffled = self.problem.reg_loader_subset_range(0, self.args.num_shaving) unsupiter = iter(unsuploader_shuffled) for batch_idx, (inputs, targets) in enumerate(train_loader_subset): num_batches += 1 if self.use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = Variable(inputs), Variable(targets, requires_grad=False) # outputs used to calculate the loss of the supervised model # must be done with the model prior to regularization: self.net.train() self.optimizer_training.zero_grad() outputs = self.net(inputs) if train_ureg: # obtain an unsupervised sample, put it in uinputs autograd Variable: try: # first, read a minibatch from the unsupervised dataset: ufeatures, ulabels = next(unsupiter) except StopIteration: unsupiter = iter(unsuploader_shuffled) ufeatures, ulabels = next(unsupiter) if self.use_cuda: ufeatures = ufeatures.cuda() # then use it to calculate the unsupervised regularization contribution to the loss: uinputs = Variable(ufeatures) performance_estimators.set_metric(batch_idx, "ureg_alpha", self.ureg._alpha) if train_ureg: ureg_loss = self.ureg.train_ureg(inputs, uinputs) if (ureg_loss is not None): performance_estimators.set_metric(batch_idx, "ureg_loss", ureg_loss.data[0]) performance_estimators.set_metric(batch_idx, "ureg_accuracy", self.ureg.ureg_accuracy()) # adjust ureg model learning rate as needed: self.ureg.schedule(ureg_loss.data[0], epoch) if train_supervised_model: alpha = self.args.ureg_alpha # if self.ureg._which_one_model is not None: # self.ureg.estimate_example_weights(inputs) supervised_loss = self.criterion(outputs, targets) regularization_loss = self.estimate_regularization_loss(inputs, uinputs, 1., 1.) if regularization_loss is not None: optimized_loss = supervised_loss * (1. - alpha) + regularization_loss * alpha performance_estimators.set_metric(batch_idx, "reg_loss", regularization_loss.data[0]) else: optimized_loss = supervised_loss optimized_loss.backward() supervised_grad_norm = grad_norm(self.net.parameters()) performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm) self.optimizer_training.step() performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", optimized_loss.data[0], outputs, targets) performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", optimized_loss.data[0], outputs, targets) progress_bar(batch_idx * self.mini_batch_size, min(self.max_regularization_examples, self.max_training_examples), " ".join([performance_estimator.progress_message() for performance_estimator in performance_estimators])) if (batch_idx + 1) * self.mini_batch_size > self.max_regularization_examples: break if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples: break print("\n") return performance_estimators