def test(self, epoch, performance_estimators=(LossHelper("test_loss"), AccuracyHelper("test_"))):
print('\nTesting, epoch: %d' % epoch)
self.net.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs, volatile=True), Variable(targets)
outputs = self.net(inputs)
loss = self.criterion(outputs, targets)
for performance_estimator in performance_estimators:
performance_estimator.observe_performance_metric(batch_idx, loss.data[0], outputs, targets)
self.ureg.estimate_accuracy(inputs)
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
print()
# Apply learning rate schedule:
test_accuracy = self.get_metric(performance_estimators, "test_accuracy")
assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_accuracy, epoch)
self.scheduler_reg.step(test_accuracy, epoch)
return performance_estimators
def test(self,
performance_estimators=(LossHelper("test_loss"),
AccuracyHelper("test_"))):
self.model.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
for batch_idx, (inputs, targets) in enumerate(
self.problem.test_loader_range(0, self.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs,
volatile=True), Variable(targets,
volatile=True)
outputs = self.model(inputs)
loss = self.criterion(outputs, targets)
for performance_estimator in performance_estimators:
performance_estimator.observe_performance_metric(
batch_idx, loss.data[0], outputs, targets)
progress_bar(
batch_idx * self.mini_batch_size, self.max_validation_examples,
" ".join([
performance_estimator.progress_message()
for performance_estimator in performance_estimators
]))
if ((batch_idx + 1) *
self.mini_batch_size) > self.max_validation_examples:
break
print()
return performance_estimators
def test_acc(self, epoch, performance_estimators=None):
print('\nTesting, epoch: %d' % epoch)
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")]
self.net.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
cm = ConfusionMeter(self.problem.num_classes(), normalized=False)
for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
image1, image2 = half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda)
encoded = self.image_encoder(image1)
unsup_image = self.image_generator(encoded)
if self.args.mode == "separate":
inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True)
outputs = self.net(inputs)
elif self.args.mode == "average":
inputs = (inputs + unsup_image.data) / 2
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
elif self.args.mode=="uonly":
inputs = unsup_image.data
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
loss = self.criterion(outputs, targets)
# accumulate the confusion matrix:
_, predicted = torch.max(outputs.data, 1)
cm.add(predicted=predicted, target=targets.data)
performance_estimators.set_metric_with_outputs(batch_idx, "test_loss", loss.data[0], outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "test_accuracy", loss.data[0], outputs, targets)
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
performance_estimators.progress_message(["test_loss", "test_accuracy"]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
# print()
# Apply learning rate schedule:
test_accuracy = performance_estimators.get_metric("test_accuracy")
assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_accuracy, epoch)
self.confusion_matrix = cm.value().transpose()
return performance_estimators
def test(self, epoch, performance_estimators=None):
print('\nTesting, epoch: %d' % epoch)
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")]
self.net.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
cm = ConfusionMeter(self.problem.num_classes(), normalized=False)
for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True)
if not hasattr(self.net, 'is_dual'):
outputs = self.net(inputs)
else:
outputs, _, _ =self.net(inputs,None)
if self.args.mode=="capsules":
one_hot_targets=Variable(self.problem.one_hot(targets.data),volatile=True)
loss, capsule_loss, _ =self.net.loss(inputs, outputs, one_hot_targets)
# ||vc|| also known as norm:
v_c = torch.sqrt((outputs**2).sum(dim=2, keepdim=True))
outputs=v_c.view(v_c.size()[0],-1)
# recover index of predicted class:
#_, outputs =torch.max(v_c.view(10,-1),dim=0)
else:
loss = self.criterion(outputs, targets)
# accumulate the confusion matrix:
_, predicted = torch.max(outputs.data, 1)
cm.add(predicted=predicted, target=targets.data)
performance_estimators.set_metric_with_outputs(batch_idx, "test_loss", loss.data[0], outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "test_accuracy", loss.data[0], outputs, targets)
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
performance_estimators.progress_message(["test_loss", "test_accuracy"]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
# print()
# Apply learning rate schedule:
test_accuracy = performance_estimators.get_metric("test_accuracy")
assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_accuracy, epoch)
self.confusion_matrix = cm.value().transpose()
return performance_estimators
def test(self, epoch, performance_estimators=None):
criterion = MSELoss()
print('\nTesting, epoch: %d' % epoch)
self.net.eval()
self.image_generator.eval()
self.image_encoder.eval()
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")]
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
# we used unsup set to train, use training to validate:
for batch_idx, (inputs, _) in enumerate(self.problem.train_loader_subset(range(0, self.args.num_validation))):
if self.use_cuda:
inputs = inputs.cuda()
image1, image2= half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda)
# train the discriminator/generator pair on the first half of the image:
encoded = self.image_encoder(image1)
output = self.image_generator(encoded)
if batch_idx == 0:
self.save_images(epoch, image1, image2, generated_image2=output)
full_image = Variable(inputs, requires_grad=False)
loss = criterion(output, full_image)
performance_estimators.set_metric(batch_idx, "test_loss", loss.data[0])
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
performance_estimators.progress_message(["test_loss"]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
# print()
# Apply learning rate schedule:
test_loss = performance_estimators.get_metric("test_loss")
assert test_loss is not None, "test_loss must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_loss, epoch)
return performance_estimators
def train_capsules(self, epoch,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("capsule_loss")]
performance_estimators += [LossHelper("reconstruction_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
#performance_estimators += [FloatHelper("reconstruct_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs,requires_grad=True), Variable(targets, requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.net.zero_grad()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
one_hot_targets = Variable(self.problem.one_hot(targets.data), requires_grad=False)
if self.published_reconstruction_loss:
(optimized_loss, capsule_loss, reconstruction_loss) = self.net.loss(inputs, outputs, one_hot_targets)
optimized_loss.backward()
self.optimizer_training.step()
reconstruct_grad_norm=0
else:
margin_loss = self.net.margin_loss(outputs, one_hot_targets)
margin_loss = margin_loss.mean()
margin_loss.backward(retain_graph=True)
#reconstruct_grad_norm = grad_norm(inputs.grad)
reconstruction_loss = self.net.focused_reconstruction_loss(inputs, inputs.grad, outputs, one_hot_targets)
reconstruction_loss.backward()
self.optimizer_training.step()
optimized_loss=margin_loss+reconstruction_loss
capsule_loss=margin_loss
supervised_grad_norm = grad_norm(self.net.decoder.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
#performance_estimators.set_metric(batch_idx, "reconstruct_grad_norm", reconstruct_grad_norm)
performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0])
performance_estimators.set_metric(batch_idx,"reconstruction_loss", reconstruction_loss.data[0])
performance_estimators.set_metric(batch_idx,"capsule_loss", capsule_loss.data[0])
progress_bar(batch_idx * self.mini_batch_size,
self.max_training_examples,
performance_estimators.progress_message(["optimized_loss","capsule_loss","reconstruction_loss"]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train_with_fm_loss(self, epoch,
gamma=1E-5, performance_estimators=None):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [FloatHelper("fm_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
#sec_train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
unsuploader_shuffled = self.problem.reg_loader_subset_range(0, self.args.num_shaving)
unsupiter = itertools.cycle(unsuploader_shuffled)
for batch_idx, ((inputs, targets),
(uinputs, _)) in enumerate(zip(train_loader_subset ,
unsupiter)):
num_batches += 1
if self.use_cuda:
inputs = inputs.cuda()
uinputs = uinputs.cuda()
targets = targets.cuda()
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.net.zero_grad()
self.optimizer_training.zero_grad()
inputs, targets, uinputs = Variable(inputs), Variable(targets, requires_grad=False), Variable(uinputs, requires_grad=True)
if self.use_cuda:
inputs, targets, uinputs=inputs.cuda(),targets.cuda(), uinputs.cuda()
outputs, outputu, fm_loss = self.net(inputs,uinputs)
supervised_loss = self.criterion(outputs, targets)
optimized_loss = supervised_loss+gamma*fm_loss
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0])
performance_estimators.set_metric(batch_idx, "fm_loss", fm_loss.data[0])
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
# performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
# outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
min(self.max_regularization_examples, self.max_training_examples),
performance_estimators.progress_message(["optimized_loss","train_loss","train_accuracy"]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train(self, epoch,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
if train_supervised_model and self.args.mode=="supervised":
# if self.ureg._which_one_model is not None:
# self.ureg.estimate_example_weights(inputs)
supervised_loss = self.criterion(outputs, targets)
optimized_loss = supervised_loss
optimized_loss.backward()
self.optimizer_training.step()
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0],
outputs, targets)
elif self.args.mode=="capsules":
one_hot_targets= Variable(self.problem.one_hot(targets.data),requires_grad=False)
(optimized_loss, capsule_loss, reconstruction_loss) = self.net.loss(inputs, outputs, one_hot_targets)
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
performance_estimators.set_metric_with_outputs(batch_idx, "optimized_loss", optimized_loss.data[0],
outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
self.max_training_examples,
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train(
self,
epoch,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
unsup_examples = numpy.random.random_integers(
0, self.args.num_shaving - 1,
int(self.args.unsup_proportion * self.args.num_training))
if self.args.label_strategy == "RANDOM_UNIFORM":
made_up_label = lambda index: randint(
0,
self.problem.num_classes() - 1)
else:
print("Unsupported --label-strategy: " + self.args.label_strategy +
" only RANDOM_UNIFORM is supported with this mode.")
exit(1)
training_dataset = ConcatDataset(datasets=[
SubsetDataset(self.problem.train_set(),
range(0, self.args.num_training)),
SubsetDataset(self.problem.unsup_set(),
unsup_examples,
get_label=made_up_label)
])
length = len(training_dataset)
train_loader_subset = torch.utils.data.DataLoader(
training_dataset,
batch_size=self.problem.mini_batch_size(),
shuffle=True,
num_workers=0)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets,
requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
if train_supervised_model:
supervised_loss = self.criterion(outputs, targets)
optimized_loss = supervised_loss
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm",
supervised_grad_norm)
performance_estimators.set_metric_with_outputs(
batch_idx, "optimized_loss", optimized_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(
batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(
batch_idx, "train_loss", supervised_loss.data[0], outputs,
targets)
progress_bar(
batch_idx * self.mini_batch_size, length,
performance_estimators.progress_message(
["train_loss", "train_accuracy"]))
if (batch_idx +
1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train_with_two_halves(self, epoch, optimizer_training,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("supervised_loss")]
if self.args.mode == "separate":
performance_estimators += [LossHelper("unsup_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("supervised_grad_norm")]
if self.args.mode == "separate":
performance_estimators += [FloatHelper("unsup_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
self.image_encoder.eval()
self.image_generator.eval()
self.net.train()
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
self.net.zero_grad()
optimizer_training.zero_grad()
image1, image2 = half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda)
encoded = self.image_encoder(image1)
unsup_image = self.image_generator(encoded)
if self.args.mode=="separate":
# train the discriminator/generator pair on the first half of the image:
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(unsup_image.detach())
unsup_loss = self.criterion(outputs, targets)
unsup_loss.backward()
unsup_grad_norm = grad_norm(self.net.parameters())
optimizer_training.step()
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.zero_grad()
optimizer_training.zero_grad()
outputs = self.net(inputs)
supervised_loss = self.criterion(outputs, targets)
supervised_grad_norm = grad_norm(self.net.parameters())
supervised_loss.backward()
unsup_grad_norm = grad_norm(self.net.parameters())
optimizer_training.step()
elif self.args.mode=="average":
inputs = (inputs + unsup_image.data) / 2
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
supervised_loss = self.criterion(outputs, targets)
supervised_grad_norm = grad_norm(self.net.parameters())
supervised_loss.backward()
optimizer_training.step()
elif self.args.mode=="uonly":
inputs = unsup_image.data
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
supervised_loss = self.criterion(outputs, targets)
supervised_grad_norm = grad_norm(self.net.parameters())
supervised_loss.backward()
optimizer_training.step()
performance_estimators.set_metric(batch_idx, "supervised_grad_norm", supervised_grad_norm)
if self.args.mode == "separate":
performance_estimators.set_metric(batch_idx, "unsup_grad_norm", unsup_grad_norm)
performance_estimators.set_metric_with_outputs(batch_idx, "unsup_loss", unsup_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "supervised_loss", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0],
outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
min(self.max_regularization_examples, self.max_training_examples),
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train_linear_combination(self, epoch,
performance_estimators=None,
train_supervised_model=True,
train_ureg=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("train_loss"), AccuracyHelper("train_")]
performance_estimators += [LossHelper("reg_loss")]
if train_ureg:
performance_estimators += [LossHelper("ureg_loss"), FloatHelper("ureg_accuracy")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
unsuploader_shuffled = self.problem.reg_loader_subset_range(0, self.args.num_shaving)
unsupiter = iter(unsuploader_shuffled)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
if train_ureg:
# obtain an unsupervised sample, put it in uinputs autograd Variable:
try:
# first, read a minibatch from the unsupervised dataset:
ufeatures, ulabels = next(unsupiter)
except StopIteration:
unsupiter = iter(unsuploader_shuffled)
ufeatures, ulabels = next(unsupiter)
if self.use_cuda: ufeatures = ufeatures.cuda()
# then use it to calculate the unsupervised regularization contribution to the loss:
uinputs = Variable(ufeatures)
performance_estimators.set_metric(batch_idx, "ureg_alpha", self.ureg._alpha)
if train_ureg:
ureg_loss = self.ureg.train_ureg(inputs, uinputs)
if (ureg_loss is not None):
performance_estimators.set_metric(batch_idx, "ureg_loss", ureg_loss.data[0])
performance_estimators.set_metric(batch_idx, "ureg_accuracy", self.ureg.ureg_accuracy())
# adjust ureg model learning rate as needed:
self.ureg.schedule(ureg_loss.data[0], epoch)
if train_supervised_model:
alpha = self.args.ureg_alpha
# if self.ureg._which_one_model is not None:
# self.ureg.estimate_example_weights(inputs)
supervised_loss = self.criterion(outputs, targets)
regularization_loss = self.estimate_regularization_loss(inputs, uinputs, 1., 1.)
if regularization_loss is not None:
optimized_loss = supervised_loss * (1. - alpha) + regularization_loss * alpha
performance_estimators.set_metric(batch_idx, "reg_loss", regularization_loss.data[0])
else:
optimized_loss = supervised_loss
optimized_loss.backward()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
self.optimizer_training.step()
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", optimized_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", optimized_loss.data[0],
outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
min(self.max_regularization_examples, self.max_training_examples),
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if (batch_idx + 1) * self.mini_batch_size > self.max_regularization_examples:
break
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
print("\n")
return performance_estimators