def test(self,
performance_estimators=(LossHelper("test_loss"),
AccuracyHelper("test_"))):
self.model.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
for batch_idx, (inputs, targets) in enumerate(
self.problem.test_loader_range(0, self.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs,
volatile=True), Variable(targets,
volatile=True)
outputs = self.model(inputs)
loss = self.criterion(outputs, targets)
for performance_estimator in performance_estimators:
performance_estimator.observe_performance_metric(
batch_idx, loss.data[0], outputs, targets)
progress_bar(
batch_idx * self.mini_batch_size, self.max_validation_examples,
" ".join([
performance_estimator.progress_message()
for performance_estimator in performance_estimators
]))
if ((batch_idx + 1) *
self.mini_batch_size) > self.max_validation_examples:
break
print()
return performance_estimators
def test(self, epoch, performance_estimators=(LossHelper("test_loss"), AccuracyHelper("test_"))):
print('\nTesting, epoch: %d' % epoch)
self.net.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs, volatile=True), Variable(targets)
outputs = self.net(inputs)
loss = self.criterion(outputs, targets)
for performance_estimator in performance_estimators:
performance_estimator.observe_performance_metric(batch_idx, loss.data[0], outputs, targets)
self.ureg.estimate_accuracy(inputs)
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
print()
# Apply learning rate schedule:
test_accuracy = self.get_metric(performance_estimators, "test_accuracy")
assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_accuracy, epoch)
self.scheduler_reg.step(test_accuracy, epoch)
return performance_estimators
def predict(self, max_examples=sys.maxsize, max_queue_size=10):
val_losses = self.validation_losses
pq = PriorityQueues(val_losses, max_queue_size=max_queue_size)
args = self.args
problem = self.problem
self.model.eval()
decoder = {}
num_classes = problem.num_classes()
for i in range(num_classes):
for j in range(num_classes):
decoder[class_label(num_classes, i, j)] = (i, j)
unsup_set_length = len(problem.unsup_set())
image_index = 0
for batch_idx, tensors in enumerate(batch(problem.unsup_set(), args.mini_batch_size)):
batch_size = min(len(tensors), args.mini_batch_size)
image_index = batch_idx * len(tensors)
tensor_images = (torch.stack([ti for ti, _ in tensors], dim=0))
image_input = Variable(torch.stack(tensor_images, dim=0), volatile=True)
if self.use_cuda:
image_input = image_input.cuda()
for training_loss in val_losses:
training_loss_input = torch.zeros(batch_size, 1)
trained_with_input = torch.zeros(batch_size, 1)
for index in range(batch_size):
training_loss_input[index] = training_loss
trained_with_input[index] = 0 # we are predicting on a set never seen by the model
trained_with_input = Variable(trained_with_input, volatile=True)
training_loss_input = Variable(training_loss_input, volatile=True)
if self.use_cuda:
training_loss_input = training_loss_input.cuda()
trained_with_input = trained_with_input.cuda()
outputs = self.model(training_loss_input, trained_with_input, image_input)
max_values, indices = torch.max(outputs.data, dim=1)
for index in range(batch_size):
(predicted_index, true_index) = decoder[indices[index]]
probability = max_values[index]
if predicted_index != true_index:
# off diagonal prediction, predicting an error on this image:
unsup_index = image_index + index
# print("training_loss={} probability={} predicted_index={}, true_index={} unsup_index={}".format(
# training_loss, probability, predicted_index, true_index, unsup_index))
pq.put(training_loss, probability, (unsup_index, predicted_index, true_index))
if args.progress_bar:
progress_bar(batch_idx * batch_size,
unsup_set_length)
if batch_idx * args.mini_batch_size > max_examples: break
return pq
def test_acc(self, epoch, performance_estimators=None):
print('\nTesting, epoch: %d' % epoch)
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")]
self.net.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
cm = ConfusionMeter(self.problem.num_classes(), normalized=False)
for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
image1, image2 = half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda)
encoded = self.image_encoder(image1)
unsup_image = self.image_generator(encoded)
if self.args.mode == "separate":
inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True)
outputs = self.net(inputs)
elif self.args.mode == "average":
inputs = (inputs + unsup_image.data) / 2
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
elif self.args.mode=="uonly":
inputs = unsup_image.data
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
loss = self.criterion(outputs, targets)
# accumulate the confusion matrix:
_, predicted = torch.max(outputs.data, 1)
cm.add(predicted=predicted, target=targets.data)
performance_estimators.set_metric_with_outputs(batch_idx, "test_loss", loss.data[0], outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "test_accuracy", loss.data[0], outputs, targets)
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
performance_estimators.progress_message(["test_loss", "test_accuracy"]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
# print()
# Apply learning rate schedule:
test_accuracy = performance_estimators.get_metric("test_accuracy")
assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_accuracy, epoch)
self.confusion_matrix = cm.value().transpose()
return performance_estimators
def train(self, epoch, confusion_data):
args = self.args
optimizer = self.optimizer
problem = self.problem
performance_estimators = PerformanceList()
performance_estimators += [FloatHelper("train_loss")]
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
self.model.train()
shuffle(confusion_data)
max = min(args.max_training, len(confusion_data))
confusion_data = confusion_data[0:max]
for batch_idx, confusion_list in enumerate(batch(confusion_data, args.mini_batch_size)):
batch_size = min(len(confusion_list), args.mini_batch_size)
images = [None] * batch_size
targets = torch.zeros(batch_size)
optimizer.zero_grad()
training_loss_input = torch.zeros(batch_size, 1)
trained_with_input = torch.zeros(batch_size, 1)
for index, confusion in enumerate(confusion_list):
num_classes = problem.num_classes()
targets[index] = class_label(num_classes,
confusion.predicted_label, confusion.true_label)
dataset = problem.train_set() if confusion.trained_with else problem.test_set()
images[index], _ = dataset[confusion.example_index]
training_loss_input[index] = confusion.train_loss
trained_with_input[index] = 1.0 if confusion.trained_with else 0.0
image_input = Variable(torch.stack(images, dim=0), requires_grad=True)
training_loss_input = Variable(training_loss_input, requires_grad=True)
trained_with_input = Variable(trained_with_input, requires_grad=True)
targets = Variable(targets, requires_grad=False).type(torch.LongTensor)
if self.use_cuda:
image_input = image_input.cuda()
training_loss_input = training_loss_input.cuda()
trained_with_input = trained_with_input.cuda()
targets = targets.cuda()
outputs = self.model(training_loss_input, trained_with_input, image_input)
loss = self.criterion(outputs, targets)
loss.backward()
optimizer.step()
performance_estimators.set_metric(batch_idx, "train_loss", loss.data[0])
if args.progress_bar:
progress_bar(batch_idx * batch_size,
len(confusion_data),
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
return performance_estimators
def test(epoch):
global best_acc
net.eval()
test_loss_accumulator = 0
correct = 0
total = 0
test_accuracy = None
test_loss = None
ureg.new_epoch(epoch)
for batch_idx, (inputs, targets) in enumerate(testloader):
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs, volatile=True), Variable(targets)
outputs = net(inputs)
loss = criterion(outputs, targets)
test_loss_accumulator += loss.data[0]
_, predicted = torch.max(outputs.data, 1)
total += targets.size(0)
correct += predicted.eq(targets.data).cpu().sum()
ureg.estimate_accuracy(inputs)
test_accuracy = 100. * correct / total
test_loss = test_loss_accumulator / (batch_idx + 1)
progress_bar(
batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' %
(test_loss, test_accuracy, correct, total))
if ((batch_idx + 1) * mini_batch_size) > max_validation_examples:
break
print()
# Apply learning rate schedules:
scheduler_training.step(test_loss, epoch=epoch)
ureg.schedule(test_loss, epoch)
# Save checkpoint.
acc = 100. * correct / total
if acc > best_acc:
print('Saving..')
state = {
'net': net.module if is_parallel else net,
'acc': acc,
'epoch': epoch,
'ureg': ureg_enabled,
'ureg_model': ureg._which_one_model
}
if not os.path.isdir('checkpoint'):
os.mkdir('checkpoint')
torch.save(state,
'./checkpoint/ckpt_{}.t7'.format(args.checkpoint_key))
best_acc = acc
return (test_loss, test_accuracy, ureg.ureg_accuracy(), ureg._alpha)
def test(self, epoch, performance_estimators=None):
print('\nTesting, epoch: %d' % epoch)
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")]
self.net.eval()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
cm = ConfusionMeter(self.problem.num_classes(), normalized=False)
for batch_idx, (inputs, targets) in enumerate(self.problem.test_loader_range(0, self.args.num_validation)):
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True)
if not hasattr(self.net, 'is_dual'):
outputs = self.net(inputs)
else:
outputs, _, _ =self.net(inputs,None)
if self.args.mode=="capsules":
one_hot_targets=Variable(self.problem.one_hot(targets.data),volatile=True)
loss, capsule_loss, _ =self.net.loss(inputs, outputs, one_hot_targets)
# ||vc|| also known as norm:
v_c = torch.sqrt((outputs**2).sum(dim=2, keepdim=True))
outputs=v_c.view(v_c.size()[0],-1)
# recover index of predicted class:
#_, outputs =torch.max(v_c.view(10,-1),dim=0)
else:
loss = self.criterion(outputs, targets)
# accumulate the confusion matrix:
_, predicted = torch.max(outputs.data, 1)
cm.add(predicted=predicted, target=targets.data)
performance_estimators.set_metric_with_outputs(batch_idx, "test_loss", loss.data[0], outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "test_accuracy", loss.data[0], outputs, targets)
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
performance_estimators.progress_message(["test_loss", "test_accuracy"]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
# print()
# Apply learning rate schedule:
test_accuracy = performance_estimators.get_metric("test_accuracy")
assert test_accuracy is not None, "test_accuracy must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_accuracy, epoch)
self.confusion_matrix = cm.value().transpose()
return performance_estimators
def test(self, epoch, performance_estimators=None):
criterion = MSELoss()
print('\nTesting, epoch: %d' % epoch)
self.net.eval()
self.image_generator.eval()
self.image_encoder.eval()
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("test_loss"), AccuracyHelper("test_")]
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
# we used unsup set to train, use training to validate:
for batch_idx, (inputs, _) in enumerate(self.problem.train_loader_subset(range(0, self.args.num_validation))):
if self.use_cuda:
inputs = inputs.cuda()
image1, image2= half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda)
# train the discriminator/generator pair on the first half of the image:
encoded = self.image_encoder(image1)
output = self.image_generator(encoded)
if batch_idx == 0:
self.save_images(epoch, image1, image2, generated_image2=output)
full_image = Variable(inputs, requires_grad=False)
loss = criterion(output, full_image)
performance_estimators.set_metric(batch_idx, "test_loss", loss.data[0])
progress_bar(batch_idx * self.mini_batch_size, self.max_validation_examples,
performance_estimators.progress_message(["test_loss"]))
if ((batch_idx + 1) * self.mini_batch_size) > self.max_validation_examples:
break
# print()
# Apply learning rate schedule:
test_loss = performance_estimators.get_metric("test_loss")
assert test_loss is not None, "test_loss must be found among estimated performance metrics"
if not self.args.constant_learning_rates:
self.scheduler_train.step(test_loss, epoch)
return performance_estimators
def eval(epoch):
"""
Evaluate performance on the test phase after a shaving phase.
:param epoch:
:return: namedtuple('Eval','training_shaved_loss', 'training_shaved_accuracy')
"""
global best_acc
net.eval()
test_loss_accumulator = 0
correct = 0
total = 0
test_accuracy = None
test_loss = None
for batch_idx, (inputs, targets) in enumerate(trainloader):
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs, volatile=True), Variable(targets)
outputs = net(inputs)
loss = criterion(outputs, targets)
test_loss_accumulator += loss.data[0]
_, predicted = torch.max(outputs.data, 1)
total += targets.size(0)
correct += predicted.eq(targets.data).cpu().sum()
test_accuracy = 100. * correct / total
test_loss = test_loss_accumulator / (batch_idx + 1)
progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
% (test_loss, test_accuracy, correct, total))
if ((batch_idx + 1) * mini_batch_size) > max_training_examples:
break
print()
return EvalPerf(test_loss, test_accuracy)
def train_linear_combination(self, epoch,
performance_estimators=None,
train_supervised_model=True,
train_ureg=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("train_loss"), AccuracyHelper("train_")]
performance_estimators += [LossHelper("reg_loss")]
if train_ureg:
performance_estimators += [LossHelper("ureg_loss"), FloatHelper("ureg_accuracy")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
unsuploader_shuffled = self.problem.reg_loader_subset_range(0, self.args.num_shaving)
unsupiter = iter(unsuploader_shuffled)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
if train_ureg:
# obtain an unsupervised sample, put it in uinputs autograd Variable:
try:
# first, read a minibatch from the unsupervised dataset:
ufeatures, ulabels = next(unsupiter)
except StopIteration:
unsupiter = iter(unsuploader_shuffled)
ufeatures, ulabels = next(unsupiter)
if self.use_cuda: ufeatures = ufeatures.cuda()
# then use it to calculate the unsupervised regularization contribution to the loss:
uinputs = Variable(ufeatures)
performance_estimators.set_metric(batch_idx, "ureg_alpha", self.ureg._alpha)
if train_ureg:
ureg_loss = self.ureg.train_ureg(inputs, uinputs)
if (ureg_loss is not None):
performance_estimators.set_metric(batch_idx, "ureg_loss", ureg_loss.data[0])
performance_estimators.set_metric(batch_idx, "ureg_accuracy", self.ureg.ureg_accuracy())
# adjust ureg model learning rate as needed:
self.ureg.schedule(ureg_loss.data[0], epoch)
if train_supervised_model:
alpha = self.args.ureg_alpha
# if self.ureg._which_one_model is not None:
# self.ureg.estimate_example_weights(inputs)
supervised_loss = self.criterion(outputs, targets)
regularization_loss = self.estimate_regularization_loss(inputs, uinputs, 1., 1.)
if regularization_loss is not None:
optimized_loss = supervised_loss * (1. - alpha) + regularization_loss * alpha
performance_estimators.set_metric(batch_idx, "reg_loss", regularization_loss.data[0])
else:
optimized_loss = supervised_loss
optimized_loss.backward()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
self.optimizer_training.step()
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", optimized_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", optimized_loss.data[0],
outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
min(self.max_regularization_examples, self.max_training_examples),
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if (batch_idx + 1) * self.mini_batch_size > self.max_regularization_examples:
break
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
print("\n")
return performance_estimators
def train_capsules(self, epoch,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("capsule_loss")]
performance_estimators += [LossHelper("reconstruction_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
#performance_estimators += [FloatHelper("reconstruct_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs,requires_grad=True), Variable(targets, requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.net.zero_grad()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
one_hot_targets = Variable(self.problem.one_hot(targets.data), requires_grad=False)
if self.published_reconstruction_loss:
(optimized_loss, capsule_loss, reconstruction_loss) = self.net.loss(inputs, outputs, one_hot_targets)
optimized_loss.backward()
self.optimizer_training.step()
reconstruct_grad_norm=0
else:
margin_loss = self.net.margin_loss(outputs, one_hot_targets)
margin_loss = margin_loss.mean()
margin_loss.backward(retain_graph=True)
#reconstruct_grad_norm = grad_norm(inputs.grad)
reconstruction_loss = self.net.focused_reconstruction_loss(inputs, inputs.grad, outputs, one_hot_targets)
reconstruction_loss.backward()
self.optimizer_training.step()
optimized_loss=margin_loss+reconstruction_loss
capsule_loss=margin_loss
supervised_grad_norm = grad_norm(self.net.decoder.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
#performance_estimators.set_metric(batch_idx, "reconstruct_grad_norm", reconstruct_grad_norm)
performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0])
performance_estimators.set_metric(batch_idx,"reconstruction_loss", reconstruction_loss.data[0])
performance_estimators.set_metric(batch_idx,"capsule_loss", capsule_loss.data[0])
progress_bar(batch_idx * self.mini_batch_size,
self.max_training_examples,
performance_estimators.progress_message(["optimized_loss","capsule_loss","reconstruction_loss"]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train_with_fm_loss(self, epoch,
gamma=1E-5, performance_estimators=None):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [FloatHelper("fm_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
#sec_train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
unsuploader_shuffled = self.problem.reg_loader_subset_range(0, self.args.num_shaving)
unsupiter = itertools.cycle(unsuploader_shuffled)
for batch_idx, ((inputs, targets),
(uinputs, _)) in enumerate(zip(train_loader_subset ,
unsupiter)):
num_batches += 1
if self.use_cuda:
inputs = inputs.cuda()
uinputs = uinputs.cuda()
targets = targets.cuda()
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.net.zero_grad()
self.optimizer_training.zero_grad()
inputs, targets, uinputs = Variable(inputs), Variable(targets, requires_grad=False), Variable(uinputs, requires_grad=True)
if self.use_cuda:
inputs, targets, uinputs=inputs.cuda(),targets.cuda(), uinputs.cuda()
outputs, outputu, fm_loss = self.net(inputs,uinputs)
supervised_loss = self.criterion(outputs, targets)
optimized_loss = supervised_loss+gamma*fm_loss
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0])
performance_estimators.set_metric(batch_idx, "fm_loss", fm_loss.data[0])
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
# performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
# outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
min(self.max_regularization_examples, self.max_training_examples),
performance_estimators.progress_message(["optimized_loss","train_loss","train_accuracy"]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train_mixup(self, epoch,
performance_estimators=None,
train_supervised_model=True,
alpha=0.5,
ratio_unsup=0,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
# performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
performance_estimators += [FloatHelper("alpha")]
performance_estimators += [FloatHelper("unsup_proportion")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
sec_train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
unsuploader_shuffled = self.problem.reg_loader_subset_range(0, self.args.num_shaving)
unsupiter = itertools.cycle(unsuploader_shuffled)
performance_estimators.set_metric(epoch, "alpha", alpha)
performance_estimators.set_metric(epoch, "unsup_proportion", ratio_unsup)
for batch_idx, ((inputs1, targets1),
(inputs2, targets2),
(uinputs1, _)) in enumerate(zip(train_loader_subset,
sec_train_loader_subset,
unsupiter)):
num_batches += 1
use_unsup = random() < ratio_unsup
if use_unsup:
# use an example from the unsupervised set to mixup with inputs1:
inputs2 = uinputs1
if self.use_cuda:
inputs1 = inputs1.cuda()
inputs2 = inputs2.cuda()
inputs, targets = self.mixup_inputs_targets(alpha, inputs1, inputs2, targets1, targets2)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.net.zero_grad()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
if train_supervised_model:
supervised_loss = self.criterion_multi_label(outputs, targets)
optimized_loss = supervised_loss
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
performance_estimators.set_metric(batch_idx, "optimized_loss", optimized_loss.data[0])
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0],
outputs, targets)
# performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
# outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
min(self.max_regularization_examples, self.max_training_examples),
performance_estimators.progress_message(["train_loss","train_accuracy"]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train(self, epoch,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
if train_supervised_model and self.args.mode=="supervised":
# if self.ureg._which_one_model is not None:
# self.ureg.estimate_example_weights(inputs)
supervised_loss = self.criterion(outputs, targets)
optimized_loss = supervised_loss
optimized_loss.backward()
self.optimizer_training.step()
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0],
outputs, targets)
elif self.args.mode=="capsules":
one_hot_targets= Variable(self.problem.one_hot(targets.data),requires_grad=False)
(optimized_loss, capsule_loss, reconstruction_loss) = self.net.loss(inputs, outputs, one_hot_targets)
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm", supervised_grad_norm)
performance_estimators.set_metric_with_outputs(batch_idx, "optimized_loss", optimized_loss.data[0],
outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
self.max_training_examples,
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train_ureg_to_convergence(self,
problem,
train_dataset,
unsup_dataset,
performance_estimators=None,
epsilon=0.01,
max_epochs=30,
max_examples=None):
"""Train the ureg model for a number of epochs until improvements in the loss
are minor.
:param supervised_loader loader for supervised examples.
:param unsupervised_loader loader for unsupervised examples.
:param max_epochs maximum number of epochs before stopping
:param epsilon used to determine convergence.
:param max_examples maximum number of examples to scan per epoch.
:return list of performance estimators
"""
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [
LossHelper("ureg_loss"),
FloatHelper("ureg_accuracy")
]
len_supervised = len(train_dataset)
len_unsupervised = len(unsup_dataset)
print(
"Training ureg to convergence with {} training and {} unsupervised samples,"
" using at most {} shuffled combinations of examples per training epoch"
.format(len_supervised * self._mini_batch_size,
len_unsupervised * self._mini_batch_size, max_examples))
self._adjust_learning_rate(self._learning_rate)
previous_average_loss = sys.maxsize
for ureg_epoch in range(0, max_epochs):
# reset metric at each ureg training epoch (we use the loss average as stopping condition):
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
from itertools import cycle
length = 0
if len_supervised < len_unsupervised:
supervised_iter = iter(
cycle(self.shuffling_iter(problem, train_dataset)))
length = len_unsupervised
else:
supervised_iter = iter(
self.shuffling_iter(problem, train_dataset))
length = len_supervised
if len_unsupervised < len_supervised:
unsupervised_iter = iter(
cycle(self.shuffling_iter(problem, train_dataset)))
else:
unsupervised_iter = iter(
self.shuffling_iter(problem, unsup_dataset))
if max_examples is None:
max_examples = length * self._mini_batch_size
length = max_examples / self._mini_batch_size
num_batches = 0
for (batch_idx, ((s_input, s_labels), (u_input, _))) in enumerate(
zip(supervised_iter, unsupervised_iter)):
xs = Variable(s_input)
xu = Variable(u_input)
if self._use_cuda:
xs = xs.cuda()
xu = xu.cuda()
weight_s, weight_u = self.loss_weights(None, None)
loss = self.train_ureg(xs, xu, weight_s, weight_u)
if loss is not None:
# print("ureg batch {} average loss={} ".format(batch_idx, loss.data[0]))
num_batches += 1
performance_estimators.set_metric_with_outputs(
batch_idx, "ureg_loss", loss.data[0], None, None)
performance_estimators.set_metric(batch_idx,
"ureg_accuracy",
self.ureg_accuracy())
epoch_ = "epoch " + str(ureg_epoch) + " "
progress_bar(
batch_idx * self._mini_batch_size, max_examples,
epoch_ + " ".join([
performance_estimator.progress_message()
for performance_estimator in performance_estimators
]))
if ((batch_idx + 1) * self._mini_batch_size > max_examples):
break
average_loss = performance_estimators[0].estimates_of_metric()[0]
# print("ureg epoch {} average loss={} ".format(ureg_epoch, average_loss))
if average_loss > previous_average_loss:
if self._scheduler is not None:
self.schedule(epoch=ureg_epoch, val_loss=average_loss)
else:
break
if average_loss < previous_average_loss and abs(
average_loss - previous_average_loss) < epsilon:
break
previous_average_loss = average_loss
return performance_estimators
def training_supervised(self, unsup_only=False):
"""Train the model in a completely supervised manner. Returns the performance obtained
at the end of the configured training run.
:param unsup_only Set to true to train with dreamed-up labels on the unsupervised examples only.
:return list of performance estimators that observed performance on the last epoch run.
"""
header_written = False
lr_train_helper = LearningRateHelper(scheduler=self.scheduler_train,
learning_rate_name="train_lr")
previous_test_perfs = None
perfs = PerformanceList()
best_test_loss = sys.maxsize
num_rollbacks = 0
epochs_since_rollback = 0
if unsup_only:
assert self.best_model is not None, "best model cannot be None to continue training with unsup only."
# scan the unsupervised set to calculate labels using the previously trained best model:
print("Calculating labels for unsupervised set..")
unsup_index_to_labels = {}
unsup_set_loader = self.problem.loader_for_dataset(
self.problem.unsup_set())
for batch_idx, (inputs, _) in enumerate(unsup_set_loader):
if self.use_cuda:
inputs = inputs.cuda()
inputs = Variable(inputs, volatile=True)
predicted = self.best_model(inputs)
if predicted.size()[1] == self.problem.num_classes():
# need to take the argmax to find the index of predicted class.
_, predicted = torch.max(predicted.data, 1)
predicted = predicted.type(
torch.cuda.LongTensor
) if self.use_cuda else predicted.type(torch.LongTensor)
select = torch.index_select(self.best_model_confusion_matrix,
dim=0,
index=predicted)
select = select.type(
torch.cuda.FloatTensor) if self.use_cuda else select.type(
torch.FloatTensor)
confusion_labels = torch.renorm(select, p=1, dim=1, maxnorm=1)
start_of_range = batch_idx * self.problem.mini_batch_size()
for example_index in range(
start_of_range,
start_of_range + self.problem.mini_batch_size()):
label_for_example = confusion_labels[example_index -
start_of_range]
unsup_index_to_labels[example_index] = label_for_example
progress_bar(
batch_idx,
self.args.num_shaving / self.problem.mini_batch_size())
if batch_idx * self.problem.mini_batch_size(
) > self.args.num_shaving:
break
self.net = self.best_model
print("Training with unsupervised set..")
for epoch in range(self.start_epoch,
self.start_epoch + self.args.num_epochs):
perfs = PerformanceList()
perfs += self.train_unsup_only(epoch, unsup_index_to_label=unsup_index_to_labels) if unsup_only else \
self.train(epoch, train_supervised_model=True)
perfs += [lr_train_helper]
if previous_test_perfs is None or self.epoch_is_test_epoch(epoch):
perfs += self.test(epoch)
test_loss = perfs.get_metric("test_loss")
if (not header_written):
header_written = True
self.log_performance_header(perfs)
early_stop, perfs = self.log_performance_metrics(epoch, perfs)
if early_stop:
# early stopping requested.
return perfs
if self.args.rollback_when_worse and epochs_since_rollback > 5 and test_loss > best_test_loss:
self.net = self.load_checkpoint()
if self.use_cuda: self.net.cuda()
print("Rolled-back")
num_rollbacks += 1
epochs_since_rollback = 0
else:
best_test_loss = test_loss
epochs_since_rollback += 1
print("best test loss={} rolled-back {} times.".format(
best_test_loss, num_rollbacks))
return perfs
def train_unsup_only(self,
epoch,
unsup_index_to_label,
performance_estimators=None):
""" Continue training a model on the unsupervised set with labels.
:param epoch:
:param unsup_index_to_label: map from index of the unsupervised example to label (in one hot encoding format, one element per class)
:param performance_estimators:
:param train_supervised_model:
:return:
"""
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [FloatHelper("train_grad_norm")]
# reset the model before training:
#init_params(self.net)
print('\nTraining, epoch: %d' % epoch)
self.net.train()
train_supervised_model = True
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
num_batches = 0
training_dataset = SubsetDataset(
self.problem.unsup_set(),
range(0, self.args.num_shaving),
get_label=lambda index: unsup_index_to_label[index])
length = len(training_dataset)
train_loader_subset = torch.utils.data.DataLoader(
training_dataset,
batch_size=self.problem.mini_batch_size(),
shuffle=False,
num_workers=0)
self.optimizer_training = torch.optim.SGD(self.net.parameters(),
lr=self.args.lr,
momentum=self.args.momentum,
weight_decay=self.args.L2)
# we use binary cross-entropy for single label with smoothing.
self.net.train()
criterion = BCELoss()
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets,
requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
# renormalize outputs by example, from multi-label to single label prediction::
outputs = torch.renorm(torch.exp(outputs), p=1, maxnorm=1, dim=1)
supervised_loss = criterion(outputs, targets)
optimized_loss = supervised_loss
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm",
supervised_grad_norm)
performance_estimators.set_metric_with_outputs(
batch_idx, "optimized_loss", optimized_loss.data[0], outputs,
targets)
performance_estimators.set_metric_with_outputs(
batch_idx, "train_loss", supervised_loss.data[0], outputs,
targets)
progress_bar(
batch_idx * self.mini_batch_size, length,
performance_estimators.progress_message(
["train_loss", "train_accuracy"]))
return performance_estimators
def train(
self,
epoch,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("optimized_loss")]
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("train_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
unsup_examples = numpy.random.random_integers(
0, self.args.num_shaving - 1,
int(self.args.unsup_proportion * self.args.num_training))
if self.args.label_strategy == "RANDOM_UNIFORM":
made_up_label = lambda index: randint(
0,
self.problem.num_classes() - 1)
else:
print("Unsupported --label-strategy: " + self.args.label_strategy +
" only RANDOM_UNIFORM is supported with this mode.")
exit(1)
training_dataset = ConcatDataset(datasets=[
SubsetDataset(self.problem.train_set(),
range(0, self.args.num_training)),
SubsetDataset(self.problem.unsup_set(),
unsup_examples,
get_label=made_up_label)
])
length = len(training_dataset)
train_loader_subset = torch.utils.data.DataLoader(
training_dataset,
batch_size=self.problem.mini_batch_size(),
shuffle=True,
num_workers=0)
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs), Variable(targets,
requires_grad=False)
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.train()
self.optimizer_training.zero_grad()
outputs = self.net(inputs)
if train_supervised_model:
supervised_loss = self.criterion(outputs, targets)
optimized_loss = supervised_loss
optimized_loss.backward()
self.optimizer_training.step()
supervised_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "train_grad_norm",
supervised_grad_norm)
performance_estimators.set_metric_with_outputs(
batch_idx, "optimized_loss", optimized_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(
batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(
batch_idx, "train_loss", supervised_loss.data[0], outputs,
targets)
progress_bar(
batch_idx * self.mini_batch_size, length,
performance_estimators.progress_message(
["train_loss", "train_accuracy"]))
if (batch_idx +
1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def train(epoch, unsupiter):
print('\nTraining, epoch: %d' % epoch)
net.train()
average_total_loss = 0
average_supervised_loss = 0
average_unsupervised_loss = 0
correct = 0
total = 0
average_total_loss = 0
training_accuracy = 0
for batch_idx, (inputs, targets) in enumerate(trainloader):
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
optimizer_training.zero_grad()
inputs, targets = Variable(inputs), Variable(targets)
outputs = net(inputs)
supervised_loss = criterion(outputs, targets)
supervised_loss.backward()
optimizer_training.step()
# the unsupervised regularization part goes here:
try:
# first, read a minibatch from the unsupervised dataset:
ufeatures, ulabels = next(unsupiter)
except StopIteration:
unsupiter = iter(unsuploader)
ufeatures, ulabels = next(unsupiter)
if use_cuda: ufeatures = ufeatures.cuda()
# then use it to calculate the unsupervised regularization contribution to the loss:
uinputs = Variable(ufeatures)
ureg.train_ureg(inputs, uinputs)
optimized_loss = supervised_loss
average_total_loss += optimized_loss.data[0]
average_supervised_loss += supervised_loss.data[0]
_, predicted = torch.max(outputs.data, 1)
total += targets.size(0)
correct += predicted.eq(targets.data).cpu().sum()
denominator = batch_idx + 1
average_total_loss = average_total_loss / denominator
average_supervised_loss = average_supervised_loss / denominator
average_unsupervised_loss = average_unsupervised_loss / denominator
training_accuracy = 100. * correct / total
progress_bar(batch_idx, len(trainloader),
('loss: %.3f s: %.3f u: %.3f | Acc: %.3f%% (%d/%d)'
% (average_total_loss,
average_supervised_loss,
average_unsupervised_loss,
training_accuracy,
correct,
total)))
if (batch_idx + 1) * mini_batch_size > max_training_examples:
break
scheduler_reg.step(average_unsupervised_loss, epoch)
print()
return TrainingPerf(average_total_loss, average_supervised_loss, training_accuracy)
def train_with_two_halves(self, epoch, optimizer_training,
performance_estimators=None,
train_supervised_model=True,
):
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("supervised_loss")]
if self.args.mode == "separate":
performance_estimators += [LossHelper("unsup_loss")]
performance_estimators += [AccuracyHelper("train_")]
performance_estimators += [FloatHelper("supervised_grad_norm")]
if self.args.mode == "separate":
performance_estimators += [FloatHelper("unsup_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
self.net.train()
supervised_grad_norm = 1.
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
unsupervised_loss_acc = 0
num_batches = 0
train_loader_subset = self.problem.train_loader_subset_range(0, self.args.num_training)
self.image_encoder.eval()
self.image_generator.eval()
self.net.train()
for batch_idx, (inputs, targets) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
self.net.zero_grad()
optimizer_training.zero_grad()
image1, image2 = half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda)
encoded = self.image_encoder(image1)
unsup_image = self.image_generator(encoded)
if self.args.mode=="separate":
# train the discriminator/generator pair on the first half of the image:
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(unsup_image.detach())
unsup_loss = self.criterion(outputs, targets)
unsup_loss.backward()
unsup_grad_norm = grad_norm(self.net.parameters())
optimizer_training.step()
# outputs used to calculate the loss of the supervised model
# must be done with the model prior to regularization:
self.net.zero_grad()
optimizer_training.zero_grad()
outputs = self.net(inputs)
supervised_loss = self.criterion(outputs, targets)
supervised_grad_norm = grad_norm(self.net.parameters())
supervised_loss.backward()
unsup_grad_norm = grad_norm(self.net.parameters())
optimizer_training.step()
elif self.args.mode=="average":
inputs = (inputs + unsup_image.data) / 2
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
supervised_loss = self.criterion(outputs, targets)
supervised_grad_norm = grad_norm(self.net.parameters())
supervised_loss.backward()
optimizer_training.step()
elif self.args.mode=="uonly":
inputs = unsup_image.data
inputs, targets = Variable(inputs), Variable(targets, requires_grad=False)
outputs = self.net(inputs)
supervised_loss = self.criterion(outputs, targets)
supervised_grad_norm = grad_norm(self.net.parameters())
supervised_loss.backward()
optimizer_training.step()
performance_estimators.set_metric(batch_idx, "supervised_grad_norm", supervised_grad_norm)
if self.args.mode == "separate":
performance_estimators.set_metric(batch_idx, "unsup_grad_norm", unsup_grad_norm)
performance_estimators.set_metric_with_outputs(batch_idx, "unsup_loss", unsup_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "supervised_loss", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_accuracy", supervised_loss.data[0],
outputs, targets)
performance_estimators.set_metric_with_outputs(batch_idx, "train_loss", supervised_loss.data[0],
outputs, targets)
progress_bar(batch_idx * self.mini_batch_size,
min(self.max_regularization_examples, self.max_training_examples),
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if (batch_idx + 1) * self.mini_batch_size > self.max_training_examples:
break
return performance_estimators
def regularize(self, epoch, performance_estimators=None,
previous_ureg_loss=1.0, previous_training_loss=1.0):
"""
Performs training vs test regularization/shaving phase.
:param epoch:
:param performance_estimators: estimators for performance metrics to collect.
:return:
"""
print('\nRegularizing, epoch: %d' % epoch)
self.net.train()
if performance_estimators is None:
performance_estimators=PerformanceList()
performance_estimators.append(FloatHelper("reg_grad_norm"))
performance_estimators.append(LossHelper("reg_loss"))
performance_estimators.append(FloatHelper("ureg_alpha"))
trainiter = iter(self.trainloader)
train_examples_used = 0
use_max_shaving_records = self.args.num_shaving
# make sure we process the entire training set, but limit how many regularization_examples we scan (randomly
# from the entire set):
if self.max_examples_per_epoch > self.args.num_training:
max_loop_index = min(self.max_examples_per_epoch, self.max_regularization_examples)
else:
max_loop_index = self.args.num_training
performance_estimators.init_performance_metrics()
unsuper_records_to_be_seen = min(max_loop_index, use_max_shaving_records)
# max_loop_index is the number of times training examples are seen,
# use_max_shaving_records is the number of times unsupervised examples are seen,
# estimate weights:
a = unsuper_records_to_be_seen / max_loop_index
b = 1
weight_s = a / (a + b)
weight_u = 1 / (a + b)
print("weight_s={} weight_u={} unsuper_records_to_be_seen={} max_loop_index={}".format(
weight_s, weight_u, unsuper_records_to_be_seen, max_loop_index))
for shaving_index in range(self.num_shaving_epochs):
print("Shaving step {}".format(shaving_index))
# produce a random subset of the unsupervised samples, exactly matching the number of training examples:
unsupsampler = self.problem.reg_loader_subset_range(0, use_max_shaving_records)
performance_estimators.init_performance_metrics()
performance_estimators.set_metric(1, "ureg_alpha",self.ureg._alpha)
for batch_idx, (inputs, targets) in enumerate(unsupsampler):
if self.use_cuda:
inputs = inputs.cuda()
self.optimizer_reg.zero_grad()
uinputs = Variable(inputs)
# don't use more training examples than allowed (-n) even if we don't use
# their labels:
if train_examples_used > self.args.num_training:
trainiter = iter(self.trainloader)
train_examples_used = 0
try:
# first, read a minibatch from the unsupervised dataset:
features, _ = next(trainiter)
except StopIteration:
trainiter = iter(self.trainloader)
features, _ = next(trainiter)
train_examples_used += 1
if self.use_cuda: features = features.cuda()
# then use it to calculate the unsupervised regularization contribution to the loss:
inputs = Variable(features)
regularization_loss = self.estimate_regularization_loss(inputs, uinputs,
weight_s=weight_s,
weight_u=weight_u)
if regularization_loss is not None:
regularization_loss = regularization_loss * self.args.ureg_alpha
reg_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "reg_grad_norm", reg_grad_norm)
optimized_loss = regularization_loss.data[0]
performance_estimators.set_metric(batch_idx, "reg_loss", optimized_loss)
regularization_loss.backward()
self.optimizer_reg.step()
else:
print("Found None in regularize")
optimized_loss = 0
performance_estimators[0].observe_performance_metric(batch_idx, optimized_loss,
inputs, uinputs)
# keep training the ureg model while regularizing, this is needed to keep ureg relevant to the
# regularized weights:
self.ureg.train_ureg(inputs, uinputs)
progress_bar(batch_idx * self.mini_batch_size, max_loop_index,
" ".join([performance_estimator.progress_message() for performance_estimator in
performance_estimators]))
if ((batch_idx + 1) * self.mini_batch_size) > max_loop_index:
break
print()
return performance_estimators
def train_unsup_only(self, epoch,
performance_estimators=None
):
""" Continue training a model on the unsupervised set with labels.
:param epoch:
:param unsup_index_to_label: map from index of the unsupervised example to label (in one hot encoding format, one element per class)
:param performance_estimators:
:param train_supervised_model:
:return:
"""
if performance_estimators is None:
performance_estimators = PerformanceList()
performance_estimators += [LossHelper("train_loss")]
performance_estimators += [FloatHelper("encoder_grad_norm")]
performance_estimators += [FloatHelper("generator_grad_norm")]
performance_estimators += [FloatHelper("net_grad_norm")]
print('\nTraining, epoch: %d' % epoch)
train_supervised_model = True
for performance_estimator in performance_estimators:
performance_estimator.init_performance_metrics()
num_batches = 0
training_dataset = SubsetDataset(self.problem.unsup_set(),
range(0, self.args.num_shaving), get_label=lambda index: 1)
length = len(training_dataset)
train_loader_subset = torch.utils.data.DataLoader(training_dataset,
batch_size=self.problem.mini_batch_size(),
shuffle=True,
num_workers=0)
# we use binary cross-entropy for single label with smoothing.
criterion = MSELoss()
self.net.train()
self.image_generator.train()
self.image_encoder.train()
for batch_idx, (inputs, _) in enumerate(train_loader_subset):
num_batches += 1
if self.use_cuda:
inputs = inputs.cuda()
self.optimizer.zero_grad()
image1, image2 = half_images(inputs, slope=get_random_slope(), cuda=self.use_cuda)
# train the discriminator/generator pair on the first half of the image:
encoded = self.image_encoder(image1)
# norm_encoded=encoded.norm(p=1)
output = self.image_generator(encoded,)
full_image=Variable(inputs,requires_grad=False)
optimized_loss = criterion(output, full_image)
optimized_loss.backward()
self.optimizer.step()
if batch_idx == 0:
self.save_images(epoch, image1, image2, generated_image2=output, prefix="train")
encoder_grad_norm = grad_norm(self.image_encoder.parameters())
generator_grad_norm = grad_norm(self.image_generator.parameters())
net_grad_norm = grad_norm(self.net.parameters())
performance_estimators.set_metric(batch_idx, "encoder_grad_norm", encoder_grad_norm)
performance_estimators.set_metric(batch_idx, "generator_grad_norm", generator_grad_norm)
performance_estimators.set_metric(batch_idx, "net_grad_norm", net_grad_norm)
performance_estimators.set_metric(batch_idx, "train_loss", optimized_loss.data[0])
progress_bar(batch_idx * self.mini_batch_size,
length,
performance_estimators.progress_message(["train_loss", "train_accuracy"]))
return performance_estimators
def regularize(epoch, unsupiter):
"""
There is an issue with this method when the training set is larger than the regularization
set, then only up to the length of the unsup set is used in the training set. This limits
the amount of regularization possible. For instance, in images, if 50,000 training samples
with 10,000 unsup (test set examples), then only the first 10,000 training samples are
considered in the regularization, not a random pairing of the complete training and the unsup
samples.
:param epoch:
:param unsupiter:
:return:
"""
print('\nRegularizing, epoch: %d' % epoch)
net.train()
average_total_loss = 0
supervised_loss = 0
trainiter = iter(trainloader)
train_examples_used = 0
for shaving_index in range(args.shaving_epochs):
print("Shaving step {}".format(shaving_index))
average_unsupervised_loss = 0
denominator = 0
for batch_idx, (inputs, targets) in enumerate(unsuploader):
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
optimizer_reg.zero_grad()
uinputs, _ = Variable(inputs), Variable(targets)
# don't use more training examples than allowed (-n) even if we don't use
# their labels:
if train_examples_used > args.num_training:
trainiter = iter(trainloader)
train_examples_used = 0
try:
# first, read a minibatch from the unsupervised dataset:
features, ulabels = next(trainiter)
except StopIteration:
trainiter = iter(trainloader)
features, _ = next(trainiter)
train_examples_used += 1
if use_cuda: features = features.cuda()
# then use it to calculate the unsupervised regularization contribution to the loss:
inputs = Variable(features)
regularization_loss = ureg.regularization_loss(inputs, uinputs)
if regularization_loss is not None:
regularization_loss.backward()
optimizer_reg.step()
optimized_loss = regularization_loss
average_total_loss += optimized_loss.data[0]
else:
optimized_loss = supervised_loss
average_unsupervised_loss += (0 if regularization_loss is None
else regularization_loss.data[0])
denominator = batch_idx + 1
average_total_loss = average_total_loss / denominator
average_unsupervised_loss = average_unsupervised_loss / denominator
progress_bar(batch_idx, len(unsuploader), ' u: %.3f'
% (average_unsupervised_loss,))
if ((batch_idx + 1) * mini_batch_size) > max_regularization_examples:
break
print()
return RegularizePerf(average_unsupervised_loss,)
