def create_which_one_model(self, xs):
num_features = self._num_features
if self._which_one_model is None:
if (self._checkpointModel):
self._which_one_model = self._checkpointModel
self._checkpointModel = None
else:
activation_list = self.extract_activation_list(xs)
self._which_one_model = self.model_assembler.assemble(
activation_list)
self.chosen_activations = self.model_assembler.get_collect_output(
)
# (we will use BCEWithLogitsLoss on top of linar to get the cross entropy after
# applying a sigmoid).
self.model_assembler.init_params()
print("done building which_one_model:" +
str(self._which_one_model))
if self._use_cuda: self._which_one_model.cuda()
# self.optimizer = torch.optim.Adam(self.which_one_model.parameters(),
# lr=self.learning_rate)
self._optimizer = torch.optim.SGD(
self._which_one_model.parameters(),
lr=self._learning_rate,
momentum=self.momentum,
weight_decay=self.L2)
if self._use_scheduler:
self._scheduler = construct_scheduler(
self._optimizer,
direction="min",
lr_patience=1,
extra_patience=10,
ureg_reset_every_n_epoch=self._reset_every_epochs,
factor=0.8,
min_lr=[1E-7])
self.loss_ys = torch.nn.BCELoss() # 0 is supervised
self.loss_yu = torch.nn.BCELoss() # 1 is unsupervised
if self._use_cuda:
self.loss_ys = self.loss_ys.cuda()
self.loss_yu = self.loss_yu.cuda()
zeros = torch.zeros(self._mini_batch_size)
ones = torch.ones(self._mini_batch_size)
self.ys_true = Variable(torch.transpose(torch.stack([ones, zeros]),
0, 1),
requires_grad=False)
self.yu_true = Variable(torch.transpose(torch.stack([zeros, ones]),
0, 1),
requires_grad=False)
uncertain_values = torch.zeros(self._mini_batch_size, 2)
uncertain_values.fill_(0.5)
self.ys_uncertain = Variable(uncertain_values, requires_grad=False)
if self._use_cuda:
self.ys_true = self.ys_true.cuda()
self.yu_true = self.yu_true.cuda()
self.ys_uncertain = self.ys_uncertain.cuda()
def init_model(self, create_model_function):
"""Resume training if necessary (args.--resume flag is True), or call the
create_model_function to initialize a new model. This function must be called
before train.
The create_model_function takes one argument: the name of the model to be
created.
"""
args = self.args
mini_batch_size = self.mini_batch_size
# restrict limits to actual size of datasets:
training_set_length = (len(self.problem.train_loader())) * mini_batch_size
if hasattr(args, 'num_training') and args.num_training > training_set_length:
args.num_training = training_set_length
unsup_set_length = (len(self.problem.reg_loader())) * mini_batch_size
if hasattr(args, 'num_shaving') and args.num_shaving > unsup_set_length:
args.num_shaving = unsup_set_length
test_set_length = (len(self.problem.test_loader())) * mini_batch_size
if hasattr(args, 'num_validation') and args.num_validation > test_set_length:
args.num_validation = test_set_length
self.max_regularization_examples = args.num_shaving if hasattr(args, 'num_shaving') else 0
self.max_validation_examples = args.num_validation if hasattr(args, 'num_validation') else 0
self.max_training_examples = args.num_training if hasattr(args, 'num_training') else 0
self.unsuploader = self.problem.reg_loader()
model_built = False
self.best_performance_metrics = None
self.best_model = None
def rmse(y, y_hat):
"""Compute root mean squared error"""
return torch.sqrt(torch.mean((y - y_hat).pow(2)))
self.agreement_loss = MSELoss()
if hasattr(args, 'resume') and args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = None
try:
checkpoint = torch.load('./checkpoint/ckpt_{}.t7'.format(args.checkpoint_key))
except FileNotFoundError:
pass
if checkpoint is not None:
self.net = checkpoint['net']
self.best_acc = checkpoint['acc']
self.start_epoch = checkpoint['epoch']
self.best_model = checkpoint['best-model']
self.best_model_confusion_matrix = checkpoint['confusion-matrix']
model_built = True
else:
print("Could not load model checkpoint, unable to --resume.")
model_built = False
if not model_built:
print('==> Building model {}'.format(args.model))
self.net = create_model_function(args.model, self.problem, dual=self.args.mode=="fm_loss")
if self.use_cuda:
self.net.cuda()
cudnn.benchmark = True
self.optimizer_training = torch.optim.SGD(self.net.parameters(), lr=args.lr, momentum=args.momentum,
weight_decay=args.L2)
self.scheduler_train = \
construct_scheduler(self.optimizer_training, 'max', factor=0.5,
lr_patience=self.args.lr_patience if hasattr(self.args, 'lr_patience') else 10,
ureg_reset_every_n_epoch=self.args.reset_lr_every_n_epochs
if hasattr(self.args, 'reset_lr_every_n_epochs')
else None)
def init_model(self, create_model_function):
"""Resume training if necessary (args.--resume flag is True), or call the
create_model_function to initialize a new model. This function must be called
before train.
The create_model_function takes one argument: the name of the model to be
created.
"""
args = self.args
mini_batch_size = self.mini_batch_size
# restrict limits to actual size of datasets:
training_set_length = (len(self.problem.train_loader())) * mini_batch_size
if hasattr(args, 'num_training') and args.num_training > training_set_length:
args.num_training = training_set_length
unsup_set_length = (len(self.problem.reg_loader())) * mini_batch_size
if hasattr(args, 'num_shaving') and args.num_shaving > unsup_set_length:
args.num_shaving = unsup_set_length
test_set_length = (len(self.problem.test_loader())) * mini_batch_size
if hasattr(args, 'num_validation') and args.num_validation > test_set_length:
args.num_validation = test_set_length
self.max_regularization_examples = args.num_shaving if hasattr(args, 'num_shaving') else 0
self.max_validation_examples = args.num_validation if hasattr(args, 'num_validation') else 0
self.max_training_examples = args.num_training if hasattr(args, 'num_training') else 0
self.unsuploader = self.problem.reg_loader()
model_built = False
self.best_performance_metrics = None
self.best_model = None
self.optimizer = None
# try loading a pre-trained model:
encoder, generator, test_loss, epoch = self.load_pretrained()
if not args.pretrain and encoder is None:
print("You did not select --pretrain, and was not able to load pre-trained model with pretrained-key="+args.pretrained_key)
exit(1)
if hasattr(args, 'resume') and args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = None
try:
checkpoint = torch.load('./checkpoint/ckpt_{}.t7'.format(args.checkpoint_key))
except FileNotFoundError:
pass
if checkpoint is not None:
self.net = checkpoint['net']
# use the pretrained data for encoder and generator if available:
self.image_encoder = checkpoint['encoder'] if encoder is None else encoder
self.image_generator = checkpoint['generator'] if generator is None else generator
if self.args.pretrain:
self.best_loss = checkpoint['test_loss']
else:
self.best_acc = checkpoint['acc']
self.start_epoch = checkpoint['epoch'] if not self.args.pretrain else epoch
# rebuild a model fresh, we only
model_built = True
else:
print("Could not load model checkpoint, unable to --resume.")
model_built = False
self.best_loss = test_loss
self.start_epoch = epoch
if not model_built:
print('==> Building model {}'.format(args.model))
self.net = create_model_function(args.model, self.problem)
if self.use_cuda:
self.net.cuda()
self.image_encoder = ImageEncoder(model=self.net, number_encoder_features=self.args.num_encoder_features,
number_representation_features=self.args.num_representation_features,
input_shape=self.problem.example_size(), use_cuda=self.use_cuda,
ngpu=self.args.n_gpus) \
if encoder is None else encoder
self.image_generator = ImageGenerator(number_encoded_features=self.args.num_representation_features,
number_of_generator_features=self.args.num_generator_features,
output_shape=self.problem.example_size(),
use_cuda=self.use_cuda,
ngpu=self.args.n_gpus) \
if generator is None else generator
print(self.image_encoder)
if self.use_cuda:
self.image_encoder.cuda()
self.image_generator.cuda()
if self.best_model is not None:
self.best_model.cuda()
self.best_model_confusion_matrix = self.best_model_confusion_matrix.cuda()
if self.start_epoch is None:
self.start_epoch=self.args.start_epoch
cudnn.benchmark = True
all_params = []
all_params += list(self.image_generator.main.parameters())
all_params += list(self.image_encoder.projection.parameters())
all_params += list(self.image_encoder.main.parameters())
self.optimizer = torch.optim.Adam(all_params, lr=self.args.lr, betas=(0.5, 0.999), weight_decay=self.args.L2)
self.scheduler_train = \
construct_scheduler(self.optimizer, 'min', factor=0.5,
lr_patience=self.args.lr_patience if hasattr(self.args, 'lr_patience') else 10,
ureg_reset_every_n_epoch=self.args.reset_lr_every_n_epochs
if hasattr(self.args, 'reset_lr_every_n_epochs')
else None)
def init_model(self, create_model_function):
"""Resume training if necessary (args.--resume flag is True), or call the
create_model_function to initialize a new model. This function must be called
before train.
The create_model_function takes one argument: the name of the model to be
created.
"""
args = self.args
mini_batch_size = self.mini_batch_size
# restrict limits to actual size of datasets:
training_set_length = (len(self.problem.train_loader())) * mini_batch_size
if args.num_training > training_set_length:
args.num_training = training_set_length
unsup_set_length = (len(self.problem.reg_loader())) * mini_batch_size
if args.num_shaving > unsup_set_length:
args.num_shaving = unsup_set_length
test_set_length = (len(self.problem.test_loader())) * mini_batch_size
if args.num_validation > test_set_length:
args.num_validation = test_set_length
self.max_regularization_examples = args.num_shaving
self.max_validation_examples = args.num_validation
self.max_training_examples = args.num_training
self.unsuploader = self.problem.reg_loader()
model_built = False
# setup the function which decides which layer outputs to use for ureg training:
if args.threshold_activation_size is not None:
include_output_function = lambda layer_index, activations: True \
if activations.size()[1] <= args.threshold_activation_size else False
elif args.include_layer_indices is not None:
layer_indices = []
for index in map(int, args.include_layer_indices.split(",")):
layer_indices.append(index)
include_output_function = lambda layer_index, activations: True \
if layer_index in layer_indices else False
else:
include_output_function = None
if args.threshold_activation_size is not None and args.include_layer_indices is not None:
print("--include-layer-indices and --threshold-activation-size are mutually exclusive, please pick one.")
exit(1)
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = None
try:
checkpoint = torch.load('./checkpoint/ckpt_{}.t7'.format(args.checkpoint_key))
except FileNotFoundError:
pass
if checkpoint is not None:
self.net = checkpoint['net']
self.best_acc = checkpoint['acc']
self.start_epoch = checkpoint['epoch']
ureg_enabled = checkpoint['ureg']
if ureg_enabled:
ureg = URegularizer(self.net, mini_batch_size, args.ureg_num_features,
args.ureg_alpha, args.ureg_learning_rate,
reset_every_epochs=args.ureg_reset_every_n_epoch,
do_not_use_scheduler=self.args.constant_learning_rates,
include_output_function=include_output_function)
ureg.set_num_examples(min(len(self.trainloader), args.num_training),
min(len(self.unsuploader), args.num_shaving))
ureg.enable()
if not args.drop_ureg_model:
ureg.resume(checkpoint['ureg_model'])
model_built = True
else:
print("Could not load model checkpoint, unable to --resume.")
model_built = False
if not model_built:
print('==> Building model {}'.format(args.model))
self.net = create_model_function(args.model)
if self.use_cuda:
self.net.cuda()
cudnn.benchmark = True
self.optimizer_training = torch.optim.SGD(self.net.parameters(), lr=args.lr, momentum=args.momentum,
weight_decay=args.L2)
self.optimizer_reg = torch.optim.SGD(self.net.parameters(), lr=args.shave_lr, momentum=args.momentum,
weight_decay=args.L2)
self.ureg = URegularizer(self.net, self.mini_batch_size, num_features=args.ureg_num_features,
alpha=args.ureg_alpha,
learning_rate=args.ureg_learning_rate,
reset_every_epochs=args.ureg_reset_every_n_epoch,
do_not_use_scheduler=self.args.constant_learning_rates,
include_output_function=include_output_function)
if args.ureg:
self.ureg.enable()
self.ureg.set_num_examples(min(len(self.trainloader) * mini_batch_size, args.num_training),
min(len(self.unsuploader) * mini_batch_size, args.num_shaving))
print("weights: {} ".format(self.ureg.loss_weights(None, None)))
self.ureg.forget_model(args.ureg_reset_every_n_epoch)
print(
"ureg is enabled with alpha={}, reset every {} epochs. ".format(args.ureg_alpha,
args.ureg_reset_every_n_epoch))
else:
self.ureg.disable()
self.ureg.set_num_examples(min(len(self.trainloader) * mini_batch_size, args.num_training),
min(len(self.unsuploader) * mini_batch_size, args.num_shaving))
print("ureg is disabled")
self.scheduler_train = \
construct_scheduler(self.optimizer_training, 'max', factor=0.9,
lr_patience=self.args.lr_patience,
ureg_reset_every_n_epoch=self.args.ureg_reset_every_n_epoch)
# the regularizer aims to increase uncertainty between training and unsup set. Larger accuracies are closer to
# success, optimize for max of the accuracy.
self.scheduler_reg = \
construct_scheduler(self.optimizer_reg,
'max', extra_patience=0 if args.mode == "one_pass" else 5,
lr_patience=self.args.lr_patience, factor=0.9,
ureg_reset_every_n_epoch=self.args.ureg_reset_every_n_epoch)
self.num_shaving_epochs = self.args.shaving_epochs
if self.args.num_training > self.args.num_shaving:
self.num_shaving_epochs = round((self.args.num_training + 1) / self.args.num_shaving)
print("--shaving-epochs overridden to " + str(self.num_shaving_epochs))
else:
print("shaving-epochs set to " + str(self.num_shaving_epochs))