def propose_H(self, dataset):
config = self.get_base_config(dataset)
# Wrap the class in KWLWrapper
original_class_name = config.model.__class__.__name__
config.model = KWayLogisticWrapper(config.model)
config.model = config.model.to(self.args.device)
h_path = Models.get_ref_model_path(self.args, original_class_name, dataset.name, suffix_str='KLogistic')
best_h_path = path.join(h_path, 'model.best.pth')
trainer = IterativeTrainer(config, self.args)
if not path.isfile(best_h_path):
raise NotImplementedError("Please use setup_model to pretrain the networks first!")
else:
print(colored('Loading H1 model from %s'%best_h_path, 'red'))
config.model.load_state_dict(torch.load(best_h_path))
trainer.run_epoch(0, phase='all')
test_average_acc = config.logger.get_measure('all_accuracy').mean_epoch(epoch=0)
print("All average accuracy %s"%colored('%.4f%%'%(test_average_acc*100), 'red'))
self.base_model = config.model
self.base_model.eval()
def propose_H(self, dataset):
config = self.get_base_config(dataset)
import models as Models
if self.default_model == 0:
config.model.netid = "BCE." + config.model.netid
else:
config.model.netid = "MSE." + config.model.netid
home_path = Models.get_ref_model_path(self.args, config.model.__class__.__name__, dataset.name, suffix_str=config.model.netid)
hbest_path = path.join(home_path, 'model.best.pth')
best_h_path = hbest_path
trainer = IterativeTrainer(config, self.args)
if not path.isfile(best_h_path):
raise NotImplementedError("%s not found!, Please use setup_model to pretrain the networks first!"%best_h_path)
else:
print(colored('Loading H1 model from %s'%best_h_path, 'red'))
config.model.load_state_dict(torch.load(best_h_path))
trainer.run_epoch(0, phase='all')
test_loss = config.logger.get_measure('all_loss').mean_epoch(epoch=0)
print("All average loss %s"%colored('%.4f'%(test_loss), 'red'))
self.base_model = config.model
self.base_model.eval()
def propose_H(self, dataset):
config = self.get_base_config(dataset)
from models import get_ref_model_path
h_path = get_ref_model_path(self.args, config.model.__class__.__name__,
dataset.name)
best_h_path = path.join(h_path, 'model.best.pth')
trainer = IterativeTrainer(config, self.args)
if not path.isfile(best_h_path):
raise NotImplementedError(
"Please use model_setup to pretrain the networks first!")
else:
print(colored('Loading H1 model from %s' % best_h_path, 'red'))
config.model.load_state_dict(torch.load(best_h_path))
trainer.run_epoch(0, phase='all')
test_average_acc = config.logger.get_measure(
'all_accuracy').mean_epoch(epoch=0)
print("All average accuracy %s" %
colored('%.4f%%' % (test_average_acc * 100), 'red'))
self.base_model = config.model
self.base_model.eval()
def train_autoencoder(args, model, dataset, BCE_Loss):
if BCE_Loss:
model.netid = "BCE." + model.netid
else:
model.netid = "MSE." + model.netid
home_path = Models.get_ref_model_path(args, model.__class__.__name__, dataset.name, model_setup=True, suffix_str=model.netid)
hbest_path = os.path.join(home_path, 'model.best.pth')
hlast_path = os.path.join(home_path, 'model.last.pth')
if not os.path.isdir(home_path):
os.makedirs(home_path)
if not os.path.isfile(hbest_path+".done"):
config = get_ae_config(args, model, dataset, BCE_Loss=BCE_Loss)
trainer = IterativeTrainer(config, args)
print(colored('Training from scratch', 'green'))
best_loss = 999999999
for epoch in range(1, config.max_epoch+1):
# Track the learning rates.
lrs = [float(param_group['lr']) for param_group in config.optim.param_groups]
config.logger.log('LRs', lrs, epoch)
config.logger.get_measure('LRs').legend = ['LR%d'%i for i in range(len(lrs))]
# One epoch of train and test.
trainer.run_epoch(epoch, phase='train')
trainer.run_epoch(epoch, phase='test')
train_loss = config.logger.get_measure('train_loss').mean_epoch()
test_loss = config.logger.get_measure('test_loss').mean_epoch()
config.scheduler.step(train_loss)
if config.visualize:
# Show the average losses for all the phases in one figure.
config.logger.visualize_average_keys('.*_loss', 'Average Loss', trainer.visdom)
config.logger.visualize_average_keys('.*_accuracy', 'Average Accuracy', trainer.visdom)
config.logger.visualize_average('LRs', trainer.visdom)
# Save the logger for future reference.
torch.save(config.logger.measures, os.path.join(home_path, 'logger.pth'))
# Saving a checkpoint. Enable if needed!
# if args.save and epoch % 10 == 0:
# print('Saving a %s at iter %s'%(colored('snapshot', 'yellow'), colored('%d'%epoch, 'yellow')))
# torch.save(config.model.state_dict(), os.path.join(home_path, 'model.%d.pth'%epoch))
if args.save and test_loss < best_loss:
print('Updating the on file model with %s'%(colored('%.4f'%test_loss, 'red')))
best_loss = test_loss
torch.save(config.model.state_dict(), hbest_path)
torch.save({'finished':True}, hbest_path+".done")
torch.save(config.model.state_dict(), hlast_path)
if config.visualize:
trainer.visdom.save([trainer.visdom.env])
else:
print("Skipping %s"%(colored(home_path, 'yellow')))
def train_classifier(args, model, dataset):
config = None
for mid in range(5):
home_path = Models.get_ref_model_path(args,
model.__class__.__name__,
dataset.name,
model_setup=True,
suffix_str='DE.%d' % mid)
hbest_path = os.path.join(home_path, 'model.best.pth')
if not os.path.isdir(home_path):
os.makedirs(home_path)
else:
if os.path.isfile(hbest_path + ".done"):
print("Skipping %s" % (colored(home_path, 'yellow')))
continue
config = get_classifier_config(args,
model.__class__(),
dataset,
mid=mid)
trainer = IterativeTrainer(config, args)
if not os.path.isfile(hbest_path + ".done"):
print(colored('Training from scratch', 'green'))
best_accuracy = -1
for epoch in range(1, config.max_epoch + 1):
# Track the learning rates.
lrs = [
float(param_group['lr'])
for param_group in config.optim.param_groups
]
config.logger.log('LRs', lrs, epoch)
config.logger.get_measure('LRs').legend = [
'LR%d' % i for i in range(len(lrs))
]
# One epoch of train and test.
trainer.run_epoch(epoch, phase='train')
trainer.run_epoch(epoch, phase='test')
train_loss = config.logger.get_measure(
'train_loss').mean_epoch()
config.scheduler.step(train_loss)
if config.visualize:
# Show the average losses for all the phases in one figure.
config.logger.visualize_average_keys(
'.*_loss', 'Average Loss', trainer.visdom)
config.logger.visualize_average_keys(
'.*_accuracy', 'Average Accuracy', trainer.visdom)
config.logger.visualize_average('LRs', trainer.visdom)
test_average_acc = config.logger.get_measure(
'test_accuracy').mean_epoch()
# Save the logger for future reference.
torch.save(config.logger.measures,
os.path.join(home_path, 'logger.pth'))
# Saving a checkpoint. Enable if needed!
# if args.save and epoch % 10 == 0:
# print('Saving a %s at iter %s'%(colored('snapshot', 'yellow'), colored('%d'%epoch, 'yellow')))
# torch.save(config.model.state_dict(), os.path.join(home_path, 'model.%d.pth'%epoch))
if args.save and best_accuracy < test_average_acc:
print('Updating the on file model with %s' %
(colored('%.4f' % test_average_acc, 'red')))
best_accuracy = test_average_acc
torch.save(config.model.state_dict(), hbest_path)
torch.save({'finished': True}, hbest_path + ".done")
if config.visualize:
trainer.visdom.save([trainer.visdom.env])
else:
print("Skipping %s" % (colored(home_path, 'yellow')))
print("Loading the best model.")
config.model.load_state_dict(torch.load(hbest_path))
config.model.eval()
trainer.run_epoch(0, phase='all')
test_average_acc = config.logger.get_measure(
'all_accuracy').mean_epoch(epoch=0)
print("All average accuracy %s" %
colored('%.4f%%' % (test_average_acc * 100), 'red'))
def train_H(self, dataset):
# Wrap the (mixture)dataset in SubDataset so to easily
# split it later.
dataset = SubDataset('%s-%s' % (self.args.D1, self.args.D2), dataset,
torch.arange(len(dataset)).int())
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if self.args.D1 in Global.mirror_augment:
print(colored("Mirror augmenting %s" % self.args.D1, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# As suggested by the authors.
all_temperatures = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000]
all_epsilons = torch.linspace(0, 0.004, 21)
total_params = len(all_temperatures) * len(all_epsilons)
best_accuracy = -1
h_path = path.join(self.args.experiment_path,
'%s' % (self.__class__.__name__),
'%d' % (self.default_model),
'%s-%s.pth' % (self.args.D1, self.args.D2))
h_parent = path.dirname(h_path)
if not path.isdir(h_parent):
os.makedirs(h_parent)
done_path = h_path + '.done'
trainer, h_config = None, None
if self.args.force_train_h or not path.isfile(done_path):
# Grid search over the temperature and the epsilons.
for i_eps, eps in enumerate(all_epsilons):
for i_temp, temp in enumerate(all_temperatures):
so_far = i_eps * len(all_temperatures) + i_temp + 1
print(
colored(
'Checking eps=%.2e temp=%.1f (%d/%d)' %
(eps, temp, so_far, total_params), 'green'))
start_time = timeit.default_timer()
h_config = self.get_H_config(train_ds=train_ds,
valid_ds=valid_ds,
epsilon=eps,
temperature=temp)
trainer = IterativeTrainer(h_config, self.args)
print(colored('Training from scratch', 'green'))
trainer.run_epoch(0, phase='test')
for epoch in range(1, h_config.max_epoch + 1):
trainer.run_epoch(epoch, phase='train')
trainer.run_epoch(epoch, phase='test')
train_loss = h_config.logger.get_measure(
'train_loss').mean_epoch()
h_config.scheduler.step(train_loss)
# Track the learning rates and threshold.
lrs = [
float(param_group['lr'])
for param_group in h_config.optim.param_groups
]
h_config.logger.log('LRs', lrs, epoch)
h_config.logger.get_measure('LRs').legend = [
'LR%d' % i for i in range(len(lrs))
]
if hasattr(h_config.model, 'H') and hasattr(
h_config.model.H, 'threshold'):
h_config.logger.log(
'threshold',
h_config.model.H.threshold.cpu().numpy(),
epoch - 1)
h_config.logger.get_measure('threshold').legend = [
'threshold'
]
if h_config.visualize:
h_config.logger.get_measure(
'threshold').visualize_all_epochs(
trainer.visdom)
if h_config.visualize:
# Show the average losses for all the phases in one figure.
h_config.logger.visualize_average_keys(
'.*_loss', 'Average Loss', trainer.visdom)
h_config.logger.visualize_average_keys(
'.*_accuracy', 'Average Accuracy',
trainer.visdom)
h_config.logger.visualize_average(
'LRs', trainer.visdom)
test_average_acc = h_config.logger.get_measure(
'test_accuracy').mean_epoch()
if best_accuracy < test_average_acc:
print('Updating the on file model with %s' %
(colored('%.4f' % test_average_acc, 'red')))
best_accuracy = test_average_acc
torch.save(h_config.model.H.state_dict(), h_path)
elapsed = timeit.default_timer() - start_time
print('Hyper-param check (%.2e, %.1f) in %.2fs' %
(eps, temp, elapsed))
torch.save({'finished': True}, done_path)
# If we load the pretrained model directly, we will have to initialize these.
if trainer is None or h_config is None:
h_config = self.get_H_config(train_ds=train_ds,
valid_ds=valid_ds,
epsilon=0,
temperature=1,
will_train=False)
# don't worry about the values of epsilon or temperature. it will be overwritten.
trainer = IterativeTrainer(h_config, self.args)
# Load the best model.
print(colored('Loading H model from %s' % h_path, 'red'))
state_dict = torch.load(h_path)
for key, val in state_dict.items():
if val.shape == torch.Size([]):
state_dict[key] = val.view((1, ))
h_config.model.H.load_state_dict(state_dict)
h_config.model.set_eval_direct(False)
print('Temperature %s Epsilon %s' %
(colored(h_config.model.H.temperature.item(), 'red'),
colored(h_config.model.H.epsilon.item(), 'red')))
trainer.run_epoch(0, phase='testU')
test_average_acc = h_config.logger.get_measure(
'testU_accuracy').mean_epoch(epoch=0)
print("Valid/Test average accuracy %s" %
colored('%.4f%%' % (test_average_acc * 100), 'red'))
self.H_class = h_config.model
self.H_class.eval()
self.H_class.set_eval_direct(False)
return test_average_acc
def train_H(self, dataset):
# Wrap the (mixture)dataset in SubDataset so to easily
# split it later.
from datasets import SubDataset
dataset = SubDataset('%s-%s' % (self.args.D1, self.args.D2), dataset,
torch.arange(len(dataset)).int())
h_path = path.join(self.args.experiment_path,
'%s' % (self.__class__.__name__),
'%d' % (self.default_model),
'%s->%s.pth' % (self.args.D1, self.args.D2))
h_parent = path.dirname(h_path)
if not path.isdir(h_parent):
os.makedirs(h_parent)
done_path = h_path + '.done'
will_train = self.args.force_train_h or not path.isfile(done_path)
h_config = self.get_H_config(dataset, will_train)
trainer = IterativeTrainer(h_config, self.args)
if will_train:
print(colored('Training from scratch', 'green'))
best_accuracy = -1
trainer.run_epoch(0, phase='test')
for epoch in range(1, h_config.max_epoch + 1):
trainer.run_epoch(epoch, phase='train')
trainer.run_epoch(epoch, phase='test')
train_loss = h_config.logger.get_measure(
'train_loss').mean_epoch()
h_config.scheduler.step(train_loss)
# Track the learning rates and threshold.
lrs = [
float(param_group['lr'])
for param_group in h_config.optim.param_groups
]
h_config.logger.log('LRs', lrs, epoch)
h_config.logger.get_measure('LRs').legend = [
'LR%d' % i for i in range(len(lrs))
]
viz_params = ['threshold', 'transfer']
for viz_param in viz_params:
if hasattr(h_config.model, 'H') and hasattr(
h_config.model.H, viz_param):
h_config.logger.log(
viz_param,
getattr(h_config.model.H, viz_param).cpu().numpy(),
epoch - 1)
h_config.logger.get_measure(viz_param).legend = [
viz_param
]
if h_config.visualize:
h_config.logger.get_measure(
viz_param).visualize_all_epochs(trainer.visdom)
if h_config.visualize:
# Show the average losses for all the phases in one figure.
h_config.logger.visualize_average_keys(
'.*_loss', 'Average Loss', trainer.visdom)
h_config.logger.visualize_average_keys(
'.*_accuracy', 'Average Accuracy', trainer.visdom)
h_config.logger.visualize_average('LRs', trainer.visdom)
test_average_acc = h_config.logger.get_measure(
'test_accuracy').mean_epoch()
# Save the logger for future reference.
torch.save(
h_config.logger.measures,
path.join(
h_parent,
'logger.%s->%s.pth' % (self.args.D1, self.args.D2)))
if best_accuracy < test_average_acc:
print('Updating the on file model with %s' %
(colored('%.4f' % test_average_acc, 'red')))
best_accuracy = test_average_acc
torch.save(h_config.model.H.state_dict(), h_path)
torch.save({'finished': True}, done_path)
if h_config.visualize:
trainer.visdom.save([trainer.visdom.env])
# Load the best model.
print(colored('Loading H model from %s' % h_path, 'red'))
h_config.model.H.load_state_dict(torch.load(h_path))
h_config.model.set_eval_direct(False)
trainer.run_epoch(0, phase='testU')
test_average_acc = h_config.logger.get_measure(
'testU_accuracy').mean_epoch(epoch=0)
print("Valid/Test average accuracy %s" %
colored('%.4f%%' % (test_average_acc * 100), 'red'))
self.H_class = h_config.model
self.H_class.eval()
self.H_class.set_eval_direct(False)
return test_average_acc
def train_variational_autoencoder(args, model, dataset, BCE_Loss=True):
if BCE_Loss:
model.netid = "BCE." + model.netid
else:
model.netid = "MSE." + model.netid
home_path = Models.get_ref_model_path(args,
model.__class__.__name__,
dataset.name,
model_setup=True,
suffix_str=model.netid)
hbest_path = os.path.join(home_path, 'model.best.pth')
hlast_path = os.path.join(home_path, 'model.last.pth')
if not os.path.isdir(home_path):
os.makedirs(home_path)
if not os.path.isfile(hbest_path + ".done"):
config = get_vae_config(args, model, dataset, home_path, BCE_Loss)
trainer = IterativeTrainer(config, args)
print(colored('Training from scratch', 'green'))
best_loss = 999999999
for epoch in range(1, config.max_epoch + 1):
# Track the learning rates.
lrs = [
float(param_group['lr'])
for param_group in config.optim.param_groups
]
config.logger.log('LRs', lrs, epoch)
config.logger.get_measure('LRs').legend = [
'LR%d' % i for i in range(len(lrs))
]
# One epoch of train and test.
trainer.run_epoch(epoch, phase='train')
trainer.run_epoch(epoch, phase='test')
train_loss = config.logger.get_measure('train_loss').mean_epoch()
test_loss = config.logger.get_measure('test_loss').mean_epoch()
config.logger.writer.add_scalar('train_loss', train_loss, epoch)
config.logger.writer.add_scalar('test_loss', test_loss, epoch)
config.scheduler.step(train_loss)
# vis in tensorboard
for (image, label) in config.valid_loader:
prediction = model(image.cuda()).data.cpu().squeeze().numpy()
prediction = (prediction - prediction.min()) / (
prediction.max() - prediction.min())
if len(prediction.shape) > 3 and prediction.shape[1] == 3:
prediction = prediction.transpose(
(0, 2, 3, 1)) # change to N W H C
N = min(prediction.shape[0], 5)
fig, ax = plt.subplots(N, 2)
image = image.data.squeeze().numpy()
image = (image - image.min()) / (image.max() - image.min())
if len(image.shape) > 3 and image.shape[1] == 3:
image = image.transpose((0, 2, 3, 1))
for i in range(N):
ax[i, 0].imshow(prediction[i])
ax[i, 1].imshow(image[i])
config.logger.writer.add_figure('Vis', fig, epoch)
plt.close(fig)
break
if config.visualize:
# Show the average losses for all the phases in one figure.
config.logger.visualize_average_keys('.*_loss', 'Average Loss',
trainer.visdom)
config.logger.visualize_average_keys('.*_accuracy',
'Average Accuracy',
trainer.visdom)
config.logger.visualize_average('LRs', trainer.visdom)
# Save the logger for future reference.
torch.save(config.logger.measures,
os.path.join(home_path, 'logger.pth'))
# Saving a checkpoint. Enable if needed!
# if args.save and epoch % 10 == 0:
# print('Saving a %s at iter %s'%(colored('snapshot', 'yellow'), colored('%d'%epoch, 'yellow')))
# torch.save(config.model.state_dict(), os.path.join(home_path, 'model.%d.pth'%epoch))
if args.save and test_loss < best_loss:
print('Updating the on file model with %s' %
(colored('%.4f' % test_loss, 'red')))
best_loss = test_loss
torch.save(config.model.state_dict(), hbest_path)
torch.save({'finished': True}, hbest_path + ".done")
torch.save(config.model.state_dict(), hlast_path)
if config.visualize:
trainer.visdom.save([trainer.visdom.env])
else:
print("Skipping %s" % (colored(home_path, 'yellow')))