def propose_H(self, dataset):
assert self.default_model > 0, 'KNN needs K>0'
if self.base_model is not None:
self.base_model.base_data = None
self.base_model = None
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s" % dataset.name, 'green'))
new_train_ds = dataset + MirroredDataset(dataset)
dataset = new_train_ds
n_data = len(dataset)
n_dim = dataset[0][0].numel()
self.base_data = torch.zeros(n_data, n_dim, dtype=torch.float32)
with tqdm(total=n_data,
disable=bool(os.environ.get("DISABLE_TQDM", False))) as pbar:
pbar.set_description('Caching X_train for %d-nn' %
self.default_model)
for i, (x, _) in enumerate(dataset):
self.base_data[i].copy_(x.view(-1))
pbar.update()
# self.base_data = torch.cat([x.view(1, -1) for x,_ in dataset])
self.base_model = KNNModel(self.base_data,
k=self.default_model).to(self.args.device)
self.base_model.eval()
def get_ae_config(args, model, dataset, BCE_Loss):
print("Preparing training D1 for %s"%(dataset.name))
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s"%dataset.name, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True)
valid_loader = DataLoader(valid_ds, batch_size=args.batch_size, num_workers=args.workers, pin_memory=True)
all_loader = DataLoader(dataset, batch_size=args.batch_size, num_workers=args.workers, pin_memory=True)
# Set up the model
model = model.to(args.device)
# Set up the criterion
criterion = None
if BCE_Loss:
criterion = nn.BCEWithLogitsLoss().to(args.device)
else:
criterion = nn.MSELoss().to(args.device)
model.default_sigmoid = True
# Set up the config
config = IterativeTrainerConfig()
config.name = 'autoencoder_%s_%s'%(dataset.name, model.preferred_name())
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {'dataset' : train_loader, 'backward': True},
'test': {'dataset' : valid_loader, 'backward': False},
'all': {'dataset' : all_loader, 'backward': False},
}
config.criterion = criterion
config.classification = False
config.cast_float_label = False
config.autoencoder_target = True
config.stochastic_gradient = True
config.visualize = not args.no_visualize
config.sigmoid_viz = BCE_Loss
config.model = model
config.logger = Logger()
config.optim = optim.Adam(model.parameters(), lr=1e-3)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim, patience=10, threshold=1e-3, min_lr=1e-6, factor=0.1, verbose=True)
config.max_epoch = 120
if hasattr(model, 'train_config'):
model_train_config = model.train_config()
for key, value in model_train_config.iteritems():
print('Overriding config.%s'%key)
config.__setattr__(key, value)
return config
def get_H_config(self, dataset, will_train=True):
print("Preparing training D1+D2 (H)")
print("Mixture size: %s"%colored('%d'%len(dataset), 'green'))
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if self.args.D1 in Global.mirror_augment:
print(colored("Mirror augmenting %s"%self.args.D1, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds, batch_size=self.args.batch_size, shuffle=True, num_workers=self.args.workers, pin_memory=True)
valid_loader = DataLoader(valid_ds, batch_size=self.args.batch_size, shuffle=True, num_workers=self.args.workers, pin_memory=True)
# To make the threshold learning, actually threshold learning
# the margin must be set to 0.
criterion = SVMLoss(margin=0.0).to(self.args.device)
# Set up the model
model = MCDropoutModelWrapper(self.base_model).to(self.args.device)
old_valid_loader = valid_loader
# By definition, this approach is uncacheable :(
# Set up the config
config = IterativeTrainerConfig()
base_model_name = self.base_model.__class__.__name__
if hasattr(self.base_model, 'preferred_name'):
base_model_name = self.base_model.preferred_name()
config.name = '_%s[%s](%s-%s)'%(self.__class__.__name__, base_model_name, self.args.D1, self.args.D2)
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {'dataset' : train_loader, 'backward': True},
'test': {'dataset' : valid_loader, 'backward': False},
'testU': {'dataset' : old_valid_loader, 'backward': False},
}
config.criterion = criterion
config.classification = True
config.cast_float_label = True
config.stochastic_gradient = True
config.visualize = not self.args.no_visualize
config.model = model
config.optim = optim.Adagrad(model.H.parameters(), lr=1e-1, weight_decay=0)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim, patience=10, threshold=1e-1, min_lr=1e-8, factor=0.1, verbose=True)
h_path = path.join(self.args.experiment_path, '%s' % (self.__class__.__name__),
'%d' % (self.default_model),
'%s-%s.pth' % (self.args.D1, self.args.D2))
h_parent = path.dirname(h_path)
config.logger = Logger(h_parent)
config.max_epoch = 100
return config
def propose_H(self, dataset):
assert self.default_model > 0, 'KNN needs K>0'
if self.base_model is not None:
self.base_model.base_data = None
self.base_model = None
# Set up the base0-model
base_model = Global.get_ref_classifier(dataset.name)[0]().to(
self.args.device)
from models import get_ref_model_path
home_path = get_ref_model_path(self.args,
base_model.__class__.__name__,
dataset.name)
hbest_path = path.join(home_path, 'model.best.pth')
best_h_path = hbest_path
print(colored('Loading H1 model from %s' % best_h_path, 'red'))
base_model.load_state_dict(torch.load(best_h_path))
base_model.eval()
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s" % dataset.name, 'green'))
new_train_ds = dataset + MirroredDataset(dataset)
dataset = new_train_ds
# Initialize the multi-threaded loaders.
all_loader = DataLoader(dataset,
batch_size=self.args.batch_size,
num_workers=1,
pin_memory=True)
n_data = len(dataset)
n_dim = base_model.partial_forward(dataset[0][0].to(
self.args.device).unsqueeze(0)).numel()
print('nHidden %d' % (n_dim))
self.base_data = torch.zeros(n_data, n_dim, dtype=torch.float32)
base_ind = 0
with torch.set_grad_enabled(False):
with tqdm(total=len(all_loader),
disable=bool(os.environ.get("DISABLE_TQDM",
False))) as pbar:
pbar.set_description('Caching X_train for %d-nn' %
self.default_model)
for i, (x, _) in enumerate(all_loader):
n_data = x.size(0)
output = base_model.partial_forward(x.to(
self.args.device)).data
self.base_data[base_ind:base_ind + n_data].copy_(output)
base_ind = base_ind + n_data
pbar.update()
# self.base_data = torch.cat([x.view(1, -1) for x,_ in dataset])
self.base_model = AEKNNModel(base_model,
self.base_data,
k=self.default_model,
SV=True).to(self.args.device)
self.base_model.eval()
def get_classifier_config(args, model, dataset, mid=0):
print("Preparing training D1 for %s" % (dataset.name))
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s" % dataset.name, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds,
batch_size=args.batch_size / 2,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
valid_loader = DataLoader(valid_ds,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True)
all_loader = DataLoader(dataset,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True)
import methods.deep_ensemble as DE
# Set up the model
model = DE.DeepEnsembleWrapper(model).to(args.device)
# Set up the criterion
criterion = DE.DeepEnsembleLoss(ensemble_network=model).to(args.device)
# Set up the config
config = IterativeTrainerConfig()
base_model_name = model.__class__.__name__
if hasattr(model, 'preferred_name'):
base_model_name = model.preferred_name()
config.name = 'DeepEnsemble_%s_%s(%d)' % (dataset.name, base_model_name,
mid)
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {
'dataset': train_loader,
'backward': True
},
'test': {
'dataset': valid_loader,
'backward': False
},
'all': {
'dataset': all_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.stochastic_gradient = True
config.visualize = not args.no_visualize
config.model = model
config.logger = Logger()
config.optim = optim.Adam(model.parameters(), lr=1e-3)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim,
patience=10,
threshold=1e-2,
min_lr=1e-6,
factor=0.1,
verbose=True)
config.max_epoch = 120
if hasattr(model.model, 'train_config'):
model_train_config = model.model.train_config()
for key, value in model_train_config.iteritems():
print('Overriding config.%s' % key)
config.__setattr__(key, value)
return config
def get_H_config(self, dataset, will_train=True):
print("Preparing training D1+D2 (H)")
print("Mixture size: %s" % colored('%d' % len(dataset), 'green'))
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if self.args.D1 in Global.mirror_augment:
print(colored("Mirror augmenting %s" % self.args.D1, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds,
batch_size=self.args.batch_size,
shuffle=True,
num_workers=self.args.workers,
pin_memory=True)
valid_loader = DataLoader(valid_ds,
batch_size=self.args.batch_size,
shuffle=True,
num_workers=self.args.workers,
pin_memory=True)
# To make the threshold learning, actually threshold learning
# the margin must be set to 0.
criterion = SVMLoss(margin=0.0).to(self.args.device)
# Set up the model
model = DeepEnsembleModelWrapper(self.base_model).to(self.args.device)
old_valid_loader = valid_loader
if will_train:
# cache the subnetwork for faster optimization.
from methods import get_cached
from torch.utils.data.dataset import TensorDataset
trainX, trainY = get_cached(model, train_loader, self.args.device)
validX, validY = get_cached(model, valid_loader, self.args.device)
new_train_ds = TensorDataset(trainX, trainY)
new_valid_ds = TensorDataset(validX, validY)
# Initialize the new multi-threaded loaders.
train_loader = DataLoader(new_train_ds,
batch_size=2048,
shuffle=True,
num_workers=0,
pin_memory=False)
valid_loader = DataLoader(new_valid_ds,
batch_size=2048,
shuffle=True,
num_workers=0,
pin_memory=False)
# Set model to direct evaluation (for cached data)
model.set_eval_direct(True)
# Set up the config
config = IterativeTrainerConfig()
base_model_name = self.base_model.preferred_name()
config.name = '_%s[%s](%s->%s)' % (self.__class__.__name__,
base_model_name, self.args.D1,
self.args.D2)
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {
'dataset': train_loader,
'backward': True
},
'test': {
'dataset': valid_loader,
'backward': False
},
'testU': {
'dataset': old_valid_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.cast_float_label = True
config.stochastic_gradient = True
config.visualize = not self.args.no_visualize
config.model = model
config.optim = optim.Adagrad(model.H.parameters(),
lr=1e-1,
weight_decay=0)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim,
patience=10,
threshold=1e-1,
min_lr=1e-8,
factor=0.1,
verbose=True)
config.logger = Logger()
config.max_epoch = 100
return config
def train_H(self, dataset):
# Wrap the (mixture)dataset in SubDataset so to easily
# split it later.
dataset = SubDataset('%s-%s' % (self.args.D1, self.args.D2), dataset,
torch.arange(len(dataset)).int())
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if self.args.D1 in Global.mirror_augment:
print(colored("Mirror augmenting %s" % self.args.D1, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# As suggested by the authors.
all_temperatures = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000]
all_epsilons = torch.linspace(0, 0.004, 21)
total_params = len(all_temperatures) * len(all_epsilons)
best_accuracy = -1
h_path = path.join(self.args.experiment_path,
'%s' % (self.__class__.__name__),
'%d' % (self.default_model),
'%s-%s.pth' % (self.args.D1, self.args.D2))
h_parent = path.dirname(h_path)
if not path.isdir(h_parent):
os.makedirs(h_parent)
done_path = h_path + '.done'
trainer, h_config = None, None
if self.args.force_train_h or not path.isfile(done_path):
# Grid search over the temperature and the epsilons.
for i_eps, eps in enumerate(all_epsilons):
for i_temp, temp in enumerate(all_temperatures):
so_far = i_eps * len(all_temperatures) + i_temp + 1
print(
colored(
'Checking eps=%.2e temp=%.1f (%d/%d)' %
(eps, temp, so_far, total_params), 'green'))
start_time = timeit.default_timer()
h_config = self.get_H_config(train_ds=train_ds,
valid_ds=valid_ds,
epsilon=eps,
temperature=temp)
trainer = IterativeTrainer(h_config, self.args)
print(colored('Training from scratch', 'green'))
trainer.run_epoch(0, phase='test')
for epoch in range(1, h_config.max_epoch + 1):
trainer.run_epoch(epoch, phase='train')
trainer.run_epoch(epoch, phase='test')
train_loss = h_config.logger.get_measure(
'train_loss').mean_epoch()
h_config.scheduler.step(train_loss)
# Track the learning rates and threshold.
lrs = [
float(param_group['lr'])
for param_group in h_config.optim.param_groups
]
h_config.logger.log('LRs', lrs, epoch)
h_config.logger.get_measure('LRs').legend = [
'LR%d' % i for i in range(len(lrs))
]
if hasattr(h_config.model, 'H') and hasattr(
h_config.model.H, 'threshold'):
h_config.logger.log(
'threshold',
h_config.model.H.threshold.cpu().numpy(),
epoch - 1)
h_config.logger.get_measure('threshold').legend = [
'threshold'
]
if h_config.visualize:
h_config.logger.get_measure(
'threshold').visualize_all_epochs(
trainer.visdom)
if h_config.visualize:
# Show the average losses for all the phases in one figure.
h_config.logger.visualize_average_keys(
'.*_loss', 'Average Loss', trainer.visdom)
h_config.logger.visualize_average_keys(
'.*_accuracy', 'Average Accuracy',
trainer.visdom)
h_config.logger.visualize_average(
'LRs', trainer.visdom)
test_average_acc = h_config.logger.get_measure(
'test_accuracy').mean_epoch()
if best_accuracy < test_average_acc:
print('Updating the on file model with %s' %
(colored('%.4f' % test_average_acc, 'red')))
best_accuracy = test_average_acc
torch.save(h_config.model.H.state_dict(), h_path)
elapsed = timeit.default_timer() - start_time
print('Hyper-param check (%.2e, %.1f) in %.2fs' %
(eps, temp, elapsed))
torch.save({'finished': True}, done_path)
# If we load the pretrained model directly, we will have to initialize these.
if trainer is None or h_config is None:
h_config = self.get_H_config(train_ds=train_ds,
valid_ds=valid_ds,
epsilon=0,
temperature=1,
will_train=False)
# don't worry about the values of epsilon or temperature. it will be overwritten.
trainer = IterativeTrainer(h_config, self.args)
# Load the best model.
print(colored('Loading H model from %s' % h_path, 'red'))
state_dict = torch.load(h_path)
for key, val in state_dict.items():
if val.shape == torch.Size([]):
state_dict[key] = val.view((1, ))
h_config.model.H.load_state_dict(state_dict)
h_config.model.set_eval_direct(False)
print('Temperature %s Epsilon %s' %
(colored(h_config.model.H.temperature.item(), 'red'),
colored(h_config.model.H.epsilon.item(), 'red')))
trainer.run_epoch(0, phase='testU')
test_average_acc = h_config.logger.get_measure(
'testU_accuracy').mean_epoch(epoch=0)
print("Valid/Test average accuracy %s" %
colored('%.4f%%' % (test_average_acc * 100), 'red'))
self.H_class = h_config.model
self.H_class.eval()
self.H_class.set_eval_direct(False)
return test_average_acc
def get_pcnn_config(args, model, home_path, dataset):
print("Preparing training D1 for %s" % (dataset.name))
sample_im, _ = dataset[0]
obs = sample_im.size()
obs = [int(d) for d in obs]
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s" % dataset.name, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
valid_loader = DataLoader(valid_ds,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True)
all_loader = DataLoader(dataset,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True)
# Set up the model
model = model.to(args.device)
# Set up the criterion
criterion = pcnn_utils.PCNN_Loss(one_d=(model.input_channels == 1))
# Set up the config
config = IterativeTrainerConfig()
config.name = 'PCNN_%s_%s' % (dataset.name, model.preferred_name())
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {
'dataset': train_loader,
'backward': True
},
'test': {
'dataset': valid_loader,
'backward': False
},
'all': {
'dataset': all_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = False
config.cast_float_label = False
config.autoencoder_target = True
config.stochastic_gradient = True
config.visualize = not args.no_visualize
config.model = model
config.logger = Logger(home_path)
config.sampler = lambda x: sample(x.model, 32, obs)
config.optim = optim.Adam(model.parameters(), lr=1e-3)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim,
patience=10,
threshold=1e-2,
min_lr=1e-5,
factor=0.1,
verbose=True)
config.max_epoch = 60
if hasattr(model, 'train_config'):
model_train_config = model.train_config()
for key, value in model_train_config.items():
print('Overriding config.%s' % key)
config.__setattr__(key, value)
return config
def get_H_config(self, dataset, will_train=True):
print("Preparing training D1+D2 (H)")
print("Mixture size: %s" % colored('%d' % len(dataset), 'green'))
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if self.args.D1 in Global.mirror_augment:
print(colored("Mirror augmenting %s" % self.args.D1, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds,
batch_size=self.args.batch_size,
shuffle=True,
num_workers=self.args.workers,
pin_memory=True)
valid_loader = DataLoader(valid_ds,
batch_size=self.args.batch_size,
shuffle=True,
num_workers=self.args.workers,
pin_memory=True)
# Set up the criterion
# margin must be non-zero.
criterion = SVMLoss(margin=1.0).cuda()
# Set up the model
model = OTModelWrapper(self.base_model, self.mav,
self.weib_models).to(self.args.device)
old_valid_loader = valid_loader
if will_train:
# cache the subnetwork for faster optimization.
from methods import get_cached
from torch.utils.data.dataset import TensorDataset
trainX, trainY = get_cached(model, train_loader, self.args.device)
validX, validY = get_cached(model, valid_loader, self.args.device)
trainX_notnan = trainX[torch.logical_not(
torch.isnan(trainX)[:, 0]).nonzero().squeeze(1)]
trainY_notnan = trainY[torch.logical_not(
torch.isnan(trainX)[:, 0]).nonzero().squeeze(1)]
validX_notnan = validX[torch.logical_not(
torch.isnan(validX)[:, 0]).nonzero().squeeze(1)]
validY_notnan = validY[torch.logical_not(
torch.isnan(validX)[:, 0]).nonzero().squeeze(1)]
new_train_ds = TensorDataset(trainX_notnan, trainY_notnan)
new_valid_ds = TensorDataset(validX_notnan, validY_notnan)
# Initialize the new multi-threaded loaders.
train_loader = DataLoader(new_train_ds,
batch_size=2048,
shuffle=True,
num_workers=0,
pin_memory=False)
valid_loader = DataLoader(new_valid_ds,
batch_size=2048,
shuffle=True,
num_workers=0,
pin_memory=False)
# Set model to direct evaluation (for cached data)
model.set_eval_direct(True)
# Set up the config
config = IterativeTrainerConfig()
base_model_name = self.base_model.__class__.__name__
if hasattr(self.base_model, 'preferred_name'):
base_model_name = self.base_model.preferred_name()
config.name = '_%s[%s](%s->%s)' % (self.__class__.__name__,
base_model_name, self.args.D1,
self.args.D2)
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {
'dataset': train_loader,
'backward': True
},
'test': {
'dataset': valid_loader,
'backward': False
},
'testU': {
'dataset': old_valid_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.cast_float_label = True
config.stochastic_gradient = True
config.visualize = not self.args.no_visualize
config.model = model
config.optim = optim.SGD(model.H.parameters(),
lr=1e-2,
weight_decay=0.0) #1.0/len(train_ds))
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim,
patience=10,
threshold=1e-1,
min_lr=1e-8,
factor=0.1,
verbose=True)
h_path = path.join(self.args.experiment_path,
'%s' % (self.__class__.__name__),
'%d' % (self.default_model),
'%s-%s.pth' % (self.args.D1, self.args.D2))
h_parent = path.dirname(h_path)
config.logger = Logger(h_parent)
config.max_epoch = 100
return config
def get_classifier_config(args, model, dataset, balanced=False):
print("Preparing training D1 for %s" % (dataset.name))
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s" % dataset.name, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
if balanced:
y_train = []
for x, y in train_ds:
y_train.append(y.numpy())
y_train = np.array(y_train)
class_sample_count = np.array(
[len(np.where(y_train == t)[0]) for t in np.unique(y_train)])
print(class_sample_count)
weight = 1. / class_sample_count
samples_weight = np.array([weight[t] for t in y_train])
samples_weight = torch.from_numpy(samples_weight)
sampler = WeightedRandomSampler(
samples_weight.type('torch.DoubleTensor'), len(samples_weight))
train_loader = DataLoader(train_ds,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True,
sampler=sampler)
y_val = []
for x, y in valid_ds:
y_val.append(y.numpy())
y_val = np.array(y_val)
class_sample_count = np.array(
[len(np.where(y_val == t)[0]) for t in np.unique(y_val)])
print(class_sample_count)
weight = 1. / class_sample_count
samples_weight = np.array([weight[t] for t in y_val])
samples_weight = torch.from_numpy(samples_weight)
sampler = WeightedRandomSampler(
samples_weight.type('torch.DoubleTensor'), len(samples_weight))
valid_loader = DataLoader(valid_ds,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True,
sampler=sampler)
else:
train_loader = DataLoader(train_ds,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
valid_loader = DataLoader(valid_ds,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True)
all_loader = DataLoader(dataset,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True)
# Set up the criterion
criterion = nn.NLLLoss().to(args.device)
# Set up the model
model = model.to(args.device)
# Set up the config
config = IterativeTrainerConfig()
config.name = 'classifier_%s_%s' % (dataset.name, model.__class__.__name__)
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {
'dataset': train_loader,
'backward': True
},
'test': {
'dataset': valid_loader,
'backward': False
},
'all': {
'dataset': all_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.stochastic_gradient = True
config.visualize = not args.no_visualize
config.model = model
home_path = Models.get_ref_model_path(args,
config.model.__class__.__name__,
dataset.name,
model_setup=True,
suffix_str='base0')
config.logger = Logger(home_path)
config.optim = optim.Adam(model.parameters(), lr=1e-3)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim,
patience=10,
threshold=1e-2,
min_lr=1e-6,
factor=0.1,
verbose=True)
config.max_epoch = 120
if hasattr(model, 'train_config'):
model_train_config = model.train_config()
for key, value in model_train_config.items():
print('Overriding config.%s' % key)
config.__setattr__(key, value)
return config
def get_H_config(self, dataset, will_train=True):
print("Preparing training D1+D2 (H)")
print("Mixture size: %s" % colored('%d' % len(dataset), 'green'))
import global_vars as Global
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if self.args.D1 in Global.mirror_augment:
print(colored("Mirror augmenting %s" % self.args.D1, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds,
batch_size=self.args.batch_size,
shuffle=True,
num_workers=self.args.workers,
pin_memory=True,
drop_last=True)
valid_loader = DataLoader(valid_ds,
batch_size=self.args.batch_size,
num_workers=self.args.workers,
pin_memory=True)
all_loader = DataLoader(dataset,
batch_size=self.args.batch_size,
num_workers=self.args.workers,
pin_memory=True)
# Set up the criterion
criterion = nn.BCEWithLogitsLoss().cuda()
# Set up the model
model = Global.get_ref_classifier(
self.args.D1)[self.default_model]().to(self.args.device)
self.add_identifier = model.__class__.__name__
if hasattr(model, 'preferred_name'):
self.add_identifier = model.preferred_name()
model = BinaryModelWrapper(model).to(self.args.device)
# Set up the config
config = IterativeTrainerConfig()
base_model_name = model.__class__.__name__
if hasattr(model, 'preferred_name'):
base_model_name = model.preferred_name()
config.name = '_%s[%s](%s->%s)' % (self.__class__.__name__,
base_model_name, self.args.D1,
self.args.D2)
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {
'dataset': train_loader,
'backward': True
},
'test': {
'dataset': valid_loader,
'backward': False
},
'testU': {
'dataset': all_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.cast_float_label = True
config.stochastic_gradient = True
config.visualize = not self.args.no_visualize
config.model = model
config.logger = Logger()
config.optim = optim.Adam(model.parameters(), lr=1e-3)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim,
patience=5,
threshold=1e-2,
min_lr=1e-6,
factor=0.1,
verbose=True)
config.max_epoch = 30
if hasattr(model, 'train_config'):
model_train_config = model.train_config()
for key, value in model_train_config.iteritems():
print('Overriding config.%s' % key)
config.__setattr__(key, value)
return config
def propose_H(self, dataset):
assert self.default_model > 0, 'KNN needs K>0'
if self.base_model is not None:
self.base_model.base_data = None
self.base_model = None
# Set up the base-model
if isinstance(self, BCEKNNSVM) or isinstance(self, MSEKNNSVM):
base_model = Global.get_ref_autoencoder(dataset.name)[0]().to(
self.args.device)
if isinstance(self, BCEKNNSVM):
base_model.netid = "BCE." + base_model.netid
else:
base_model.netid = "MSE." + base_model.netid
home_path = Models.get_ref_model_path(
self.args,
base_model.__class__.__name__,
dataset.name,
suffix_str=base_model.netid)
elif isinstance(self, VAEKNNSVM):
base_model = Global.get_ref_vae(dataset.name)[0]().to(
self.args.device)
home_path = Models.get_ref_model_path(
self.args,
base_model.__class__.__name__,
dataset.name,
suffix_str=base_model.netid)
else:
raise NotImplementedError()
hbest_path = path.join(home_path, 'model.best.pth')
best_h_path = hbest_path
print(colored('Loading H1 model from %s' % best_h_path, 'red'))
base_model.load_state_dict(torch.load(best_h_path))
base_model.eval()
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s" % dataset.name, 'green'))
new_train_ds = dataset + MirroredDataset(dataset)
dataset = new_train_ds
# Initialize the multi-threaded loaders.
all_loader = DataLoader(dataset,
batch_size=self.args.batch_size,
num_workers=1,
pin_memory=True)
n_data = len(dataset)
n_dim = base_model.encode(dataset[0][0].to(
self.args.device).unsqueeze(0)).numel()
print('nHidden %d' % (n_dim))
self.base_data = torch.zeros(n_data, n_dim, dtype=torch.float32)
base_ind = 0
with torch.set_grad_enabled(False):
with tqdm(total=len(all_loader)) as pbar:
pbar.set_description('Caching X_train for %d-nn' %
self.default_model)
for i, (x, _) in enumerate(all_loader):
n_data = x.size(0)
output = base_model.encode(x.to(self.args.device)).data
self.base_data[base_ind:base_ind + n_data].copy_(output)
base_ind = base_ind + n_data
pbar.update()
# self.base_data = torch.cat([x.view(1, -1) for x,_ in dataset])
self.base_model = AEKNNModel(base_model,
self.base_data,
k=self.default_model).to(self.args.device)
self.base_model.eval()