def get_base_config(self, dataset):
print("Preparing training D1 for %s" %
(dataset.parent_dataset.__class__.__name__))
all_loader = DataLoader(dataset,
batch_size=self.args.batch_size,
num_workers=self.args.workers,
pin_memory=True)
# Set up the criterion
criterion = nn.NLLLoss().cuda()
# Set up the model
model_class = Global.get_ref_classifier(
dataset.name)[self.default_model]
self.add_identifier = model_class.__name__
# We must create 5 instances of this class.
from models import get_ref_model_path
all_models = []
for mid in range(5):
model = model_class()
model = DeepEnsembleWrapper(model)
model = model.to(self.args.device)
h_path = get_ref_model_path(self.args,
model_class.__name__,
dataset.name,
suffix_str='DE.%d' % mid)
best_h_path = path.join(h_path, 'model.best.pth')
if not path.isfile(best_h_path):
raise NotImplementedError(
"Please use setup_model to pretrain the networks first! Can't find %s"
% best_h_path)
else:
print(colored('Loading H1 model from %s' % best_h_path, 'red'))
model.load_state_dict(torch.load(best_h_path))
model.eval()
all_models.append(model)
master_model = DeepEnsembleMasterWrapper(all_models)
# Set up the config
config = IterativeTrainerConfig()
config.name = '%s-CLS' % (self.args.D1)
config.phases = {
'all': {
'dataset': all_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.cast_float_label = False
config.stochastic_gradient = True
config.model = master_model
config.optim = None
config.autoencoder_target = False
config.visualize = False
config.logger = Logger()
return config
def propose_H(self, dataset):
assert self.default_model > 0, 'KNN needs K>0'
if self.base_model is not None:
self.base_model.base_data = None
self.base_model = None
# Set up the base0-model
base_model = Global.get_ref_classifier(dataset.name)[0]().to(
self.args.device)
from models import get_ref_model_path
home_path = get_ref_model_path(self.args,
base_model.__class__.__name__,
dataset.name)
hbest_path = path.join(home_path, 'model.best.pth')
best_h_path = hbest_path
print(colored('Loading H1 model from %s' % best_h_path, 'red'))
base_model.load_state_dict(torch.load(best_h_path))
base_model.eval()
if dataset.name in Global.mirror_augment:
print(colored("Mirror augmenting %s" % dataset.name, 'green'))
new_train_ds = dataset + MirroredDataset(dataset)
dataset = new_train_ds
# Initialize the multi-threaded loaders.
all_loader = DataLoader(dataset,
batch_size=self.args.batch_size,
num_workers=1,
pin_memory=True)
n_data = len(dataset)
n_dim = base_model.partial_forward(dataset[0][0].to(
self.args.device).unsqueeze(0)).numel()
print('nHidden %d' % (n_dim))
self.base_data = torch.zeros(n_data, n_dim, dtype=torch.float32)
base_ind = 0
with torch.set_grad_enabled(False):
with tqdm(total=len(all_loader),
disable=bool(os.environ.get("DISABLE_TQDM",
False))) as pbar:
pbar.set_description('Caching X_train for %d-nn' %
self.default_model)
for i, (x, _) in enumerate(all_loader):
n_data = x.size(0)
output = base_model.partial_forward(x.to(
self.args.device)).data
self.base_data[base_ind:base_ind + n_data].copy_(output)
base_ind = base_ind + n_data
pbar.update()
# self.base_data = torch.cat([x.view(1, -1) for x,_ in dataset])
self.base_model = AEKNNModel(base_model,
self.base_data,
k=self.default_model,
SV=True).to(self.args.device)
self.base_model.eval()
def get_base_config(self, dataset):
print("Preparing training D1 for %s" % (dataset.name))
all_loader = DataLoader(dataset,
batch_size=self.args.batch_size,
num_workers=self.args.workers,
pin_memory=True)
# Set up the criterion
criterion = nn.NLLLoss().to(self.args.device)
# Set up the model
import global_vars as Global
model = Global.get_ref_classifier(
dataset.name)[self.default_model]().to(self.args.device)
self.add_identifier = model.__class__.__name__
if hasattr(model, 'preferred_name'):
self.add_identifier = model.preferred_name()
# Set up the config
config = IterativeTrainerConfig()
config.name = '%s-CLS' % (self.args.D1)
config.phases = {
'all': {
'dataset': all_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.cast_float_label = False
config.stochastic_gradient = True
config.model = model
config.optim = None
config.autoencoder_target = False
config.visualize = False
config.logger = Logger()
return config
def get_H_config(self, dataset, will_train=True):
print("Preparing training D1+D2 (H)")
print("Mixture size: %s" % colored('%d' % len(dataset), 'green'))
import global_vars as Global
# 80%, 20% for local train+test
train_ds, valid_ds = dataset.split_dataset(0.8)
if self.args.D1 in Global.mirror_augment:
print(colored("Mirror augmenting %s" % self.args.D1, 'green'))
new_train_ds = train_ds + MirroredDataset(train_ds)
train_ds = new_train_ds
# Initialize the multi-threaded loaders.
train_loader = DataLoader(train_ds,
batch_size=self.args.batch_size,
shuffle=True,
num_workers=self.args.workers,
pin_memory=True,
drop_last=True)
valid_loader = DataLoader(valid_ds,
batch_size=self.args.batch_size,
num_workers=self.args.workers,
pin_memory=True)
all_loader = DataLoader(dataset,
batch_size=self.args.batch_size,
num_workers=self.args.workers,
pin_memory=True)
# Set up the criterion
criterion = nn.BCEWithLogitsLoss().cuda()
# Set up the model
model = Global.get_ref_classifier(
self.args.D1)[self.default_model]().to(self.args.device)
self.add_identifier = model.__class__.__name__
if hasattr(model, 'preferred_name'):
self.add_identifier = model.preferred_name()
model = BinaryModelWrapper(model).to(self.args.device)
# Set up the config
config = IterativeTrainerConfig()
base_model_name = model.__class__.__name__
if hasattr(model, 'preferred_name'):
base_model_name = model.preferred_name()
config.name = '_%s[%s](%s->%s)' % (self.__class__.__name__,
base_model_name, self.args.D1,
self.args.D2)
config.train_loader = train_loader
config.valid_loader = valid_loader
config.phases = {
'train': {
'dataset': train_loader,
'backward': True
},
'test': {
'dataset': valid_loader,
'backward': False
},
'testU': {
'dataset': all_loader,
'backward': False
},
}
config.criterion = criterion
config.classification = True
config.cast_float_label = True
config.stochastic_gradient = True
config.visualize = not self.args.no_visualize
config.model = model
config.logger = Logger()
config.optim = optim.Adam(model.parameters(), lr=1e-3)
config.scheduler = optim.lr_scheduler.ReduceLROnPlateau(config.optim,
patience=5,
threshold=1e-2,
min_lr=1e-6,
factor=0.1,
verbose=True)
config.max_epoch = 30
if hasattr(model, 'train_config'):
model_train_config = model.train_config()
for key, value in model_train_config.iteritems():
print('Overriding config.%s' % key)
config.__setattr__(key, value)
return config