def __init__(self, pretrain_params, *args, **kwds):
super().__init__(pretrain_params, *args, **kwds)
# only on negative example
# self.params.train_only_on_negative_examples = True
assert self.params.penalty in ["lasso", "ridge"]
assert self.params.type_penalty in ["last", "group"]
assert self.params.yoshua
self.bin_classif = self.params.version == 7
self.max_label = 2.0 * self.params.threshold # 1.0 if threshold = 0.5, 5.0 if threshold=2.5
if self.bin_classif:
self.params.threshold = 0.5
self.max_label = 1.0
else:
self.max_label = 5.0
pass
self.params.pretrain_type = 1 # for evaluation (text gen)
self.params.eval_pretrainer = False # for evaluation (classification)
self.alpha, self.beta = [
float(coef) for coef in self.params.deb_alpha_beta.split(",")
]
self.denoising_ae = self.pre_trainer.params.ae_steps != []
self.lambda_ = self.params.threshold
#self.on_init(pretrain_params)
self.deb_optimizer = get_optimizer(self.deb.parameters(),
self.params.deb_optimizer)
self.deb_criterion = LossDebias(penalty=self.params.penalty)
self.after_init()
if self.params.fgim:
self.train = self.fgim_algorithm
self.eval = self.fgim_algorithm
def run_train(self):
params = self.params
# optimizers
self.optimizer_e = get_optimizer(
list(self.model.module.get_parameters(params.finetune_layers)),
params.optimizer_e)
self.optimizer_p = get_optimizer(self.model.module.proj.parameters(),
params.optimizer_p)
# criterion
self.criterion = nn.CrossEntropyLoss().cuda(self.gpu)
# train and evaluate the model
for epoch in range(params.n_epochs):
self.sampler['train'].set_epoch(
epoch) # 因为train的需要shuffle, valid不用
# update epoch
self.epoch = epoch
# training
logger.info("GPU %i - XLM - Training epoch %i ..." %
(self.gpu, epoch))
self.train()
def main(seed, pretrain, resume, evaluate, print_runtime, epochs, disable_tqdm,
visdom_port, ckpt_path, make_plot, cuda):
device = torch.device("cuda" if cuda else "cpu")
callback = None if visdom_port is None else VisdomLogger(port=visdom_port)
if seed is not None:
random.seed(seed)
torch.manual_seed(seed)
cudnn.deterministic = True
torch.cuda.set_device(0)
# create model
print("=> Creating model '{}'".format(
ex.current_run.config['model']['arch']))
model = torch.nn.DataParallel(get_model()).cuda()
print('Number of model parameters: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
optimizer = get_optimizer(model)
if pretrain:
pretrain = os.path.join(pretrain, 'checkpoint.pth.tar')
if os.path.isfile(pretrain):
print("=> loading pretrained weight '{}'".format(pretrain))
checkpoint = torch.load(pretrain)
model_dict = model.state_dict()
params = checkpoint['state_dict']
params = {k: v for k, v in params.items() if k in model_dict}
model_dict.update(params)
model.load_state_dict(model_dict)
else:
print(
'[Warning]: Did not find pretrained model {}'.format(pretrain))
if resume:
resume_path = ckpt_path + '/checkpoint.pth.tar'
if os.path.isfile(resume_path):
print("=> loading checkpoint '{}'".format(resume_path))
checkpoint = torch.load(resume_path)
start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
# scheduler.load_state_dict(checkpoint['scheduler'])
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
resume_path, checkpoint['epoch']))
else:
print('[Warning]: Did not find checkpoint {}'.format(resume_path))
else:
start_epoch = 0
best_prec1 = -1
cudnn.benchmark = True
# Data loading code
evaluator = Evaluator(device=device, ex=ex)
if evaluate:
print("Evaluating")
results = evaluator.run_full_evaluation(model=model,
model_path=ckpt_path,
callback=callback)
#MYMOD
#,model_tag='best',
#shots=[5],
#method="tim-gd")
return results
# If this line is reached, then training the model
trainer = Trainer(device=device, ex=ex)
scheduler = get_scheduler(optimizer=optimizer,
num_batches=len(trainer.train_loader),
epochs=epochs)
tqdm_loop = warp_tqdm(list(range(start_epoch, epochs)),
disable_tqdm=disable_tqdm)
for epoch in tqdm_loop:
# Do one epoch
trainer.do_epoch(model=model,
optimizer=optimizer,
epoch=epoch,
scheduler=scheduler,
disable_tqdm=disable_tqdm,
callback=callback)
# Evaluation on validation set
prec1 = trainer.meta_val(model=model,
disable_tqdm=disable_tqdm,
epoch=epoch,
callback=callback)
print('Meta Val {}: {}'.format(epoch, prec1))
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if not disable_tqdm:
tqdm_loop.set_description('Best Acc {:.2f}'.format(best_prec1 *
100.))
# Save checkpoint
save_checkpoint(state={
'epoch': epoch + 1,
'arch': ex.current_run.config['model']['arch'],
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict()
},
is_best=is_best,
folder=ckpt_path)
if scheduler is not None:
scheduler.step()
# Final evaluation on test set
results = evaluator.run_full_evaluation(model=model, model_path=ckpt_path)
return results