def train(self, train_loader, epoch):
print('Training ...')
self.model1.train() # Change model to 'train' mode.
self.model2.train() # Change model to 'train' mode
if self.adjust_lr == 1:
self.adjust_learning_rate(self.optimizer, epoch)
train_total = 0
train_correct = 0
train_total2 = 0
train_correct2 = 0
pure_ratio_1_list = []
pure_ratio_2_list = []
for i, (images, labels, indexes) in enumerate(train_loader):
ind = indexes.cpu().numpy().transpose()
if i > self.num_iter_per_epoch:
break
images = Variable(images).to(self.device)
labels = Variable(labels).to(self.device)
# Forward + Backward + Optimize
logits1 = self.model1(images)
prec1 = accuracy(logits1, labels, topk=(1, ))
train_total += 1
train_correct += prec1
logits2 = self.model2(images)
prec2 = accuracy(logits2, labels, topk=(1, ))
train_total2 += 1
train_correct2 += prec2
loss_1, loss_2, pure_ratio_1, pure_ratio_2 = self.loss_fn(
logits1, logits2, labels, self.rate_schedule[epoch], ind,
self.noise_or_not, self.co_lambda)
self.optimizer.zero_grad()
loss_1.backward()
self.optimizer.step()
pure_ratio_1_list.append(100 * pure_ratio_1)
pure_ratio_2_list.append(100 * pure_ratio_2)
if (i + 1) % self.print_freq == 0:
print(
'Epoch [%d/%d], Iter [%d/%d] Training Accuracy1: %.4F, Training Accuracy2: %.4f, Loss1: %.4f, Loss2: %.4f, Pure Ratio1 %.4f %% Pure Ratio2 %.4f %%'
% (epoch + 1, self.n_epoch, i + 1,
len(self.train_dataset) // self.batch_size, prec1,
prec2, loss_1.data.item(), loss_2.data.item(),
sum(pure_ratio_1_list) / len(pure_ratio_1_list),
sum(pure_ratio_2_list) / len(pure_ratio_2_list)))
train_acc1 = float(train_correct) / float(train_total)
train_acc2 = float(train_correct2) / float(train_total2)
return train_acc1, train_acc2, pure_ratio_1_list, pure_ratio_2_list
def valid(args, model, device, valid_loader, criterion):
model.eval()
val_losses = AverageMeter()
val_top1 = AverageMeter()
with torch.no_grad():
for data, target in valid_loader:
data, target = data.to(device), target.to(device)
output = model(data)
val_loss = criterion(output, target)
val_losses.update(val_loss.item(), data.size(0))
prec1 = accuracy(output, target, topk=(1,))
val_top1.update(prec1[0], data.size(0))
print('\nTest set: Average loss: {:.4f}, Accuracy: {:.2f} %\n'.format(
val_losses.avg, val_top1.avg.item()))
return val_losses, val_top1
def train(args, model, device, train_loader, optimizer, criterion, epoch):
model.train()
losses = AverageMeter()
top1 = AverageMeter()
for batch_idx, (data, target) in enumerate(train_loader):
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
losses.update(loss.item(), data.size(0))
prec1 = accuracy(output, target, topk=(1,))
top1.update(prec1[0], data.size(0))
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == (args.log_interval - 1):
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), losses.avg))
return losses, top1
loss = F.cross_entropy(output, target)
avg_loss += loss.item()
"""
alpha_loss = F.relu(model.Lblock.alpha - 2) + F.relu(-model.Lblock.alpha)
loss += alpha_loss.sum()
l1_loss = 0
for i in range(model.Lblock.alpha.size()[0]):
line_i = model.Lblock.alpha[i,1:]
l1_loss += F.l1_loss(line_i, torch.zeros_like(line_i))
"""
loss.backward()
optimizer.step()
accuracies += utils.accuracy(output, target, topk=topk)
acc_score = accuracies / (1 + batch_idx)
tqdm_log = prefix + "Epoch {}/{}, LR: {:.1E}, Train_Loss: {:.3f}, Test_loss: {:.3f}, ".format(
epoch, args.epochs, lr, avg_loss / (1 + batch_idx), test_loss)
for i, k in enumerate(topk):
tqdm_log += "Train_acc(top{}): {:.3f}, Test_acc(top{}): {:.3f}, ".format(
k, acc_score[i], k, test_acc[i])
tqdm.write(tqdm_log)
logger.update({"epoch_time": (time.time() - t0) / 60})
logger.update({"train_loss": loss.item()})
for i, k in enumerate(topk):
logger.update({"train_acc(top{})".format(k): acc_score[i]})
## TESTING
position=1,
leave=False,
ncols=100,
unit="batch"):
x_batch = x_batch.to(device)
y_batch = y_batch.to(device)
y_pred = model(x_batch)
iteration_nfes = get_and_reset_nfes(model)
epoch_nfes += iteration_nfes
loss = F.cross_entropy(y_pred, y_batch)
loss.backward()
avg_loss += loss.item()
optimizer.step()
accuracies += utils.accuracy(y_pred, y_batch, topk=topk)
acc_score = accuracies / (1 + batch_idx)
iteration_backward_nfes = get_and_reset_nfes(model)
epoch_backward_nfes += iteration_backward_nfes
tqdm_log = prefix + "Epoch {}/{}, LR: {:.1E}, Train_Loss: {:.3f}, Test_loss: {:.3f}, NFE: {}, bkwdNFE : {}, ".format(
epoch, args.epochs, lr, avg_loss / (1 + batch_idx), test_loss,
iteration_nfes, iteration_backward_nfes)
for i, k in enumerate(topk):
tqdm_log += "Train_acc(top{}): {:.3f}, Test_acc(top{}): {:.3f}, ".format(
k, acc_score[i], k, test_acc[i])
tqdm.write(tqdm_log)
logger.update({"epoch_time": (time.time() - t0) / 60})
logger.update({"train_loss": loss.item()})