def test(self, epoch):
"""
testing
"""
ori_top1_error = utils.AverageMeter()
ori_top1_loss = utils.AverageMeter()
ori_top5_error = utils.AverageMeter()
quan_top1_error = []
quan_top1_loss = []
quan_top5_error = []
for j in range(len(self.quan_model)):
quan_top1_error.append(utils.AverageMeter())
quan_top1_loss.append(utils.AverageMeter())
quan_top5_error.append(utils.AverageMeter())
self.quan_model[j].eval()
self.ori_model.eval()
iters = len(self.test_loader)
start_time = time.time()
end_time = start_time
with torch.no_grad():
for i, (images, labels) in enumerate(self.test_loader):
start_time = time.time()
data_time = start_time - end_time
labels = labels.cuda()
if self.settings.nGPU == 1:
images = images.cuda()
ori_output, ori_loss = self.forward(self.ori_model, images,
labels)
ori_single_error, ori_single_loss, ori_single5_error = utils.compute_singlecrop(
outputs=ori_output,
loss=ori_loss,
labels=labels,
top5_flag=True,
mean_flag=True)
for j in range(len(self.quan_model)):
quan_output, quan_loss = self.forward(
self.quan_model[j], images, labels)
quan_single_error, quan_single_loss, quan_single5_error = utils.compute_singlecrop(
outputs=quan_output,
loss=quan_loss,
labels=labels,
top5_flag=True,
mean_flag=True)
quan_top1_error[j].update(quan_single_error,
images.size(0))
quan_top1_loss[j].update(quan_single_loss, images.size(0))
quan_top5_error[j].update(quan_single5_error,
images.size(0))
ori_top1_error.update(ori_single_error, images.size(0))
ori_top1_loss.update(ori_single_loss, images.size(0))
ori_top5_error.update(ori_single5_error, images.size(0))
end_time = time.time()
iter_time = end_time - start_time
log_str = ">>> {}: [{:0>3d}|{:0>3d}], Iter: [{:0>3d}|{:0>3d}], LR: {:.6f}, DataTime: {:.4f}, IterTime: {:.4f}, ".format(
"Test", epoch + 1, self.settings.nEpochs, i + 1, iters,
self.ori_lr_master.lr, data_time, iter_time)
log_str += "Loss: {:.4f}, Error: {:.4f}, Top5 Error: {:.4f}, ".format(
ori_single_loss, ori_single_error, ori_single5_error)
time_str, _, _ = utils.compute_remain_time(
epoch,
self.settings.nEpochs,
i + 1,
iters,
data_time,
iter_time,
mode="Train")
self.logger.info(log_str + time_str)
log_str = ">>> {}: [{:0>3d}|{:0>3d}], Iter: [{:0>3d}|{:0>3d}], LR: {:.6f}, DataTime: {:.4f}, IterTime: {:.4f}, ".format(
"Test", epoch + 1, self.settings.nEpochs, i + 1, iters,
self.quan_lr_master.lr, data_time, iter_time)
for j in range(len(self.quan_model)):
log_str += "Loss_{:d}: {:.4f}, Error_{:d}: {:.4f}, Top5 Error_{:d}: {:.4f}, ".format(
j, quan_top1_loss[j].val, j, quan_top1_error[j].val, j,
quan_top5_error[j].val)
self.logger.info(log_str)
if self.settings.nEpochs == 1 and i + 1 >= 50:
self.logger.info(
"|===>Program testing for only 50 iterations")
break
self.scalar_info['ori_testing_top1error'] = ori_top1_error.avg
self.scalar_info['ori_testing_top5error'] = ori_top5_error.avg
self.scalar_info['ori_testing_loss'] = ori_top1_loss.avg
for j in range(len(self.quan_model)):
self.scalar_info['quan_testing_top1error_{}'.format(
j)] = quan_top1_error[j].avg
self.scalar_info['quan_testing_top5error_{}'.format(
j)] = quan_top5_error[j].avg
self.scalar_info['quan_testing_loss_{}'.format(
j)] = quan_top1_loss[j].avg
if self.tensorboard_logger is not None:
for tag, value in self.scalar_info.items():
self.tensorboard_logger.scalar_summary(tag, value,
self.run_count)
self.scalar_info = {}
self.run_count += 1
self.logger.info(
"|===>Original Model Testing Error: {:.4f} Loss: {:.4f}, Top5 Error: {:.4f}"
.format(ori_top1_error.avg, ori_top1_loss.avg, ori_top5_error.avg))
log_str = "|===>Quantized Model "
for j in range(len(self.quan_model)):
log_str += "Training Error_{}: {:.4f}, Loss_{}: {:.4f}, Top5 Error_{}: {:.4f}, ".format(
j, quan_top1_error[j].avg, j, quan_top1_loss[j].avg, j,
quan_top5_error[j].avg)
self.logger.info(log_str)
return ori_top1_error.avg, ori_top1_loss.avg, ori_top5_error.avg, quan_top1_error, quan_top1_loss, quan_top5_error
def test(self, epoch):
"""
testing
"""
top1_error = utils.AverageMeter()
top1_loss = utils.AverageMeter()
top5_error = utils.AverageMeter()
self.model.eval()
iters = len(self.test_loader)
start_time = time.time()
end_time = start_time
with torch.no_grad():
for i, (images, labels) in enumerate(self.test_loader):
start_time = time.time()
data_time = start_time - end_time
labels = labels.cuda()
if self.settings.tenCrop:
image_size = images.size()
images = images.view(image_size[0] * 10,
image_size[1] / 10, image_size[2],
image_size[3])
images_tuple = images.split(image_size[0])
output = None
for img in images_tuple:
if self.settings.nGPU == 1:
img = img.cuda()
img_var = Variable(img, volatile=True)
temp_output, _ = self.forward(img_var)
if output is None:
output = temp_output.data
else:
output = torch.cat((output, temp_output.data))
single_error, single_loss, single5_error = utils.compute_tencrop(
outputs=output, labels=labels)
else:
if self.settings.nGPU == 1:
images = images.cuda()
output, loss = self.forward(images, labels)
single_error, single_loss, single5_error = utils.compute_singlecrop(
outputs=output,
loss=loss,
labels=labels,
top5_flag=True,
mean_flag=True)
top1_error.update(single_error, images.size(0))
top1_loss.update(single_loss, images.size(0))
top5_error.update(single5_error, images.size(0))
end_time = time.time()
iter_time = end_time - start_time
utils.print_result(epoch,
self.settings.nEpochs,
i + 1,
iters,
self.lr_master.lr,
data_time,
iter_time,
single_error,
single_loss,
top5error=single5_error,
mode="Test",
logger=self.logger)
if self.settings.nEpochs == 1 and i + 1 >= 50:
self.logger.info(
"|===>Program testing for only 50 iterations")
break
self.scalar_info['testing_top1error'] = top1_error.avg
self.scalar_info['testing_top5error'] = top5_error.avg
self.scalar_info['testing_loss'] = top1_loss.avg
if self.tensorboard_logger is not None:
for tag, value in self.scalar_info.items():
self.tensorboard_logger.scalar_summary(tag, value,
self.run_count)
self.scalar_info = {}
self.run_count += 1
self.logger.info(
"|===>Testing Error: {:.4f} Loss: {:.4f}, Top5 Error: {:.4f}".
format(top1_error.avg, top1_loss.avg, top5_error.avg))
return top1_error.avg, top1_loss.avg, top5_error.avg
def train(self, epoch):
"""
training
"""
ori_top1_error = utils.AverageMeter()
ori_top1_loss = utils.AverageMeter()
ori_kl_loss = utils.AverageMeter()
ori_total_loss = utils.AverageMeter()
ori_top5_error = utils.AverageMeter()
quan_top1_error = []
quan_top1_loss = []
quan_kl_loss = []
quan_total_loss = []
quan_top5_error = []
for i in range(len(self.quan_model)):
quan_top1_error.append(utils.AverageMeter())
quan_top1_loss.append(utils.AverageMeter())
quan_kl_loss.append(utils.AverageMeter())
quan_total_loss.append(utils.AverageMeter())
quan_top5_error.append(utils.AverageMeter())
iters = len(self.train_loader)
self.update_ori_lr(epoch)
self.update_quan_lr(epoch)
# Switch to train mode
self.ori_model.train()
for i in range(len(self.quan_model)):
self.quan_model[i].train()
start_time = time.time()
end_time = start_time
for i, (images, labels) in enumerate(self.train_loader):
start_time = time.time()
data_time = start_time - end_time
# if we use multi-gpu, its more efficient to send input
# to different gpu, instead of send it to the master gpu.
if self.settings.nGPU == 1:
images = images.cuda()
labels = labels.cuda()
quan_output_list = []
ori_output, _ = self.forward(self.ori_model, images)
for j in range(self.num_model):
quan_output, _ = self.forward(self.quan_model[j], images)
quan_loss = self.criterion(quan_output, labels)
quan_kl_loss_ = (self.settings.T[j] * self.settings.T[j]) * \
(self.criterion_kl(self.log_softmax(quan_output / self.settings.T[j]),
self.softmax(ori_output.detach() / self.settings.T[j])) / quan_output.shape[0])
quan_total_loss_ = self.settings.student_lambda * quan_loss + self.settings.sloss_lambda * quan_kl_loss_
self.quan_optimizer[j].zero_grad()
quan_total_loss_.backward()
self.quan_optimizer[j].step()
quan_output_list.append(quan_output.detach())
quan_single_error, quan_single_loss, quan_single5_error = utils.compute_singlecrop(
outputs=quan_output,
labels=labels,
loss=quan_loss,
top5_flag=True,
mean_flag=True)
quan_top1_error[j].update(quan_single_error, images.size(0))
quan_top1_loss[j].update(quan_single_loss, images.size(0))
quan_kl_loss[j].update(quan_kl_loss_.item(), images.size(0))
quan_total_loss[j].update(quan_total_loss_.item(),
images.size(0))
quan_top5_error[j].update(quan_single5_error, images.size(0))
# avg_p = 0
# for j in range(len(self.quan_model)):
# avg_p += self.softmax(quan_output_list[j])
# avg_p = avg_p / self.num_model
# ori_log_p = self.log_softmax(ori_output)
# ori_loss = self.criterion(ori_output, labels)
# ori_kl_loss_ = self.settings.loss_lambda * \
# self.criterion_kl(ori_log_p, self.softmax(avg_p.detach()))
ori_kl_loss_ = 0
ori_loss = self.criterion(ori_output, labels)
for j in range(self.num_model):
ori_kl_loss_ += (self.settings.T[j] * self.settings.T[j]) * \
(self.criterion_kl(self.log_softmax(ori_output / self.settings.T[j]),
self.softmax(quan_output_list[j].detach() / self.settings.T[j])) / ori_output.shape[0])
# Add in 2018.12.30
ori_kl_loss_ = ori_kl_loss_ / self.num_model
ori_total_loss_ = self.settings.teacher_lambda * ori_loss + self.settings.tloss_lambda * ori_kl_loss_
self.ori_optimizer.zero_grad()
ori_total_loss_.backward()
self.ori_optimizer.step()
ori_single_error, ori_single_loss, ori_single5_error = utils.compute_singlecrop(
outputs=ori_output,
labels=labels,
loss=ori_loss,
top5_flag=True,
mean_flag=True)
ori_top1_error.update(ori_single_error, images.size(0))
ori_top1_loss.update(ori_single_loss, images.size(0))
ori_top5_error.update(ori_single5_error, images.size(0))
ori_kl_loss.update(ori_kl_loss_.item(), images.size(0))
ori_total_loss.update(ori_total_loss_.item(), images.size(0))
end_time = time.time()
iter_time = end_time - start_time
log_str = ">>> {}: [{:0>3d}|{:0>3d}], Iter: [{:0>3d}|{:0>3d}], LR: {:.6f}, DataTime: {:.4f}, IterTime: {:.4f}, ".format(
"Train", epoch + 1, self.settings.nEpochs, i + 1, iters,
self.ori_lr_master.lr, data_time, iter_time)
log_str += "KLD Loss: {:.4f}, Softmax Loss: {:.4f}, Total Loss: {:.4f}, Error: {:.4f}, Top5 Error: {:.4f}, ".format(
ori_kl_loss_.item(), ori_single_loss, ori_total_loss_.item(),
ori_single_error, ori_single5_error)
time_str, _, _ = utils.compute_remain_time(epoch,
self.settings.nEpochs,
i + 1,
iters,
data_time,
iter_time,
mode="Train")
self.logger.info(log_str + time_str)
log_str = ">>> {}: [{:0>3d}|{:0>3d}], Iter: [{:0>3d}|{:0>3d}], LR: {:.6f}, DataTime: {:.4f}, IterTime: {:.4f}, ".format(
"Train", epoch + 1, self.settings.nEpochs, i + 1, iters,
self.quan_lr_master.lr, data_time, iter_time)
for j in range(len(self.quan_model)):
log_str += "KLD Loss_{:d}: {:.4f}, Softmax Loss_{:d}: {:.4f}, Total Loss_{:d}: {:.4f}, Error_{:d}: {:.4f}, Top5 Error_{:d}: {:.4f}, ".format(
j, quan_kl_loss[j].val, j, quan_top1_loss[j].val, j,
quan_total_loss[j].val, j, quan_top1_error[j].val, j,
quan_top5_error[j].val)
self.logger.info(log_str)
if self.settings.nEpochs == 1 and i + 1 >= 50:
self.logger.info("|===>Program testing for only 50 iterations")
break
self.scalar_info['ori_training_top1error'] = ori_top1_error.avg
self.scalar_info['ori_training_top5error'] = ori_top5_error.avg
self.scalar_info['ori_training_loss'] = ori_top1_loss.avg
self.scalar_info['ori_total_loss'] = ori_total_loss.avg
self.scalar_info['ori_kl_loss'] = ori_kl_loss.avg
for j in range(len(self.quan_model)):
self.scalar_info['quan_training_top1error_{}'.format(
j)] = quan_top1_error[j].avg
self.scalar_info['quan_training_top5error_{}'.format(
j)] = quan_top5_error[j].avg
self.scalar_info['quan_training_loss_{}'.format(
j)] = quan_top1_loss[j].avg
self.scalar_info['quan_total_loss_{}'.format(
j)] = quan_total_loss[j].avg
self.scalar_info['quan_kl_loss_{}'.format(j)] = quan_kl_loss[j].avg
if self.tensorboard_logger is not None:
for tag, value in list(self.scalar_info.items()):
self.tensorboard_logger.scalar_summary(tag, value,
self.run_count)
self.scalar_info = {}
self.logger.info(
"|===>Original Model Training Error: {:.4f}, Loss: {:.4f}, Top5 Error: {:.4f}"
.format(ori_top1_error.avg, ori_top1_loss.avg, ori_top5_error.avg))
log_str = "|===>Quantized Model "
for j in range(len(self.quan_model)):
log_str += "Training Error_{}: {:.4f}, Loss_{}: {:.4f}, Top5 Error_{}: {:.4f}, ".format(
j, quan_top1_error[j].avg, j, quan_top1_loss[j].avg, j,
quan_top5_error[j].avg)
log_str += "\n"
self.logger.info(log_str)
return ori_top1_error.avg, ori_top1_loss.avg, ori_top5_error.avg, quan_top1_error, quan_top1_loss, quan_top5_error
def train(self, epoch):
"""
training
"""
top1_error = utils.AverageMeter()
top1_loss = utils.AverageMeter()
top5_error = utils.AverageMeter()
iters = len(self.train_loader)
self.update_lr(epoch)
# Switch to train mode
self.model.train()
start_time = time.time()
end_time = start_time
for i, (images, labels) in enumerate(self.train_loader):
start_time = time.time()
data_time = start_time - end_time
# if we use multi-gpu, its more efficient to send input
# to different gpu, instead of send it to the master gpu.
if self.settings.nGPU == 1:
images = images.cuda()
labels = labels.cuda()
output, loss = self.forward(images, labels)
self.backward(loss)
single_error, single_loss, single5_error = utils.compute_singlecrop(
outputs=output,
labels=labels,
loss=loss,
top5_flag=True,
mean_flag=True)
top1_error.update(single_error, images.size(0))
top1_loss.update(single_loss, images.size(0))
top5_error.update(single5_error, images.size(0))
end_time = time.time()
iter_time = end_time - start_time
utils.print_result(epoch,
self.settings.nEpochs,
i + 1,
iters,
self.lr_master.lr,
data_time,
iter_time,
single_error,
single_loss,
top5error=single5_error,
mode="Train",
logger=self.logger)
if self.settings.nEpochs == 1 and i + 1 >= 50:
self.logger.info("|===>Program testing for only 50 iterations")
break
self.scalar_info['training_top1error'] = top1_error.avg
self.scalar_info['training_top5error'] = top5_error.avg
self.scalar_info['training_loss'] = top1_loss.avg
if self.tensorboard_logger is not None:
for tag, value in list(self.scalar_info.items()):
self.tensorboard_logger.scalar_summary(tag, value,
self.run_count)
self.scalar_info = {}
self.logger.info(
"|===>Training Error: {:.4f} Loss: {:.4f}, Top5 Error: {:.4f}".
format(top1_error.avg, top1_loss.avg, top5_error.avg))
return top1_error.avg, top1_loss.avg, top5_error.avg