def validate(val_loader, model, epoch, args):
global writer
batch_time = SumEpochMeter('Time', ':6.3f')
losses = AverageEpochMeter('Loss', ':.4e')
top1 = AverageEpochMeter('Top-1 Classification Acc')
top5 = AverageEpochMeter('Top-5 Classification Acc')
progress = ProgressEpochMeter(len(val_loader),
[batch_time, losses, top1, top5],
prefix="\nValidation Phase: ")
# switch to evaluate mode
model.eval()
with torch.no_grad():
end = time.time()
for i, (images, target, _) in enumerate(val_loader):
if args.gpu is not None:
images = images.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
# compute output
output = model(images, target)
# loss = criterion(output, target)
if args.mode == 'base':
loss = model.module.get_loss(target)
elif args.mode == 'ACoL':
loss_raw, loss_erase = model.module.get_loss(target)
loss = loss_raw + loss_erase
else:
loss = model.module.get_loss(target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), images.size(0))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.gpu == 0:
progress.display(epoch + 1)
return top1.avg, top5.avg, losses.avg
def evaluate_loc(val_loader, model, epoch, log_folder, args):
batch_time = SumEpochMeter('Time')
losses = AverageEpochMeter('Loss')
top1 = AverageEpochMeter('Top-1 Classification Acc')
top5 = AverageEpochMeter('Top-5 Classification Acc')
GT_loc = AverageEpochMeter('Top-1 GT-Known Localization Acc')
top1_loc = AverageEpochMeter('Top-1 Localization Acc')
progress = ProgressEpochMeter(
len(val_loader), [batch_time, losses, GT_loc, top1_loc, top1, top5],
prefix="\nValidation Phase: ")
# image requeired for individual storage
image_names = get_image_name(args.test_list)
gt_bbox = load_bbox_size(dataset_path=args.data_list,
resize_size=args.resize_size,
crop_size=args.crop_size)
cnt = 0
cnt_false = 0
hit_known = 0
hit_top1 = 0
means = [0.485, .456, .406]
stds = [.229, .224, .225]
means = torch.reshape(torch.tensor(means), (1, 3, 1, 1))
stds = torch.reshape(torch.tensor(stds), (1, 3, 1, 1))
model.module.eval()
with torch.no_grad():
end = time.time()
if args.gpu == 0:
for i, (images, target, image_ids) in tqdm(enumerate(val_loader)):
# imagenet is actually not used here
if args.gpu is not None:
images = images.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
image_ids = image_ids.data.cpu().numpy()
output = model.module(images, target)
# loss = criterion(output, target)
loss = model.module.get_loss(target)
# Get acc1, acc5 and update
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), images.size(0))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
_, pred = output.topk(1, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
wrongs = [c == 0 for c in correct.cpu().numpy()][0]
# original image in tensor format
if args.cam_curve == False and args.evaluate == True:
evaluate_image_ = images.clone().detach().cpu()
tensor_image = images.clone().detach().cpu() * stds + means
image_ = images.clone().detach().cpu() * stds + means
b = image_.shape[0]
# cam image in tensor format
cam = get_cam(model=model, target=target, args=args)
# generate bbox
blend_tensor = torch.zeros_like(image_)
wrong_blend_tensor = torch.zeros_like(image_)
# channel first to channel last, 3*W*H->W*H*3 and normaliza the image to 0-1
image_ = image_.clone().detach().cpu().numpy().transpose(
0, 2, 3, 1)
# print("cam shape",cam.shape)
cam_ = cam.clone().detach().cpu().numpy().transpose(0, 2, 3, 1)
# reverse the color representation(RGB -> BGR) and Opencv format
image_ = image_[:, :, :, ::-1] * 255
# cam_ = cam_[:, :, :, ::-1]
for j in range(images.size(0)):
# TODO: paint some image here
# 1 save original image
# 2 save total feature map
# 3 save erased feature map
# 4 save blend feature map
# blend bbox: box and original image
estimated_bbox, blend_bbox = generate_bbox(
image_[j], cam_[j], gt_bbox[image_ids[j]],
args.cam_thr, args)
# reverse the color representation(RGB -> BGR) and reshape
if args.gpu == 0:
blend_bbox = blend_bbox[:, :, ::-1] / 255.
blend_bbox = blend_bbox.transpose(2, 0, 1)
blend_tensor[j] = torch.tensor(blend_bbox)
# calculate IOU for WSOL
IOU = calculate_IOU(gt_bbox[image_ids[j]], estimated_bbox)
if IOU >= 0.5:
hit_known += 1
if not wrongs[j]:
hit_top1 += 1
else:
if args.evaluate and wrongs[j]:
wrong_blend_tensor[j] = torch.tensor(blend_bbox)
if wrongs[j]:
cnt_false += 1
cnt += 1
# that's say, only if you run the code in evaluate mode will give a visualization of images.
if args.gpu == 0 and i < 1 and not args.cam_curve and not args.evaluate:
save_images(log_folder, 'results', epoch, i, blend_tensor,
args)
loc_gt = hit_known / cnt * 100
loc_top1 = hit_top1 / cnt * 100
GT_loc.update(loc_gt, images.size(0))
top1_loc.update(loc_top1, images.size(0))
batch_time.update(time.time() - end)
end = time.time()
if args.gpu == 0:
progress.display(epoch + 1)
torch.cuda.empty_cache()
# dist.barrier()
cuda_tensor = torch.cuda.FloatTensor(
[top1.avg, top5.avg, losses.avg, GT_loc.avg, top1_loc.avg])
# sync validation result to all processes
if args.task != 'wsol_eval':
dist.broadcast(cuda_tensor, src=0)
# return top1.avg, top5.avg, losses.avg, GT_loc.avg, top1_loc.avg
return tuple(cuda_tensor.tolist())
def train(train_loader, model, criterion, optimizer, epoch, args):
batch_time = SumEpochMeter('Time', ':6.3f')
data_time = SumEpochMeter('Data', ':6.3f')
losses = AverageEpochMeter('Loss', ':.4e')
top1 = AverageEpochMeter('Top-1 Classification Acc')
top5 = AverageEpochMeter('Top-5 Classification Acc')
learning_rate = AverageEpochMeter('Learning Rate:', ':.1e')
progress = ProgressEpochMeter(
len(train_loader),
[batch_time, data_time, losses, learning_rate, top1, top5],
prefix="\nTraining Phase: ")
for param_group in optimizer.param_groups:
learning_rate.update(param_group['lr'])
break
# switch to train mode
model.train()
end = time.time()
means = [.485, .456, .406]
stds = [.229, .224, .225]
means = torch.reshape(torch.tensor(means), (1, 3, 1, 1)).cuda(args.gpu)
stds = torch.reshape(torch.tensor(stds), (1, 3, 1, 1)).cuda(args.gpu)
for i, (images, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
if args.gpu is not None:
images = images.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
# compute output
output = model(images)
loss = criterion(output, target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), images.size(0))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
adl_maps = list()
with torch.no_grad():
if 'ADL' in args.arch:
if i < 5:
image_ = images.clone().detach() * stds + means
for name, module in model.module.named_modules():
if 'resnet' in args.arch:
if isinstance(module, resnet.ADL):
adl_maps.append(module.get_maps())
elif 'vgg' in args.arch:
if isinstance(module, vgg.ADL):
adl_maps.append(module.get_maps())
b, _, h, w = image_.shape
image_ = F.interpolate(image_, (h // 2, w // 2))
image_set = vutils.make_grid(image_[:16], nrow=16)
for idx, (attention, drop_mask) in enumerate(adl_maps):
attention = F.interpolate(attention, (h // 2, w // 2),
mode='bilinear',
align_corners=False)
drop_mask = F.interpolate(drop_mask, (h // 2, w // 2),
mode='bilinear',
align_corners=False)
attention = vutils.make_grid(attention[:16],
nrow=16,
normalize=True,
scale_each=True)
drop_mask = vutils.make_grid(drop_mask[:16],
nrow=16,
normalize=True,
scale_each=True)
if idx == 0:
compare_set = torch.cat(
(image_set, attention, drop_mask), dim=1)
else:
compare_set = torch.cat(
(compare_set, attention, drop_mask), dim=1)
if args.gpu == 0:
writer.add_image(args.name + '/ADL_' + str(args.gpu),
compare_set, epoch * 5 + i)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
return top1.avg, losses.avg, progress
def evaluate_loc(val_loader, model, criterion, epoch, args):
batch_time = SumEpochMeter('Time')
losses = AverageEpochMeter('Loss')
top1 = AverageEpochMeter('Top-1 Classification Acc')
top5 = AverageEpochMeter('Top-5 Classification Acc')
GT_loc = AverageEpochMeter('Top-1 GT-Known Localization Acc')
top1_loc = AverageEpochMeter('Top-1 Localization Acc')
progress = ProgressEpochMeter(
len(val_loader), [batch_time, losses, GT_loc, top1_loc, top1, top5],
prefix="\nValidation Phase: ")
# image 개별 저장할 때 필요
image_names = get_image_name(args.test_list)
gt_bbox = load_bbox_size(dataset_path=args.data_list,
resize_size=args.resize_size,
crop_size=args.crop_size)
cnt = 0
cnt_false = 0
hit_known = 0
hit_top1 = 0
means = [0.485, .456, .406]
stds = [.229, .224, .225]
means = torch.reshape(torch.tensor(means), (1, 3, 1, 1))
stds = torch.reshape(torch.tensor(stds), (1, 3, 1, 1))
model.eval()
with torch.no_grad():
end = time.time()
for i, (images, target, image_ids) in enumerate(val_loader):
if args.gpu is not None:
images = images.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
image_ids = image_ids.data.cpu().numpy()
output = model(images)
loss = criterion(output, target)
# Get acc1, acc5 and update
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), images.size(0))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
_, pred = output.topk(1, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
wrongs = [c == 0 for c in correct.cpu().numpy()][0]
# original image in tensor format
image_ = images.clone().detach().cpu() * stds + means
# cam image in tensor format
cam = get_cam(model=model, target=target, args=args)
# generate tensor base blend image
# blend_tensor = generate_blend_tensor(image_, cam)
# generate bbox
blend_tensor = torch.zeros_like(image_)
image_ = image_.clone().detach().cpu().numpy().transpose(
0, 2, 3, 1)
cam_ = cam.clone().detach().cpu().numpy().transpose(0, 2, 3, 1)
# reverse the color representation(RGB -> BGR) and Opencv format
image_ = image_[:, :, :, ::-1] * 255
# cam_ = cam_[:, :, :, ::-1]
for j in range(images.size(0)):
estimated_bbox, blend_bbox = generate_bbox(
image_[j], cam_[j], gt_bbox[image_ids[j]], args.cam_thr)
# reverse the color representation(RGB -> BGR) and reshape
if args.gpu == 0:
blend_bbox = blend_bbox[:, :, ::-1] / 255.
blend_bbox = blend_bbox.transpose(2, 0, 1)
blend_tensor[j] = torch.tensor(blend_bbox)
# calculate IOU for WSOL
IOU = calculate_IOU(gt_bbox[image_ids[j]], estimated_bbox)
if IOU >= 0.5:
hit_known += 1
if not wrongs[j]:
hit_top1 += 1
if wrongs[j]:
cnt_false += 1
cnt += 1
# save the tensor
if args.gpu == 0 and i < 1 and not args.cam_curve:
save_images('results', epoch, i, blend_tensor, args)
if args.gpu == 0 and args.evaluate and not args.cam_curve:
save_images('results_best', epoch, i, blend_tensor, args)
loc_gt = hit_known / cnt * 100
loc_top1 = hit_top1 / cnt * 100
GT_loc.update(loc_gt, images.size(0))
top1_loc.update(loc_top1, images.size(0))
batch_time.update(time.time() - end)
end = time.time()
if args.gpu == 0:
progress.display(epoch + 1)
torch.cuda.empty_cache()
return top1.avg, top5.avg, losses.avg, GT_loc.avg, top1_loc.avg