def visualize_label(self, img, label):
# visualize result
unique_labels, label_counts = np.unique(label, return_counts=True)
print('- labels:')
label_titles = {}
for label_value, label_count in zip(unique_labels, label_counts):
label_region = label_count / label.size
if label_region < 0.001:
continue
title = '{0}:{1} = {2:.1%}'.format(label_value,
self.label_names[label_value],
label_region)
label_titles[label_value] = title
print(' - {0}'.format(title))
labelviz = utils.draw_label(label,
img,
n_class=len(self.label_names),
label_titles=label_titles)
# save result
return utils.get_tile_image([img, labelviz])
def validate(self):
visualizations = []
val_metrics = runningScore(self.n_classes)
val_loss_meter = averageMeter()
with torch.no_grad():
self.model.eval()
for rgb, ir, target in tqdm.tqdm(
self.val_loader, total=len(self.val_loader),
desc=f'Valid epoch={self.epoch}', ncols=80, leave=False):
rgb, ir, target = rgb.to(self.device), ir.to(self.device), target.to(self.device)
score = self.model(rgb, ir)
# score = self.model(rgb)
weight = self.val_loader.dataset.class_weight
if weight:
weight = torch.Tensor(weight).to(self.device)
loss = CrossEntropyLoss(score, target, weight=weight, reduction='mean', ignore_index=-1)
loss_data = loss.data.item()
if np.isnan(loss_data):
raise ValueError('loss is nan while validating')
val_loss_meter.update(loss_data)
rgbs = rgb.data.cpu()
irs = ir.data.cpu()
if isinstance(score, (tuple, list)):
lbl_pred = score[0].data.max(1)[1].cpu().numpy()
else:
lbl_pred = score.data.max(1)[1].cpu().numpy()
lbl_true = target.data.cpu()
for rgb, ir, lt, lp in zip(rgbs, irs, lbl_true, lbl_pred):
rgb, ir, lt = self.val_loader.dataset.untransform(rgb, ir, lt)
val_metrics.update(lt, lp)
if len(visualizations) < 9:
viz = visualize_segmentation(
lbl_pred=lp, lbl_true=lt, img=rgb, ir=ir,
n_classes=self.n_classes, dataloader=self.train_loader)
visualizations.append(viz)
acc, acc_cls, mean_iou, fwavacc, cls_iu = val_metrics.get_scores()
metrics = [acc, acc_cls, mean_iou, fwavacc]
print(f'\nEpoch: {self.epoch}', f'loss: {val_loss_meter.avg}, mIoU: {mean_iou}')
out = osp.join(self.out, 'visualization_viz')
if not osp.exists(out):
os.makedirs(out)
out_file = osp.join(out, 'epoch{:0>5d}.jpg'.format(self.epoch))
scipy.misc.imsave(out_file, get_tile_image(visualizations))
with open(osp.join(self.out, 'log.csv'), 'a') as f:
elapsed_time = (
datetime.datetime.now(pytz.timezone('UTC')) -
self.timestamp_start).total_seconds()
log = [self.epoch] + [''] * 5 + \
[val_loss_meter.avg] + metrics + [elapsed_time]
log = map(str, log)
f.write(','.join(log) + '\n')
mean_iu = metrics[2]
is_best = mean_iu > self.best_mean_iu
if is_best:
self.best_mean_iu = mean_iu
torch.save({
'epoch': self.epoch,
'arch': self.model.__class__.__name__,
'optim_state_dict': self.optim.state_dict(),
'model_state_dict': self.model.state_dict(),
'best_mean_iu': self.best_mean_iu,
}, osp.join(self.out, 'checkpoint.pth.tar'))
if is_best:
shutil.copy(osp.join(self.out, 'checkpoint.pth.tar'),
osp.join(self.out, 'model_best.pth.tar'))
val_loss_meter.reset()
val_metrics.reset()
class_name = self.val_loader.dataset.class_names
if class_name is not None:
for index, value in enumerate(cls_iu.values()):
offset = 20 - len(class_name[index])
print(class_name[index] + ' ' * offset + f'{value * 100:>.2f}')
else:
print("\nyou don't specify class_names, use number instead")
for key, value in cls_iu.items():
print(key, f'{value * 100:>.2f}')
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--model', type=str, default='deeplab-largefov')
parser.add_argument(
'--model_file',
type=str,
default=
'/home/ecust/lx/Semantic-Segmentation-PyTorch/logs/deeplab-largefov_20190417_230357/model_best.pth.tar',
help='Model path')
parser.add_argument('--dataset_type',
type=str,
default='voc',
help='type of dataset')
parser.add_argument(
'--dataset',
type=str,
default='/home/ecust/Datasets/PASCAL VOC/VOCdevkit/VOC2012',
help='path to dataset')
parser.add_argument('--img_size',
type=tuple,
default=None,
help='resize images using bilinear interpolation')
parser.add_argument('--crop_size',
type=tuple,
default=None,
help='crop images')
parser.add_argument('--n_classes',
type=int,
default=21,
help='number of classes')
parser.add_argument('--pretrained',
type=bool,
default=True,
help='should be set the same as train.py')
args = parser.parse_args()
model_file = args.model_file
root = args.dataset
n_classes = args.n_classes
crop = None
# crop = Compose([RandomCrop(args.crop_size)])
loader = get_loader(args.dataset_type)
val_loader = DataLoader(loader(root,
n_classes=n_classes,
split='val',
img_size=args.img_size,
augmentations=crop,
pretrained=args.pretrained),
batch_size=1,
shuffle=False,
num_workers=4)
model, _, _ = Models.model_loader(args.model, n_classes, resume=None)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(device)
print('==> Loading {} model file: {}'.format(model.__class__.__name__,
model_file))
model_data = torch.load(model_file)
try:
model.load_state_dict(model_data)
except Exception:
model.load_state_dict(model_data['model_state_dict'])
model.eval()
print('==> Evaluating with {} dataset'.format(args.dataset_type))
visualizations = []
metrics = runningScore(n_classes)
for data, target in tqdm.tqdm(val_loader,
total=len(val_loader),
ncols=80,
leave=False):
data, target = data.to(device), target.to(device)
score = model(data)
imgs = data.data.cpu()
lbl_pred = score.data.max(1)[1].cpu().numpy()
lbl_true = target.data.cpu()
for img, lt, lp in zip(imgs, lbl_true, lbl_pred):
img, lt = val_loader.dataset.untransform(img, lt)
metrics.update(lt, lp)
if len(visualizations) < 9:
viz = visualize_segmentation(lbl_pred=lp,
lbl_true=lt,
img=img,
n_classes=n_classes,
dataloader=val_loader)
visualizations.append(viz)
acc, acc_cls, mean_iu, fwavacc, cls_iu = metrics.get_scores()
print('''
Accuracy: {0:.2f}
Accuracy Class: {1:.2f}
Mean IoU: {2:.2f}
FWAV Accuracy: {3:.2f}'''.format(acc * 100, acc_cls * 100, mean_iu *
100, fwavacc * 100) + '\n')
class_name = val_loader.dataset.class_names
if class_name is not None:
for index, value in enumerate(cls_iu.values()):
offset = 20 - len(class_name[index])
print(class_name[index] + ' ' * offset + f'{value * 100:>.2f}')
else:
print("\nyou don't specify class_names, use number instead")
for key, value in cls_iu.items():
print(key, f'{value * 100:>.2f}')
viz = get_tile_image(visualizations)
# img = Image.fromarray(viz)
# img.save('viz_evaluate.png')
scipy.misc.imsave('viz_evaluate.png', viz)
def main():
# parser = argparse.ArgumentParser(
# formatter_class=argparse.ArgumentDefaultsHelpFormatter
# )
# parser.add_argument('--model', type=str, default='multi-gnn1')
# parser.add_argument('--model_file', type=str, default='/home/ecust/lx/Multimodal/logs/multi-gnn1_FS/model_best.pth.tar',help='Model path')
# parser.add_argument('--dataset_type', type=str, default='b',help='type of dataset')
# parser.add_argument('--dataset', type=str, default='/home/ecust/Datasets/数据库B(541)',help='path to dataset')
# parser.add_argument('--base_size', type=tuple, default=(300, 300), help='resize images using bilinear interpolation')
# parser.add_argument('--crop_size', type=tuple, default=None, help='crop images')
# parser.add_argument('--n_classes', type=int, default=13, help='number of classes')
# parser.add_argument('--pretrained', type=bool, default=True, help='should be set the same as train.py')
# args = parser.parse_args()
args = argparser()
model_file = '/home/ecust/lx/Multimodal/logs/resnet_20190916_093026/model_best.pth.tar'
root = args.dataset_root
crop=None
# crop = Compose([RandomCrop(args.crop_size)])
loader = get_loader(args.dataset)
val_loader = DataLoader(
loader(root, split='val', base_size=args.base_size, augmentations=crop),
batch_size=1, shuffle=False, num_workers=4)
args.n_classes = loader.NUM_CLASS
model = Models.model_loader(args.model, args.n_classes,
backbone=args.backbone, norm_layer=nn.BatchNorm2d,
multi_grid=args.multi_grid,
multi_dilation=args.multi_dilation)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(device)
print('==> Loading {} model file: {}'.format(model.__class__.__name__, model_file))
model_data = torch.load(model_file)
try:
model.load_state_dict(model_data)
except Exception:
model.load_state_dict(model_data['model_state_dict'])
model.eval()
print('==> Evaluating with {} dataset'.format(args.dataset))
visualizations = []
metrics = runningScore(args.n_classes)
i = 0
for rgb, ir, target in tqdm.tqdm(val_loader, total=len(val_loader), ncols=80, leave=False):
rgb, ir, target = rgb.to(device), ir.to(device), target.to(device)
score = model(rgb, ir)
# score = model(ir)
rgbs = rgb.data.cpu()
irs = ir.data.cpu()
lbl_pred = score[0].data.max(1)[1].cpu().numpy()
lbl_true = target.data.cpu()
for rgb, ir, lt, lp in zip(rgbs, irs, lbl_true, lbl_pred):
rgb, ir, lt = val_loader.dataset.untransform(rgb, ir, lt)
metrics.update(lt, lp)
i += 1
if i % 5 == 0:
if len(visualizations) < 9:
viz = visualize_segmentation(
lbl_pred=lp, lbl_true=lt, img=rgb, ir=ir,
n_classes=args.n_classes, dataloader=val_loader)
visualizations.append(viz)
acc, acc_cls, mean_iu, fwavacc, cls_iu = metrics.get_scores()
print('''
Accuracy: {0:.2f}
Accuracy Class: {1:.2f}
Mean IoU: {2:.2f}
FWAV Accuracy: {3:.2f}'''.format(acc * 100,
acc_cls * 100,
mean_iu * 100,
fwavacc * 100) + '\n')
class_name = val_loader.dataset.class_names
if class_name is not None:
for index, value in enumerate(cls_iu.values()):
offset = 20 - len(class_name[index])
print(class_name[index] + ' ' * offset + f'{value * 100:>.2f}')
else:
print("\nyou don't specify class_names, use number instead")
for key, value in cls_iu.items():
print(key, f'{value * 100:>.2f}')
viz = get_tile_image(visualizations)
# img = Image.fromarray(viz)
# img.save('viz_evaluate.png')
scipy.misc.imsave('viz_evaluate.png', viz)
def validate(self):
visualizations = []
val_metrics = runningScore(self.n_classes)
val_loss_meter = averageMeter()
with torch.no_grad():
self.model.eval()
for data, target in tqdm.tqdm(self.val_loader,
total=len(self.val_loader),
desc=f'Valid epoch={self.epoch}',
ncols=80,
leave=False):
data, target = data.to(self.device), target.to(self.device)
score = self.model(data)
weight = self.val_loader.dataset.class_weight
if weight:
weight = torch.Tensor(weight).to(self.device)
# target = resize_labels(target, (score.size()[2], score.size()[3]))
# target = target.to(self.device)
loss = CrossEntropyLoss(score,
target,
weight=weight,
reduction='mean',
ignore_index=-1)
loss_data = loss.data.item()
if np.isnan(loss_data):
raise ValueError('loss is nan while validating')
val_loss_meter.update(loss_data)
# if not isinstance(score, tuple):
# lbl_pred = score.data.max(1)[1].cpu().numpy()
# else:
# lbl_pred = score[-1].data.max(1)[1].cpu().numpy()
# lbl_pred, lbl_true = get_multiscale_results(score, target, upsample_logits=False)
imgs = data.data.cpu()
if isinstance(score, tuple):
lbl_pred = score[-1].data.max(1)[1].cpu().numpy()
else:
lbl_pred = score.data.max(1)[1].cpu().numpy()
lbl_true = target.data.cpu()
for img, lt, lp in zip(imgs, lbl_true, lbl_pred):
img, lt = self.val_loader.dataset.untransform(img, lt)
val_metrics.update(lt, lp)
# img = Image.fromarray(img).resize((lt.shape[1], lt.shape[0]), Image.BILINEAR)
# img = np.array(img)
if len(visualizations) < 9:
viz = visualize_segmentation(
lbl_pred=lp,
lbl_true=lt,
img=img,
n_classes=self.n_classes,
dataloader=self.train_loader)
visualizations.append(viz)
acc, acc_cls, mean_iou, fwavacc, _ = val_metrics.get_scores()
metrics = [acc, acc_cls, mean_iou, fwavacc]
print(f'\nEpoch: {self.epoch}',
f'loss: {val_loss_meter.avg}, mIoU: {mean_iou}')
out = osp.join(self.out, 'visualization_viz')
if not osp.exists(out):
os.makedirs(out)
out_file = osp.join(out, 'epoch{:0>5d}.jpg'.format(self.epoch))
scipy.misc.imsave(out_file, get_tile_image(visualizations))
with open(osp.join(self.out, 'log.csv'), 'a') as f:
elapsed_time = (datetime.datetime.now(pytz.timezone('UTC')) -
self.timestamp_start).total_seconds()
log = [self.epoch] + [''] * 5 + \
[val_loss_meter.avg] + metrics + [elapsed_time]
log = map(str, log)
f.write(','.join(log) + '\n')
mean_iu = metrics[2]
is_best = mean_iu > self.best_mean_iu
if is_best:
self.best_mean_iu = mean_iu
torch.save(
{
'epoch': self.epoch,
'arch': self.model.__class__.__name__,
'optim_state_dict': self.optim.state_dict(),
'model_state_dict': self.model.state_dict(),
'best_mean_iu': self.best_mean_iu,
}, osp.join(self.out, 'checkpoint.pth.tar'))
if is_best:
shutil.copy(osp.join(self.out, 'checkpoint.pth.tar'),
osp.join(self.out, 'model_best.pth.tar'))
val_loss_meter.reset()
val_metrics.reset()
def validate(self):
# -----------------------------------------------------------------------------
training = self.model.training
self.model.eval()
MAX_NUM = 100 # HACK: stop after 100 images
n_class = self.val_loader.dataset.num_hc_bins
val_loss = 0
visualizations = []
label_trues, label_preds = [], []
for batch_idx, (data, (target)) in tqdm.tqdm(
enumerate(self.val_loader), total=len(self.val_loader),
desc='Valid iteration=%d' % self.iteration, ncols=80,
leave=True):
# Computing val losses
if self.train_loader.dataset.bins == 'one-hot':
target_hue, target_chroma = target
if self.cuda:
data, target_hue, target_chroma = \
data.cuda(), target_hue.cuda(), target_chroma.cuda()
data, target_hue, target_chroma = \
Variable(data, volatile=True), Variable(target_hue, volatile=True), Variable(target_chroma, volatile=True)
(score_hue, score_chroma) = self.model(data)
loss_hue = cross_entropy2d(score_hue, target_hue,
size_average=self.size_average)
loss_chroma = cross_entropy2d(score_chroma, target_chroma,
size_average=self.size_average)
if np.isnan(float(loss_hue.data[0])):
raise ValueError('hue loss is NaN while validation')
if np.isnan(float(loss_chroma.data[0])):
raise ValueError('chroma loss is NaN while validation')
loss = loss_chroma + loss_hue
val_loss += float(loss.data[0]) / len(data)
del loss_hue, loss_chroma
elif self.train_loader.dataset.bins == 'soft' \
or self.train_loader.dataset.bins == 'uniform':
if self.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
score = self.model(data)
loss = kl_div2d(score, target, size_average=self.size_average) # DEBUG: MSE loss
if np.isnan(float(loss.data[0])):
raise ValueError('loss is NaN while validation')
val_loss += float(loss.data[0]) / len(data)
else:
raise ValueError('Bins can be one-hot, soft or uniform')
# Visualization
imgs = data.data.cpu()
if self.train_loader.dataset.bins == 'one-hot':
# visualize only hue predictions
lbl_pred = score_hue.data.max(1)[1].cpu().numpy()[:, :, :]
lbl_true = target_hue.data.cpu()
del score_hue, target_hue, score_chroma, target_chroma
elif self.train_loader.dataset.bins == 'soft' \
or self.train_loader.dataset.bins == 'uniform':
lbl_pred = score.data.max(1)[1].cpu().numpy()[:, :, :]
_, lbl_true = target.data.max(dim=3) # EDIT - .data
lbl_true = lbl_true.cpu()
if len(visualizations) < 9:
# HACK: use 1st image from each batch to visualize
if score.data.size()[0] > 9:
for i in range(9):
img = \
PIL.Image.open(self.val_loader.dataset.files['val'][i])
img = self.val_loader.dataset.rescale(img) # orig RGB image
viz = utils.visualize_colorization(
lbl_pred=score.data[i].cpu().numpy(),
lbl_true=target.data[i].cpu().numpy(),
img_orig=img, im_l=np.squeeze(imgs[i].numpy()),
gmm=self.val_loader.dataset.gmm,
mean_l=self.val_loader.dataset.mean_l)
visualizations.append(viz)
else:
# HACK: batch-size 1
img = \
PIL.Image.open(self.val_loader.dataset.files['val'][batch_idx])
img = self.val_loader.dataset.rescale(img) # orig RGB image
viz = utils.visualize_colorization(
lbl_pred=score.data[0].cpu().numpy(),
lbl_true=target.data[0].cpu().numpy(),
img_orig=img, im_l=np.squeeze(imgs[0].numpy()),
gmm=self.val_loader.dataset.gmm,
mean_l=self.val_loader.dataset.mean_l)
visualizations.append(viz)
del target, score
else:
raise ValueError('Bins can be one-hot, soft or uniform')
lbl_pred = lbl_pred.squeeze()
lbl_true = np.squeeze(lbl_true.numpy())
label_trues.append(lbl_true)
label_preds.append(lbl_pred)
del lbl_true, lbl_pred, data, loss, imgs
if batch_idx > MAX_NUM:
break
out = osp.join(self.out, 'visualization_viz')
if not osp.exists(out):
os.makedirs(out)
out_file = osp.join(out, 'iter%012d.jpg' % self.iteration)
scipy.misc.imsave(out_file, utils.get_tile_image(visualizations))
del visualizations
# Computing metrics
metrics = utils.label_accuracy_score(
label_trues, label_preds, n_class)
val_loss /= len(self.val_loader)
with open(osp.join(self.out, 'log.csv'), 'a') as f:
elapsed_time = (
datetime.datetime.now(pytz.timezone('US/Eastern')) -
self.timestamp_start).total_seconds()
log = [self.epoch, self.iteration] + [''] * 5 + \
[val_loss] + list(metrics) + [elapsed_time]
log = map(str, log)
f.write(','.join(log) + '\n')
del label_trues, label_preds
gc.collect()
mean_iu = metrics[2]
# TODO - save best model based on Validation **Loss**
is_best = val_loss > self.best_val_loss
if is_best:
self.best_val_loss = val_loss
torch.save({
'epoch': self.epoch,
'iteration': self.iteration,
'arch': self.model.__class__.__name__,
'optim_state_dict': self.optim.state_dict(),
'model_state_dict': self.model.state_dict(),
'best_mean_iu': None,
'best_val_loss': self.best_val_loss, # changed
}, osp.join(self.out, 'checkpoint.pth.tar'))
if is_best:
shutil.copy(osp.join(self.out, 'checkpoint.pth.tar'),
osp.join(self.out, 'model_best.pth.tar'))
if training:
self.model.train()