def eval(model,dataloader, args, label_info):
print('start test!')
with torch.no_grad():
model.eval()
precision_record = []
tq = tqdm.tqdm(total=len(dataloader) * args.batch_size)
tq.set_description('test')
hist = np.zeros((args.num_classes, args.num_classes))
for i, (data, label) in enumerate(dataloader):
tq.update(args.batch_size)
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda()
predict = model(data).squeeze()
predict = reverse_one_hot(predict)
predict = np.array(predict)
# predict = colour_code_segmentation(np.array(predict), label_info)
label = label.squeeze()
label = reverse_one_hot(label)
label = np.array(label)
# label = colour_code_segmentation(np.array(label), label_info)
precision = compute_global_accuracy(predict, label)
hist += fast_hist(label.flatten(), predict.flatten(), args.num_classes)
precision_record.append(precision)
precision = np.mean(precision_record)
miou = np.mean(per_class_iu(hist))
tq.close()
print('precision for test: %.3f' % precision)
print('mIoU for validation: %.3f' % miou)
return precision
def ensemble_boost_evaluate(models, dataloader, device, num_classes=10, boost_type=None, boost_model=None):
hist_sum = np.zeros((num_classes, num_classes))
with torch.no_grad():
for batch, item in tqdm(enumerate(dataloader)):
data, label = item
data = data.to(device)
out_all = None
for model in models:
temp_out = model(data)
temp_out = torch.nn.functional.softmax(temp_out, dim=1) # 转成概率
temp_out = torch.unsqueeze(temp_out, dim=1) # batch*model*chnl*w*h
if out_all is None:
out_all = temp_out
else:
out_all = torch.cat((out_all, temp_out), dim=1) # batch*model*chnl*w*h
# 把模型输出转成预测的标签pred
# out_all = torch.argmax(out_all, dim=2)
# out_all = out_all.permute(0, 2, 3, 1)
# out_all = out_all.reshape((-1, 14))
out_all = out_all.permute(0, 3, 4, 1, 2) # batch*w*h*model*chnl
out_all = out_all.reshape((-1, out_all.shape[-2] * out_all.shape[-1])) # (batch*w*h)*(model*chnl)
out_all = out_all.cpu().numpy()
if boost_type == 'adaBoost':
pred = boost_model.predict(out_all) # (batch*w*h)
elif boost_type == 'XGBoost':
pred = boost_model.predict(xgboost.DMatrix(data=out_all)) # (batch*w*h)
pred = pred.reshape((-1, 256, 256)).astype(np.int64) # batch*w*h
label = label.cpu().numpy()
for i in range(len(pred)):
hist = fast_hist(label[i], pred[i], num_classes)
hist_sum += hist
miou = compute_miou(hist_sum)
return miou
def evaluate(model, loader, n_class, device, dtype, iter_idx, writer):
hist = np.zeros((n_class, n_class))
for batch_idx, (data, target) in enumerate(loader):
data = data.to(device=device, dtype=dtype)
with torch.no_grad():
output = model(data)
_, h, w = target.shape
output = torch.nn.functional.interpolate(output,
size=(h, w),
mode='bilinear',
align_corners=True)
output, target = output.data.cpu().numpy(), target.data.cpu().numpy()
output = np.argmax(output, axis=1)
hist += fast_hist(target.flatten(), output.flatten(), n_class)
if batch_idx == 0:
writer.add_image(
'val/input',
vutils.make_grid(data,
normalize=True,
scale_each=True,
padding=0), iter_idx)
writer.add_image('val/output', decode_labels(output[0]), iter_idx)
writer.add_image('val/gt', decode_labels(target[0]), iter_idx)
m_iou = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
return np.sum(m_iou) / len(m_iou)
def val(args, model, dataloader, csv_path):
print('start val!')
# label_info = get_label_info(csv_path)
with torch.no_grad():
model.eval()
precision_record = []
hist = np.zeros((args.num_classes, args.num_classes))
for i, (data, label) in enumerate(dataloader):
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda()
# get RGB predict image
predict = model(data).squeeze()
predict = reverse_one_hot(predict)
predict = np.array(predict)
# get RGB label image
label = label.squeeze()
label = reverse_one_hot(label)
label = np.array(label)
# compute per pixel accuracy
precision = compute_global_accuracy(predict, label)
hist += fast_hist(label.flatten(), predict.flatten(), args.num_classes)
# there is no need to transform the one-hot array to visual RGB array
# predict = colour_code_segmentation(np.array(predict), label_info)
# label = colour_code_segmentation(np.array(label), label_info)
precision_record.append(precision)
dice = np.mean(precision_record)
miou = np.mean(per_class_iu(hist))
print('precision per pixel for validation: %.3f' % dice)
print('mIoU for validation: %.3f' % miou)
return dice
def val(args, model, dataloader):
print('start val!')
# label_info = get_label_info(csv_path)
'''
model.eval() is a kind of switch for some specific layers/parts of the model that
behave differently during training and inference (evaluating) time. For example,
Dropouts Layers, BatchNorm Layers etc. You need to turn off them during model evaluation,
and .eval() will do it for you. In addition, the common practice for evaluating/validation
is using torch.no_grad() in pair with model.eval() to turn off gradients computation.
After that we call model.train() to reswitch them on.
'''
with torch.no_grad():
model.eval()
# lista delle precisioni
precision_record = []
# inizializziamo una matrice per calcolare successivamente mIoU
hist = np.zeros((args.num_classes, args.num_classes))
# Itarates over what? Single images or batches?
for i, (data, label) in enumerate(dataloader):
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda().long()
output = model(data)
# get RGB predict image
_, prediction = output.max(dim=1) # B, H, W
label = label.cpu().numpy()
prediction = prediction.cpu().numpy()
# compute per pixel accuracy
precision = compute_global_accuracy(prediction, label)
# Cosa fa fast_hist????
hist += fast_hist(label.flatten(), prediction.flatten(),
args.num_classes)
# there is no need to transform the one-hot array to visual RGB array
# predict = colour_code_segmentation(np.array(predict), label_info)
# label = colour_code_segmentation(np.array(label), label_info)
precision_record.append(precision)
# compute the mean precision:
precision = np.mean(precision_record)
# miou = np.mean(per_class_iu(hist))
miou_list = per_class_iu(hist)[:
-1] # preché tutti tranne l'ultimo?????
# miou_dict, miou = cal_miou(miou_list, csv_path)
miou = np.mean(miou_list)
print('precision per pixel for test: %.3f' % precision)
print('mIoU for validation: %.3f' % miou)
# miou_str = ''
# for key in miou_dict:
# miou_str += '{}:{},\n'.format(key, miou_dict[key])
# print('mIoU for each class:')
# print(miou_str)
return precision, miou
def eval(model, dataloader, args, csv_path):
print('start test!')
with torch.no_grad():
model.eval()
precision_record = []
tq = tqdm.tqdm(total=len(dataloader) * args.batch_size)
tq.set_description('test')
hist = np.zeros((args.num_classes, args.num_classes))
for i, (data, label) in enumerate(dataloader):
tq.update(args.batch_size)
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda()
predict = model(data).squeeze()
predict = reverse_one_hot(predict)
predict = np.array(predict)
# predict = colour_code_segmentation(np.array(predict), label_info)
label = label.squeeze()
if args.loss == 'dice':
label = reverse_one_hot(label)
label = np.array(label)
# label = colour_code_segmentation(np.array(label), label_info)
#saving some images
if args.save_images_path is not None and i < 40:
current_image = transforms.functional.to_pil_image(data[0])
current_label = Image.fromarray(colorize_label(label))
current_predi = Image.fromarray(colorize_label(predict))
current_image.save(args.save_images_path + f"/image{i}.jpg")
current_label.save(args.save_images_path + f"/label{i}.jpeg")
current_predi.save(args.save_images_path +
f"/prediction{i}.jpeg")
precision = compute_global_accuracy(predict, label)
hist += fast_hist(label.flatten(), predict.flatten(),
args.num_classes)
precision_record.append(precision)
precision = np.mean(precision_record)
miou_list = per_class_iu(hist)[:-1]
miou_dict, miou = cal_miou(miou_list, csv_path)
print('IoU for each class:')
for key in miou_dict:
print('{}:{},'.format(key, miou_dict[key]))
tq.close()
print('precision for test: %.3f' % precision)
print('mIoU for validation: %.3f' % miou)
return precision
def val(args, model, dataloader):
print('start val!')
# label_info = get_label_info(csv_path)
with torch.no_grad():
model.eval()
precision_record = []
hist = np.zeros((args.num_classes, args.num_classes))
for i, (data, label) in enumerate(dataloader):
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda()
# get RGB predict image
predict = model(data).squeeze()
predict = reverse_one_hot(predict)
predict = np.array(predict.cpu())
# get RGB label image
label = label.squeeze()
if args.loss == 'dice':
label = reverse_one_hot(label)
label = np.array(label.cpu())
# compute per pixel accuracy
precision = compute_global_accuracy(predict, label)
hist += fast_hist(label.flatten(), predict.flatten(),
args.num_classes)
# there is no need to transform the one-hot array to visual RGB array
# predict = colour_code_segmentation(np.array(predict), label_info)
# label = colour_code_segmentation(np.array(label), label_info)
precision_record.append(precision)
precision = np.mean(precision_record)
# miou = np.mean(per_class_iu(hist))
miou_list = per_class_iu(hist)[:-1]
# miou_dict, miou = cal_miou(miou_list, csv_path)
miou = np.mean(miou_list)
print('precision per pixel for test: %.3f' % precision)
print('mIoU for validation: %.3f' % miou)
# miou_str = ''
# for key in miou_dict:
# miou_str += '{}:{},\n'.format(key, miou_dict[key])
# print('mIoU for each class:')
# print(miou_str)
return precision, miou
def validate(val_loader, model, criterion):
batch_time = utils.AverageMeter()
data_time = utils.AverageMeter()
losses = utils.AverageMeter()
acc = utils.AverageMeter()
model.eval()
hist = np.zeros((cfg.data.classes, cfg.data.classes))
with torch.no_grad():
end = time.time()
for i, (input, target) in enumerate(val_loader):
data_time.update(time.time() - end)
target = target.cuda(non_blocking=True)
output = model(input)
loss = criterion(output, target)
losses.update(loss.item(), input.size(0))
acc.update(utils.accuracy(output, target), input.size(0))
_, pred = output.max(1)
hist += utils.fast_hist(pred.cpu().data.numpy().flatten(),
target.cpu().numpy().flatten(),
cfg.data.classes)
batch_time.update(time.time() - end)
end = time.time()
if i % cfg.training.print_freq == 0:
print(
'Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Accuracy {accuracy.val:.4f} ({accuracy.avg:.4f})'.format(
i,
len(val_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
accuracy=acc))
ious = utils.per_class_iou(hist) * 100
return batch_time.avg, data_time.avg, losses.avg, acc.avg, ious
def evaluate(args):
# device
device = torch.device('cuda:{}'.format(args.gpu) if args.gpu >= 0 and torch.cuda.is_available() else 'cpu')
if args.gpu >= 0 and torch.cuda.is_available():
cudnn.benchmark = True
# dtype
if args.type == 'float64':
dtype = torch.float64
elif args.type == 'float32':
dtype = torch.float32
elif args.type == 'float16':
dtype = torch.float16
else:
raise ValueError('Wrong type!')
# model
mask = np.load(args.mask_path)
model = SparseMask(mask, backbone_name=args.backbone_name, depth=args.depth, in_channels=3, num_classes=args.n_class)
# dataset
eval_loader = get_loader(args.im_path, args.gt_path, args.eval_list, 1, 1, training=False)
# to device
if args.gpu >= 0 is not None:
model = torch.nn.DataParallel(model, [args.gpu])
model.to(device=device, dtype=dtype)
# load weight
checkpoint = torch.load(args.pretrained_model, map_location=device)
model.load_state_dict(checkpoint['state_dict'], strict=True)
model.eval()
with torch.no_grad():
hist = np.zeros((args.n_class, args.n_class))
for batch_idx, (data, target) in enumerate(tqdm(eval_loader)):
data = data.to(device=device, dtype=dtype)
output = model(data)
_, h, w = target.shape
output = torch.nn.functional.interpolate(output, size=(h, w), mode='bilinear', align_corners=True)
output, target = output.data.cpu().numpy(), target.data.cpu().numpy()
output = np.argmax(output, axis=1)
hist += fast_hist(target.flatten(), output.flatten(), args.n_class)
m_iou = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
print(np.sum(m_iou) / len(m_iou))
def test(epoch_idx, net, test_loader, logger, n_class):
net.cuda()
net.eval()
len_batch = len(test_loader)
visualizations = []
hist = np.zeros((n_class, n_class))
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(test_loader):
inputs, targets = inputs.cuda(), targets.cuda()
output = net(inputs)
_, predicted = output.max(1)
predicted, targets = to_np(predicted), to_np(targets)
print(('test', batch_idx, epoch_idx))
hist += fast_hist(targets.flatten(), predicted.flatten(), n_class)
miou = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
miou = np.sum(miou) / len(miou)
logger.scalar_summary('Mean iou', miou, epoch_idx)
print(('Mean iou: ', miou))
def val(args, model, dataloader, loss_func):
# print("start val!")
# label_info = get_label_info(csv_path)
tq = tqdm(total=len(dataloader) * args.batch_size)
tq.set_description("validating:")
with torch.no_grad():
model.eval()
precision_record = []
loss_record = []
hist = np.zeros((args.num_classes, args.num_classes))
for i, (data, label) in enumerate(dataloader):
tq.update(args.batch_size)
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda().long()
output = model(data)
loss = loss_func(output, label)
loss_record.append(loss.item())
# get RGB predict image
_, prediction = output.max(dim=1) # B, H, W
label = label.cpu().numpy()
prediction = prediction.cpu().numpy()
# compute per pixel accuracy
precision = compute_global_accuracy(prediction, label)
hist += fast_hist(label.flatten(),
prediction.flatten(), args.num_classes)
# there is no need to transform the one-hot array to visual RGB array
# predict = colour_code_segmentation(np.array(predict), label_info)
# label = colour_code_segmentation(np.array(label), label_info)
precision_record.append(precision)
tq.close()
loss_mean = np.mean(loss_record)
precision = np.mean(precision_record)
miou_list = per_class_iu(hist)[:-1]
miou = np.mean(miou_list)
return precision, miou, loss_mean
def ensemble_evaluate(models, dataloader, ensemble_weight, device, num_classes=10):
hist_sum = np.zeros((num_classes, num_classes))
with torch.no_grad():
for batch, item in tqdm(enumerate(dataloader)):
data, label = item
data = data.to(device)
print(data)
out_avg = torch.zeros(size=(len(data), 10, 256, 256)).to(device)
model_num = 0
for model in models:
temp_out = model(data)
temp_out = torch.nn.functional.softmax(temp_out, dim=1) # 转成概率
out_avg += ensemble_weight[model_num] * temp_out
model_num += 1
print(sum(sum(torch.argmax(temp_out, dim=1)[0] + 1 == 2)) / 256 / 256)
pred = torch.argmax(out_avg, dim=1).cpu().numpy()
label = label.cpu().numpy()
for i in range(len(pred)):
hist = fast_hist(label[i], pred[i], num_classes)
hist_sum += hist
miou = compute_miou(hist_sum)
return miou
def eval(model, dataloader, args, csv_path):
print('start test!')
with torch.no_grad():
total_pred = np.array([0])
total_label = np.array([0])
total_cm = np.zeros((6, 6))
model.eval()
precision_record = []
tq = tqdm.tqdm(total=len(dataloader) * args.batch_size)
tq.set_description('test')
hist = np.zeros((args.num_classes, args.num_classes))
total_time = 0
total_cks, total_f1 = 0.0, 0.0
length = len(dataloader)
print('length: %d' % length)
for i, (data, label) in enumerate(dataloader):
tq.update(args.batch_size)
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda()
start = time.clock()
predict = model(data).squeeze()
end = time.clock()
# 转为类别矩阵
predict = reverse_one_hot(predict)
predict = np.array(predict)
# predict = colour_code_segmentation(np.array(predict), label_info)
# end = time.clock()
# 测试花费时间
total_time += (end - start)
label = label.squeeze()
# 转换为类别矩阵
if args.loss == 'dice':
label = reverse_one_hot(label)
label = np.array(label)
# 计算cm
# total_pred = np.append(total_pred, predict.flatten())
# total_label = np.append(total_label, label.flatten())
# if (i+1) % 8 == 0:
# total_cm += confusion_matrix(total_label[1:], total_pred[1:])
# total_label = np.array([0])
# total_pred = np.array([0])
# 计算kappa，总的算平均
cks = cohen_kappa_score(label.flatten(), predict.flatten())
total_cks += cks
f1 = f1_score(label.flatten(), predict.flatten(), average='macro')
total_f1 += f1
# label = colour_code_segmentation(np.array(label), label_info)
# 计算oa
precision = compute_global_accuracy(predict, label)
hist += fast_hist(label.flatten(), predict.flatten(),
args.num_classes)
# 记录总的精度
precision_record.append(precision)
# 保存cm
# np.savetxt('cm.txt', total_cm)
precision = np.mean(precision_record)
miou_list = per_class_iu(hist)
miou_dict, miou = cal_miou(miou_list, csv_path)
print('IoU for each class:')
for key in miou_dict:
print('{}:{},'.format(key, miou_dict[key]))
tq.close()
print('oa for test: %.3f' % precision)
print('mIoU for test: %.3f' % miou)
# 计算cm, kappa, cr //作废
cm, cks, cr = compute_cm_cks_cr(predict, label)
# print('cm for test:\n', cm)
total_cks /= length
print('kappa for test: %.4f' % total_cks)
total_f1 /= length
print('f1 for test: %.4f' % total_f1)
fps = length / total_time
print('fps: %.2f' % fps)
return precision, cm, total_cks, cr
def eval_masks(self,
dataset_val,
eval_dir,
save_images,
save_stats,
gui,
sort_class_name,
verbose=True):
eval_dir = eval_dir.replace('\\', '/')
assert self.mode == 'inference'
num_images = len(dataset_val.image_ids)
def _vprint(*args):
if verbose:
print(*args)
_vprint('num_images: %d' % num_images)
eval_class_ids = []
eval_names = []
for k, inds in self.options['eval_classes'].items():
eval_names.append(k)
eval_class_ids.append(inds)
if k == sort_class_name:
sort_class_ids = inds
assert len(eval_class_ids) == len(eval_names)
# evaluate
def _collect_wrong_pixel_num_for_sort(hist_vals):
intersected = 0
label_ids = sort_class_ids
for label_id1 in label_ids:
for label_id2 in label_ids:
intersected += hist_vals[label_id1, label_id2]
A = hist_vals[label_ids, :].sum()
B = hist_vals[:, label_ids].sum()
return A + B - 2 * intersected
def _collect_f1(hist_vals):
f1s = dict()
for i, label_ids in enumerate(eval_class_ids):
name = eval_names[i]
_vprint('ids of {} is {}'.format(name, label_ids))
intersected = 0
for label_id1 in label_ids:
for label_id2 in label_ids:
intersected += hist_vals[label_id1, label_id2]
A = hist_vals[label_ids, :].sum()
B = hist_vals[:, label_ids].sum()
f1 = 2 * intersected / (A + B)
f1s[name] = f1
return f1s
# including background
batch_size = self.options['images_per_gpu'] * self.gpu_count
generator = data_generator(dataset_val,
self.required_data_names(self.mode),
shuffle=False,
batch_size=batch_size)
gt_generator = data_generator(dataset_val, ['image', 'masks'],
shuffle=False,
batch_size=batch_size)
all_hists = [None] * num_images
all_gt_labels = [None] * num_images
all_pred_labels = [None] * num_images
all_pred_masks = [None] * num_images
all_pred_head_boxes = [None] * num_images
all_pred_labels_in_original = [None] * num_images
all_pred_masks_in_original = [None] * num_images
start_id = 0
for (inputs, _), ((_, gt_masks_exist, images, gt_masks),
_) in zip(generator, gt_generator):
stop_id = start_id + batch_size
_vprint('processing %d-%d' % (start_id, stop_id))
start_time = time.time()
preds = self.keras_model.predict(inputs, verbose=0)
elapsed_time = time.time() - start_time
print('time cost %f sec' % elapsed_time)
if not isinstance(preds, list):
preds = [preds]
pred_masks = np.clip(preds[0], 0.0, 1.0)
if len(preds) >= 2:
pred_molded_head_boxes = preds[1]
pred_head_boxes = pred_molded_head_boxes * \
self.options['image_size']
assert (pred_masks.shape[0], pred_masks.shape[1]) == \
(gt_masks.shape[0], gt_masks.shape[1])
assert pred_masks.shape == gt_masks.shape
for k in range(batch_size):
ind = (start_id + k) % dataset_val.num_images
# first collect labels
all_pred_masks[ind] = pred_masks[k]
all_gt_labels[ind] = self._flatten_gt_masks(gt_masks[k])
all_pred_labels[ind] = self._flatten_predicted_masks(
pred_masks[k])
align_matrix, align_matrix_exists = dataset_val.load_align_matrix(
ind)
original_image, original_image_exists = dataset_val.load_original_image(
ind)
has_original_data = align_matrix_exists and original_image_exists
if has_original_data:
all_pred_masks_in_original[ind] = utils.reverse_align(
pred_masks[k],
align_matrix,
original_image.shape,
is_bhwc=False)
all_pred_labels_in_original[
ind] = self._flatten_predicted_masks(
all_pred_masks_in_original[ind])
# second compute hists
num_masks = len(self.options['class_names'])
assert pred_masks.shape[1] == num_masks
assert gt_masks.shape[1] == num_masks
all_hists[ind] = utils.fast_hist(all_gt_labels[ind],
all_pred_labels[ind],
num_masks + 1)
if 'pred_head_boxes' in locals():
all_pred_head_boxes[ind] = pred_head_boxes[k]
# calc individual wrong pixel nums
wpn_here = _collect_wrong_pixel_num_for_sort(all_hists[ind])
_vprint('# of wrong classified %s pixels = %d' %
(sort_class_name, wpn_here))
# write to files
image = images[k]
blended_labels = utils.blend_labels(image,
all_pred_labels[ind])
blended_alphas = utils.blend_alphas(image, all_pred_masks[ind])
blended_alphas = (blended_alphas - np.min(blended_alphas)) / \
(np.max(blended_alphas) - np.min(blended_alphas))
blended_labels_gt = utils.blend_labels(image,
all_gt_labels[ind])
if has_original_data:
blended_labels_in_original = utils.blend_labels(
original_image, all_pred_labels_in_original[ind])
blended_alphas_in_original = utils.blend_alphas(
original_image, all_pred_masks_in_original[ind])
blended_alphas_in_original = (blended_alphas_in_original - np.min(blended_alphas_in_original)) / \
(np.max(blended_alphas_in_original) -
np.min(blended_alphas_in_original))
if gui:
#plt.imshow(blended_alphas)
#plt.show()
pass
if save_images:
im_fname = os.path.basename(
dataset_val.source_image_link(ind))
folder = os.path.join(
eval_dir, '%.4f_%05d_%s' %
(wpn_here /
(self.options['image_size']**2), ind, im_fname))
# folder = os.path.join(eval_dir, os.path.basename(dataset_val.source_image_link(ind)))
folder = folder.replace('\\', '/')
if not os.path.exists(folder):
os.mkdir(folder)
assert os.path.exists(folder)
io.imsave(
os.path.join(folder, 'input.jpg').replace('\\', '/'),
image)
if 'pred_head_boxes' in locals():
boxes = pred_head_boxes[k] # k x (y1 x1 y2 x2)
boxes = np.clip(boxes, 0,
self.options['image_size'] - 1)
labels_with_boxes = utils.blend_labels(
None, all_pred_labels[ind])
for kk in range(boxes.shape[0]):
y1, x1, y2, x2 = boxes[kk].astype(int)
# draw boxes
labels_with_boxes[draw.line(y1, x1, y1, x2)] = 1.0
labels_with_boxes[draw.line(y1, x2, y2, x2)] = 1.0
labels_with_boxes[draw.line(y2, x2, y2, x1)] = 1.0
labels_with_boxes[draw.line(y2, x1, y1, x1)] = 1.0
io.imsave(
os.path.join(folder,
'labels_with_boxes.jpg').replace(
'\\', '/'), labels_with_boxes)
io.imsave(
os.path.join(folder,
'blended_labels.png').replace('\\', '/'),
blended_labels)
io.imsave(
os.path.join(folder, 'blended_labels_gt.png').replace(
'\\', '/'), blended_labels_gt)
io.imsave(
os.path.join(folder,
'blended_alphas.png').replace('\\', '/'),
blended_alphas)
# io.imsave(os.path.join(
# folder, 'blended_labels128.jpg'), blended_labels128)
# io.imsave(os.path.join(
# folder, 'blended_labels_gt128.jpg'), blended_labels_gt128)
# io.imsave(os.path.join(
# folder, 'blended_alphas128.jpg'), blended_alphas128)
io.imsave(
os.path.join(folder,
'blended_labels_in_original.jpg').replace(
'\\', '/'),
blended_labels_in_original)
# io.imsave(os.path.join(
# folder, 'blended_labels_gt_in_original.jpg').replace('\\', '/'), blended_labels_gt_in_original)
io.imsave(
os.path.join(folder,
'blended_alphas_in_original.jpg').replace(
'\\', '/'),
blended_alphas_in_original)
start_id += batch_size
if stop_id >= num_images:
break
all_hists = np.stack(all_hists, axis=0)[:num_images]
all_gt_labels = np.stack(all_gt_labels, axis=0)[:num_images]
all_pred_labels = np.stack(all_pred_labels, axis=0)[:num_images]
# all_pred_masks = np.stack(all_pred_masks, axis=0)[:num_images]
# all_pred_landmark68_pts = np.stack(
# all_pred_landmark68_pts, axis=0)[:num_images]
# all_hists128 = np.stack(all_hists128, axis=0)[:num_images]
# all_hists_in_original = np.stack(
# all_hists_in_original, axis=0)[:num_images]
# all_gt_labels_in_original = np.stack(
# all_gt_labels_in_original, axis=0)[:num_images]
# all_pred_labels_in_original = np.stack(
# all_pred_labels_in_original, axis=0)[:num_images]
# all_pred_masks_in_original = np.stack(
# all_pred_masks_in_original, axis=0)[:num_images]
# all_pred_landmark68_pts_in_original = np.stack(
# all_pred_landmark68_pts_in_original, axis=0)[:num_images]
hists = np.sum(all_hists, axis=0)
# hists128 = np.sum(all_hists128, axis=0)
# hists_in_original = np.sum(all_hists_in_original, axis=0)
f1s = _collect_f1(hists)
for name, f1 in f1s.items():
print('#f1.aligned of %s\t\t=%f' % (name, f1))
# f1s128 = _collect_f1(hists128)
# for name, f1 in f1s128.items():
# print('#f1.aligned128 of %s\t\t=%f' % (name, f1))
# f1s_in_original = _collect_f1(hists_in_original)
# for name, f1 in f1s_in_original.items():
# print('#f1.original of %s\t\t=%f' % (name, f1))
if save_stats:
with open(os.path.join(eval_dir, 'f1s.csv'), 'w') as csv_file:
for name in f1s.keys():
csv_file.write(',%s' % name)
csv_file.write('\n')
csv_file.write('f1s aligned')
for name in f1s.keys():
csv_file.write(',%f' % f1s[name])
csv_file.write('\n')
# csv_file.write('f1s 128 aligned')
# for name in f1s.keys():
# csv_file.write(',%f' % f1s128[name])
# csv_file.write('\n')
# csv_file.write('f1s original')
# for name in f1s.keys():
# csv_file.write(',%f' % f1s_in_original[name])
scio.savemat(
os.path.join(eval_dir, 'data.mat'),
{
'all_hists': all_hists,
# 'all_hists128': all_hists128,
# 'all_hists_in_original': all_hists_in_original,
'f1s': f1s,
# 'f1s128': f1s128,
# 'f1s_in_original': f1s_in_original,
'all_gt_labels': all_gt_labels,
'all_pred_labels': all_pred_labels
},
do_compression=True)
return f1s[sort_class_name]
def test(self):
print self.list_path + ' evaluation begin!'
print 'Total number ' + str(len(self.pair_list))
loc = dmac_vgg.DMAC_VGG(self.nolabel, self.gpu, self.input_scale)
loc_saved_state_dict = torch.load(self.model_path)
loc.load_state_dict(loc_saved_state_dict)
loc.cuda(self.gpu)
loc.eval()
start = time.time()
count = 0
for piece in self.pair_list:
img1, _ = imreadtonumpy(self.data_path, piece[0], self.im_scale)
img2, _ = imreadtonumpy(self.data_path, piece[1], self.im_scale)
im1_pred = torch.zeros(
[self.grid_num + 1, self.im_scale, self.im_scale],
dtype=torch.float).cuda(self.gpu)
im2_pred = torch.zeros(
[self.grid_num + 1, self.im_scale, self.im_scale],
dtype=torch.float).cuda(self.gpu)
label_tmp = int(piece[2])
if label_tmp == 1:
gt_temp = cv2.imread(os.path.join(self.data_path,
piece[3]))[:, :, 0]
gt_temp[gt_temp == 255] = 1
gt_temp = cv2.resize(gt_temp, (self.im_scale, self.im_scale),
interpolation=cv2.INTER_NEAREST)
gt1 = gt_temp
gt_temp = cv2.imread(os.path.join(self.data_path,
piece[4]))[:, :, 0]
gt_temp[gt_temp == 255] = 1
gt_temp = cv2.resize(gt_temp, (self.im_scale, self.im_scale),
interpolation=cv2.INTER_NEAREST)
gt2 = gt_temp
else:
gt1 = np.zeros((self.im_scale, self.im_scale))
gt2 = np.zeros((self.im_scale, self.im_scale))
for im1_grid_idx_h in range(self.grid_edge_num):
for im1_grid_idx_w in range(self.grid_edge_num):
image1 = torch.from_numpy(
img1[np.newaxis, im1_grid_idx_h *
self.stride:(im1_grid_idx_h * self.stride +
self.input_scale), im1_grid_idx_w *
self.stride:(im1_grid_idx_w * self.stride +
self.input_scale), :].transpose(
0, 3, 1,
2)).float().cuda(self.gpu)
for im2_grid_idx_h in range(self.grid_edge_num):
for im2_grid_idx_w in range(self.grid_edge_num):
image2 = torch.from_numpy(
img2[np.newaxis, im2_grid_idx_h * self.stride:(
im2_grid_idx_h * self.stride +
self.input_scale),
im2_grid_idx_w * self.stride:(
im2_grid_idx_w * self.stride +
self.input_scale), :].transpose(
0, 3, 1,
2)).float().cuda(self.gpu)
output = loc(image1, image2)
output1 = self.softmax_mask(
self.interp256(output[0])).detach()
output2 = self.softmax_mask(
self.interp256(output[1])).detach()
torch.cuda.empty_cache()
tmp1 = im1_pred[
im2_grid_idx_h * self.grid_edge_num +
im2_grid_idx_w, im1_grid_idx_h *
self.stride:(im1_grid_idx_h * self.stride +
self.input_scale),
im1_grid_idx_w *
self.stride:(im1_grid_idx_w * self.stride +
self.input_scale)].unsqueeze(0)
tmp1 = torch.cat([tmp1, output1], 0)
tmp1, _ = torch.max(tmp1, dim=0, keepdim=True)
im1_pred[im2_grid_idx_h * self.grid_edge_num +
im2_grid_idx_w,
im1_grid_idx_h * self.stride:(
im1_grid_idx_h * self.stride +
self.input_scale),
im1_grid_idx_w * self.stride:(
im1_grid_idx_w * self.stride +
self.input_scale)] = tmp1
tmp2 = im2_pred[
im1_grid_idx_h * self.grid_edge_num +
im1_grid_idx_w, im2_grid_idx_h *
self.stride:(im2_grid_idx_h * self.stride +
self.input_scale),
im2_grid_idx_w *
self.stride:(im2_grid_idx_w * self.stride +
self.input_scale)].unsqueeze(0)
tmp2 = torch.cat([tmp2, output2], 0)
tmp2, _ = torch.max(tmp2, dim=0, keepdim=True)
im2_pred[im1_grid_idx_h * self.grid_edge_num +
im1_grid_idx_w,
im2_grid_idx_h * self.stride:(
im2_grid_idx_h * self.stride +
self.input_scale),
im2_grid_idx_w * self.stride:(
im2_grid_idx_w * self.stride +
self.input_scale)] = tmp2
img1_, _ = imreadtonumpy(self.data_path, piece[0],
self.input_scale)
img2_, _ = imreadtonumpy(self.data_path, piece[1],
self.input_scale)
image1 = torch.from_numpy(img1_[np.newaxis, :].transpose(
0, 3, 1, 2)).float().cuda(self.gpu)
image2 = torch.from_numpy(img2_[np.newaxis, :].transpose(
0, 3, 1, 2)).float().cuda(self.gpu)
output = loc(image1, image2)
output1 = self.softmax_mask(self.interp512(output[0])).detach()
output2 = self.softmax_mask(self.interp512(output[1])).detach()
torch.cuda.empty_cache()
im1_pred[self.grid_num, :, :] = output1
im2_pred[self.grid_num, :, :] = output2
final_output1, _ = torch.max(im1_pred, dim=0)
final_output2, _ = torch.max(im2_pred, dim=0)
output_1 = final_output1.cpu().data.numpy()
output_1 = output_1[:self.im_scale, :self.im_scale]
output_2 = final_output2.cpu().data.numpy()
output_2 = output_2[:self.im_scale, :self.im_scale]
if self.vis_flag == True:
vis_output = [output_1, output_2]
self.vis_fun(piece, img1, img2, gt1, gt2, vis_output)
output_1[output_1 > 0.5] = 1
output_1[output_1 <= 0.5] = 0
output_2[output_2 > 0.5] = 1
output_2[output_2 <= 0.5] = 0
output_1 = output_1.astype(int)
output_2 = output_2.astype(int)
hist1 = fast_hist(gt1.flatten(), output_1.flatten(),
self.nolabel).astype(float)
hist2 = fast_hist(gt2.flatten(), output_2.flatten(),
self.nolabel).astype(float)
self.NMM1 += get_NMM(hist1, gt1)
self.NMM2 += get_NMM(hist2, gt2)
self.MCC1 += get_MCC(hist1)
self.MCC2 += get_MCC(hist2)
iou1_tmp = np.diag(hist1) / (hist1.sum(1) + hist1.sum(0) -
np.diag(hist1))
iou2_tmp = np.diag(hist2) / (hist2.sum(1) + hist2.sum(0) -
np.diag(hist2))
self.iou1 += iou1_tmp[1]
self.iou2 += iou2_tmp[1]
print 'item ' + str(count) + ' processed!'
count += 1
stop = time.time()
print(stop - start) / float(count)
self.iou = (self.iou1 + self.iou2) / float(count * 2)
self.iou1 = self.iou1 / float(count)
self.iou2 = self.iou2 / float(count)
self.NMM = (self.NMM1 + self.NMM2) / float(count * 2)
self.NMM1 = self.NMM1 / float(count)
self.NMM2 = self.NMM2 / float(count)
self.MCC = (self.MCC1 + self.MCC2) / float(count * 2)
self.MCC1 = self.MCC1 / float(count)
self.MCC2 = self.MCC2 / float(count)
self.printscores()
def test(self):
self.loc.eval()
print self.list_path + ' evaluation begin!'
print 'Total number ' + str(len(self.pair_list))
count = 0
for piece in self.pair_list:
img1, _ = imreadtonumpy(self.data_path, piece[0], self.input_scale)
img2, _ = imreadtonumpy(self.data_path, piece[1], self.input_scale)
label_tmp = int(piece[2])
if label_tmp == 1:
gt_temp = cv2.imread(os.path.join(self.data_path,
piece[3]))[:, :, 0]
gt_temp[gt_temp == 255] = 1
gt_temp = cv2.resize(gt_temp,
(self.input_scale, self.input_scale),
interpolation=cv2.INTER_NEAREST)
gt1 = gt_temp
gt_temp = cv2.imread(os.path.join(self.data_path,
piece[4]))[:, :, 0]
gt_temp[gt_temp == 255] = 1
gt_temp = cv2.resize(gt_temp,
(self.input_scale, self.input_scale),
interpolation=cv2.INTER_NEAREST)
gt2 = gt_temp
else:
gt1 = np.zeros((self.input_scale, self.input_scale))
gt2 = np.zeros((self.input_scale, self.input_scale))
image1 = torch.from_numpy(img1[np.newaxis, :].transpose(
0, 3, 1, 2)).float().cuda(self.gpu)
image2 = torch.from_numpy(img2[np.newaxis, :].transpose(
0, 3, 1, 2)).float().cuda(self.gpu)
output = self.loc(image1, image2)
output0 = self.interp256(output[0]).cpu().data[0].numpy()
output0 = output0[:, :self.input_scale, :self.input_scale]
output1 = self.interp256(output[1]).cpu().data[0].numpy()
output1 = output1[:, :self.input_scale, :self.input_scale]
output0 = output0.transpose(1, 2, 0)
output0 = np.argmax(output0, axis=2)
output1 = output1.transpose(1, 2, 0)
output1 = np.argmax(output1, axis=2)
hist1 = fast_hist(gt1.flatten(), output0.flatten(),
self.nolabel).astype(float)
hist2 = fast_hist(gt2.flatten(), output1.flatten(),
self.nolabel).astype(float)
self.NMM1 += get_NMM(hist1, gt1)
self.NMM2 += get_NMM(hist2, gt2)
self.MCC1 += get_MCC(hist1)
self.MCC2 += get_MCC(hist2)
iou1_tmp = np.diag(hist1) / (hist1.sum(1) + hist1.sum(0) -
np.diag(hist1))
iou2_tmp = np.diag(hist2) / (hist2.sum(1) + hist2.sum(0) -
np.diag(hist2))
self.iou1 += iou1_tmp[1]
self.iou2 += iou2_tmp[1]
if self.vis_flag == True:
self.vis_fun(piece, img1, img2, gt1, gt2, output)
print 'item ' + str(count) + ' processed!'
count += 1
self.iou = (self.iou1 + self.iou2) / float(count * 2)
self.iou1 = self.iou1 / float(count)
self.iou2 = self.iou2 / float(count)
self.NMM = (self.NMM1 + self.NMM2) / float(count * 2)
self.NMM1 = self.NMM1 / float(count)
self.NMM2 = self.NMM2 / float(count)
self.MCC = (self.MCC1 + self.MCC2) / float(count * 2)
self.MCC1 = self.MCC1 / float(count)
self.MCC2 = self.MCC2 / float(count)
self.printscores()
def val(args, model, dataloader, data_name):
print('start val!')
# label_info = get_label_info(csv_path)
total_cks, total_f1 = 0.0, 0.0
total_pred = np.array([0])
total_label = np.array([0])
length = len(dataloader)
with torch.no_grad():
model.eval()
precision_record = []
hist = np.zeros((args.num_classes, args.num_classes))
for i, (data, label) in enumerate(dataloader):
# print('label size: ', label.size())
# print('data size: ', data.size())
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda()
# get RGB predict image
# print('label_cuda size: ', label.size())
# print('data_cuda size: ', data.size())
predict = model(data).squeeze()
# print('predict size: ', predict.size())
predict = reverse_one_hot(predict)
predict = np.array(predict)
# get RGB label image
label = label.squeeze()
if args.loss == 'dice':
label = reverse_one_hot(label)
label = np.array(label)
total_pred = np.append(total_pred, predict.flatten())
total_label = np.append(total_label, label.flatten())
if (i + 1) % 8 == 0:
# total_cm += confusion_matrix(total_label[1:], total_pred[1:])
cks = cohen_kappa_score(total_label[1:], total_pred[1:])
total_label = np.array([0])
total_pred = np.array([0])
total_cks += cks
# cks = cohen_kappa_score(label.flatten(), predict.flatten())
# total_cks += cks
f1 = f1_score(label.flatten(), predict.flatten(), average='macro')
total_f1 += f1
precision = compute_global_accuracy(predict, label)
hist += fast_hist(label.flatten(), predict.flatten(),
args.num_classes)
# there is no need to transform the one-hot array to visual RGB array
# predict = colour_code_segmentation(np.array(predict), label_info)
# label = colour_code_segmentation(np.array(label), label_info)
precision_record.append(precision)
precision = np.mean(precision_record)
# miou = np.mean(per_class_iu(hist))
miou_list = per_class_iu(hist)[:-1]
# miou_dict, miou = cal_miou(miou_list, csv_path)
miou = np.mean(miou_list)
# print('precision per pixel for test: %.3f' % precision)
print('oa for %s: %.3f' % (data_name, precision))
# print('mIoU for validation: %.3f' % miou)
print('mIoU for %s: %.3f' % (data_name, miou))
cm, cks, cr = compute_cm_cks_cr(predict, label)
total_f1 /= length
total_cks = total_cks / (length // 8)
# print('cm:\n', cm)
print('kappa for %s: %.4f' % (data_name, total_cks))
print('f1 for {}:\n'.format(data_name), total_f1)
# miou_str = ''
# for key in miou_dict:
# miou_str += '{}:{},\n'.format(key, miou_dict[key])
# print('mIoU for each class:')
# print(miou_str)
return precision, miou, cm, total_cks, total_f1
