def validate(config, testloader, model, writer_dict):
model.eval()
ave_loss = AverageMeter()
confusion_matrix = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES))
with torch.no_grad():
for _, batch in enumerate(testloader):
image, label, _, _ = batch
size = label.size()
label = label.long().cuda()
losses, pred = model(image, label)
pred = F.upsample(input=pred,
size=(size[-2], size[-1]),
mode='bilinear')
loss = losses.mean()
ave_loss.update(loss.item())
confusion_matrix += get_confusion_matrix(
label, pred, size, config.DATASET.NUM_CLASSES,
config.TRAIN.IGNORE_LABEL)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
writer = writer_dict['writer']
global_steps = writer_dict['valid_global_steps']
writer.add_scalar('valid_loss', ave_loss.average(), global_steps)
writer.add_scalar('valid_mIoU', mean_IoU, global_steps)
writer_dict['valid_global_steps'] = global_steps + 1
return ave_loss.average(), mean_IoU, IoU_array
def validate(config, testloader, model, writer_dict):
model.eval()
ave_loss = AverageMeter()
nums = config.MODEL.NUM_OUTPUTS
confusion_matrix = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES, nums))
with torch.no_grad():
for idx, batch in enumerate(testloader):
image, label, _, _ = batch
size = label.size()
image = image.cuda()
label = label.long().cuda()
losses, pred = model(image, label)
if not isinstance(pred, (list, tuple)):
pred = [pred]
for i, x in enumerate(pred):
x = F.interpolate(
input=x, size=size[-2:],
mode='bilinear', align_corners=config.MODEL.ALIGN_CORNERS
)
confusion_matrix[..., i] += get_confusion_matrix(
label,
x,
size,
config.DATASET.NUM_CLASSES,
config.TRAIN.IGNORE_LABEL
)
if idx % 10 == 0:
print(idx)
loss = losses.mean()
if dist.is_distributed():
reduced_loss = reduce_tensor(loss)
else:
reduced_loss = loss
ave_loss.update(reduced_loss.item())
if dist.is_distributed():
confusion_matrix = torch.from_numpy(confusion_matrix).cuda()
reduced_confusion_matrix = reduce_tensor(confusion_matrix)
confusion_matrix = reduced_confusion_matrix.cpu().numpy()
for i in range(nums):
pos = confusion_matrix[..., i].sum(1)
res = confusion_matrix[..., i].sum(0)
tp = np.diag(confusion_matrix[..., i])
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
if dist.get_rank() <= 0:
logging.info('{} {} {}'.format(i, IoU_array, mean_IoU))
writer = writer_dict['writer']
global_steps = writer_dict['valid_global_steps']
writer.add_scalar('valid_loss', ave_loss.average(), global_steps)
writer.add_scalar('valid_mIoU', mean_IoU, global_steps)
writer_dict['valid_global_steps'] = global_steps + 1
return ave_loss.average(), mean_IoU, IoU_array
def testval(config, test_dataset, testloader, model,
# sv_dir='', sv_pred=False):
sv_dir='', sv_pred=True):
model.eval()
confusion_matrix = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES))
with torch.no_grad():
for index, batch in enumerate(tqdm(testloader)):
image, label, _, name, *border_padding = batch
size = label.size()
pred = test_dataset.multi_scale_inference(
config,
model,
image,
scales=config.TEST.SCALE_LIST,
flip=config.TEST.FLIP_TEST)
if len(border_padding) > 0:
border_padding = border_padding[0]
pred = pred[:, :, 0:pred.size(2) - border_padding[0], 0:pred.size(3) - border_padding[1]]
if pred.size()[-2] != size[-2] or pred.size()[-1] != size[-1]:
pred = F.interpolate(
pred, size[-2:],
mode='bilinear', align_corners=config.MODEL.ALIGN_CORNERS
)
confusion_matrix += get_confusion_matrix(
label,
pred,
size,
config.DATASET.NUM_CLASSES,
config.TRAIN.IGNORE_LABEL)
if sv_pred:
sv_path = os.path.join(sv_dir, 'test_results_debug_1217')
if not os.path.exists(sv_path):
os.mkdir(sv_path)
test_dataset.save_pred(pred, sv_path, name)
if index % 100 == 0:
logging.info('processing: %d images' % index)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
logging.info('mIoU: %.4f' % (mean_IoU))
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
pixel_acc = tp.sum()/pos.sum()
mean_acc = (tp/np.maximum(1.0, pos)).mean()
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
return mean_IoU, IoU_array, pixel_acc, mean_acc
def validate(config, testloader, model, writer_dict, device):
rank = get_rank()
world_size = get_world_size()
model.eval()
ave_loss = AverageMeter()
confusion_matrix = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES))
confusion_matrix_sum = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES))
with torch.no_grad():
for _, batch in enumerate(testloader):
image, label, boundary_gt, _, _ = batch
size = label.size()
image = image.to(device)
boundary_gt = boundary_gt.to(device)
label = label.long().to(device)
losses, aux_loss, error_loss, losses_2, aux_loss_2, error_loss_2, preds = model(
image, label, boundary_gt.float())
pred = F.upsample(input=preds[0],
size=(size[-2], size[-1]),
mode='bilinear')
loss = (losses + 0.4 * aux_loss + 4 * error_loss + losses_2 +
0.4 * aux_loss_2 + 4 * error_loss_2).mean()
reduced_loss = reduce_tensor(loss)
ave_loss.update(reduced_loss.item())
confusion_matrix += get_confusion_matrix(
label, pred, size, config.DATASET.NUM_CLASSES,
config.TRAIN.IGNORE_LABEL)
confusion_matrix = torch.from_numpy(confusion_matrix).to(device)
reduced_confusion_matrix = reduce_tensor(confusion_matrix)
confusion_matrix = reduced_confusion_matrix.cpu().numpy()
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
print_loss = ave_loss.average() / world_size
if rank == 0:
writer = writer_dict['writer']
global_steps = writer_dict['valid_global_steps']
writer.add_scalar('valid_loss', print_loss, global_steps)
writer.add_scalar('valid_mIoU', mean_IoU, global_steps)
writer_dict['valid_global_steps'] = global_steps + 1
# cv2.imwrite(str(global_steps)+'_boundary.png', (preds[0][0][0].data.cpu().numpy()*255).astype(np.uint8))
# cv2.imwrite(str(global_steps) + '_error.png', (preds[2][0][0].data.cpu().numpy() * 255).astype(np.uint8))
cv2.imwrite(
str(global_steps) + '_error.png',
(preds[2][0][0].data.cpu().numpy() * 255).astype(np.uint8))
return print_loss, mean_IoU, IoU_array
def testval(config, test_dataset, testloader, model,
sv_dir='', sv_pred=False):
model.eval()
confusion_matrix = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES))
with torch.no_grad():
for index, batch in enumerate(tqdm(testloader)):
image, label, _, name = batch
size = label.size()
pred = test_dataset.multi_scale_inference(
model,
image,
scales=config.TEST.SCALE_LIST,
flip=config.TEST.FLIP_TEST)
if pred.size()[-2] != size[-2] or pred.size()[-1] != size[-1]:
pred = F.upsample(pred, (size[-2], size[-1]),
mode='bilinear')
confusion_matrix += get_confusion_matrix(
label,
pred,
size,
config.DATASET.NUM_CLASSES,
config.TRAIN.IGNORE_LABEL)
if sv_pred:
sv_path = os.path.join(sv_dir,'test_val_results')
if not os.path.exists(sv_path):
os.mkdir(sv_path)
test_dataset.save_pred(pred, sv_path, name)
if index % 100 == 0:
logging.info('processing: %d images' % index)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
logging.info('mIoU: %.4f' % (mean_IoU))
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
pixel_acc = tp.sum()/pos.sum()
mean_acc = (tp/np.maximum(1.0, pos)).mean()
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
return mean_IoU, IoU_array, pixel_acc, mean_acc
def log_confusion_matrix(epoch, logs):
# Use the model to predict the values from the validation dataset.
y_pred_raw = model.predict(test_batches)
y_pred = np.argmax(y_pred_raw, axis=1)
y_true = test_batches.classes
# Get the confusion
cm_image = get_confusion_matrix(y_true,
y_pred,
class_names=class_names,
normalize='true')
# Log the confusion matrix as an image summary.
with file_writer_cm.as_default():
tf.summary.image("Confusion Matrix", cm_image, step=epoch)
def validate(config, testloader, model, writer_dict, device):
rank = get_rank()
world_size = get_world_size()
model.eval()
ave_loss = AverageMeter()
confusion_matrix = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES))
with torch.no_grad():
for _, batch in enumerate(testloader):
image, label, _, _ = batch
size = label.size()
image = image.to(device)
label = label.long().to(device)
losses, pred = model(image, label)
pred = F.upsample(input=pred, size=(
size[-2], size[-1]), mode='bilinear')
loss = losses.mean()
reduced_loss = reduce_tensor(loss)
ave_loss.update(reduced_loss.item())
confusion_matrix += get_confusion_matrix(
label,
pred,
size,
config.DATASET.NUM_CLASSES,
config.TRAIN.IGNORE_LABEL)
confusion_matrix = torch.from_numpy(confusion_matrix).to(device)
reduced_confusion_matrix = reduce_tensor(confusion_matrix)
confusion_matrix = reduced_confusion_matrix.cpu().numpy()
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
print_loss = ave_loss.average()/world_size
if rank == 0:
writer = writer_dict['writer']
global_steps = writer_dict['valid_global_steps']
writer.add_scalar('valid_loss', print_loss, global_steps)
writer.add_scalar('valid_mIoU', mean_IoU, global_steps)
writer_dict['valid_global_steps'] = global_steps + 1
return print_loss, mean_IoU, IoU_array
logits, dim=1) # (shape: (batch_size, num_classes, h, w))
p = p + p_value / M_float
p_numpy = p.cpu().data.numpy().transpose(
0, 2, 3, 1) # (array of shape: (batch_size, h, w, num_classes))
seg_pred = np.argmax(p_numpy, axis=3).astype(np.uint8)
m_seg_pred = np.ma.masked_array(seg_pred, mask=torch.eq(label, 255))
np.ma.set_fill_value(m_seg_pred, 20)
seg_pred = m_seg_pred
seg_gt = label.numpy().astype(np.int)
ignore_index = seg_gt != 255
seg_gt = seg_gt[ignore_index]
seg_pred = seg_pred[ignore_index]
confusion_matrix += get_confusion_matrix(seg_gt, seg_pred, num_classes)
entropy = -np.sum(p_numpy * np.log(p_numpy),
axis=3) # (shape: (batch_size, h, w))
pred_label_imgs_raw = np.argmax(p_numpy, axis=3).astype(np.uint8)
for i in range(image.size(0)):
if i == 0:
img = image[i].data.cpu().numpy()
img = np.transpose(img,
(1, 2, 0)) # (shape: (img_h, img_w, 3))
img = img + np.array([102.9801, 115.9465, 122.7717])
img = img[:, :, ::-1]
cv2.imwrite(output_path + "/" + name[i] + "_img.png", img)
label_img = label[i].data.cpu().numpy()
label_img = label_img.astype(np.uint8)
def main():
args.time = get_currect_time()
visualizer = Visualizer(args)
log = Log(args)
log.record_sys_param()
log.record_file()
"""Set GPU Environment"""
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
trainloader = data.DataLoader(NYUDataset_crop_fast(args.data_list, args.random_scale, args.random_mirror, args.random_crop,
args.batch_size, args.colorjitter),batch_size=args.batch_size,
shuffle=True, num_workers=4, pin_memory=True)
valloader = data.DataLoader(NYUDataset_val_full(args.data_val_list, args.random_scale, args.random_mirror, args.random_crop,
1), batch_size=8, shuffle=False, pin_memory=True)
"""Create Network"""
deeplab = Res_Deeplab(num_classes=args.num_classes)
print(deeplab)
"""Load pretrained Network"""
saved_state_dict = torch.load(args.restore_from)
print(args.restore_from)
new_params = deeplab.state_dict().copy()
for i in saved_state_dict:
# Scale.layer5.conv2d_list.3.weight
i_parts = i.split('.')
# print i_parts
# if not i_parts[1]=='layer5':
if not i_parts[0] == 'fc':
new_params['.'.join(i_parts[0:])] = saved_state_dict[i]
deeplab.load_state_dict(new_params)
model = deeplab
model.cuda()
model.train()
model = model.float()
model = DataParallelModel(model, device_ids=[0, 1])
criterion = CriterionDSN()
criterion = DataParallelCriterion(criterion)
optimizer = optim.SGD([{'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.learning_rate }],
lr=args.learning_rate, momentum=args.momentum,weight_decay=args.weight_decay)
optimizer.zero_grad()
i_iter = 0
args.num_steps = len(trainloader) * args.epoch
best_iou = 0.0
total = sum([param.nelement() for param in model.parameters()])
print(' + Number of params: %.2fM' % (total / 1e6))
for epoch in range(args.epoch):
## Train one epoch
model.train()
for batch in trainloader:
start = timeit.default_timer()
i_iter = i_iter + 1
images = batch['image'].cuda()
labels = batch['seg'].cuda()
HHAs = batch['HHA'].cuda()
depths = batch['depth'].cuda()
labels = torch.squeeze(labels,1).long()
if (images.size(0) != args.batch_size):
break
optimizer.zero_grad()
preds = model(images, HHAs, depths)
loss = criterion(preds, labels)
loss.backward()
optimizer.step()
if i_iter % 100 == 0:
visualizer.add_scalar('learning_rate', args.learning_rate, i_iter)
visualizer.add_scalar('loss', loss.data.cpu().numpy(), i_iter)
current_lr = optimizer.param_groups[0]['lr']
end = timeit.default_timer()
log.log_string(
'====================> epoch=%03d/%d, iter=%05d/%05d, loss=%.3f, %.3fs/iter, %02d:%02d:%02d, lr=%.6f' % (
epoch, args.epoch, i_iter, len(trainloader)*args.epoch, loss.data.cpu().numpy(), (end - start),
(int((end - start) * (args.num_steps - i_iter)) // 3600),
(int((end - start) * (args.num_steps - i_iter)) % 3600 // 60),
(int((end - start) * (args.num_steps - i_iter)) % 3600 % 60), current_lr))
if (epoch+1) % 40 == 0:
adjust_learning_rate(optimizer, i_iter, args)
if epoch % 5 == 0:
model.eval()
confusion_matrix = np.zeros((args.num_classes, args.num_classes))
loss_val = 0
log.log_string("====================> evaluating")
for batch_val in valloader:
images_val = batch_val['image'].cuda()
labels_val = batch_val['seg'].cuda()
labels_val = torch.squeeze(labels_val,1).long()
HHAs_val = batch_val['HHA'].cuda()
depths_val = batch_val['depth'].cuda()
with torch.no_grad():
preds_val = model(images_val, HHAs_val, depths_val)
loss_val += criterion(preds_val, labels_val)
preds_val = torch.cat([preds_val[i][0] for i in range(len(preds_val))], 0)
preds_val = F.upsample(input=preds_val, size=(480, 640), mode='bilinear', align_corners=True)
preds_val = np.asarray(np.argmax(preds_val.cpu().numpy(), axis=1), dtype=np.uint8)
labels_val = np.asarray(labels_val.cpu().numpy(), dtype=np.int)
ignore_index = labels_val != 255
labels_val = labels_val[ignore_index]
preds_val = preds_val[ignore_index]
confusion_matrix += get_confusion_matrix(labels_val, preds_val, args.num_classes)
loss_val = loss_val / len(valloader)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IU = IU_array.mean()
# getConfusionMatrixPlot(confusion_matrix)
log.log_string('val loss' + ' ' + str(loss_val.cpu().numpy()) + ' ' + 'meanIU' + str(mean_IU) + 'IU_array' + str(IU_array))
visualizer.add_scalar('val loss', loss_val.cpu().numpy(), epoch)
visualizer.add_scalar('meanIU', mean_IU, epoch)
if mean_IU > best_iou:
best_iou = mean_IU
log.log_string('save best model ...')
torch.save(deeplab.state_dict(),
osp.join(args.snapshot_dir, 'model', args.dataset + NAME + 'best_iu' + '.pth'))
if epoch % 5 == 0:
log.log_string('save model ...')
torch.save(deeplab.state_dict(),osp.join(args.snapshot_dir,'model', args.dataset+ NAME + str(epoch)+'.pth'))
def predict(self):
## Get image-label mapping
image_label_dict = {}
dataset_labels_file_path = 'datasets/densenet/ISIC2018_Task3_Training_GroundTruth.csv'
with open(dataset_labels_file_path) as csvfile:
read_csv = csv.reader(csvfile, delimiter=',')
for index, row in enumerate(read_csv):
## Skip header
if index == 0:
continue
label_one_hot_encoding = [
int(round(float(row[i + 1]), 0)) for i in range(7)
]
image_label_dict[row[0]] = np.argmax(label_one_hot_encoding)
## Get image paths
image_paths = utils_image.get_images_path_list_from_dir(
self.config.tfrecords_path_train, img_format='jpg')
# image_paths = utils_image.get_images_path_list_from_dir(self.config.tfrecords_path_test, img_format='jpg')
# image_paths = utils_image.get_images_path_list_from_dir(self.config.tfrecords_path_val, img_format='jpg')
## Sample n images
random.shuffle(image_paths)
image_paths = image_paths[0:self.config.predict_num_images]
## Get gt_labels
gt_labels = []
for image_path in image_paths:
image_name = os.path.basename(image_path).rsplit('.', 1)[0]
gt_labels.append(image_label_dict[image_name])
images = []
for image_path in image_paths:
## Load image
image = Image.open(image_path)
## Resize and center crop image. size: (width, height)
image = ImageOps.fit(
image,
(self.config.tfr_image_width, self.config.tfr_image_height),
Image.LANCZOS, 0, (0.5, 0.5))
# img = cv2.imread(image_paths[i])
# img = cv2.resize(img, (224, 224))
## Preprocess images
image = np.float32(np.array(image))
image = self.data.preprocess_data(image)
images.append(image)
images = np.array(images)
logging.debug('model_name {}'.format(self.config.model_name))
logging.debug('images {}'.format(images.shape))
# TODO: Don't shuffle else labels will mismatch
x_key = self.config.model_name + '_input'
predict_input_fn = tf.estimator.inputs.numpy_input_fn(
x={x_key: images},
y=None,
batch_size=self.config.batch_size_pred,
num_epochs=1,
shuffle=False,
queue_capacity=1000,
# In order to have predicted and repeatable order of reading and enqueueing,
# such as in prediction and evaluation mode, num_threads should be 1.
num_threads=1)
# with tf.Session() as sess:
# init_op = tf.global_variables_initializer()
# sess.run(init_op)
# # sess.run(self.model.model.output, feed_dict={self.model.model.input:images})
# # sess.run(self.model.model['class_prob'], feed_dict={self.model.model.input:images})
# # op = self.model.model.get_layer('class_prob')
# # op = self.model.model.get_layer('class_prob')
# # op = self.model.model.layers[0]
# op = self.model.model.get_layer('dense_1')
# out = sess.run(op,
# feed_dict={self.model.model.input:images})
# logging.debug('out {}'.format(out))
# exit(0)
checkpoint_path = None
if not self.config.predict_weights_path:
checkpoint_path = self.config.predict_weights_path
# NOTE: predictions is <generator object Estimator.predict> and hence (maybe) we can dereference it only once.
# TODO: Check predict_keys
predictions = self.estimator.predict(input_fn=predict_input_fn,
checkpoint_path=checkpoint_path,
hooks=None,
predict_keys=None,
yield_single_examples=True)
class_prob = [p['class_prob'] for p in predictions]
# class_prob = [p['dense_4'] for p in predictions]
pred_labels = np.argmax(np.array(class_prob), axis=1)
for gt_label, pred_label in zip(gt_labels, pred_labels):
print('GT, PRED: [{}, {}]'.format(gt_label, pred_label))
## Confusion matrix
# https://stackoverflow.com/questions/41617463/tensorflow-confusion-matrix-in-tensorboard
confusion = tf.confusion_matrix(labels=gt_labels,
predictions=pred_labels,
num_classes=self.config.num_classes)
logging.debug('Row(GT), Col(Pred)')
with tf.Session() as sess:
print(sess.run(confusion))
# Plot and save confusion matrix
utils.get_confusion_matrix(self.config, gt_labels, pred_labels)
def predict_cascade(self, mode_ds='test_ds'):
# TODO: Hardcoding
labels_binary = {"REST": 0, "NV": 1}
labels_multi6 = {
"MEL": 0,
"NV": 1,
"BCC": 2,
"AKIEC": 3,
"BKL": 4,
"DF": 5,
"VASC": 6
}
labels_binary_inv = {v: k for k, v in labels_binary.items()}
labels_multi6_inv = {v: k for k, v in labels_multi6.items()}
print('aaa')
self.config.exp_cascade_name = 'binary'
pkl_path = os.path.join(self.config.output_path, self.config.exp_name,
self.config.exp_cascade_name, 'pkl')
with open(os.path.join(pkl_path, 'labels_gt.pkl'), 'rb') as handle:
labels_gt_binary = pickle.load(handle)
with open(os.path.join(pkl_path, 'labels_pred_prob.pkl'),
'rb') as handle:
labels_pred_prob_binary = pickle.load(handle)
with open(os.path.join(pkl_path, 'labels_pred_cls.pkl'),
'rb') as handle:
labels_pred_cls_binary = pickle.load(handle)
self.config.exp_cascade_name = 'multi6'
pkl_path = os.path.join(self.config.output_path, self.config.exp_name,
self.config.exp_cascade_name, 'pkl')
with open(os.path.join(pkl_path, 'labels_gt.pkl'), 'rb') as handle:
labels_gt_multi6 = pickle.load(handle)
with open(os.path.join(pkl_path, 'labels_pred_prob.pkl'),
'rb') as handle:
labels_pred_prob_multi6 = pickle.load(handle)
with open(os.path.join(pkl_path, 'labels_pred_cls.pkl'),
'rb') as handle:
labels_pred_cls_multi6 = pickle.load(handle)
labels_pred_prob = []
labels_pred_cls = []
labels_gt = labels_gt_binary
# for lgb, lpb, lcb, lgm, lpm, lcm in zip(labels_gt_binary, labels_pred_prob_binary, labels_pred_cls_binary, labels_gt_multi6, labels_pred_prob_multi6, labels_pred_cls_multi6):
for i in range(len(labels_gt_binary)):
# binary
#if (labels_pred_cls_binary[i] == labels_binary['NV']) and (np.max(labels_pred_prob_binary[i]) > 0.5):
if (labels_pred_cls_binary[i] == labels_binary['NV']):
labels_pred_cls.append(labels_pred_cls_binary[i])
prob = 7 * [
labels_pred_prob_binary[i][labels_binary['REST']] / 6.0
]
prob[labels_binary['NV']] = labels_pred_prob_binary[i][
labels_binary['NV']]
labels_pred_prob.append(prob)
#labels_pred_prob.append(np.max(labels_pred_prob_binary[i]))
# multi6
else:
labels_pred_cls.append(labels_pred_cls_multi6[i])
labels_pred_prob.append(labels_pred_prob_multi6[i])
## Plot ROC curve
#utils.gen_roc_curve(self.config, labels_gt, labels_pred_prob, mode_ds)
## Plot PR Curve
utils.gen_precision_recall_curve(self.config, labels_pred_cls,
labels_pred_prob, mode_ds)
## Confusion matrix
# confusion = tf.confusion_matrix(labels=labels_gt, predictions=labels_pred_cls, num_classes=self.config.num_classes)
# logging.debug('Row(GT), Col(Pred)')
# with tf.Session() as sess:
# print(sess.run(confusion))
## Plot and save confusion matrix
utils.get_confusion_matrix(self.config, labels_gt, labels_pred_cls,
mode_ds)
## Print PR and F1
utils.summary_pr_fscore(self.config, labels_gt, labels_pred_cls,
self.config.labels)
## Plot Metrics
utils.get_metrics(self.config, labels_gt, labels_pred_cls, mode_ds)
def predict(self, mode_ds='test_ds', mixed=False):
if mode_ds == 'train_ds':
image_paths = utils_image.get_images_path_list_from_dir(
self.config.tfrecords_path_train, img_format='jpg')
elif mode_ds == 'val_ds':
images_path = utils_image.get_images_path_list_from_dir(
self.config.tfrecords_path_val, img_format='jpg')
elif mode_ds == 'test_ds':
images_path = utils_image.get_images_path_list_from_dir(
self.config.tfrecords_path_test, img_format='jpg')
else:
logging.error('Unknown mode_ds {}', mode_ds)
exit(1)
## Get image-label mapping
image_label_dict = {}
dataset_labels_file_path = 'datasets/densenet/ISIC2018_Task3_Training_GroundTruth.csv'
with open(dataset_labels_file_path) as csvfile:
read_csv = csv.reader(csvfile, delimiter=',')
for index, row in enumerate(read_csv):
## Skip header
if index == 0:
continue
label_one_hot_encoding = [
int(round(float(row[i + 1]), 0)) for i in range(7)
]
image_label_dict[row[0]] = np.argmax(label_one_hot_encoding)
## Sample n images
# random.shuffle(images_path)
images_path = images_path[0:self.config.predict_num_images]
## Get labels_gt
labels_gt = []
for image_path in images_path:
# TODO: Image name should have no dot
# image_name = os.path.basename(image_path).split('.', 1)[0]
image_name = os.path.basename(image_path).rsplit('.', 1)[0]
labels_gt.append(image_label_dict[image_name])
images_path = images_path[0:self.config.predict_num_images]
images = []
for image_path in images_path:
## Load image
image = Image.open(image_path)
## Resize and center crop image. size: (width, height)
image = ImageOps.fit(
image,
(self.config.tfr_image_width, self.config.tfr_image_height),
Image.LANCZOS, 0, (0.5, 0.5))
## Preprocess images
image = np.float32(np.array(image))
image = self.data.preprocess_data(image)
images.append(image)
features = np.array(images)
logging.debug('model_name {}'.format(self.config.model_name))
logging.debug('features {}'.format(features.shape))
## Predict in batches
num_images = len(images_path)
batch_size = self.config.batch_size_pred
iters = int(num_images / batch_size)
logging.debug('num_images {}'.format(num_images))
logging.debug('batch_size {}'.format(batch_size))
labels_pred_cls = []
labels_pred_prob = []
idx_start = 0
idx_end = 0
for iter_no in range(iters):
idx_start = iter_no * batch_size
idx_end = idx_start + batch_size
logging.debug('idx:[{}-{}]'.format(idx_start, idx_end))
labels_pred_prob_batch, labels_pred_cls_batch = self.sess.run(
[
self.model.labels_pred_prob,
self.model.labels_pred_cls,
],
feed_dict={
self.model.features: features[idx_start:idx_end],
})
logging.debug('labels_gt {}'.format(
np.array(labels_gt[idx_start:idx_end])))
logging.debug(
'labels_pred_cls_batch {}'.format(labels_pred_cls_batch))
# logging.debug('labels_pred_prob_batch {}'.format(labels_pred_prob_batch))
labels_pred_cls = labels_pred_cls + labels_pred_cls_batch.tolist()
labels_pred_prob = labels_pred_prob + labels_pred_prob_batch.tolist(
)
## For images < batch_size and For images which do not fit the last batch
idx_start = iters * batch_size
idx_end = idx_start + (num_images % batch_size)
logging.debug('idx:[{}-{}]'.format(idx_start, idx_end))
if (num_images % batch_size):
labels_pred_prob_batch, labels_pred_cls_batch = self.sess.run(
[
self.model.labels_pred_prob,
self.model.labels_pred_cls,
],
feed_dict={
self.model.features: features[idx_start:idx_end],
})
logging.debug('labels_gt {}'.format(
labels_gt[idx_start:idx_end]))
logging.debug(
'labels_pred_cls_batch {}'.format(labels_pred_cls_batch))
# logging.debug('labels_pred_prob_batch {}'.format(labels_pred_prob_batch))
labels_pred_cls = labels_pred_cls + labels_pred_cls_batch.tolist()
labels_pred_prob = labels_pred_prob + labels_pred_prob_batch.tolist(
)
for label_gt, label_pred_cls in zip(labels_gt, labels_pred_cls):
print('GT, PRED: [{}, {}]'.format(label_gt, label_pred_cls))
pkl_path = os.path.join(self.config.output_path, self.config.exp_name,
self.config.exp_cascade_name, 'pkl')
os.makedirs(pkl_path, exist_ok=True)
with open(os.path.join(pkl_path, 'labels_gt.pkl'), 'wb') as handle:
pickle.dump(labels_gt, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(pkl_path, 'labels_pred_prob.pkl'),
'wb') as handle:
pickle.dump(labels_pred_prob,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(pkl_path, 'labels_pred_cls.pkl'),
'wb') as handle:
pickle.dump(labels_pred_cls,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
### ANALYSIS ###
## Plot ROC curve
utils.gen_roc_curve(self.config, labels_gt, labels_pred_prob, mode_ds)
## Plot PR Curve
utils.gen_precision_recall_curve(self.config, labels_pred_cls,
labels_pred_prob, mode_ds)
## Confusion matrix
# confusion = tf.confusion_matrix(labels=labels_gt, predictions=labels_pred_cls, num_classes=self.config.num_classes)
# logging.debug('Row(GT), Col(Pred)')
# with tf.Session() as sess:
# print(sess.run(confusion))
## Plot and save confusion matrix
utils.get_confusion_matrix(self.config, labels_gt, labels_pred_cls,
mode_ds)
## Print PR and F1
utils.summary_pr_fscore(self.config, labels_gt, labels_pred_cls,
self.config.labels)
## Plot Metrics
utils.get_metrics(self.config, labels_gt, labels_pred_cls, mode_ds)
def testval(config, test_dataset, testloader, model, sv_dir='', sv_pred=False):
model.eval()
confusion_matrix = np.zeros(
(config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES))
with torch.no_grad():
for index, batch in enumerate(tqdm(testloader)):
image, label, _, name, *border_padding = batch
size = label.size()
pred = test_dataset.multi_scale_inference(
config,
model,
image,
scales=config.TEST.SCALE_LIST,
flip=config.TEST.FLIP_TEST)
if len(border_padding) > 0:
border_padding = border_padding[0]
pred = pred[:, :, 0:pred.size(2) - border_padding[0],
0:pred.size(3) - border_padding[1]]
if pred.size()[-2] != size[-2] or pred.size()[-1] != size[-1]:
pred = F.interpolate(pred, size[-2:], mode='nearest')
# # crf used for post-processing
# postprocessor = DenseCRF( )
# # image
# mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225]
# timage = image.squeeze(0)
# timage = timage.numpy().copy().transpose((1,2,0))
# timage *= std
# timage += mean
# timage *= 255.0
# timage = timage.astype(np.uint8)
# # pred
# tprob = torch.softmax(pred, dim=1)[0].cpu().numpy()
# pred = postprocessor(np.array(timage, dtype=np.uint8), tprob)
# pred = torch.from_numpy(pred).unsqueeze(0)
confusion_matrix += get_confusion_matrix(
label, pred, size, config.DATASET.NUM_CLASSES,
config.TRAIN.IGNORE_LABEL)
if sv_pred:
sv_path = os.path.join(sv_dir, 'test_results')
if not os.path.exists(sv_path):
os.mkdir(sv_path)
test_dataset.save_pred2(image, pred, sv_path, name)
if index % 100 == 0:
logging.info('processing: %d images' % index)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
logging.info('mIoU: %.4f' % (mean_IoU))
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
pixel_acc = tp.sum() / pos.sum()
mean_acc = (tp / np.maximum(1.0, pos)).mean()
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
return mean_IoU, IoU_array, pixel_acc, mean_acc
pallete="Set2",
title="Leave-One-Subject-Out Cross Validation",
showmeans=True,
width=800,
height=500)
st.plotly_chart(fig)
"""
### F-measure
"""
fig = pu.plotly_multi_boxplot(f_measure,
x="Target",
y="F-measure",
hue="Model",
pallet="Set2",
title="F-measure per Target",
showmeans=True,
width=1000,
height=500)
st.plotly_chart(fig)
"""
### Confusion Matrix
"""
model_select = st.selectbox("Model", accuracy["Model"].unique())
conf_mat = utils.get_confusion_matrix(model_select)
fig = pu.plotly_heatmap(conf_mat)
st.plotly_chart(fig)
if st.checkbox("Show Confusion Matrix via DataFrame"):
st.write(conf_mat)
def testval(num_class,
ignore_label,
rowSize,
test_dataset,
testloader,
model,
sv_dir='',
sv_pred=False):
model.eval()
confusion_matrix = np.zeros((num_class + 1, num_class + 1))
with torch.no_grad():
for index, batch in enumerate(tqdm(testloader)):
image, label, _, name = batch
size = label.size()
pred = torch.rand(size[1], size[2], num_class)
for j in range(0, size[1], rowSize): # ¶àÐвâÊÔ
row_size = rowSize
if size[1] - j < rowSize:
row_size = size[1] - j
imageCubes, labelCubes = createTestCube(image[0],
label[0],
windowSize=11,
r=j,
size=row_size)
hsiDataset = datasets.hsicube(imageCubes, labelCubes)
hsiDataloader = torch.utils.data.DataLoader(
hsiDataset, batch_size=size[2] * row_size)
for _, (inputCube, _) in enumerate(hsiDataloader):
inputCube = inputCube.cuda()
with torch.no_grad():
output = model.forward(inputCube)
pred[j:j + row_size, :, :] = output.reshape(
(row_size, size[2], num_class))
print('\r' + '{}'.format(j), end='', flush=True)
confusion_matrix += get_confusion_matrix(label, pred, size,
num_class + 1,
ignore_label)
if sv_pred:
sv_path = os.path.join(sv_dir, 'test_val_results')
if not os.path.exists(sv_path):
os.mkdir(sv_path)
test_dataset.save_pred(pred, sv_path, name)
# test_dataset.save_pred_gray(pred, sv_path, name)
# if index % 100 == 0:
logging.info('processing: %d images' % index)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
logging.info('mIoU: %.4f' % (mean_IoU))
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
pixel_acc = tp.sum() / pos.sum()
mean_acc = (tp / np.maximum(1.0, pos)).mean()
IoU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IoU = IoU_array.mean()
return mean_IoU, IoU_array, pixel_acc, mean_acc
