def evaluate(args, model, criterion, val_loader):
model.eval()
losses = AverageMeter()
jaccars = AverageMeter()
dices = AverageMeter()
eva = Evaluation()
for i, (images, labels) in enumerate(val_loader):
if args.cuda:
images = images.cuda()
labels = labels.cuda()
criterion = criterion.cuda()
images = Variable(images)
labels = Variable(labels)
outputs = model(images)
loss = criterion(outputs, labels)
losses.update(loss.data.cpu().numpy())
jacc_index = eva.jaccard_similarity_coefficient(
outputs.cpu().data.numpy().squeeze(),
labels.cpu().data.numpy())
dice_index = eva.dice_coefficient(outputs.cpu().data.numpy().squeeze(),
labels.cpu().data.numpy())
jaccars.update(jacc_index)
dices.update(dice_index)
return losses.avg, jaccars.avg, dices.avg
def evaluate(args, model, criterion, val_loader):
model.eval()
losses = AverageMeter()
jaccars = AverageMeter()
dices = AverageMeter()
eva = Evaluation()
for i, (images, labels) in enumerate(val_loader):
if args.cuda:
images = images.cuda()
labels = labels.cuda()
criterion = criterion.cuda()
images = Variable(images)
labels = Variable(labels)
outputs = model(images)
# print('images(eval) : ', images.shape)
# print('labels(eval) : ', labels.shape)
# print('outputs(eval) : ', outputs.shape)
outputs = outputs.view(1, -1)
# outputs = outputs.view(1, *outputs.size()[-2:])
# print('output_size_changed(eval) : ', outputs.size())
labels = labels.view(1, -1)
# labels = labels.view(1, *labels.size())
# print('labels.view(changed) : ', labels.size())
jacc_index = eva.jaccard_similarity_coefficient(
outputs.cpu().data.numpy().squeeze(),
labels.cpu().data.numpy())
dice_index = eva.dice_coefficient(outputs.cpu().data.numpy().squeeze(),
labels.cpu().data.numpy())
#loss = criterion(outputs, labels)
crit = criterion(outputs, labels)
loss = crit #- jacc_index
# print('no jacc loss : ', crit)
# print(' jacc loss : ', loss)
losses.update(loss.data.cpu().numpy())
jaccars.update(jacc_index)
dices.update(dice_index)
return losses.avg, jaccars.avg, dices.avg
class Experiments():
def __init__(self):
self.eval = Evaluation()
self.board = Dashboad(8098)
def error_hist(self, gtdir, resdir, imgprefix, plot=False):
listGTFiles = [
k.split('/')[-1].split('.')[0]
for k in glob.glob(os.path.join(gtdir, '*.bmp'))
]
filename_jacc = dict()
filename_dice = dict()
filename_sens = dict()
filename_spec = dict()
for currfile in tqdm(listGTFiles):
if currfile.count('_') == 2:
continue
gt = np.array(Image.open(os.path.join(gtdir,
currfile + '.bmp'))) / 255
res = np.array(
Image.open(
os.path.join(resdir, currfile + '_' + imgprefix + '.bmp')))
res[res > 10] = 255
res /= 255
jacc_index = self.eval.jaccard_similarity_coefficient(
gt.squeeze(), res.squeeze())
dice = self.eval.dice_coefficient(gt.squeeze(), res.squeeze())
spec, sens, _ = self.eval.specificity_sensitivity(
gt.squeeze(), res.squeeze())
filename_jacc[currfile] = jacc_index
filename_dice[currfile] = dice
filename_sens[currfile] = sens
filename_spec[currfile] = spec
if plot:
self.board.metric_bar(filename_jacc.values(),
'Jaccard_' + imgprefix,
nbins=20)
self.board.metric_bar(filename_dice.values(),
'Dice_' + imgprefix,
nbins=20)
self.board.metric_bar(filename_sens.values(),
'Sens_' + imgprefix,
nbins=20)
self.board.metric_bar(filename_spec.values(),
'Spec_' + imgprefix,
nbins=20)
return filename_jacc, filename_dice, filename_sens, filename_spec
def get_failure_cases(self, args, threshold=0.5):
list_dics = []
for m in args.methods:
result, _, _, _ = self.error_hist(args.gtdir,
args.resdir,
m,
plot=False)
list_dics.append(result)
# Remove the images with jaccard greater than 0.5
for d in list_dics:
for k, v in d.items():
if v > threshold:
del d[k]
# Find the failure cases common between all methods
common_failures = set.intersection(*tuple(
set(d.keys()) for d in list_dics))
return common_failures
# TODO Remove val argument in function
def make_grid(self, args, val=True, selected_filenames=None):
bordersize = 2
batch = np.empty((0, 3, 244, 324))
num2sample = 60
if selected_filenames is not None:
filenames = list(selected_filenames)
num2sample = len(filenames)
else:
if val:
filenames = [
k.split('.')[-2].split('/')[-1]
for k in glob.glob(os.path.join(args.imgdir, "val_*"))
]
if args.test:
filenames = [
k.split('.')[-2].split('/')[-1]
for k in glob.glob(os.path.join(args.imgdir, "test_*.bmp"))
]
else:
filenames = [
k.split('.')[-2].split('/')[-1]
for k in glob.glob(os.path.join(args.imgdir, "*.bmp"))
]
train_ind = np.random.choice(np.arange(0, len(filenames)),
num2sample,
replace=False)
for i in range(train_ind.shape[0]):
currfile = filenames[train_ind[i]]
im = np.array(
Image.open(os.path.join(args.imgdir,
currfile + ".bmp")).convert('RGB'))
# Applies a border on the top of the image
im = cv2.copyMakeBorder(im,
top=bordersize,
bottom=bordersize,
left=bordersize,
right=bordersize,
borderType=cv2.BORDER_CONSTANT,
value=[255, 0, 0])
im = cv2.putText(im,
currfile, (10, 20),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=0.75,
color=(255, 255, 255),
thickness=2)
im = im.transpose((2, 0, 1))
batch = np.append(batch, im[np.newaxis, :, :, :], axis=0)
if val:
im = np.array(
Image.open(os.path.join(args.gtdir,
currfile + ".bmp")).convert('L'))
im = np.repeat(im[:, :, np.newaxis], 3, axis=2)
im = cv2.copyMakeBorder(im,
top=bordersize,
bottom=bordersize,
left=bordersize,
right=bordersize,
borderType=cv2.BORDER_CONSTANT,
value=[0, 255, 0])
im = im.transpose((2, 0, 1))
batch = np.append(batch, im[np.newaxis, :, :, :], axis=0)
for m in args.methods:
res = np.array(
Image.open(
os.path.join(args.resdir, currfile + "_" + m +
".bmp")).convert('L'))
res = np.repeat(res[:, :, np.newaxis], 3, axis=2)
res = cv2.copyMakeBorder(res,
top=bordersize,
bottom=bordersize,
left=bordersize,
right=bordersize,
borderType=cv2.BORDER_CONSTANT,
value=[0, 255, 200])
# Writes the name of the models.
res = cv2.putText(res,
m, (10, 20),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=0.75,
color=(255, 255, 255),
thickness=2)
res = res.transpose((2, 0, 1))
batch = np.append(batch, res[np.newaxis, :, :, :], axis=0)
return batch
def train(args, model):
# board = Dashboad(args.visdom_port) #visdom
tr_losses = AverageMeter()
tLoader, vLoader = load_data(args)
criterion = nn.BCELoss()
# criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay,
momentum=0.99)
scheduler = lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)
for epoch in range(1, args.num_epochs + 1):
scheduler.step()
if epoch == 1:
tr_loss, _, _ = evaluate(args, model, criterion, tLoader)
vl_loss, vl_jacc, vl_dice = evaluate(args, model, criterion,
vLoader)
# Draw the loss curves
win = None
# win = board.loss_curves([tr_loss, vl_loss], epoch, win=win) #visdom
print('[Initial TrainLoss: {0:.4f}]'
'\t[Initial ValidationLoss: {1:.4f}]'
'\t[Initial ValidationJaccard: {2:.4f}]'
'\t[Initial ValidationDice: {3:.4f}]'.format(
tr_loss, vl_loss, vl_jacc, vl_dice))
print(
'----------------------------------------------------------------------------------------------------'
'--------------')
for step, (images, labels) in enumerate(tLoader):
model.train(True)
if args.cuda:
images = images.cuda()
labels = labels.cuda()
criterion = criterion.cuda()
inputs = Variable(images)
targets = Variable(labels)
optimizer.zero_grad()
outputs = model(inputs)
eva = Evaluation()
# jacc = eva.jaccard_similarity_coefficient(outputs.cpu().data.numpy().squeeze(),
# labels.cpu().data.numpy())
output_np = outputs.cpu().data.numpy().squeeze()
targets_np = targets.cpu().data.numpy()
dice = eva.dice_coefficient(output_np, targets_np)
crit = criterion(outputs, targets)
loss = crit - dice
print('bceloss : ', crit.data)
print('bceloss - dice : ', loss.data)
loss.backward()
optimizer.step()
tr_losses.update(loss.data.cpu().numpy())
if epoch % args.log_step == 0:
vl_loss, vl_jacc, vl_dice = evaluate(args, model, criterion,
vLoader)
print('[Epoch: {0:02}/{1:02}]'
'\t[TrainLoss: {2:.4f}]'
'\t[ValidationLoss: {3:.4f}]'
'\t[ValidationJaccard: {4:.4f}]'
'\t[ValidationDice: {5:.4f}]'.format(epoch, args.num_epochs,
tr_losses.avg, vl_loss,
vl_jacc, vl_dice)),
filename = "weights/{0}-{1:02}.pth".format(args.model, epoch)
torch.save(model.state_dict(), filename)
print(' [Snapshot]')
else:
vl_loss, vl_jacc, vl_dice = evaluate(args, model, criterion,
vLoader)
print('[Epoch: {0:02}/{1:02}]'
'\t[TrainLoss: {2:.4f}]'
'\t[ValidationLoss: {3:.4f}]'
'\t[ValidationJaccard: {4:.4f}]'
'\t[ValidationDice: {5:.4f}]'.format(epoch, args.num_epochs,
tr_losses.avg, vl_loss,
vl_jacc, vl_dice))