def valid(self, epoch, val_loader):
self.G.eval()
with torch.no_grad():
# (tn, fp, fn, tp)
cm = utils.ConfusionMatrix()
for i, (input_, target_, _) in enumerate(val_loader):
input_ = input_.to(self.torch_device)
output_ = self.G(input_)
target_ = target_.to(self.torch_device)
ground_truth = target_.int().squeeze(1)
prediction = torch.argmax(output_, dim=1).int()
cm.update(
utils.confusion_matrix(prediction,
ground_truth,
reduce=False))
metric = 1.5 * cm.f2 + cm.accuracy
if metric > self.best_metric:
self.best_metric = metric
self.save(epoch)
self.logger.write(
"[Val] epoch: %d accuracy: %f f05: %f f1: %f f2: %f" %
(epoch, cm.accuracy, cm.f05, cm.f1, cm.f2))
def train_one_epoch(model, criterion, optimizer, data_loader, lr_scheduler,
device, epoch, print_freq, num_classes):
epoch_loss = list()
model.train()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
metric_logger.add_meter('lr',
utils.SmoothedValue(window_size=1, fmt='{value}'))
header = 'Epoch: [{}]'.format(epoch)
for image, target in metric_logger.log_every(data_loader, print_freq,
header):
image, target = image.to(device), target.to(device)
output = model(image)
loss = criterion(output, target)
epoch_loss.append(loss.item())
confmat.update(target.flatten(), output['out'].argmax(1).flatten())
optimizer.zero_grad()
loss.backward()
optimizer.step()
lr_scheduler.step()
metric_logger.update(loss=loss.item(),
lr=optimizer.param_groups[0]["lr"])
confmat.reduce_from_all_processes()
acc_global, _, _ = confmat.compute()
epoch_acc = acc_global.item()
return epoch_acc, sum(epoch_loss) / len(epoch_loss) if len(
epoch_loss) else 0.0
def valid(self, epoch, val_loader):
self.G.eval()
with torch.no_grad():
# (tn, fp, fn, tp)
cm = utils.ConfusionMatrix()
sd = utils.SurfaceDSC()
for i, (input_, target_, _) in enumerate(val_loader):
_, output_, target_ = self.forward_for_test(input_, target_)
sd.update(utils.surface_DSC_batch(target_, output_),
n=output_.shape[0])
cm.update(utils.confusion_matrix_2d(output_,
target_,
0.5,
reduce=False),
n=output_.shape[0])
metric = sd.sDSC.avg
if metric > self.best_metric:
self.best_metric = metric
self.save(epoch)
self.logger.write(
"[Val] epoch: %d f05: %f f1: %f f2: %f jacard: %f dice: %f surf_dice: %f"
% (epoch, cm.f05, cm.f1, cm.f2, cm.jcc.avg, cm.dice.avg,
sd.sDSC.avg))
def evaluate(model, data_loader, device, num_classes):
model.eval()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
header = "Test:"
num_processed_samples = 0
with torch.inference_mode():
for image, target in metric_logger.log_every(data_loader, 100, header):
image, target = image.to(device), target.to(device)
output = model(image)
output = output["out"]
confmat.update(target.flatten(), output.argmax(1).flatten())
# FIXME need to take into account that the datasets
# could have been padded in distributed setup
num_processed_samples += image.shape[0]
confmat.reduce_from_all_processes()
num_processed_samples = utils.reduce_across_processes(
num_processed_samples)
if (hasattr(data_loader.dataset, "__len__")
and len(data_loader.dataset) != num_processed_samples
and torch.distributed.get_rank() == 0):
# See FIXME above
warnings.warn(
f"It looks like the dataset has {len(data_loader.dataset)} samples, but {num_processed_samples} "
"samples were used for the validation, which might bias the results. "
"Try adjusting the batch size and / or the world size. "
"Setting the world size to 1 is always a safe bet.")
return confmat
def evaluate(model, data_loader, device, num_classes):
model.eval()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
class_iou_image = list()
img_list = list()
target_list = list()
prediction_list = list()
header = "Evaluate:"
with torch.inference_mode():
for image, target in metric_logger.log_every(data_loader, 100, header):
image, target = image.to(device), target.to(device)
confmat_image = utils.ConfusionMatrix(num_classes)
output = model(image)
output = output["out"]
inv_normalize = transforms.Normalize(mean=(-0.485, -0.456, -0.406),
std=(1 / 0.229, 1 / 0.224,
1 / 0.225))
img_npy = inv_normalize(image[0], target)[0].cpu().detach().numpy()
target_npy = target.cpu().detach().numpy()
prediction_npy = output.cpu().detach().numpy()
img_list.append(img_npy)
target_list.append(target_npy)
prediction_list.append(prediction_npy)
confmat.update(target.flatten(), output.argmax(1).flatten())
confmat_image.update(target.flatten(), output.argmax(1).flatten())
class_iou_image.append(confmat_image.get_class_iou())
confmat_image.reduce_from_all_processes()
confmat.reduce_from_all_processes()
return confmat, class_iou_image, img_list, target_list, prediction_list
def test(self, test_loader, val_loader):
print("\nStart Test")
self.G.eval()
with torch.no_grad():
sd = utils.SurfaceDSC()
cm = utils.ConfusionMatrix()
y_true = np.array([])
y_pred = np.array([])
for i, (input_, target_, f_name) in enumerate(test_loader):
input_, output_, target_ = self.forward_for_test(
input_, target_)
sd.update(utils.surface_DSC_batch(target_, output_),
n=output_.shape[0])
cm.update(utils.confusion_matrix_2d(output_,
target_,
0.5,
reduce=False),
n=output_.shape[0])
input_np = input_.type(torch.FloatTensor).numpy()
target_np = target_.type(torch.FloatTensor).numpy()
output_np = output_.type(torch.FloatTensor).numpy()
y_true = np.concatenate([y_true, target_np.flatten()], axis=0)
y_pred = np.concatenate([y_pred, output_np.flatten()], axis=0)
for batch_idx in range(0, input_.shape[0]):
input_b = input_np[batch_idx, 0, :, :]
target_b = target_np[batch_idx, 0, :, :]
output_b = output_np[batch_idx, 0, :, :]
save_path = "%s/fold%s/%s" % (self.save_path, self.fold,
f_name[batch_idx][:-4])
utils.image_save(save_path, input_b, target_b, output_b)
self.logger.will_write(
"[Save] fname:%s dice:%f jss:%f surf_dice:%f" %
(f_name[batch_idx][:-4], cm.dice.val[batch_idx],
cm.jcc.val[batch_idx], sd.sDSC.val[batch_idx]))
pr_values = np.array(precision_recall_curve(y_true, y_pred))
roc_auc = roc_auc_score(y_true, y_pred)
pr_auc = auc(pr_values[0], pr_values[1], reorder=True)
self.logger.write(
"Best dice:%f surf_dice:%f jcc:%f f05:%f f1:%f f2:%f roc:%f pr:%f"
% (cm.dice.avg, sd.sDSC.avg, cm.jcc.avg, cm.f05, cm.f1, cm.f2,
roc_auc, pr_auc))
print("End Test\n")
def evaluate(model, data_loader, device, num_classes):
model.eval()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'
with torch.no_grad():
for image, target in metric_logger.log_every(data_loader, 100, header):
target = target.to(device)
output = model(image)
output = output['out']
confmat.update(target.flatten(), output.argmax(1).flatten())
confmat.reduce_from_all_processes()
return confmat
def test(self, test_loader, val_loader):
print("\nStart Test")
self.G.eval()
with torch.no_grad():
cm = utils.ConfusionMatrix()
y_true = np.array([])
y_pred = np.array([])
for i, (input_, target_, clinic_,
f_name) in enumerate(test_loader):
input_, clinic_, output_, target_ = self.forward_for_test(
input_, target_, clinic_)
ground_truth = target_.int().squeeze(1)
prediction = torch.argmax(output_, dim=1).int()
cm.update(
utils.confusion_matrix(prediction,
ground_truth,
reduce=False))
prediction_np = prediction.type(torch.FloatTensor).numpy()
ground_truth_np = ground_truth.type(torch.FloatTensor).numpy()
y_true = np.concatenate([y_true, ground_truth_np], axis=0)
y_pred = np.concatenate([y_pred, prediction_np], axis=0)
for batch_idx in range(0, input_.shape[0]):
output_b = prediction_np[batch_idx]
target_b = ground_truth_np[batch_idx]
self.logger.will_write(
"[Test] fname:%s true_label:%f prediction:%f" %
(f_name[batch_idx][:-4], target_b, output_b))
pr_values = np.array(precision_recall_curve(y_true, y_pred))
roc_auc = roc_auc_score(y_true, y_pred)
pr_auc = auc(pr_values[0], pr_values[1], reorder=True)
self.logger.write("accuracy:%f f05:%f f1:%f f2:%f roc:%f pr:%f" %
(cm.accuracy, cm.f05, cm.f1, cm.f2, roc_auc, pr_auc))
print("End Test\n")
def evaluate(model, criterion, data_loader, device, num_classes):
epoch_loss = list()
model.eval()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'
with torch.no_grad():
for image, target in metric_logger.log_every(data_loader, 100, header):
image, target = image.to(device), target.to(device)
output = model(image)
loss = criterion(output, target)
epoch_loss.append(loss.item())
output = output['out']
confmat.update(target.flatten(), output.argmax(1).flatten())
confmat.reduce_from_all_processes()
print('eval confmat', confmat)
acc_global, _, _ = confmat.compute()
epoch_acc = acc_global.item()
return epoch_acc, sum(epoch_loss) / len(epoch_loss) if len(
epoch_loss) else 0.0
def main(args):
if args.output_dir:
utils.mkdir(args.output_dir)
utils.init_distributed_mode(args)
print(args)
device = torch.device(args.device)
# CRF post-processor
postprocessor = DenseCRF(
iter_max=10,
pos_xy_std=1,
pos_w=3,
bi_xy_std=67,
bi_rgb_std=3,
bi_w=4,
)
folder_name = str(args.model) + str(args.backbone)
# Path to prediction images
prediction_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"prediction",
)
# Path to logits
logit_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"logit",
)
print("Logit src:", logit_dir)
if not os.path.isdir(logit_dir):
print("Logit not found, run first: python main.py test [OPTIONS]")
quit()
# Path to save scores
save_dir = os.path.join(
args.output_dir,
"scores",
"voc12",
folder_name.lower(),
"val",
)
save_path = os.path.join(save_dir, "scores_crf.json")
print("Score dst:", save_path)
dataset_test, num_classes = get_dataset(args.data_path, args.dataset, "val", get_transform(train=False))
confmat = utils.ConfusionMatrix(num_classes)
# Process per sample
def process(i):
image, target = dataset_test.__getitem__(i)
filename = os.path.join(str(logit_dir), str(i) + ".npy")
logit = np.load(filename)[0]
_, H, W = image.shape
logit = torch.FloatTensor(logit)[None, ...]
logit = F.interpolate(logit, size=(H, W), mode="bilinear", align_corners=False)
prob = F.softmax(logit, dim=1)[0].numpy()
image = image.numpy().astype(np.uint8).transpose(1, 2, 0)
prob = postprocessor(image, prob)
label = np.argmax(prob, axis=0)
return label, target
for i in tqdm(range(len(dataset_test))):
image, target = process(i)
writer.add_image('Images/image', image, i, dataformats='HW')
writer.add_image('Images/target', target, i, dataformats='HW')
confmat.update(target.flatten(), image.flatten())
writer.add_scalar("Mean IoU/val", confmat.get_IoU(), i)
writer.flush()
confmat.reduce_from_all_processes()
with open(save_path, "w") as f:
print(confmat, file=f)
def __init__(self, learn, n_classes, pred_func, obj, target_func=None):
super().__init__(learn, ['P.A.', 'mIoU', 'FP/FN'])
self.target_func = target_func if target_func else utils.cm_target_func
self.pred_func = pred_func
self.cm = utils.ConfusionMatrix(n_classes)
self.obj = obj
def evaluate(model, data_loader, device, num_classes, output_dir, save=False):
model.eval()
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'
confmat = utils.ConfusionMatrix(num_classes)
per_mean_iou = list()
image_epoch_dir = os.path.join(output_dir, "images/")
utils.mkdir(image_epoch_dir)
image_dir = os.path.join(output_dir, "images/image/")
utils.mkdir(image_dir)
target_dir = os.path.join(output_dir, "images/target/")
utils.mkdir(target_dir)
prediction_dir = os.path.join(output_dir, "images/prediction/")
utils.mkdir(prediction_dir)
mean_iou_file = os.path.join(output_dir, "mean_iou.csv")
per_mean_iou_file = os.path.join(output_dir, "per_image_mean_iou.csv")
with torch.no_grad():
start_time = time.time()
for idx, (image, target) in enumerate(
metric_logger.log_every(data_loader, 10, header)):
image, target = image.to(device), target.to(device)
inv_normalize = T.Normalize(mean=(-0.485, -0.456, -0.406),
std=(1 / 0.229, 1 / 0.224, 1 / 0.225))
#if idx<10:
# save_image(inv_normalize(image[0], target)[0], "img{}.png".format(idx))
confmat_image = utils.ConfusionMatrix(num_classes)
output = model(image)
output = output['out']
if save:
image_path = os.path.join(image_dir, '{}.npy'.format(idx))
target_path = os.path.join(target_dir, '{}.npy'.format(idx))
prediction_path = os.path.join(prediction_dir,
'{}.npy'.format(idx))
utils.save_on_master(
inv_normalize(image[0], target)[0], image_path)
utils.save_on_master(target, target_path)
utils.save_on_master(output, prediction_path)
confmat.update(target.flatten(), output.argmax(1).flatten())
confmat_image.update(target.flatten(), output.argmax(1).flatten())
acc_global, acc, iu = confmat_image.compute()
confmat_image.reduce_from_all_processes()
image_mean_iou = list((iu * 100).tolist())
per_mean_iou.append(image_mean_iou)
confmat.reduce_from_all_processes()
total_time = time.time() - start_time
total_time_str = str(datetime.timedelta(seconds=int(total_time)))
print('Validation time {}'.format(total_time_str))
with open(per_mean_iou_file, 'w', newline='') as f:
wr = csv.writer(f, quoting=csv.QUOTE_ALL)
wr.writerow(per_mean_iou)
with open(mean_iou_file, 'w', newline='') as f:
wr = csv.writer(f, quoting=csv.QUOTE_ALL)
wr.writerow([confmat.get_mean_iou()])
return confmat
def train(train_dir, val_dir, checkpoint_file=None):
train_loader, val_loader = load_data(train_dir, val_dir)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = models.segmentation.deeplabv3_resnet50(pretrained=False,
num_classes=2)
for param in model.parameters():
param.requires_grad = True
model = nn.Sequential(model, CRF(n_spatial_dims=2))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.002)
model.to(device)
writer = SummaryWriter('runs/deeplab_experiment_11')
epochs = 100
print_freq = 500
save_freq = 25
lr_scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer, lambda x: (1 - x / (len(train_loader) * epochs))**0.9)
if checkpoint_file is not None:
checkpoint = torch.load(checkpoint_file)
model.load_state_dict(checkpoint['model'])
optimizer.load_state_dict(checkpoint['optimizer'])
lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
last_epoch = checkpoint['epoch']
acc_global = checkpoint['acc_global']
acc = checkpoint['acc']
iou = checkpoint['iou']
miou = checkpoint['miou']
dice_coef = checkpoint['dice_coef']
mcc = checkpoint['mcc']
else:
last_epoch = 0
for epoch in range(last_epoch + 1, last_epoch + epochs + 1):
train_mcc = 0.0
train_confmat = utils.ConfusionMatrix(num_classes=2)
for inputs, labels in train_loader:
model.train()
inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad()
try:
logps = model(inputs)
#logps = logps['out'].squeeze(1)
except:
continue
loss = criterion(logps, labels)
loss.backward()
optimizer.step()
lr_scheduler.step(loss.cpu().data.numpy())
train_pred = torch.argmax(logps, dim=1)
train_confmat.update(labels.flatten().long(), train_pred.flatten())
train_mcc += matthews_corrcoef(labels.cpu().numpy().flatten(),
train_pred.cpu().numpy().flatten())
train_acc_global, train_acc, train_iou, train_miou = train_confmat.compute(
)
train_mcc = train_mcc / len(train_loader)
print("Train loss: ", loss.item(), " ... ", "Train acc: ",
train_acc_global, " ... ", "Train mIOU: ", train_miou, " ... ",
"Train MCC: ", train_mcc)
writer.add_scalar('Train/Loss', loss.item(), epoch)
writer.add_scalar('Train/Global Accuracy', train_acc_global, epoch)
writer.add_scalar('Train/Accuracy/nontumor', train_acc[0], epoch)
writer.add_scalar('Train/Accuracy/tumor', train_acc[1], epoch)
writer.add_scalar('Train/IoU/nontumor', train_iou[0], epoch)
writer.add_scalar('Train/IoU/tumor', train_iou[1], epoch)
writer.add_scalar('Train/mIoU', train_miou, epoch)
writer.add_scalar('Train/MCC', train_mcc, epoch)
# Evaluate validation loss and confusion matrix after every epoch
model.eval()
val_loss = 0.0
val_mcc = 0.0
confmat = utils.ConfusionMatrix(num_classes=2)
with torch.no_grad():
for val_inputs, val_labels in val_loader:
val_inputs, val_labels = val_inputs.to(device), val_labels.to(
device)
try:
val_logps = model(val_inputs)
#val_logps = val_logps['out'].squeeze(1)
except:
continue
val_preds = torch.argmax(val_logps, dim=1)
probability = torch.sigmoid(val_logps)
predicted = (probability > 0.5).int()
confmat.update(val_labels.flatten().long(),
val_preds.flatten())
val_mcc += matthews_corrcoef(
val_labels.cpu().numpy().flatten(),
val_preds.cpu().numpy().flatten())
batch_loss = criterion(val_logps, val_labels)
val_loss += batch_loss.item()
val_loss = val_loss / len(val_loader)
acc_global, acc, iou, miou = confmat.compute()
dice_coef = dice(val_preds.flatten(), val_labels.flatten())
val_mcc = val_mcc / len(val_loader)
print("Val loss: ", val_loss, " ... ", "Val acc: ", acc_global,
" ... ", "Val mIOU: ", miou, " ... ", "Val Dice coeff: ",
dice_coef, "Val MCC: ", val_mcc)
writer.add_scalar('Val/Loss', val_loss, epoch)
writer.add_scalar('Val/Global Accuracy', acc_global, epoch)
writer.add_scalar('Val/Accuracy/nontumor', acc[0], epoch)
writer.add_scalar('Val/Accuracy/tumor', acc[1], epoch)
writer.add_scalar('Val/IoU/nontumor', iou[0], epoch)
writer.add_scalar('Val/IoU/tumor', iou[1], epoch)
writer.add_scalar('Val/mIoU', miou, epoch)
writer.add_scalar('Val/Dice coeff', dice_coef, epoch)
writer.add_scalar('Val/MCC', val_mcc, epoch)
writer.close()
# Save checkpoint after every save_freq epochs
if epoch % save_freq == 0:
utils.save_on_master(
{
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'lr_scheduler': lr_scheduler.state_dict(),
'epoch': epoch,
'acc_global': acc_global,
'acc': acc,
'iou': iou,
'miou': miou,
'dice_coef': dice_coef,
'mcc': val_mcc
},
os.path.join(
'./checkpoints',
'deeplab_resnet101_experiment_11_{}.pth'.format(epoch)))
def evaluate(args,
model,
data_loader,
ref_ids,
refer,
bert_model,
device,
num_classes,
display=False,
baseline_model=None,
objs_ids=None,
num_objs_list=None):
model.eval()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
refs_ids_list = []
outputs = []
# dict to save results for DAVIS
total_outputs = {}
# evaluation variables
cum_I, cum_U = 0, 0
eval_seg_iou_list = [.5, .6, .7, .8, .9]
seg_correct = np.zeros(len(eval_seg_iou_list), dtype=np.int32)
seg_total = 0
mean_IoU = []
header = 'Test:'
with torch.no_grad():
k = 0
l = 0
for image, target, sentences, attentions in metric_logger.log_every(
data_loader, 100, header):
image, target, sentences, attentions = image.to(device), target.to(
device), sentences.to(device), attentions.to(device)
sentences = sentences.squeeze(1)
attentions = attentions.squeeze(1)
if args.dataset == 'davis' or args.dataset == 'a2d':
sentences = sentences.unsqueeze(-1)
attentions = attentions.unsqueeze(-1)
target = target.cpu().data.numpy()
for j in range(sentences.size(-1)):
refs_ids_list.append(k)
if args.baseline_bilstm:
sent = sentences[:, :, :, j]
att = attentions[:, :, j]
num_tokens = torch.sum(att, dim=-1)
processed_seqs = sent[:num_tokens, :]
hidden_states, cell_states = baseline_model[0](
processed_seqs)
hidden_states = hidden_states[0]
processed_hidden_states = hidden_states[:num_tokens, :]
last_hidden_states = torch.mean(processed_hidden_states,
dim=0)
last_hidden_states = baseline_model[1](last_hidden_states)
embedding = last_hidden_states.unsqueeze(1)
else:
last_hidden_states = bert_model(
sentences[:, :, j], attention_mask=attentions[:, :,
j])[0]
embedding = last_hidden_states[:, 0, :]
output, _, _ = model(image, embedding.squeeze(1))
output = output['out'].cpu()
output_mask = output.argmax(1).data.numpy()
outputs.append(output_mask)
I, U = computeIoU(output_mask, target)
if U == 0:
this_iou = 0.0
else:
this_iou = I * 1.0 / U
mean_IoU.append(this_iou)
cum_I += I
cum_U += U
for n_eval_iou in range(len(eval_seg_iou_list)):
eval_seg_iou = eval_seg_iou_list[n_eval_iou]
seg_correct[n_eval_iou] += (this_iou >= eval_seg_iou)
seg_total += 1
del image, target, attentions
if display:
plt.figure()
plt.axis('off')
if args.dataset == 'refcoco' or args.dataset == 'refcoco+':
ref = refer.loadRefs(ref_ids[k])
image_info = refer.Imgs[ref[0]['image_id']]
for p in range(len(ref[0]['sentences'])):
l += 1
if args.dataset == 'refcoco' or args.dataset == 'refcoco+':
sentence = ref[0]['sentences'][p]['raw']
im_path = os.path.join(refer.IMAGE_DIR,
image_info['file_name'])
elif args.dataset == 'davis':
idx = ref_ids[k]
sentence = refer[idx]
im_path = os.path.join(args.davis_data_root,
img_list[k])
elif args.dataset == 'a2d':
sentence = refer[k]
image_name = ref_ids[k]
im_path = os.path.join(args.a2d_root_dir, image_name)
im = imread(im_path)
plt.imshow(im)
if args.dataset == 'davis':
if img_list[k] not in total_outputs:
total_outputs[img_list[k]] = {}
o_mask = output_mask.copy()
o_mask = o_mask.astype(int) * int(
idx.split('_')[-1])
total_outputs[img_list[k]] = o_mask.squeeze(0)
else:
total_outputs[img_list[k]][output_mask.squeeze(
0) == True] = int(idx.split('_')[-1])
plt.text(0, 0, sentence, fontsize=12)
ax = plt.gca()
ax.set_autoscale_on(False)
# mask definition
img = np.ones((im.shape[0], im.shape[1], 3))
color_mask = np.array([0, 255, 0]) / 255.0
for i in range(3):
img[:, :, i] = color_mask[i]
if args.dataset == 'refcoco' or args.dataset == 'refcoco+':
output_mask = outputs[-len(ref[0]['sentences']) +
p].transpose(1, 2, 0)
ax.imshow(np.dstack((img, output_mask * 0.5)))
if not os.path.isdir(results_folder):
os.makedirs(results_folder)
figname = os.path.join(args.results_folder,
str(l) + '.png')
plt.close()
k += 1
if args.dataset == 'davis':
for r in total_outputs.keys():
new_im = Image.fromarray(total_outputs[r].astype(np.uint8))
file_name = r.split('/')[-1].split('.')[0]
folder_name = r.split('/')[-2]
if not os.path.isdir(os.path.join(submission_path,
folder_name)):
os.makedirs(os.path.join(submission_path, folder_name))
new_im.save(
os.path.join(args.submission_path, folder_name,
file_name + '.png'))
mean_IoU = np.array(mean_IoU)
mIoU = np.mean(mean_IoU)
print('Final results:')
print('Mean IoU is %.2f\n' % (mIoU * 100.))
results_str = ''
for n_eval_iou in range(len(eval_seg_iou_list)):
results_str += ' precision@%s = %.2f\n' % \
(str(eval_seg_iou_list[n_eval_iou]), seg_correct[n_eval_iou] * 100. / seg_total)
results_str += ' overall IoU = %.2f\n' % (cum_I * 100. / cum_U)
print(results_str)
return refs_ids_list, outputs
def main(args):
folder_name = str(args.model) + str(args.backbone)
# Path to save logits
logit_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"logit",
)
utils.mkdir(logit_dir)
print("Logit dst:", logit_dir)
# Path to save images
image_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"image",
)
utils.mkdir(image_dir)
print("Image dst:", image_dir)
# Path to save processed images
processed_image_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"processed_image",
)
utils.mkdir(processed_image_dir)
print("Processed Image dst:", processed_image_dir)
# Path to ground truth images
ground_truth_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"ground_truth",
)
utils.mkdir(ground_truth_dir)
print("Ground truth dst:", ground_truth_dir)
# Path to prediction images
prediction_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"prediction",
)
utils.mkdir(prediction_dir)
print("Prediction dst:", prediction_dir)
# Path to target images
target_dir = os.path.join(
args.output_dir,
"features",
"voc12",
folder_name.lower(),
"val",
"target",
)
utils.mkdir(target_dir)
print("Target dst:", target_dir)
# Path to save scores
save_dir = os.path.join(
args.output_dir,
"scores",
"voc12",
folder_name.lower(),
"val",
)
utils.mkdir(save_dir)
save_path = os.path.join(save_dir, "scores.json")
print("Score dst:", save_path)
if args.output_dir:
utils.mkdir(args.output_dir)
utils.init_distributed_mode(args)
print(args)
iterator = utils.Iterator()
device = torch.device(args.device)
dataset_test, num_classes = get_dataset(args.data_path, args.dataset,
"val", get_transform(train=False))
if args.distributed:
test_sampler = torch.utils.data.distributed.DistributedSampler(
dataset_test)
else:
test_sampler = torch.utils.data.SequentialSampler(dataset_test)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
shuffle=False,
sampler=test_sampler,
num_workers=args.workers,
collate_fn=utils.collate_fn)
model = _load_model(arch_type=args.model,
backbone=args.backbone,
pretrained=False,
progress=True,
num_classes=num_classes,
aux_loss=args.aux_loss)
model.to(torch.device('cuda'))
if args.distributed:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
model_without_ddp = model
if args.distributed:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
model_without_ddp = model.module
params_to_optimize = [
{
"params": [
p for p in model_without_ddp.backbone.parameters()
if p.requires_grad
]
},
{
"params": [
p for p in model_without_ddp.classifier.parameters()
if p.requires_grad
]
},
]
if args.aux_loss:
params = [
p for p in model_without_ddp.aux_classifier.parameters()
if p.requires_grad
]
params_to_optimize.append({"params": params, "lr": args.lr * 10})
optimizer = torch.optim.SGD(params_to_optimize,
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
checkpoint = torch.load(
"/home/AD/rraina/segmentation_benchmark/semseg/output_models/checkpoint.pth",
map_location='cpu')
model_without_ddp.load_state_dict(checkpoint['model'])
optimizer.load_state_dict(checkpoint['optimizer'])
model.eval()
start_time = time.time()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'
with torch.no_grad():
for idx, (image, target) in tqdm(enumerate(dataset_test)):
image, target = image.to(device), target.to(device)
image = image.unsqueeze(0)
target = target.unsqueeze(0)
output = model(image)
output = output['out']
# Saving Logits
filename = os.path.join(logit_dir, str(idx) + ".npy")
np.save(filename, output.cpu().numpy())
# Saving Ground Truths
ground_truth = Image.open(str(data_loader_test.dataset.masks[idx]))
filename = os.path.join(ground_truth_dir, str(idx) + ".png")
ground_truth.save(str(filename))
# Saving Images
images = Image.open(str(data_loader_test.dataset.images[idx]))
filename = os.path.join(image_dir, str(idx) + ".png")
images.save(str(filename))
# Saving Processed Image
processed_image_in = image[0].cpu().numpy().transpose(1, 2, 0)
processed_image = Image.fromarray(
(processed_image_in * 255).astype(np.uint8))
filename = os.path.join(processed_image_dir, str(idx) + ".png")
processed_image.save(str(filename))
# Saving Target Image
target_in = target[0].cpu().numpy()
target_image = Image.fromarray(np.uint8(target_in * 255), 'L')
filename = os.path.join(target_dir, str(idx) + ".png")
target_image.save(str(filename))
# Saving Prediction Image
prediction_in = get_mask(output)
prediction_image = Image.fromarray(
(prediction_in * 255).astype(np.uint8))
filename = os.path.join(prediction_dir, str(idx) + ".png")
prediction_image.save(str(filename))
confmat.update(target.flatten(), output.argmax(1).flatten())
confmat.reduce_from_all_processes()
with open(save_path, "w") as f:
print(confmat, file=f)
total_time = time.time() - start_time
total_time_str = str(datetime.timedelta(seconds=int(total_time)))
print('Evaluation time {}'.format(total_time_str))
def evaluate(model, data_loader, args, bert_model, device, num_classes, epoch,
logger, baseline_model):
model.eval()
confmat = utils.ConfusionMatrix(num_classes)
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'
val_loss = 0
seg_loss = 0
cos_loss = 0
total_its = 0
acc_ious = 0
with torch.no_grad():
for data in metric_logger.log_every(data_loader, 100, header):
total_its += 1
image, target, sentences, attentions = data
image, target, sentences, attentions = image.to(device), target.to(
device), sentences.to(device), attentions.to(device)
sentences = sentences.squeeze(1)
attentions = attentions.squeeze(1)
if args.baseline_bilstm:
num_tokens = torch.sum(attentions, dim=-1)
unbinded_sequences = list(torch.unbind(sentences, dim=0))
processed_seqs = [
seq[:num_tokens[i], :]
for i, seq in enumerate(unbinded_sequences)
]
packed_sentences = torch.nn.utils.rnn.pack_sequence(
processed_seqs, enforce_sorted=False)
hidden_states, cell_states = baseline_model[0](
packed_sentences)
hidden_states = torch.nn.utils.rnn.pad_packed_sequence(
hidden_states, batch_first=True, total_length=20)
hidden_states = hidden_states[0]
unbinded_hidden_states = list(
torch.unbind(hidden_states, dim=0))
processed_hidden_states = [
seq[:num_tokens[i], :]
for i, seq in enumerate(unbinded_hidden_states)
]
mean_hidden_states = [
torch.mean(seq, dim=0).unsqueeze(0)
for seq in processed_hidden_states
]
last_hidden_states = torch.cat(mean_hidden_states, dim=0)
last_hidden_states = baseline_model[1](last_hidden_states)
last_hidden_states = last_hidden_states.unsqueeze(1)
else:
last_hidden_states = bert_model(sentences,
attention_mask=attentions)[0]
embedding = last_hidden_states[:, 0, :]
output, vis_emb, lan_emb = model(image, embedding.squeeze(1))
iou = IoU(output['out'], target)
acc_ious += iou
loss = criterion(output, target, args)
output = output['out']
confmat.update(target.flatten(), output.argmax(1).flatten())
confmat.reduce_from_all_processes()
val_loss = val_loss / total_its
iou = acc_ious / total_its
logger.scalar_summary('loss', val_loss, epoch)
logger.scalar_summary('iou', iou, epoch)
return confmat, iou
