def evaluate(model, writer, iteration, CONFIG):
"""
Evaluation on validation set
"""
device = 0
torch.set_grad_enabled(False)
model.eval()
model.to(device)
# Dataset
if CONFIG.DATASET.NAME == "h16":
CONFIG.DATASET.NAME = "vocaug"
dataset = get_dataset(CONFIG.DATASET.NAME)(
root=CONFIG.DATASET.ROOT,
split=CONFIG.DATASET.SPLIT.VAL,
ignore_label=CONFIG.DATASET.IGNORE_LABEL,
mean_bgr=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G,
CONFIG.IMAGE.MEAN.R),
augment=False,
)
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.SOLVER.BATCH_SIZE.TEST,
num_workers=CONFIG.DATALOADER.NUM_WORKERS,
shuffle=False,
)
preds, gts = [], []
for image_ids, images, gt_labels in loader:
# Image
images = images.to(device)
# Forward propagation
logits = model(images)
# Pixel-wise labeling
_, H, W = gt_labels.shape
logits = F.interpolate(logits,
size=(H, W),
mode="bilinear",
align_corners=False)
probs = F.softmax(logits, dim=1)
labels = torch.argmax(probs, dim=1)
preds += list(labels.cpu().numpy())
gts += list(gt_labels.numpy())
# Pixel Accuracy, Mean Accuracy, Class IoU, Mean IoU, Freq Weighted IoU
score = scores(gts, preds, n_class=CONFIG.DATASET.N_CLASSES)
print("MeanIoU: {:2.2f}".format(score["Mean IoU"] * 100))
writer.add_scalar("meanIoU",
score["Mean IoU"] * 100,
global_step=iteration)
def main(config, model_path, cuda, crf):
# Configuration
CONFIG = Dict(yaml.load(open(config)))
cuda = cuda and torch.cuda.is_available()
if cuda:
current_device = torch.cuda.current_device()
print('Running on', torch.cuda.get_device_name(current_device))
image_size = (
CONFIG.IMAGE.SIZE.TEST,
CONFIG.IMAGE.SIZE.TEST,
)
# Dataset
dataset = CocoStuff10k(
root=CONFIG.ROOT,
split='test',
image_size=image_size,
scale=False,
flip=False,
preload=False,
)
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.BATCH_SIZE,
num_workers=CONFIG.NUM_WORKERS,
shuffle=False,
)
state_dict = torch.load(model_path,
map_location=lambda storage, loc: storage)
# Model
model = DeepLabV2_ResNet101_MSC(n_classes=CONFIG.N_CLASSES)
model.load_state_dict(state_dict)
model = nn.DataParallel(model)
model.eval()
if cuda:
model.cuda()
targets, outputs = [], []
for data, target in tqdm(
loader,
total=len(loader),
leave=False,
dynamic_ncols=True,
):
# Image
data = data.cuda() if cuda else data
data = Variable(data, volatile=True)
# Forward propagation
output = model(data)
output = F.upsample(output, size=image_size, mode='bilinear')
output = F.softmax(output, dim=1)
output = output.data.cpu().numpy()
# Postprocessing
if crf:
crf_output = np.zeros(output.shape)
images = data.data.cpu().numpy().astype(np.uint8)
for i, (image, prob_map) in enumerate(zip(images, output)):
image = image.transpose(1, 2, 0)
crf_output[i] = dense_crf(image, prob_map)
output = crf_output
output = np.argmax(output, axis=1)
target = target.numpy()
for o, t in zip(output, target):
outputs.append(o)
targets.append(t)
score, class_iou = scores(targets, outputs, n_class=CONFIG.N_CLASSES)
for k, v in score.items():
print(k, v)
score['Class IoU'] = {}
for i in range(CONFIG.N_CLASSES):
score['Class IoU'][i] = class_iou[i]
with open(model_path.replace('.pth', '.json'), 'w') as f:
json.dump(score, f, indent=4, sort_keys=True)
def test(config_path, model_path, cuda):
"""
Evaluation on validation set
"""
# Configuration
CONFIG = Dict(yaml.load(config_path))
device = get_device(cuda)
torch.set_grad_enabled(False)
# Dataset
dataset = get_dataset(CONFIG.DATASET.NAME)(
root=CONFIG.DATASET.ROOT,
split=CONFIG.DATASET.SPLIT.VAL,
ignore_label=CONFIG.DATASET.IGNORE_LABEL,
mean_bgr=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
augment=False,
)
print(dataset)
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.SOLVER.BATCH_SIZE.TEST,
num_workers=CONFIG.DATALOADER.NUM_WORKERS,
shuffle=False,
)
# Model
model = eval(CONFIG.MODEL.NAME)(n_classes=CONFIG.DATASET.N_CLASSES)
state_dict = torch.load(model_path, map_location=lambda storage, loc: storage)
model.load_state_dict(state_dict)
model = nn.DataParallel(model)
model.eval()
model.to(device)
# Path to save logits
logit_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"features",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
"logit",
)
makedirs(logit_dir)
print("Logit dst:", logit_dir)
# Path to save scores
save_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"scores",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
)
makedirs(save_dir)
save_path = os.path.join(save_dir, "scores.json")
print("Score dst:", save_path)
preds, gts = [], []
for image_ids, images, gt_labels in tqdm(
loader, total=len(loader), dynamic_ncols=True
):
# Image
images = images.to(device)
# Forward propagation
logits = model(images)
"""
# Save on disk for CRF post-processing
for image_id, logit in zip(image_ids, logits):
filename = os.path.join(logit_dir, image_id + ".npy")
np.save(filename, logit.cpu().numpy())
# Pixel-wise labeling
_, H, W = gt_labels.shape
logits = F.interpolate(
logits, size=(H, W), mode="bilinear", align_corners=False
)
probs = F.softmax(logits, dim=1)
labels = torch.argmax(probs, dim=1)
preds += list(labels.cpu().numpy())
gts += list(gt_labels.numpy())
# Pixel Accuracy, Mean Accuracy, Class IoU, Mean IoU, Freq Weighted IoU
score = scores(gts, preds, n_class=CONFIG.DATASET.N_CLASSES)
with open(save_path, "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
@main.command()
@click.option(
"-c",
"--config-path",
type=click.File(),
required=True,
help="Dataset configuration file in YAML",
)
@click.option(
"-j",
"--n-jobs",
type=int,
default=multiprocessing.cpu_count(),
show_default=True,
help="Number of parallel jobs",
)
def crf(config_path, n_jobs):
"""
CRF post-processing on pre-computed logits
"""
# Configuration
CONFIG = Dict(yaml.load(config_path))
torch.set_grad_enabled(False)
print("# jobs:", n_jobs)
# Dataset
dataset = get_dataset(CONFIG.DATASET.NAME)(
root=CONFIG.DATASET.ROOT,
split=CONFIG.DATASET.SPLIT.VAL,
ignore_label=CONFIG.DATASET.IGNORE_LABEL,
mean_bgr=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
augment=False,
)
print(dataset)
# CRF post-processor
postprocessor = DenseCRF(
iter_max=CONFIG.CRF.ITER_MAX,
pos_xy_std=CONFIG.CRF.POS_XY_STD,
pos_w=CONFIG.CRF.POS_W,
bi_xy_std=CONFIG.CRF.BI_XY_STD,
bi_rgb_std=CONFIG.CRF.BI_RGB_STD,
bi_w=CONFIG.CRF.BI_W,
)
# Path to logit files
logit_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"features",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
"logit",
)
print("Logit src:", logit_dir)
if not os.path.isdir(logit_dir):
print("Logit not found, run first: python main.py test [OPTIONS]")
quit()
# Path to save scores
save_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"scores",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
)
makedirs(save_dir)
save_path = os.path.join(save_dir, "scores_crf.json")
print("Score dst:", save_path)
# Process per sample
def process(i):
image_id, image, gt_label = dataset.__getitem__(i)
filename = os.path.join(logit_dir, image_id + ".npy")
logit = np.load(filename)
_, H, W = image.shape
logit = torch.FloatTensor(logit)[None, ...]
logit = F.interpolate(logit, size=(H, W), mode="bilinear", align_corners=False)
prob = F.softmax(logit, dim=1)[0].numpy()
image = image.astype(np.uint8).transpose(1, 2, 0)
prob = postprocessor(image, prob)
label = np.argmax(prob, axis=0)
return label, gt_label
# CRF in multi-process
results = joblib.Parallel(n_jobs=n_jobs, verbose=10, pre_dispatch="all")(
[joblib.delayed(process)(i) for i in range(len(dataset))]
)
preds, gts = zip(*results)
# Pixel Accuracy, Mean Accuracy, Class IoU, Mean IoU, Freq Weighted IoU
score = scores(gts, preds, n_class=CONFIG.DATASET.N_CLASSES)
with open(save_path, "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
def crf(config_path, n_jobs):
"""
CRF post-processing on pre-computed logits
"""
# Configuration
CONFIG = Dict(yaml.load(config_path))
torch.set_grad_enabled(False)
print("# jobs:", n_jobs)
# Dataset
dataset = get_dataset(CONFIG.DATASET.NAME)(
root=CONFIG.DATASET.ROOT,
split=CONFIG.DATASET.SPLIT.VAL,
ignore_label=CONFIG.DATASET.IGNORE_LABEL,
mean_bgr=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G,
CONFIG.IMAGE.MEAN.R),
augment=False,
)
print(dataset)
# CRF post-processor
postprocessor = DenseCRF(
iter_max=CONFIG.CRF.ITER_MAX,
pos_xy_std=CONFIG.CRF.POS_XY_STD,
pos_w=CONFIG.CRF.POS_W,
bi_xy_std=CONFIG.CRF.BI_XY_STD,
bi_rgb_std=CONFIG.CRF.BI_RGB_STD,
bi_w=CONFIG.CRF.BI_W,
)
# Path to logit files
logit_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"features",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
"logit",
)
print("Logit src:", logit_dir)
if not os.path.isdir(logit_dir):
print("Logit not found, run first: python main.py test [OPTIONS]")
quit()
# Path to save scores
save_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"scores",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
)
makedirs(save_dir)
save_path = os.path.join(save_dir, "scores_crf.json")
print("Score dst:", save_path)
# Process per sample
def process(i):
image_id, image, gt_label = dataset.__getitem__(i)
filename = os.path.join(logit_dir, image_id + ".npy")
logit = np.load(filename)
_, H, W = image.shape
logit = torch.FloatTensor(logit)[None, ...]
logit = F.interpolate(logit,
size=(H, W),
mode="bilinear",
align_corners=False)
prob = F.softmax(logit, dim=1)[0].numpy()
image = image.astype(np.uint8).transpose(1, 2, 0)
prob = postprocessor(image, prob)
label = np.argmax(prob, axis=0)
return label, gt_label
# CRF in multi-process
results = joblib.Parallel(n_jobs=n_jobs, verbose=10, pre_dispatch="all")(
[joblib.delayed(process)(i) for i in range(len(dataset))])
preds, gts = zip(*results)
# Pixel Accuracy, Mean Accuracy, Class IoU, Mean IoU, Freq Weighted IoU
score = scores(gts, preds, n_class=CONFIG.DATASET.N_CLASSES)
with open(save_path, "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
def test(config_path, model_path, cuda):
"""
Evaluation on validation set
"""
# Configuration
CONFIG = Dict(yaml.load(config_path))
device = get_device(cuda)
torch.set_grad_enabled(False)
# Dataset
dataset = get_dataset(CONFIG.DATASET.NAME)(
root=CONFIG.DATASET.ROOT,
split=CONFIG.DATASET.SPLIT.VAL,
ignore_label=CONFIG.DATASET.IGNORE_LABEL,
mean_bgr=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G,
CONFIG.IMAGE.MEAN.R),
augment=False,
)
print(dataset)
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.SOLVER.BATCH_SIZE.TEST,
num_workers=CONFIG.DATALOADER.NUM_WORKERS,
shuffle=False,
)
# Model
model = eval(CONFIG.MODEL.NAME)(n_classes=CONFIG.DATASET.N_CLASSES)
state_dict = torch.load(model_path,
map_location=lambda storage, loc: storage)
model.load_state_dict(state_dict)
model = nn.DataParallel(model)
model.eval()
model.to(device)
# Path to save logits
logit_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"features",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
"logit",
)
makedirs(logit_dir)
print("Logit dst:", logit_dir)
# Path to save scores
save_dir = os.path.join(
CONFIG.EXP.OUTPUT_DIR,
"scores",
CONFIG.EXP.ID,
CONFIG.MODEL.NAME.lower(),
CONFIG.DATASET.SPLIT.VAL,
)
makedirs(save_dir)
save_path = os.path.join(save_dir, "scores.json")
print("Score dst:", save_path)
preds, gts = [], []
for image_ids, images, gt_labels in tqdm(loader,
total=len(loader),
dynamic_ncols=True):
# Image
images = images.to(device)
# Forward propagation
logits = model(images)
# Save on disk for CRF post-processing
for image_id, logit in zip(image_ids, logits):
filename = os.path.join(logit_dir, image_id + ".npy")
np.save(filename, logit.cpu().numpy())
# Pixel-wise labeling
_, H, W = gt_labels.shape
logits = F.interpolate(logits,
size=(H, W),
mode="bilinear",
align_corners=False)
probs = F.softmax(logits, dim=1)
labels = torch.argmax(probs, dim=1)
preds += list(labels.cpu().numpy())
gts += list(gt_labels.numpy())
# Pixel Accuracy, Mean Accuracy, Class IoU, Mean IoU, Freq Weighted IoU
score = scores(gts, preds, n_class=CONFIG.DATASET.N_CLASSES)
with open(save_path, "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
gt[gt >=128.] = 1
preds += list(pd)
gts += list(gt)
##################visualization##################
pd_vis_list = []
for i,(im,pd) in enumerate(zip(image_list,pd_list)):
im = cv2.imread(im).astype(np.float32)
im = cv2.resize(im,(321,321))
pd_vis = vis(im,pd,0.7)
pd_vis_list.append(pd_vis)
all_pd = np.concatenate(pd_vis_list,axis = 1)
cv2.imwrite(save_dir + '/' + one_class_name + '.png',all_pd)
score = scores(gts, preds, n_class=2)
print(one_class_name)
print('jaccard: ' + str(score['Class IoU'][1]))
print('precision: ' + str(score['Pixel Accuracy']))
score_list.append(score)
iou_list = [one['Class IoU'][1] for one in score_list]
print('mean jaccard: ' + str(sum(iou_list)/len(iou_list)))
pre_list = [one['Pixel Accuracy'] for one in score_list]
print('mean precision: ' + str(sum(pre_list)/len(pre_list)))
def test(config, model_path, cuda, crf):
# Disable autograd globally
torch.set_grad_enabled(False)
# Setup
device = get_device(cuda)
CONFIG = Dict(yaml.load(open(config)))
# If the image size never change,
if CONFIG.DATASET.WARP_IMAGE:
# Auto-tune cuDNN
torch.backends.cudnn.benchmark = True
# Dataset 10k or 164k
dataset = get_dataset(CONFIG.DATASET.NAME)(
root=CONFIG.DATASET.ROOT,
split=CONFIG.DATASET.SPLIT.VAL,
base_size=CONFIG.IMAGE.SIZE.TEST,
crop_size=None,
mean=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
warp=CONFIG.DATASET.WARP_IMAGE,
scale=None,
flip=False,
)
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.SOLVER.BATCH_SIZE.TEST,
num_workers=CONFIG.DATALOADER.NUM_WORKERS,
shuffle=False,
)
# Model
model = setup_model(model_path, CONFIG.DATASET.N_CLASSES, train=False)
model.to(device)
# CRF post-processor
postprocessor = DenseCRF(
iter_max=CONFIG.CRF.ITER_MAX,
pos_xy_std=CONFIG.CRF.POS_XY_STD,
pos_w=CONFIG.CRF.POS_W,
bi_xy_std=CONFIG.CRF.BI_XY_STD,
bi_rgb_std=CONFIG.CRF.BI_RGB_STD,
bi_w=CONFIG.CRF.BI_W,
)
preds, gts = [], []
for images, labels in tqdm(loader,
total=len(loader),
leave=False,
dynamic_ncols=True):
# Image
images = images.to(device)
_, H, W = labels.shape
# Forward propagation
logits = model(images)
logits = F.interpolate(logits,
size=(H, W),
mode="bilinear",
align_corners=True)
probs = F.softmax(logits, dim=1)
probs = probs.data.cpu().numpy()
# Postprocessing
if crf:
# images: (B,C,H,W) -> (B,H,W,C)
images = images.data.cpu().numpy().astype(np.uint8).transpose(
0, 2, 3, 1)
probs = joblib.Parallel(n_jobs=-1)([
joblib.delayed(postprocessor)(*pair)
for pair in zip(images, probs)
])
labelmaps = np.argmax(probs, axis=1)
preds += list(labelmaps)
gts += list(labels.numpy())
# Pixel Accuracy, Mean Accuracy, Class IoU, Mean IoU, Freq Weighted IoU
score = scores(gts, preds, n_class=CONFIG.DATASET.N_CLASSES)
with open(model_path.replace(".pth", ".json"), "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
def main(config, model_path, cuda, crf):
cuda = cuda and torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
if cuda:
current_device = torch.cuda.current_device()
print("Running on", torch.cuda.get_device_name(current_device))
else:
print("Running on CPU")
# Configuration
CONFIG = Dict(yaml.load(open(config)))
# Dataset 10k or 164k
dataset = get_dataset(CONFIG.DATASET)(
root=CONFIG.ROOT,
split=CONFIG.SPLIT.VAL,
base_size=CONFIG.IMAGE.SIZE.TEST,
mean=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
warp=CONFIG.WARP_IMAGE,
scale=None,
flip=False,
)
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.BATCH_SIZE.TEST,
num_workers=CONFIG.NUM_WORKERS,
shuffle=False,
)
torch.set_grad_enabled(False)
# Model
model = DeepLabV2_ResNet101_MSC(n_classes=CONFIG.N_CLASSES)
state_dict = torch.load(model_path,
map_location=lambda storage, loc: storage)
model.load_state_dict(state_dict)
model = nn.DataParallel(model)
model.eval()
model.to(device)
targets, outputs = [], []
for data, target in tqdm(loader,
total=len(loader),
leave=False,
dynamic_ncols=True):
# Image
data = data.to(device)
# Forward propagation
output = model(data)
output = F.interpolate(output, size=data.shape[2:], mode="bilinear")
output = F.softmax(output, dim=1)
output = output.data.cpu().numpy()
# Postprocessing
if crf:
crf_output = np.zeros(output.shape)
images = data.data.cpu().numpy().astype(np.uint8)
for i, (image, prob_map) in enumerate(zip(images, output)):
image = image.transpose(1, 2, 0)
crf_output[i] = dense_crf(image, prob_map)
output = crf_output
output = np.argmax(output, axis=1)
target = target.numpy()
for o, t in zip(output, target):
outputs.append(o)
targets.append(t)
score, class_iou = scores(targets, outputs, n_class=CONFIG.N_CLASSES)
for k, v in score.items():
print(k, v)
score["Class IoU"] = {}
for i in range(CONFIG.N_CLASSES):
score["Class IoU"][i] = class_iou[i]
with open(model_path.replace(".pth", ".json"), "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
def evaluate(model, writer, train_iter, save_class=False, save_logit=False):
"""Create the model and start the evaluation process."""
# Configuration
torch.set_grad_enabled(False)
device = get_device(torch.cuda.is_available())
model.eval()
# Path to save logits
logit_dir = os.path.join(
CONFIG.EXP.SAVE_PRED,
"logit",
)
if not os.path.exists(logit_dir):
os.makedirs(logit_dir)
print("Logit dst:", logit_dir)
# Path to save scores
save_dir = os.path.join(
CONFIG.EXP.SAVE_PRED,
"scores",
)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path = os.path.join(save_dir, "scores_xavier.json")
print("Score dst:", save_path)
# saved_state_dict = torch.load(args.test_weight_path)
# model.load_state_dict(saved_state_dict)
testloader = data.DataLoader(VOCDataSet(CONFIG.DATASET.DIRECTORY,
CONFIG.DATASET.VAL_LIST_PATH,
crop_size=(CONFIG.IMAGE.SIZE.TEST,
CONFIG.IMAGE.SIZE.TEST),
mean=IMG_MEAN,
scale=False,
mirror=False),
batch_size=CONFIG.SOLVER.BATCH_SIZE.TEST,
shuffle=False,
pin_memory=True)
t_start = time.time()
# with torch.no_grad():
preds, gts = [], []
for index, batch in enumerate(testloader):
images, gt_labels, size, image_names = batch
# Image
images = images.to(device)
# Forward propagation
logits = model(images)
# Save on disk for CRF post-processing
if save_logit:
for image_id, logit in zip(image_names, logits):
filename = os.path.join(logit_dir, image_id + ".npy")
np.save(filename, logit.cpu().numpy())
# Pixel-wise labeling
_, H, W = gt_labels.shape
logits = F.interpolate(logits,
size=(H, W),
mode="bilinear",
align_corners=False)
probs = F.softmax(logits, dim=1)
labels = torch.argmax(probs, dim=1)
preds += list(labels.cpu().numpy())
gts += list(gt_labels.numpy())
score = scores(gts, preds, n_class=CONFIG.DATASET.N_CLASSES)
""" scores()
return {
"Pixel Accuracy": acc,
"Mean Accuracy": acc_cls,
"Frequency Weighted IoU": fwavacc,
"Mean IoU": mean_iu,
"Class IoU": cls_iu,
}
"""
print("Evaluate [{}] name {} time: {:3} s".format(
index, image_names[0],
time.time() - t_start),
end='\r')
# === save predict result in .png
if save_class:
save_dir_class = os.path.join(
CONFIG.EXP.SAVE_PRED,
"classes",
)
if not os.path.exists(save_dir_class):
os.makedirs(save_dir_class)
scipy.misc.toimage(labels.cpu().numpy()[0],
cmin=0,
cmax=255,
pal=colors_map,
mode='P').save(
os.path.join(save_dir_class,
image_names[0] + '.png'))
# show_timing(t_start, time.time())
writer.add_scalar("meanIoU", score["Mean IoU"], train_iter)
def main(config, excludeval, embedding, model_path, run, cuda, crf, redo,
imagedataset, threshold):
pth_extn = '.pth.tar'
if osp.isfile(model_path.replace(
pth_extn, "_" + run + ".json")) and not threshold and not redo:
print("Already Done!")
with open(model_path.replace(pth_extn,
"_" + run + ".json")) as json_file:
data = json.load(json_file)
for key, value in data.items():
if not key == "Class IoU":
print(key, value)
sys.exit()
cuda = cuda and torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
if cuda:
current_device = torch.cuda.current_device()
print("Running on", torch.cuda.get_device_name(current_device))
else:
print("Running on CPU")
# Configuration
CONFIG = Dict(yaml.load(open(config)))
datadir = os.path.join('data/datasets', imagedataset)
print("Split dir: ", datadir)
savedir = osp.dirname(model_path)
epoch = re.findall("checkpoint_(.*)\." + pth_extn[1:],
osp.basename(model_path))[-1]
val = None
visible_classes = None
if run == 'zlss' or run == 'flss':
val = np.load(datadir + '/split/test_list.npy')
visible_classes = np.load(datadir + '/split/novel_cls.npy')
elif run == 'gzlss' or run == 'gflss':
val = np.load(datadir + '/split/test_list.npy')
if excludeval:
vals_cls = np.asarray(np.load(datadir + '/split/seen_cls.npy'),
dtype=int)
else:
vals_cls = np.asarray(np.concatenate([
np.load(datadir + '/split/seen_cls.npy'),
np.load(datadir + '/split/val_cls.npy')
]),
dtype=int)
valu_cls = np.load(datadir + '/split/novel_cls.npy')
visible_classes = np.concatenate([vals_cls, valu_cls])
else:
print("invalid run ", run)
sys.exit()
if threshold is not None and run != 'gzlss':
print("invalid run for threshold", run)
sys.exit()
cls_map = np.array([255] * 256)
for i, n in enumerate(visible_classes):
cls_map[n] = i
if threshold is not None:
savedir = osp.join(savedir, str(threshold))
if crf is not None:
savedir = savedir + '-crf'
if run == 'gzlss' or run == 'gflss':
novel_cls_map = np.array([255] * 256)
for i, n in enumerate(list(valu_cls)):
novel_cls_map[cls_map[n]] = i
seen_cls_map = np.array([255] * 256)
for i, n in enumerate(list(vals_cls)):
seen_cls_map[cls_map[n]] = i
if threshold is not None:
thresholdv = np.asarray(np.zeros((visible_classes.shape[0], 1)),
dtype=np.float)
thresholdv[np.in1d(visible_classes, vals_cls), 0] = threshold
thresholdv = torch.tensor(thresholdv).float().cuda()
visible_classesp = np.concatenate([visible_classes, [255]])
all_labels = np.genfromtxt(datadir + '/labels_2.txt',
delimiter='\t',
usecols=1,
dtype='str')
print("Visible Classes: ", visible_classes)
# Dataset
dataset = get_dataset(CONFIG.DATASET)(
train=None,
test=val,
root=CONFIG.ROOT,
split=CONFIG.SPLIT.TEST,
base_size=CONFIG.IMAGE.SIZE.TEST,
mean=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
warp=CONFIG.WARP_IMAGE,
scale=None,
flip=False,
)
if embedding == 'word2vec':
class_emb = pickle.load(
open(datadir + '/word_vectors/word2vec.pkl', "rb"))
elif embedding == 'fasttext':
class_emb = pickle.load(
open(datadir + '/word_vectors/fasttext.pkl', "rb"))
elif embedding == 'fastnvec':
class_emb = np.concatenate([
pickle.load(open(datadir + '/word_vectors/fasttext.pkl', "rb")),
pickle.load(open(datadir + '/word_vectors/word2vec.pkl', "rb"))
],
axis=1)
else:
print("invalid emb ", embedding)
sys.exit()
class_emb = class_emb[visible_classes]
class_emb = F.normalize(torch.tensor(class_emb), p=2, dim=1).cuda()
print("Embedding dim: ", class_emb.shape[1])
print("# Visible Classes: ", class_emb.shape[0])
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.BATCH_SIZE.TEST,
num_workers=CONFIG.NUM_WORKERS,
shuffle=False,
)
torch.set_grad_enabled(False)
# Model
model = DeepLabV2_ResNet101_MSC(class_emb.shape[1], class_emb)
sdir = osp.join(savedir, model_path.replace(pth_extn, ""), str(epoch), run)
state_dict = torch.load(model_path,
map_location=lambda storage, loc: storage)
model = nn.DataParallel(model)
model.load_state_dict(state_dict['state_dict'])
model.eval()
model.to(device)
imgfeat = []
targets, outputs = [], []
for data, target, img_id in tqdm(loader,
total=len(loader),
leave=False,
dynamic_ncols=True):
# Image
data = data.to(device)
# Forward propagation
output = model(data)
output = F.interpolate(output,
size=data.shape[2:],
mode="bilinear",
align_corners=False)
output = F.softmax(output, dim=1)
if threshold is not None:
output = output - thresholdv.view(1, -1, 1, 1)
target = cls_map[target.numpy()]
# Postprocessing
if crf:
output = output.data.cpu().numpy()
crf_output = np.zeros(output.shape)
images = data.data.cpu().numpy().astype(np.uint8)
for i, (image, prob_map) in enumerate(zip(images, output)):
image = image.transpose(1, 2, 0)
crf_output[i] = dense_crf(image, prob_map)
output = crf_output
output = np.argmax(output, axis=1)
else:
output = torch.argmax(output, dim=1).cpu().numpy()
for o, t in zip(output, target):
outputs.append(o)
targets.append(t)
if run == 'gzlss' or run == 'gflss':
score, class_iou = scores_gzsl(targets,
outputs,
n_class=len(visible_classes),
seen_cls=cls_map[vals_cls],
unseen_cls=cls_map[valu_cls])
else:
score, class_iou = scores(targets,
outputs,
n_class=len(visible_classes))
for k, v in score.items():
print(k, v)
score["Class IoU"] = {}
for i in range(len(visible_classes)):
score["Class IoU"][all_labels[visible_classes[i]]] = class_iou[i]
if threshold is not None:
with open(
model_path.replace(pth_extn, "_" + run + '_T' +
str(threshold) + ".json"), "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
else:
with open(model_path.replace(pth_extn, "_" + run + ".json"), "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
print(score["Class IoU"])
def main(config, cuda, show):
CONFIG = Dict(yaml.load(open(config)))
cuda = cuda and torch.cuda.is_available()
dataset = VOCSegmentation(
root=CONFIG.DATASET_ROOT, image_set="val", dataset_name="VOC2012"
)
dataloader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=1, #! DO NOT CHANGE
num_workers=CONFIG.NUM_WORKERS,
pin_memory=False,
shuffle=False,
)
# Load a model
state_dict = torch.load(CONFIG.PYTORCH_MODEL)
# Model
model = PSPNet(
n_classes=CONFIG.N_CLASSES, n_blocks=CONFIG.N_BLOCKS, pyramids=CONFIG.PYRAMIDS
)
model.load_state_dict(state_dict)
model = nn.DataParallel(model)
model.eval()
if cuda:
model.cuda()
crop_size = CONFIG.IMAGE.SIZE.TEST
targets, outputs = [], []
for image, target in tqdm(
dataloader, total=len(dataloader), leave=False, dynamic_ncols=True
):
h, w = image.size()[2:]
outputs_ = []
for scale in CONFIG.SCALES:
# Resize
long_side = int(scale * CONFIG.IMAGE.SIZE.BASE)
new_h = long_side
new_w = long_side
if h > w:
new_w = int(long_side * w / h)
else:
new_h = int(long_side * h / w)
image_ = F.upsample(image, size=(new_h, new_w), mode="bilinear").data
# Predict (w/ flipping)
if long_side <= crop_size:
# Padding evaluation
image_ = pad_image(image_, crop_size)
image_ = to_var(image_, cuda)
output = torch.cat(
(model(image_), flip(model(flip(image_)))) # C, H, W # C, H, W
)
output = F.upsample(output, size=(crop_size,) * 2, mode="bilinear")
# Revert to original size
output = output[..., 0:new_h, 0:new_w]
output = F.upsample(output, size=(h, w), mode="bilinear")
outputs_ += [o for o in output.data] # 2 x [C, H, W]
else:
# Sliced evaluation
image_ = pad_image(image_, crop_size)
output = torch.cat(
(
tile_predict(image_, model, crop_size, cuda, CONFIG.N_CLASSES),
flip(
tile_predict(
flip(image_), model, crop_size, cuda, CONFIG.N_CLASSES
)
),
)
)
# Revert to original size
output = output[..., 0:new_h, 0:new_w]
output = F.upsample(output, size=(h, w), mode="bilinear")
outputs_ += [o for o in output.data] # 2 x [C, H, W]
# Average
output = torch.stack(outputs_, dim=0) # 2x#scales, C, H, W
output = torch.mean(output, dim=0) # C, H, W
output = torch.max(output, dim=0)[1] # H, W
output = output.cpu().numpy()
target = target.squeeze(0).numpy()
if show:
res_gt = np.concatenate((output, target), 1)
mask = (res_gt >= 0)[..., None]
res_gt[res_gt < 0] = 0
res_gt = np.uint8(res_gt / float(CONFIG.N_CLASSES) * 255)
res_gt = cv2.applyColorMap(res_gt, cv2.COLORMAP_JET)
res_gt = np.uint8(res_gt * mask)
img = np.uint8(image.numpy()[0].transpose(1, 2, 0) + dataset.mean_rgb)[
..., ::-1
]
img_res_gt = np.concatenate((img, res_gt), 1)
cv2.imshow("result", img_res_gt)
cv2.waitKey(10)
outputs.append(output)
targets.append(target)
score, class_iou = scores(targets, outputs, n_class=CONFIG.N_CLASSES)
for k, v in score.items():
print(k, v)
score["Class IoU"] = {}
for i in range(CONFIG.N_CLASSES):
score["Class IoU"][i] = class_iou[i]
with open("results.json", "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
def main(config, model_path, cuda):
# Configuration
with open(config) as f:
CONFIG = yaml.load(f)
cuda = cuda and torch.cuda.is_available()
image_size = (CONFIG['IMAGE']['SIZE']['TEST'],
CONFIG['IMAGE']['SIZE']['TEST'])
n_classes = CONFIG['N_CLASSES']
# Dataset
dataset = get_dataset(CONFIG['DATASET'])(root=CONFIG['ROOT'],
split='test',
image_size=image_size,
scale=False,
flip=False,
preload=False)
# DataLoader
loader = torch.utils.data.DataLoader(dataset=dataset,
batch_size=CONFIG['BATCH_SIZE'],
num_workers=CONFIG['NUM_WORKERS'],
shuffle=False)
state_dict = torch.load(model_path,
map_location=lambda storage, loc: storage)
# Model
model = DeepLabV2_ResNet101_MSC(n_classes=n_classes)
model.load_state_dict(state_dict)
model.eval()
if cuda:
model.cuda()
targets, outputs = [], []
for data, target in tqdm(loader,
total=len(loader),
leave=False,
dynamic_ncols=True):
# Image
data = data.cuda() if cuda else data
data = Variable(data, volatile=True)
# Forward propagation
output = model(data)
output = F.upsample(output, size=image_size, mode='bilinear')
output = F.softmax(output, dim=1)
output = output.data.cpu().numpy()
crf_output = np.zeros(output.shape)
images = data.data.cpu().numpy().astype(np.uint8)
for i, (image, prob_map) in enumerate(zip(images, output)):
image = image.transpose(1, 2, 0)
crf_output[i] = dense_crf(image, prob_map)
output = crf_output
output = np.argmax(output, axis=1)
target = target.numpy()
for o, t in zip(output, target):
outputs.append(o)
targets.append(t)
score, class_iou = scores(targets, outputs, n_class=n_classes)
for k, v in score.items():
print k, v
score['Class IoU'] = {}
for i in range(n_classes):
score['Class IoU'][i] = class_iou[i]
with open('results.json', 'w') as f:
json.dump(score, f)
def main(config, model_path, cuda, crf):
cuda = cuda and torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
if cuda:
current_device = torch.cuda.current_device()
print("Running on", torch.cuda.get_device_name(current_device))
else:
print("Running on CPU")
# Configuration
CONFIG = Dict(yaml.load(open(config)))
# Dataset 10k or 164k
dataset = get_dataset(CONFIG.DATASET)(
root=CONFIG.ROOT,
split=CONFIG.SPLIT.VAL,
base_size=CONFIG.IMAGE.SIZE.TEST,
mean=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
warp=CONFIG.WARP_IMAGE,
scale=None,
flip=False,
)
# DataLoader
loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=CONFIG.BATCH_SIZE.TEST,
num_workers=CONFIG.NUM_WORKERS,
shuffle=False,
)
torch.set_grad_enabled(False)
# Model
model = DeepLabV2_ResNet101_MSC(n_classes=CONFIG.N_CLASSES)
state_dict = torch.load(model_path, map_location=lambda storage, loc: storage)
model.load_state_dict(state_dict)
model = nn.DataParallel(model)
model.eval()
model.to(device)
preds, gts = [], []
for images, labels in tqdm(
loader, total=len(loader), leave=False, dynamic_ncols=True
):
# Image
images = images.to(device)
# Forward propagation
logits = model(images)
logits = F.interpolate(
logits, size=images.shape[2:], mode="bilinear", align_corners=True
)
probs = F.softmax(logits, dim=1)
probs = probs.data.cpu().numpy()
# Postprocessing
if crf:
pool = mp.Pool(mp.cpu_count())
images = images.data.cpu().numpy().astype(np.uint8).transpose(0, 2, 3, 1)
probs = pool.map(dense_crf_wrapper, zip(images, probs))
pool.close()
preds += list(np.argmax(probs, axis=1))
gts += list(labels.numpy())
score = scores(gts, preds, n_class=CONFIG.N_CLASSES)
with open(model_path.replace(".pth", ".json"), "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
def main(config, embedding, model_path, run, imagedataset, local_rank, resnet,
bkg):
rank, world_size, device_id, device = setup(local_rank)
print("Local rank: {} Rank: {} World Size: {} Device_id: {} Device: {}".
format(local_rank, rank, world_size, device_id, device))
pth_extn = '.pth.tar'
# Configuration
CONFIG = Dict(yaml.load(open(config)))
datadir = os.path.join('data/datasets', imagedataset)
print("Split dir: ", datadir)
savedir = osp.dirname(model_path)
epoch = re.findall("checkpoint_(.*)\." + pth_extn[1:],
osp.basename(model_path))[-1]
if run == 'zlss' or run == 'flss':
val = np.load(datadir + '/split/test_list.npy')
visible_classes = np.load(datadir + '/split/novel_cls.npy')
if bkg:
visible_classes = np.asarray(np.concatenate(
[np.array([0]), visible_classes]),
dtype=int)
elif run == 'gzlss' or run == 'gflss':
val = np.load(datadir + '/split/test_list.npy')
vals_cls = np.asarray(np.concatenate([
np.load(datadir + '/split/seen_cls.npy'),
np.load(datadir + '/split/val_cls.npy')
]),
dtype=int)
if bkg:
vals_cls = np.asarray(np.concatenate([np.array([0]), vals_cls]),
dtype=int)
valu_cls = np.load(datadir + '/split/novel_cls.npy')
visible_classes = np.concatenate([vals_cls, valu_cls])
else:
print("invalid run ", run)
sys.exit()
cls_map = np.array([255] * 256)
for i, n in enumerate(visible_classes):
cls_map[n] = i
if run == 'gzlss' or run == 'gflss':
novel_cls_map = np.array([255] * 256)
for i, n in enumerate(list(valu_cls)):
novel_cls_map[cls_map[n]] = i
seen_cls_map = np.array([255] * 256)
for i, n in enumerate(list(vals_cls)):
seen_cls_map[cls_map[n]] = i
all_labels = np.genfromtxt(datadir + '/labels_2.txt',
delimiter='\t',
usecols=1,
dtype='str')
print("Visible Classes: ", visible_classes)
# Dataset
dataset = get_dataset(CONFIG.DATASET)(
train=None,
test=val,
root=CONFIG.ROOT,
split=CONFIG.SPLIT.TEST,
base_size=CONFIG.IMAGE.SIZE.TEST,
mean=(CONFIG.IMAGE.MEAN.B, CONFIG.IMAGE.MEAN.G, CONFIG.IMAGE.MEAN.R),
warp=CONFIG.WARP_IMAGE,
scale=None,
flip=False,
)
random.seed(42)
if embedding == 'word2vec':
class_emb = pickle.load(
open(datadir + '/word_vectors/word2vec.pkl', "rb"))
elif embedding == 'fasttext':
class_emb = pickle.load(
open(datadir + '/word_vectors/fasttext.pkl', "rb"))
elif embedding == 'fastnvec':
class_emb = np.concatenate([
pickle.load(open(datadir + '/word_vectors/fasttext.pkl', "rb")),
pickle.load(open(datadir + '/word_vectors/word2vec.pkl', "rb"))
],
axis=1)
else:
print("invalid emb ", embedding)
sys.exit()
class_emb = class_emb[visible_classes]
class_emb = F.normalize(torch.tensor(class_emb), p=2, dim=1).cuda()
print("Embedding dim: ", class_emb.shape[1])
print("# Visible Classes: ", class_emb.shape[0])
# DataLoader
loader = torch.utils.data.DataLoader(dataset=dataset,
batch_size=CONFIG.BATCH_SIZE.TEST,
num_workers=CONFIG.NUM_WORKERS,
shuffle=False,
sampler=DistributedSampler(
dataset,
num_replicas=world_size,
rank=rank,
shuffle=False),
pin_memory=True,
drop_last=True)
torch.set_grad_enabled(False)
# Model
model = DeepLabV2_ResNet101_MSC(class_emb.shape[1],
class_emb,
resnet=resnet)
state_dict = torch.load(model_path, map_location='cpu')
model = DistributedDataParallel(model.to(device), device_ids=[rank])
new_state_dict = OrderedDict()
if resnet == 'spnet':
for k, v in state_dict['state_dict'].items():
name = k.replace("scale", "base") # 'scale'->base
name = name.replace("stages.", "")
new_state_dict[name] = v
else:
new_state_dict = state_dict['state_dict']
model.load_state_dict(new_state_dict)
del state_dict
model.eval()
targets, outputs = [], []
loader_iter = iter(loader)
iterations = len(loader_iter)
print("Iterations: {}".format(iterations))
pbar = tqdm(loader,
total=iterations,
leave=False,
dynamic_ncols=True,
position=rank)
for iteration in pbar:
data, target, img_id = next(loader_iter)
# Image
data = data.to(device)
# Forward propagation
output = model(data)
output = F.interpolate(output,
size=data.shape[2:],
mode="bilinear",
align_corners=False)
output = F.softmax(output, dim=1)
target = cls_map[target.numpy()]
remote_target = torch.tensor(target).to(device)
if rank == 0:
remote_target = torch.zeros_like(remote_target).to(device)
output = torch.argmax(output, dim=1).cpu().numpy()
remote_output = torch.tensor(output).to(device)
if rank == 0:
remote_output = torch.zeros_like(remote_output).to(device)
for o, t in zip(output, target):
outputs.append(o)
targets.append(t)
torch.distributed.reduce(remote_output, dst=0)
torch.distributed.reduce(remote_target, dst=0)
torch.distributed.barrier()
if rank == 0:
remote_output = remote_output.cpu().numpy()
remote_target = remote_target.cpu().numpy()
for o, t in zip(remote_output, remote_target):
outputs.append(o)
targets.append(t)
if rank == 0:
if run == 'gzlss' or run == 'gflss':
score, class_iou = scores_gzsl(targets,
outputs,
n_class=len(visible_classes),
seen_cls=cls_map[vals_cls],
unseen_cls=cls_map[valu_cls])
else:
score, class_iou = scores(targets,
outputs,
n_class=len(visible_classes))
for k, v in score.items():
print(k, v)
score["Class IoU"] = {}
for i in range(len(visible_classes)):
score["Class IoU"][all_labels[visible_classes[i]]] = class_iou[i]
name = ""
name = model_path.replace(pth_extn, "_" + run + ".json")
if bkg == True:
with open(name.replace('.json', '_bkg.json'), "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
else:
with open(name, "w") as f:
json.dump(score, f, indent=4, sort_keys=True)
print(score["Class IoU"])
return
