def deploy_evaluate(lbl, gt_lbl):
pred_lbl = clean_reindex(lbl)
bestDice, FgBgDice = GetDices(pred_lbl, gt_lbl)
diffFG = DiffFGLabels(pred_lbl, gt_lbl)
MWCov, MUCov, AvgFP, AvgFN = kitti_metric(pred_lbl, gt_lbl)
rand_i = adjusted_rand_index(gt_lbl, pred_lbl)
return bestDice, FgBgDice, diffFG, MWCov, MUCov, AvgFP, AvgFN, rand_i
def evaluate(args, env):
pred_lbl = clean_reindex(env.lbl, args.minsize)
gt_lbl = env.gt_lbl
bestDice, FgBgDice = GetDices(pred_lbl, gt_lbl)
diffFG = DiffFGLabels(pred_lbl, gt_lbl)
MWCov, MUCov, AvgFP, AvgFN = kitti_metric(pred_lbl, gt_lbl)
rand_i = adjusted_rand_index(gt_lbl, pred_lbl)
return bestDice, FgBgDice, diffFG, MWCov, MUCov, AvgFP, AvgFN, rand_i
def inference(args,
logger,
model,
tests,
test_env,
gpu_id,
rng,
iter,
upsize_lbl=None):
log_img = []
render_imgs = []
if args.data in ['cvppp_eval']:
original_size = (512, 512)
elif args.data in ['kitti']:
original_size = (1224, 370)
# indices testset
if args.deploy:
idxs = np.arange(len(tests)).tolist()
else:
idxs = []
if (len(tests) <= 40):
idxs.append(0)
else:
if "3D" in args.data:
ret = []
gt_ret = []
idxs.append(
rng.choice(range(tests), min(len(tests), 5),
replace=False))
else:
idxs.append(
rng.choice(range(tests),
min(len(tests), 33),
replace=False))
for _ in range(min(len(tests), 33)):
idxs.append((idxs[-1] + 1) % len(tests))
# if args.data in ['cvppp']:
# resize = True
# else:
# resize = False
if "3D" in args.data:
ret = []
gt_ret = []
if args.deploy:
ret = []
gt_ret = []
# ---------- Define Evaluating Variables ----------
# CVPPP
bestDices = []
FgBgDices = []
diffFGs = []
# Kitti
MUCovs = []
MWCovs = []
AvgFPs = []
AvgFNs = []
rand_is = []
# ----------End Define Evaluating Variables ----------
eval_times = []
for idx in idxs:
# obs = test_env.set_sample (idx, resize)
obs = test_env.set_sample(idx)
# render_img = [np.repeat (np.expand_dims (test_env.raw, -1), 3, -1)]
done = False
if args.lstm_feats:
with torch.cuda.device(gpu_id):
cx, hx = model.lstm.init_hidden(batch_size=1, use_cuda=True)
currTime = time.time()
while (not done):
with torch.no_grad():
with torch.cuda.device(gpu_id):
t_obs = torch.tensor(obs[None],
dtype=torch.float32,
device="cuda")
if args.lstm_feats:
value, logit, (hx, cx) = model((t_obs, (hx, cx)))
else:
value, logit = model(t_obs)
prob = F.softmax(logit, dim=1)
action = prob.max(1)[1].data.cpu().numpy()
obs, _, done, _ = test_env.step_inference(action[0])
# render_img.append (test_env.pred_lbl2rgb(test_env.lbl))
# render_img = np.concatenate (render_img, 1).astype (np.uint8)
# render_imgs.append (render_img)
if not "3D" in args.data:
clean_reindex(test_env.lbl, args.minsize)
deltaTime = time.time() - currTime
eval_times.append(deltaTime)
# print ("eval time: ", deltaTime)
if not args.deploy:
img = test_env.render()
log_img.append(img[:len(img) // 2])
elif args.data in ['cvppp_eval', 'cvppp', "448_cremi", "256_cremi"]:
# ret.append(test_env.lbl)
# lbl_upsize = cv2.resize (test_env.lbl, original_size, interpolation=cv2.INTER_NEAREST)
ret.append(test_env.lbl)
# # ---------------------------------------- Evaluate ----------------------------------------
# bestDice, FgBgDice, diffFG, MWCov, MUCov, AvgFP, AvgFN, rand_i = deploy_evaluate(lbl_upsize, upsize_lbl[idx])
# # CVPPP
# bestDices.append(bestDice)
# FgBgDices.append(FgBgDice)
# diffFGs.append(abs (diffFG))
# # Kitti
# MWCovs.append(MWCov)
# MUCovs.append(MUCov)
# AvgFPs.append(AvgFP)
# AvgFNs.append(AvgFN)
# rand_is.append(rand_i)
# ---------------------------------------- End Evaluate ----------------------------------------
elif args.data == 'cvppp':
ret.append(test_env.lbl)
if "3D" in args.data:
ret.append(test_env.lbl)
gt_ret.append(test_env.gt_lbl)
print(eval_times)
print("Mean eval times: ", np.mean(eval_times))
if not args.deploy:
log_img = np.concatenate(log_img, 0)
# render_imgs = np.concatenate (render_imgs, 0)
# print (render_imgs.shape)
# log_info = {"test_samples": log_img, "render_imgs": render_imgs}
log_info = {
"test_samples": log_img,
}
for tag, img in log_info.items():
img = img[None]
logger.image_summary(tag, img, iter)
if "3D" in args.data:
for i, vol in enumerate(ret):
io.imsave(
args.log_dir + "tifs_test/" + str(iter) + "_" + str(i) +
"_test.tif", vol.astype(np.uint8))
else:
print("\n------------------------------------------------")
print((str(datetime.now()) + "\n"))
if not "3D" in args.data:
ret = np.array(ret, dtype=np.int32)
io.imsave("deploy/" + "deploy_" + args.env + ".tif", ret)
else:
pass
# print ("\nExport Images: ", "deploy/" + "deploy_" + args.data + ".tif")
# if args.data in ['kitti', 'cremi', 'cvppp_eval']:
# print("Evaluate on "+ args.eval_data + " " + args.data)
# log_txt = open("deploy/" + "log_" + str(args.env) + ".txt", "w")
# log_info = {
# "bestDice": np.mean(bestDices),
# "FgBgDice": np.mean(FgBgDices),
# "diffFG": np.mean(diffFGs),
# "MWCov": np.mean(MWCovs),
# "MUCov": np.mean(MUCovs),
# "AvgFP": np.mean(AvgFPs),
# "AvgFN": np.mean(AvgFNs),
# "rand_i": np.mean(rand_is),
# }
# print (log_info)
# log_txt.write("##########################################" + "\n")
# log_txt.write(str(datetime.now()) + "\n")
# for k, v in log_info.items():
# log_txt.write(str(k) + ": " + str(v) + "\n")
# print(str(k) + ": " + str(v) + "\n")
# log_txt.write("##########################################" + "\n")
# log_txt.close()
print("\nDone !")
print("\n------------------------------------------------")
