def flow_loader(path):
if '.pfm' in path:
data = readPFM(path)[0]
data[:, :, 2] = 1
return data
else:
return read_flow(path)
def read_gen(file_name, mode):
ext = splitext(file_name)[-1]
if mode == 'image':
assert ext in ['.png', '.jpeg', '.ppm', '.jpg']
return Image.open(file_name)
elif mode == 'flow':
assert ext in ['.flo', '.png', '.pfm']
return fl.read_flow(file_name)
elif mode == 'stereo':
assert ext in ['.png', '.pfm']
return fl.read_disp(file_name)
else:
raise ValueError('Unknown mode {}'.format(mode))
def test_rain():
rain_image_path = 'haze_rain'
prediction_file = 'flownets-pred-0000000.flo'
left_name_base = 'haze_rain_light/render_haze_left_beta'
right_name_base = 'haze_rain_light/render_haze_right_beta'
flow_file = 'haze_rain_light/flow_left.flo'
result = open('result.txt', 'wb')
sum_error = 0
for beta in range(0, 200, 5):
for contrast in range(120, 201, 5):
img_files = []
left_name = left_name_base + str(beta) + 'contrast' + str(
contrast) + '.png'
right_name = right_name_base + str(beta) + 'contrast' + str(
contrast) + '.png'
img_files.append(right_name)
img_files.append(left_name)
# sanity check
if os.path.exists(prediction_file):
os.remove(prediction_file)
FlowNet.run(this_dir, img_files, './model_simple')
epe = fl.evaluate_flow_file(flow_file, prediction_file)
flow = fl.read_flow(prediction_file)
flowpic = fl.flow_to_image(flow)
flow_image = Image.fromarray(flowpic)
flow_image.save('beta' + str(beta) + 'contrast' + str(contrast) +
'flow.png')
sum_error += epe
result.write('beta: ' + str(beta) + ' contrast: ' + str(contrast) +
' epe: ' + str(epe) + '\n')
print 'sum of average end point error: ', sum_error
result.close()
if args.dataset == 'chairs':
with open('FlyingChairs_train_val.txt', 'r') as f:
split = [int(i) for i in f.readlines()]
test_left_img = [
test_left_img[i] for i, flag in enumerate(split) if flag == 2
]
test_right_img = [
test_right_img[i] for i, flag in enumerate(split) if flag == 2
]
flow_paths = [flow_paths[i] for i, flag in enumerate(split) if flag == 2]
for i, gtflow_path in enumerate(flow_paths):
num = gtflow_path.split('/')[-1].strip().replace('flow.flo', 'img1.png')
if not 'test' in args.dataset and not 'clip' in args.dataset:
gtflow = read_flow(gtflow_path)
num = num.replace('jpg', 'png')
flow = read_flow('%s/%s/flo-%s' %
(args.path, args.dataset, num.replace('.png', '.pfm')))
if args.vis == 'yes':
flowimg = flow_to_image(flow) * np.linalg.norm(
flow[:, :, :2], 2, 2)[:, :, np.newaxis] / 100. / 255.
mkdir_p('%s/%s/flowimg' % (args.path, args.dataset))
plt.imsave('%s/%s/flowimg/%s' % (args.path, args.dataset, num),
flowimg)
if 'test' in args.dataset or 'clip' in args.dataset:
continue
gtflowimg = flow_to_image(gtflow)
mkdir_p('%s/%s/gtimg' % (args.path, args.dataset))
plt.imsave('%s/%s/gtimg/%s' % (args.path, args.dataset, num),
gtflowimg)
def main():
model.eval()
ttime_all = []
rmses = 0
nrmses = 0
inx = test_left_img.index(args.image)
print(test_left_img[inx])
flo = read_flow(test_flow[inx])
imgL_o = np.asarray(Image.open(test_left_img[inx]))
imgR_o = np.asarray(Image.open(test_right_img[inx]))
# resize
maxh = imgL_o.shape[0]*args.testres
maxw = imgL_o.shape[1]*args.testres
max_h = int(maxh // 64 * 64)
max_w = int(maxw // 64 * 64)
if max_h < maxh: max_h += 64
if max_w < maxw: max_w += 64
input_size = imgL_o.shape
imgL = cv2.resize(imgL_o,(max_w, max_h))
imgR = cv2.resize(imgR_o,(max_w, max_h))
# flip channel, subtract mean
imgL = imgL[:, :, None].copy() / 255. - np.asarray(mean_L).mean(0)[np.newaxis,np.newaxis,:]
imgR = imgR[:, :, None].copy() / 255. - np.asarray(mean_R).mean(0)[np.newaxis,np.newaxis,:]
print(imgL.shape)
imgL = np.transpose(imgL, [2,0,1])[np.newaxis]
imgR = np.transpose(imgR, [2,0,1])[np.newaxis]
# forward
imgL = Variable(torch.FloatTensor(imgL).cuda())
imgR = Variable(torch.FloatTensor(imgR).cuda())
with torch.no_grad():
imgLR = torch.cat([imgL,imgR],0)
model.eval()
torch.cuda.synchronize()
start_time = time.time()
rts = model(imgLR)
torch.cuda.synchronize()
ttime = (time.time() - start_time); print('time = %.2f' % (ttime*1000) )
ttime_all.append(ttime)
pred_disp, entropy = rts
# upsampling
pred_disp = torch.squeeze(pred_disp).data.cpu().numpy()
pred_disp = cv2.resize(np.transpose(pred_disp,(1,2,0)), (input_size[1], input_size[0]))
pred_disp[:,:,0] *= input_size[1] / max_w
pred_disp[:,:,1] *= input_size[0] / max_h
flow = np.ones([pred_disp.shape[0],pred_disp.shape[1],3])
flow[:,:,:2] = pred_disp
rmse = np.sqrt((np.linalg.norm(flow[:,:,:2] - flo[:,:,:2], ord=2, axis=-1) ** 2).mean())
rmses += rmse
nrmses += rmse / np.sqrt((np.linalg.norm(flo[:,:,:2], ord=2, axis=-1) ** 2).mean())
error = np.linalg.norm(flow[:,:,:2] - flo[:,:,:2], ord=2, axis=-1) ** 2
error = 255 - 255 * error / error.max()
entropy = torch.squeeze(entropy).data.cpu().numpy()
entropy = cv2.resize(entropy, (input_size[1], input_size[0]))
# save predictions
if args.dataset == 'mbstereo':
dirname = '%s/%s/%s'%(args.outdir, args.dataset, test_left_img[inx].split('/')[-2])
mkdir_p(dirname)
idxname = ('%s/%s')%(dirname.rsplit('/',1)[-1],test_left_img[inx].split('/')[-1])
else:
idxname = test_left_img[inx].split('/')[-1]
if args.dataset == 'mbstereo':
with open(test_left_img[inx].replace('im0.png','calib.txt')) as f:
lines = f.readlines()
#max_disp = int(int(lines[9].split('=')[-1]))
max_disp = int(int(lines[6].split('=')[-1]))
with open('%s/%s/%s'% (args.outdir, args.dataset,idxname.replace('im0.png','disp0IO.pfm')),'w') as f:
save_pfm(f,np.clip(-flow[::-1,:,0].astype(np.float32),0,max_disp) )
with open('%s/%s/%s/timeIO.txt'%(args.outdir, args.dataset,idxname.split('/')[0]),'w') as f:
f.write(str(ttime))
elif args.dataset == 'k15stereo' or args.dataset == 'k12stereo':
skimage.io.imsave('%s/%s/%s.png'% (args.outdir, args.dataset,idxname.split('.')[0]),(-flow[:,:,0].astype(np.float32)*256).astype('uint16'))
else:
# write_flow('%s/%s/%s.png'% (args.outdir, args.dataset,idxname.rsplit('.',1)[0]), flow.copy())
cv2.imwrite('%s/%s/%s.png' % (args.outdir, args.dataset,idxname.rsplit('.',1)[0]), flow_to_image(flow)[:, :, ::-1])
cv2.imwrite('%s/%s/%s-gt.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]), flow_to_image(flo)[:, :, ::-1])
arrow_pic(flo, '%s/%s/%s-vec-gt.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]))
arrow_pic(flow, '%s/%s/%s-vec.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]))
test_compressibility(
flo, flow,
'%s/%s/%s-compr.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0])
)
test_energy_spectrum(
flo, flow,
'%s/%s/%s-energy.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0])
)
test_intermittency_r(
flo, flow,
'%s/%s/%s-interm-r.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0])
)
test_intermittency_n(
flo, flow,
'%s/%s/%s-interm-n.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0])
)
cv2.imwrite('%s/%s/%s-err.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]), error)
# cv2.imwrite('%s/%s/ent-%s.png'% (args.outdir, args.dataset,idxname.rsplit('.',1)[0]), entropy*200)
torch.cuda.empty_cache()
rmses /= len(test_left_img)
nrmses /= len(test_left_img)
print(np.mean(ttime_all), rmses, nrmses)
def main():
model.eval()
ttime_all = []
rmses = 0
nrmses = 0
flo = read_flow(args.flow)
imgL_o = np.asarray(Image.open(args.left))
imgR_o = np.asarray(Image.open(args.right))
# resize
maxh = imgL_o.shape[0] * args.testres
maxw = imgL_o.shape[1] * args.testres
max_h = int(maxh // 64 * 64)
max_w = int(maxw // 64 * 64)
if max_h < maxh: max_h += 64
if max_w < maxw: max_w += 64
input_size = imgL_o.shape
imgL = cv2.resize(imgL_o, (max_w, max_h))
imgR = cv2.resize(imgR_o, (max_w, max_h))
# flip channel, subtract mean
imgL = imgL[:, :, None].copy() / 255. - np.asarray(mean_L).mean(0)[
np.newaxis, np.newaxis, :]
imgR = imgR[:, :, None].copy() / 255. - np.asarray(mean_R).mean(0)[
np.newaxis, np.newaxis, :]
print(imgL.shape)
imgL = np.transpose(imgL, [2, 0, 1])[np.newaxis]
imgR = np.transpose(imgR, [2, 0, 1])[np.newaxis]
# forward
imgL = Variable(torch.FloatTensor(imgL).cuda())
imgR = Variable(torch.FloatTensor(imgR).cuda())
with torch.no_grad():
imgLR = torch.cat([imgL, imgR], 0)
model.eval()
torch.cuda.synchronize()
start_time = time.time()
rts = model(imgLR)
torch.cuda.synchronize()
ttime = (time.time() - start_time)
print('time = %.2f' % (ttime * 1000))
ttime_all.append(ttime)
pred_disp, entropy = rts
# upsampling
pred_disp = torch.squeeze(pred_disp).data.cpu().numpy()
pred_disp = cv2.resize(np.transpose(pred_disp, (1, 2, 0)),
(input_size[1], input_size[0]))
pred_disp[:, :, 0] *= input_size[1] / max_w
pred_disp[:, :, 1] *= input_size[0] / max_h
flow = np.ones([pred_disp.shape[0], pred_disp.shape[1], 3])
flow[:, :, :2] = pred_disp
rmse = np.sqrt((np.linalg.norm(flow[:, :, :2] - flo[:, :, :2],
ord=2,
axis=-1)**2).mean())
rmses += rmse
nrmses += rmse / np.sqrt(
(np.linalg.norm(flo[:, :, :2], ord=2, axis=-1)**2).mean())
error = np.linalg.norm(flow[:, :, :2] - flo[:, :, :2], ord=2, axis=-1)**2
error = 255 * error / error.max()
entropy = torch.squeeze(entropy).data.cpu().numpy()
entropy = cv2.resize(entropy, (input_size[1], input_size[0]))
idxname = args.left.split('/')[-1]
cv2.imwrite('%s.png' % idxname.rsplit('.', 1)[0],
flow_to_image(flow)[:, :, ::-1])
cv2.imwrite('%s-gt.png' % idxname.rsplit('.', 1)[0],
flow_to_image(flo)[:, :, ::-1])
arrow_pic(flo, '%s-vec-gt.png' % idxname.rsplit('.', 1)[0])
arrow_pic(flow, '%s-vec.png' % idxname.rsplit('.', 1)[0])
cv2.imwrite('%s-err.png' % idxname.rsplit('.', 1)[0], error)
torch.cuda.empty_cache()
print(ttime_all, rmses, nrmses)
def eval_f(fp):
import warnings
warnings.filterwarnings("ignore")
# gt
if '2015' in args.dataset:
mask_gt = cv2.imread(maskp[fp], 0)
flow_gt = read_flow(maskp[fp].replace('obj_map',
'flow_occ')).astype(np.float32)
validmask = flow_gt[:, :, -1] == 1
bgmask_gt = np.logical_and(validmask, mask_gt == 0).astype(float)
else:
bgmask_gt = cv2.imread(maskp[fp], 0) > 0
shape = bgmask_gt.shape[:2]
validmask = np.ones(shape).astype(bool)
shape = bgmask_gt.shape[:2]
# pred
flowpath = flow_paths[fp]
# KITTI
if '2015' in args.dataset:
idnum = int(flowpath.split('/')[-1].split('_')[-2])
if args.method == 'ours':
maskp_pred = '%s/%s/pm-%06d_10.pfm' % (args.path, args.dataset,
idnum)
mask_pred = readPFM(maskp_pred)[0].astype(int)
bgmask_pred = mask_pred == 0
elif args.method == 'detectron2':
maskp_pred = '/data/gengshay/code/moseg/moseg-detectron2/%06d.png' % idnum
mask_pred = cv2.imread(maskp_pred, 0)
bgmask_pred = mask_pred == 0
elif args.method == 'mrflowss':
bgmask_pred = np.load(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_kitti_smodel/%06d_10.npy'
% idnum)
bgmask_pred = cv2.resize(bgmask_pred.astype(float),
shape[::-1]).astype(bool)
elif args.method == 'mrflowsk':
bgmask_pred = cv2.imread(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_kitti_kmodel/%06d_10.png'
% idnum, 0).astype(bool)
elif args.method == 'u2net':
bgmask_pred = cv2.imread(
'/data/gengshay/code/U-2-Net/test_data/u2net_results/%06d_10.png'
% idnum, 0) < 128
elif args.method == 'rtn':
bgmask_pred = ~cv2.imread(
'/data/gengshay/code/learningrigidity_mod/output_kitti/%06d_10.png'
% idnum, 0).astype(bool)
elif args.method == 'fusionseg':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/kitti/results_liu/joint_davis_val/%06d_10_mask.npy'
% idnum).astype(bool)
elif args.method == 'fusionsegm':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/kitti/results_liu/motion/%06d_10_mask.npy'
% idnum).astype(bool)
elif args.method == 'cosnet':
bgmask_pred = cv2.resize(
cv2.imread(
'/data/gengshay/code/COSNet/result/test/davis_iteration_conf/Results/kitti/%06d_10.png'
% idnum, 0), shape[::-1]) < 128
elif args.method == 'matnet':
bgmask_pred = cv2.imread(
'/data/gengshay/code/MATNet/output/kitti_liu/%06d_10.png' %
idnum, 0) < 128
elif args.method == 'cc':
bgmask_pred = cv2.resize(
np.load('/data/gengshay/code/cc/output/mask-joint/%03d.npy' %
idnum)[0], shape[::-1]) == 1
elif args.method == 'ccm':
bgmask_pred = cv2.resize(
np.load('/data/gengshay/code/cc/output/mask-motion/%03d.npy' %
idnum)[0], shape[::-1]) == 1
elif args.method == 'angle':
maskp_pred = '%s/%s/pm-%06d_10.pfm' % (args.path, args.dataset,
idnum)
mask_pred = readPFM(maskp_pred)[0].astype(int)
bgmask_pred = mask_pred == 0
else:
exit()
# Sintel
else:
passname = flowpath.split('/')[-1].split('_')[-4]
seqname1 = flowpath.split('/')[-1].split('_')[-3]
seqname2 = flowpath.split('/')[-1].split('_')[-2]
framename = int(flowpath.split('/')[-1].split('_')[-1].split('.')[0])
if args.method == 'ours':
maskp_pred = '%s/%s/pm-Sintel_%s_%s_%s_%02d.pfm' % (
args.path, args.dataset, passname, seqname1, seqname2,
framename)
mask_pred = readPFM(maskp_pred)[
0] # label map in {1,...N} given N rigid body predictions
bgmask_pred = mask_pred == 0
elif args.method == 'detectron2':
maskp_pred = '/data/gengshay/code/moseg/smoseg-detectron2/Sintel_%s_%s_%s_%02d.png' % (
passname, seqname1, seqname2, framename)
mask_pred = cv2.imread(maskp_pred, 0)
bgmask_pred = mask_pred == 0
elif args.method == 'mrflowss':
bgmask_pred = np.load(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_sintel_smodel/Sintel_%s_%s_%s_%02d.npy'
% (passname, seqname1, seqname2, framename)).astype(bool)
elif args.method == 'mrflowsk':
bgmask_pred = cv2.imread(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_sintel_kmodel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0).astype(bool)
elif args.method == 'u2net':
bgmask_pred = cv2.imread(
'/data/gengshay/code/U-2-Net/test_data/u2net_results_sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0) < 128
elif args.method == 'rtn':
bgmask_pred = ~cv2.imread(
'/data/gengshay/code/learningrigidity_mod/output_sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0).astype(bool)
elif args.method == 'fusionseg':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/sintel/results/joint_davis_val/Sintel_%s_%s_%s_%02d_mask.npy'
% (passname, seqname1, seqname2, framename)).astype(bool)
elif args.method == 'fusionsegm':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/sintel/results/motion/Sintel_%s_%s_%s_%02d_mask.npy'
% (passname, seqname1, seqname2, framename)).astype(bool)
elif args.method == 'cosnet':
bgmask_pred = cv2.resize(
cv2.imread(
'/data/gengshay/code/COSNet/result/test/davis_iteration_conf/Results/sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0),
shape[::-1]) < 128
elif args.method == 'matnet':
bgmask_pred = cv2.imread(
'/data/gengshay/code/MATNet/output/sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0) < 128
elif args.method == 'angle':
maskp_pred = '%s/%s/pm-Sintel_%s_%s_%s_%02d.pfm' % (
args.path, args.dataset, passname, seqname1, seqname2,
framename)
mask_pred = readPFM(maskp_pred)[0]
bgmask_pred = mask_pred == 0
else:
exit()
if args.method != 'ours' and args.method != 'detectron2':
_, mask_pred = cv2.connectedComponentsWithAlgorithm(
(1 - bgmask_pred.astype(np.uint8)),
connectivity=8,
ltype=cv2.CV_16U,
ccltype=cv2.CCL_WU)
# bg-iou
try:
bgiou = np.logical_and(
bgmask_gt.astype(bool)[validmask],
bgmask_pred[validmask]).sum() / np.logical_or(
bgmask_gt.astype(bool)[validmask],
bgmask_pred[validmask]).sum()
except:
pdb.set_trace()
# obj-F (penalizes false positive)
# defined in https://arxiv.org/abs/1902.03715 Sec.4
# F=2PR/P+R => 2/(1/P+1/R)
# since P = sum(|c and g|) / sum(|c|) and R = sum(|c and g|) / sum(|g|),
# where c is prediction and g is GT
# 1/P+1/R = [sum(|c|) + sum(|g|)] / sum(|c and g|)
# therefore F = 2*sum(|c and g|) / [sum(|c|) + sum(|g|)]
if '2015' in args.dataset:
gtlist = list(set(mask_gt.flatten()))
M = len(gtlist) - 1
imatrix = np.zeros((M, mask_pred.max()))
fmatrix = np.zeros((M, mask_pred.max()))
for i in range(M): # for all bg instances
objx_mask_gt = mask_gt == gtlist[i + 1]
for j in range(mask_pred.max()):
objx_mask_pred = mask_pred == j + 1
imatrix[i, j] = float(
np.logical_and(objx_mask_gt,
objx_mask_pred)[validmask].sum())
fmatrix[i,
j] = imatrix[i,
j] / (objx_mask_gt[validmask].sum() +
objx_mask_pred[validmask].sum()) * 2
ridx, cidx = scipy.optimize.linear_sum_assignment(1 - fmatrix)
objf = imatrix[ridx, cidx].sum() / ((mask_pred > 0)[validmask].sum() +
(mask_gt > 0)[validmask].sum()) * 2
else:
objf = 0.
return bgiou, objf
def default_loader(root, path_imgs, path_flo):
imgs = [os.path.join(root, path) for path in path_imgs]
flo = os.path.join(root, path_flo)
return [imread(img).astype(np.float32) for img in imgs], read_flow(flo)
def main():
model.eval()
flowl0 = "/gpfs/gpfs0/y.maximov/kolya/piv/SQG/SQG_00001_flow.flo"
iml0 = "/gpfs/gpfs0/y.maximov/kolya/piv/SQG/SQG_00001_img1.tif"
iml1 = "/gpfs/gpfs0/y.maximov/kolya/piv/SQG/SQG_00001_img2.tif"
iml0 = default_loader(iml0)
iml1 = default_loader(iml1)
iml1 = np.asarray(iml1) / 255.
iml0 = np.asarray(iml0) / 255.
iml0 = iml0[:, :, None].copy() # iml0[:,:,::-1].copy()
iml1 = iml1[:, :, None].copy() # iml1[:,:,::-1].copy()
flowl0 = read_flow(flowl0)
# flowl0 = random_incompressible_flow(
# 1,
# [256, 256],
# np.random.choice([30, 40, 50]), # 10. ** (2 * np.random.rand()),
# incompressible=False
# )
# iml0, iml1 = image_from_flow(
# ppp=np.random.uniform(0.008, 0.1),
# pip=np.random.uniform(0.95, 1.0),
# flow=flowl0,
# intensity_bounds=(0.8, 1),
# diameter_bounds=(0.35, 6)
# )
# iml0 = iml0.transpose(1, 2, 0).copy()
# iml1 = iml1.transpose(1, 2, 0).copy()
# flowl0 = flowl0[0]
# flowl0 = np.concatenate([
# flowl0,
# np.ones(flowl0.shape[:-1] + (1,), dtype=flowl0.dtype)
# ], axis=-1)
flowl0 = np.ascontiguousarray(flowl0, dtype=np.float32)
flowl0[np.isnan(flowl0)] = 1e6 # set to max
cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "flow-orig"),
flow_to_image(flowl0)[:, :, ::-1])
schedule_aug_coeff = 1.0
scl = None # 0.2 * schedule_aug_coeff
# if scl > 0:
# scl = [0.2 * schedule_aug_coeff, 0., 0.2 * schedule_aug_coeff]
# else:
# scl = None
rot = 0.17 * schedule_aug_coeff
if rot > 0:
rot = [0.17 * schedule_aug_coeff, 0.0]
else:
rot = None
trans = 0.2 * schedule_aug_coeff
if trans > 0:
trans = [0.2 * schedule_aug_coeff, 0.0]
else:
trans = None
co_transform = flow_transforms.Compose([
# flow_transforms.Scale(1, order=0),
flow_transforms.SpatialAug([256, 256],
scale=scl,
rot=rot,
trans=trans,
schedule_coeff=1,
order=0,
black=True),
# flow_transforms.PCAAug(schedule_coeff=schedule_coeff),
# flow_transforms.ChromaticAug(schedule_coeff=schedule_coeff, noise=self.noise),
])
augmented, flowl0 = co_transform([iml0, iml1], flowl0)
iml0 = augmented[0]
iml1 = augmented[1]
cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "flow"),
flow_to_image(flowl0)[:, :, ::-1])
cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "mask"),
255 * flowl0[:, :, -1])
cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "img1"),
255 * iml0[:, :, ::-1])
cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "img2"),
255 * iml1[:, :, ::-1])
def __getitem__(self, index):
(
image1_name,
image2_name,
reference_video_name,
reference_name1,
reference_name2,
reference_name3,
reference_name4,
reference_name5,
flow_forward_name,
flow_backward_name,
mask_name,
reference_gt1,
reference_gt2,
reference_gt3,
path,
) = self.image_pairs[index]
try:
I1 = Image.open(os.path.join(path, "input_pad", image1_name))
I2 = Image.open(os.path.join(path, "input_pad", image2_name))
I_reference_video = Image.open(
os.path.join(path, "reference_gt", random.choice([reference_gt1, reference_gt2, reference_gt3]))
)
I_reference_video_real = Image.open(
os.path.join(
path,
"reference",
random.choice(
[reference_name1, reference_name2, reference_name3, reference_name4, reference_name5]
),
)
)
flow_forward = read_flow(os.path.join(path, "flow", flow_forward_name)) # numpy
flow_backward = read_flow(os.path.join(path, "flow", flow_backward_name)) # numpy
mask = Image.open(os.path.join(path, "mask", mask_name))
# binary mask
mask = np.array(mask)
mask[mask < 240] = 0
mask[mask >= 240] = 1
# transform
I1 = self.image_transform(I1)
I2 = self.image_transform(I2)
I_reference_video = self.image_transform(self.CenterPad(I_reference_video))
I_reference_video_real = self.image_transform(self.CenterPad(I_reference_video_real))
flow_forward = self.ToTensor(self.CenterCrop(flow_forward))
flow_backward = self.ToTensor(self.CenterCrop(flow_backward))
mask = self.ToTensor(self.CenterCrop(mask))
if np.random.random() < self.real_reference_probability:
I_reference_output = I_reference_video_real
placeholder = torch.zeros_like(I1)
self_ref_flag = torch.zeros_like(I1)
else:
I_reference_output = I_reference_video
placeholder = I2 if np.random.random() < self.nonzero_placeholder_probability else torch.zeros_like(I1)
self_ref_flag = torch.ones_like(I1)
outputs = [
I1,
I2,
I_reference_output,
flow_forward,
flow_backward,
mask,
placeholder,
self_ref_flag,
]
except Exception as e:
print("problem in, ", path)
print(e)
return self.__getitem__(np.random.randint(0, len(self.image_pairs)))
return outputs
def main():
model.eval()
flow_pairs = []
for inx in range(inx_st, inx_fn + 1):
print(test_left_img[inx])
flo = read_flow(test_flow[inx])
imgL_o = np.asarray(Image.open(test_left_img[inx]))
imgR_o = np.asarray(Image.open(test_right_img[inx]))
# resize
maxh = imgL_o.shape[0] * args.testres
maxw = imgL_o.shape[1] * args.testres
max_h = int(maxh // 64 * 64)
max_w = int(maxw // 64 * 64)
if max_h < maxh: max_h += 64
if max_w < maxw: max_w += 64
input_size = imgL_o.shape
imgL = cv2.resize(imgL_o, (max_w, max_h))
imgR = cv2.resize(imgR_o, (max_w, max_h))
# flip channel, subtract mean
imgL = imgL[:, :, None].copy() / 255. - np.asarray(mean_L).mean(0)[
np.newaxis, np.newaxis, :]
imgR = imgR[:, :, None].copy() / 255. - np.asarray(mean_R).mean(0)[
np.newaxis, np.newaxis, :]
print(imgL.shape)
imgL = np.transpose(imgL, [2, 0, 1])[np.newaxis]
imgR = np.transpose(imgR, [2, 0, 1])[np.newaxis]
# forward
imgL = Variable(torch.FloatTensor(imgL).cuda())
imgR = Variable(torch.FloatTensor(imgR).cuda())
with torch.no_grad():
imgLR = torch.cat([imgL, imgR], 0)
model.eval()
torch.cuda.synchronize()
rts = model(imgLR)
torch.cuda.synchronize()
pred_disp, entropy = rts
# upsampling
pred_disp = torch.squeeze(pred_disp).data.cpu().numpy()
pred_disp = cv2.resize(np.transpose(pred_disp, (1, 2, 0)),
(input_size[1], input_size[0]))
pred_disp[:, :, 0] *= input_size[1] / max_w
pred_disp[:, :, 1] *= input_size[0] / max_h
flow = np.ones([pred_disp.shape[0], pred_disp.shape[1], 3])
flow[:, :, :2] = pred_disp
flow_pairs.append((flo, flow))
test_compressibility(
flow_pairs,
'%s/%s/%d-%d-compr.png' % (args.outdir, args.dataset, inx_st, inx_fn))
test_energy_spectrum(
flow_pairs,
'%s/%s/%d-%d-energy.png' % (args.outdir, args.dataset, inx_st, inx_fn))
test_intermittency_r(
flow_pairs, '%s/%s/%d-%d-interm-r.png' %
(args.outdir, args.dataset, inx_st, inx_fn))
test_intermittency_n(
flow_pairs, '%s/%s/%d-%d-interm-n.png' %
(args.outdir, args.dataset, inx_st, inx_fn))
torch.cuda.empty_cache()
def eval_f(fp):
import warnings
warnings.filterwarnings("ignore")
#gts
if '2015' in args.dataset:
gt_disp0 = disparity_loader(disp0p[fp])
gt_disp1 = disparity_loader(disp1p[fp])
gt_flow = read_flow(flow_paths[fp]).astype(np.float32)
ints = load_calib_cam_to_cam(calib[fp])
K0 = ints['K_cam2']
fl = K0[0, 0]
bl = ints['b20'] - ints['b30']
elif 'sintel' in args.dataset:
gt_disp0 = disparity_read(disp0p[fp])
gt_disp1 = disparity_read(disp1p[fp])
gt_flow = read_flow(flow_paths[fp]).astype(np.float32)
K0, _ = cam_read(calib[fp])
fl = K0[0, 0]
bl = 0.1
d1mask = gt_disp0 > 0
d2mask = gt_disp1 > 0
flowmask = gt_flow[:, :, -1] == 1
validmask = np.logical_and(np.logical_and(d1mask, d2mask), flowmask)
if '2015' in args.dataset:
fgmask = cv2.imread(flow_paths[fp].replace('flow_occ', 'obj_map'),
0) > 0
fgmask = np.logical_and(fgmask, validmask)
shape = gt_disp0.shape
# pred
idnum = expansionp[fp].split('/')[-1].split('.')[0]
if args.method == 'ours':
logdc = disparity_loader('%s/%s/mid-%s.pfm' %
(args.path, args.dataset, idnum))
pred_flow = read_flow('%s/%s/flo-%s.pfm' %
(args.path, args.dataset, idnum))
try:
pred_disp = disparity_loader('%s/%s/%s_disp.pfm' %
(args.path, args.dataset, idnum))
except:
try:
pred_disp = disparity_loader('%s/%s/%s.png' %
(args.path, args.dataset, idnum))
except:
try:
pred_disp = disparity_loader(
'%s/%s/disp-%s.pfm' % (args.path, args.dataset, idnum))
except:
pred_disp = disparity_loader(
'%s/%s/exp-%s.pfm' % (args.path, args.dataset, idnum))
pred_disp[pred_disp == np.inf] = pred_disp[pred_disp != np.inf].max()
pred_disp[np.isnan(pred_disp)] = 1e-12
pred_disp[pred_disp < 1e-12] = 1e-12
pred_disp1 = pred_disp / np.exp(logdc)
pred_flow = disparity_loader('%s/%s/flo-%s.pfm' %
(args.path, args.dataset, idnum))
elif args.method == 'monodepth2':
pred_disp = disparity_loader('%s/%s/%s_disp.pfm' %
(args.path, args.dataset, idnum))
else:
exit()
#hom_p = np.stack((pred_disp.flatten(), np.ones(pred_disp.flatten().shape))).T[validmask.flatten()]
#xx = np.linalg.inv(np.matmul(hom_p[:,:,np.newaxis],hom_p[:,np.newaxis,:]).sum(0))
#yy = (hom_p[:,:,np.newaxis]*gt_disp0.flatten()[validmask.flatten(),np.newaxis,np.newaxis]).sum(0)
#st = xx.dot(yy)
#pred_disp = pred_disp*st[0] + st[1]
scale_factor = np.median((gt_disp0 / pred_disp)[validmask])
pred_disp = scale_factor * pred_disp
pred_disp1 = scale_factor * pred_disp1
# eval
d1err = np.abs(pred_disp - gt_disp0)
d1err_map = (np.logical_and(d1err >= 3, d1err / gt_disp0 >= 0.05))
d1err = d1err_map[validmask]
d2err = np.abs(pred_disp1 - gt_disp1)
d2err_map = (np.logical_and(d2err >= 3, d2err / gt_disp1 >= 0.05))
d2err = d2err_map[validmask]
flow_epe = np.sqrt(np.power(gt_flow - pred_flow, 2).sum(-1))
gt_flow_mag = np.linalg.norm(gt_flow[:, :, :2], 2, -1)
flerr_map = np.logical_and(flow_epe > 3, flow_epe / gt_flow_mag > 0.05)
flerr = flerr_map[validmask]
flerr_map[~validmask] = False
try:
d1ferr = d1err_map[fgmask]
d2ferr = d2err_map[fgmask]
flferr = flerr_map[fgmask]
except:
d1ferr = np.zeros(1)
d2ferr = np.zeros(1)
flferr = np.zeros(1)
sferr = np.logical_or(np.logical_or(d1err, d2err), flerr)
sfferr = np.logical_or(np.logical_or(d1ferr, d2ferr), flferr)
img = cv2.imread(test_left_img[fp])[:, :, ::-1]
# cv2.imwrite('%s/%s/err-%s.png'%(args.path,args.dataset,idnum),np.vstack((gt_disp0,pred_disp,gt_disp1,pred_disp1,flerr_map.astype(float)*255)))
if '2015' in args.dataset:
flowvis = cv2.imread(test_left_img[fp].replace(
'image_2', 'viz_flow_occ'))[:, :, ::-1]
else:
flowvis = flow_to_image(gt_flow)
pred_flow[:, :, -1] = 1
cv2.imwrite(
'%s/%s/err-%s.png' % (args.path, args.dataset, idnum),
np.vstack(
(img, flowvis, 255 * visualize_flow(pred_flow, mode='RGB'),
np.tile(flerr_map.reshape(shape)[:, :, None], 3).astype(float) *
255))[:, :, ::-1])
return d1err.mean(), d2err.mean(), flerr.mean(), sferr.mean(),\
d1ferr.mean(), d2ferr.mean(), flferr.mean(), sfferr.mean(),gt_flow_mag.mean()
flow_paths = [flow_paths[i] for i, flag in enumerate(split) if flag == 2]
#pdb.set_trace()
#test_left_img = [i for i in test_left_img if 'clean' in i]
#test_right_img = [i for i in test_right_img if 'clean' in i]
#flow_paths = [i for i in flow_paths if 'clean' in i]
#for i,gtflow_path in enumerate(sorted(flow_paths)):
for i, gtflow_path in enumerate(flow_paths):
#if not 'Sintel_clean_cave_4_10' in gtflow_path:
# continue
#if i%10!=1:
# continue
num = gtflow_path.split('/')[-1].strip().replace('flow.flo', 'img1.png')
if not 'test' in args.dataset and not 'clip' in args.dataset:
gtflow = read_flow(gtflow_path)
num = num.replace('jpg', 'png')
flow = read_flow('%s/%s/%s' % (args.path, args.dataset, num))
if args.vis == 'yes':
#flowimg = flow_to_image(flow)
flowimg = flow_to_image(flow) * np.linalg.norm(
flow[:, :, :2], 2, 2)[:, :, np.newaxis] / 100. / 255.
#gtflowimg = compute_color(gtflow[:,:,0]/20, gtflow[:,:,1]/20)/255.
#flowimg = compute_color(flow[:,:,0]/20, flow[:,:,1]/20)/255.
mkdir_p('%s/%s/flowimg' % (args.path, args.dataset))
plt.imsave('%s/%s/flowimg/%s' % (args.path, args.dataset, num),
flowimg)
if 'test' in args.dataset or 'clip' in args.dataset:
continue
gtflowimg = flow_to_image(gtflow)
mkdir_p('%s/%s/gtimg' % (args.path, args.dataset))
