def flow_loader(path): if '.pfm' in path: data = readPFM(path)[0] data[:, :, 2] = 1 return data else: return read_flow(path)
def read_gen(file_name, mode): ext = splitext(file_name)[-1] if mode == 'image': assert ext in ['.png', '.jpeg', '.ppm', '.jpg'] return Image.open(file_name) elif mode == 'flow': assert ext in ['.flo', '.png', '.pfm'] return fl.read_flow(file_name) elif mode == 'stereo': assert ext in ['.png', '.pfm'] return fl.read_disp(file_name) else: raise ValueError('Unknown mode {}'.format(mode))
def test_rain(): rain_image_path = 'haze_rain' prediction_file = 'flownets-pred-0000000.flo' left_name_base = 'haze_rain_light/render_haze_left_beta' right_name_base = 'haze_rain_light/render_haze_right_beta' flow_file = 'haze_rain_light/flow_left.flo' result = open('result.txt', 'wb') sum_error = 0 for beta in range(0, 200, 5): for contrast in range(120, 201, 5): img_files = [] left_name = left_name_base + str(beta) + 'contrast' + str( contrast) + '.png' right_name = right_name_base + str(beta) + 'contrast' + str( contrast) + '.png' img_files.append(right_name) img_files.append(left_name) # sanity check if os.path.exists(prediction_file): os.remove(prediction_file) FlowNet.run(this_dir, img_files, './model_simple') epe = fl.evaluate_flow_file(flow_file, prediction_file) flow = fl.read_flow(prediction_file) flowpic = fl.flow_to_image(flow) flow_image = Image.fromarray(flowpic) flow_image.save('beta' + str(beta) + 'contrast' + str(contrast) + 'flow.png') sum_error += epe result.write('beta: ' + str(beta) + ' contrast: ' + str(contrast) + ' epe: ' + str(epe) + '\n') print 'sum of average end point error: ', sum_error result.close()
if args.dataset == 'chairs': with open('FlyingChairs_train_val.txt', 'r') as f: split = [int(i) for i in f.readlines()] test_left_img = [ test_left_img[i] for i, flag in enumerate(split) if flag == 2 ] test_right_img = [ test_right_img[i] for i, flag in enumerate(split) if flag == 2 ] flow_paths = [flow_paths[i] for i, flag in enumerate(split) if flag == 2] for i, gtflow_path in enumerate(flow_paths): num = gtflow_path.split('/')[-1].strip().replace('flow.flo', 'img1.png') if not 'test' in args.dataset and not 'clip' in args.dataset: gtflow = read_flow(gtflow_path) num = num.replace('jpg', 'png') flow = read_flow('%s/%s/flo-%s' % (args.path, args.dataset, num.replace('.png', '.pfm'))) if args.vis == 'yes': flowimg = flow_to_image(flow) * np.linalg.norm( flow[:, :, :2], 2, 2)[:, :, np.newaxis] / 100. / 255. mkdir_p('%s/%s/flowimg' % (args.path, args.dataset)) plt.imsave('%s/%s/flowimg/%s' % (args.path, args.dataset, num), flowimg) if 'test' in args.dataset or 'clip' in args.dataset: continue gtflowimg = flow_to_image(gtflow) mkdir_p('%s/%s/gtimg' % (args.path, args.dataset)) plt.imsave('%s/%s/gtimg/%s' % (args.path, args.dataset, num), gtflowimg)
def main(): model.eval() ttime_all = [] rmses = 0 nrmses = 0 inx = test_left_img.index(args.image) print(test_left_img[inx]) flo = read_flow(test_flow[inx]) imgL_o = np.asarray(Image.open(test_left_img[inx])) imgR_o = np.asarray(Image.open(test_right_img[inx])) # resize maxh = imgL_o.shape[0]*args.testres maxw = imgL_o.shape[1]*args.testres max_h = int(maxh // 64 * 64) max_w = int(maxw // 64 * 64) if max_h < maxh: max_h += 64 if max_w < maxw: max_w += 64 input_size = imgL_o.shape imgL = cv2.resize(imgL_o,(max_w, max_h)) imgR = cv2.resize(imgR_o,(max_w, max_h)) # flip channel, subtract mean imgL = imgL[:, :, None].copy() / 255. - np.asarray(mean_L).mean(0)[np.newaxis,np.newaxis,:] imgR = imgR[:, :, None].copy() / 255. - np.asarray(mean_R).mean(0)[np.newaxis,np.newaxis,:] print(imgL.shape) imgL = np.transpose(imgL, [2,0,1])[np.newaxis] imgR = np.transpose(imgR, [2,0,1])[np.newaxis] # forward imgL = Variable(torch.FloatTensor(imgL).cuda()) imgR = Variable(torch.FloatTensor(imgR).cuda()) with torch.no_grad(): imgLR = torch.cat([imgL,imgR],0) model.eval() torch.cuda.synchronize() start_time = time.time() rts = model(imgLR) torch.cuda.synchronize() ttime = (time.time() - start_time); print('time = %.2f' % (ttime*1000) ) ttime_all.append(ttime) pred_disp, entropy = rts # upsampling pred_disp = torch.squeeze(pred_disp).data.cpu().numpy() pred_disp = cv2.resize(np.transpose(pred_disp,(1,2,0)), (input_size[1], input_size[0])) pred_disp[:,:,0] *= input_size[1] / max_w pred_disp[:,:,1] *= input_size[0] / max_h flow = np.ones([pred_disp.shape[0],pred_disp.shape[1],3]) flow[:,:,:2] = pred_disp rmse = np.sqrt((np.linalg.norm(flow[:,:,:2] - flo[:,:,:2], ord=2, axis=-1) ** 2).mean()) rmses += rmse nrmses += rmse / np.sqrt((np.linalg.norm(flo[:,:,:2], ord=2, axis=-1) ** 2).mean()) error = np.linalg.norm(flow[:,:,:2] - flo[:,:,:2], ord=2, axis=-1) ** 2 error = 255 - 255 * error / error.max() entropy = torch.squeeze(entropy).data.cpu().numpy() entropy = cv2.resize(entropy, (input_size[1], input_size[0])) # save predictions if args.dataset == 'mbstereo': dirname = '%s/%s/%s'%(args.outdir, args.dataset, test_left_img[inx].split('/')[-2]) mkdir_p(dirname) idxname = ('%s/%s')%(dirname.rsplit('/',1)[-1],test_left_img[inx].split('/')[-1]) else: idxname = test_left_img[inx].split('/')[-1] if args.dataset == 'mbstereo': with open(test_left_img[inx].replace('im0.png','calib.txt')) as f: lines = f.readlines() #max_disp = int(int(lines[9].split('=')[-1])) max_disp = int(int(lines[6].split('=')[-1])) with open('%s/%s/%s'% (args.outdir, args.dataset,idxname.replace('im0.png','disp0IO.pfm')),'w') as f: save_pfm(f,np.clip(-flow[::-1,:,0].astype(np.float32),0,max_disp) ) with open('%s/%s/%s/timeIO.txt'%(args.outdir, args.dataset,idxname.split('/')[0]),'w') as f: f.write(str(ttime)) elif args.dataset == 'k15stereo' or args.dataset == 'k12stereo': skimage.io.imsave('%s/%s/%s.png'% (args.outdir, args.dataset,idxname.split('.')[0]),(-flow[:,:,0].astype(np.float32)*256).astype('uint16')) else: # write_flow('%s/%s/%s.png'% (args.outdir, args.dataset,idxname.rsplit('.',1)[0]), flow.copy()) cv2.imwrite('%s/%s/%s.png' % (args.outdir, args.dataset,idxname.rsplit('.',1)[0]), flow_to_image(flow)[:, :, ::-1]) cv2.imwrite('%s/%s/%s-gt.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]), flow_to_image(flo)[:, :, ::-1]) arrow_pic(flo, '%s/%s/%s-vec-gt.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0])) arrow_pic(flow, '%s/%s/%s-vec.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0])) test_compressibility( flo, flow, '%s/%s/%s-compr.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]) ) test_energy_spectrum( flo, flow, '%s/%s/%s-energy.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]) ) test_intermittency_r( flo, flow, '%s/%s/%s-interm-r.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]) ) test_intermittency_n( flo, flow, '%s/%s/%s-interm-n.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]) ) cv2.imwrite('%s/%s/%s-err.png' % (args.outdir, args.dataset, idxname.rsplit('.', 1)[0]), error) # cv2.imwrite('%s/%s/ent-%s.png'% (args.outdir, args.dataset,idxname.rsplit('.',1)[0]), entropy*200) torch.cuda.empty_cache() rmses /= len(test_left_img) nrmses /= len(test_left_img) print(np.mean(ttime_all), rmses, nrmses)
def main(): model.eval() ttime_all = [] rmses = 0 nrmses = 0 flo = read_flow(args.flow) imgL_o = np.asarray(Image.open(args.left)) imgR_o = np.asarray(Image.open(args.right)) # resize maxh = imgL_o.shape[0] * args.testres maxw = imgL_o.shape[1] * args.testres max_h = int(maxh // 64 * 64) max_w = int(maxw // 64 * 64) if max_h < maxh: max_h += 64 if max_w < maxw: max_w += 64 input_size = imgL_o.shape imgL = cv2.resize(imgL_o, (max_w, max_h)) imgR = cv2.resize(imgR_o, (max_w, max_h)) # flip channel, subtract mean imgL = imgL[:, :, None].copy() / 255. - np.asarray(mean_L).mean(0)[ np.newaxis, np.newaxis, :] imgR = imgR[:, :, None].copy() / 255. - np.asarray(mean_R).mean(0)[ np.newaxis, np.newaxis, :] print(imgL.shape) imgL = np.transpose(imgL, [2, 0, 1])[np.newaxis] imgR = np.transpose(imgR, [2, 0, 1])[np.newaxis] # forward imgL = Variable(torch.FloatTensor(imgL).cuda()) imgR = Variable(torch.FloatTensor(imgR).cuda()) with torch.no_grad(): imgLR = torch.cat([imgL, imgR], 0) model.eval() torch.cuda.synchronize() start_time = time.time() rts = model(imgLR) torch.cuda.synchronize() ttime = (time.time() - start_time) print('time = %.2f' % (ttime * 1000)) ttime_all.append(ttime) pred_disp, entropy = rts # upsampling pred_disp = torch.squeeze(pred_disp).data.cpu().numpy() pred_disp = cv2.resize(np.transpose(pred_disp, (1, 2, 0)), (input_size[1], input_size[0])) pred_disp[:, :, 0] *= input_size[1] / max_w pred_disp[:, :, 1] *= input_size[0] / max_h flow = np.ones([pred_disp.shape[0], pred_disp.shape[1], 3]) flow[:, :, :2] = pred_disp rmse = np.sqrt((np.linalg.norm(flow[:, :, :2] - flo[:, :, :2], ord=2, axis=-1)**2).mean()) rmses += rmse nrmses += rmse / np.sqrt( (np.linalg.norm(flo[:, :, :2], ord=2, axis=-1)**2).mean()) error = np.linalg.norm(flow[:, :, :2] - flo[:, :, :2], ord=2, axis=-1)**2 error = 255 * error / error.max() entropy = torch.squeeze(entropy).data.cpu().numpy() entropy = cv2.resize(entropy, (input_size[1], input_size[0])) idxname = args.left.split('/')[-1] cv2.imwrite('%s.png' % idxname.rsplit('.', 1)[0], flow_to_image(flow)[:, :, ::-1]) cv2.imwrite('%s-gt.png' % idxname.rsplit('.', 1)[0], flow_to_image(flo)[:, :, ::-1]) arrow_pic(flo, '%s-vec-gt.png' % idxname.rsplit('.', 1)[0]) arrow_pic(flow, '%s-vec.png' % idxname.rsplit('.', 1)[0]) cv2.imwrite('%s-err.png' % idxname.rsplit('.', 1)[0], error) torch.cuda.empty_cache() print(ttime_all, rmses, nrmses)
def eval_f(fp): import warnings warnings.filterwarnings("ignore") # gt if '2015' in args.dataset: mask_gt = cv2.imread(maskp[fp], 0) flow_gt = read_flow(maskp[fp].replace('obj_map', 'flow_occ')).astype(np.float32) validmask = flow_gt[:, :, -1] == 1 bgmask_gt = np.logical_and(validmask, mask_gt == 0).astype(float) else: bgmask_gt = cv2.imread(maskp[fp], 0) > 0 shape = bgmask_gt.shape[:2] validmask = np.ones(shape).astype(bool) shape = bgmask_gt.shape[:2] # pred flowpath = flow_paths[fp] # KITTI if '2015' in args.dataset: idnum = int(flowpath.split('/')[-1].split('_')[-2]) if args.method == 'ours': maskp_pred = '%s/%s/pm-%06d_10.pfm' % (args.path, args.dataset, idnum) mask_pred = readPFM(maskp_pred)[0].astype(int) bgmask_pred = mask_pred == 0 elif args.method == 'detectron2': maskp_pred = '/data/gengshay/code/moseg/moseg-detectron2/%06d.png' % idnum mask_pred = cv2.imread(maskp_pred, 0) bgmask_pred = mask_pred == 0 elif args.method == 'mrflowss': bgmask_pred = np.load( '/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_kitti_smodel/%06d_10.npy' % idnum) bgmask_pred = cv2.resize(bgmask_pred.astype(float), shape[::-1]).astype(bool) elif args.method == 'mrflowsk': bgmask_pred = cv2.imread( '/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_kitti_kmodel/%06d_10.png' % idnum, 0).astype(bool) elif args.method == 'u2net': bgmask_pred = cv2.imread( '/data/gengshay/code/U-2-Net/test_data/u2net_results/%06d_10.png' % idnum, 0) < 128 elif args.method == 'rtn': bgmask_pred = ~cv2.imread( '/data/gengshay/code/learningrigidity_mod/output_kitti/%06d_10.png' % idnum, 0).astype(bool) elif args.method == 'fusionseg': bgmask_pred = ~np.load( '/data/gengshay/code/fusionseg/python/kitti/results_liu/joint_davis_val/%06d_10_mask.npy' % idnum).astype(bool) elif args.method == 'fusionsegm': bgmask_pred = ~np.load( '/data/gengshay/code/fusionseg/python/kitti/results_liu/motion/%06d_10_mask.npy' % idnum).astype(bool) elif args.method == 'cosnet': bgmask_pred = cv2.resize( cv2.imread( '/data/gengshay/code/COSNet/result/test/davis_iteration_conf/Results/kitti/%06d_10.png' % idnum, 0), shape[::-1]) < 128 elif args.method == 'matnet': bgmask_pred = cv2.imread( '/data/gengshay/code/MATNet/output/kitti_liu/%06d_10.png' % idnum, 0) < 128 elif args.method == 'cc': bgmask_pred = cv2.resize( np.load('/data/gengshay/code/cc/output/mask-joint/%03d.npy' % idnum)[0], shape[::-1]) == 1 elif args.method == 'ccm': bgmask_pred = cv2.resize( np.load('/data/gengshay/code/cc/output/mask-motion/%03d.npy' % idnum)[0], shape[::-1]) == 1 elif args.method == 'angle': maskp_pred = '%s/%s/pm-%06d_10.pfm' % (args.path, args.dataset, idnum) mask_pred = readPFM(maskp_pred)[0].astype(int) bgmask_pred = mask_pred == 0 else: exit() # Sintel else: passname = flowpath.split('/')[-1].split('_')[-4] seqname1 = flowpath.split('/')[-1].split('_')[-3] seqname2 = flowpath.split('/')[-1].split('_')[-2] framename = int(flowpath.split('/')[-1].split('_')[-1].split('.')[0]) if args.method == 'ours': maskp_pred = '%s/%s/pm-Sintel_%s_%s_%s_%02d.pfm' % ( args.path, args.dataset, passname, seqname1, seqname2, framename) mask_pred = readPFM(maskp_pred)[ 0] # label map in {1,...N} given N rigid body predictions bgmask_pred = mask_pred == 0 elif args.method == 'detectron2': maskp_pred = '/data/gengshay/code/moseg/smoseg-detectron2/Sintel_%s_%s_%s_%02d.png' % ( passname, seqname1, seqname2, framename) mask_pred = cv2.imread(maskp_pred, 0) bgmask_pred = mask_pred == 0 elif args.method == 'mrflowss': bgmask_pred = np.load( '/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_sintel_smodel/Sintel_%s_%s_%s_%02d.npy' % (passname, seqname1, seqname2, framename)).astype(bool) elif args.method == 'mrflowsk': bgmask_pred = cv2.imread( '/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_sintel_kmodel/Sintel_%s_%s_%s_%02d.png' % (passname, seqname1, seqname2, framename), 0).astype(bool) elif args.method == 'u2net': bgmask_pred = cv2.imread( '/data/gengshay/code/U-2-Net/test_data/u2net_results_sintel/Sintel_%s_%s_%s_%02d.png' % (passname, seqname1, seqname2, framename), 0) < 128 elif args.method == 'rtn': bgmask_pred = ~cv2.imread( '/data/gengshay/code/learningrigidity_mod/output_sintel/Sintel_%s_%s_%s_%02d.png' % (passname, seqname1, seqname2, framename), 0).astype(bool) elif args.method == 'fusionseg': bgmask_pred = ~np.load( '/data/gengshay/code/fusionseg/python/sintel/results/joint_davis_val/Sintel_%s_%s_%s_%02d_mask.npy' % (passname, seqname1, seqname2, framename)).astype(bool) elif args.method == 'fusionsegm': bgmask_pred = ~np.load( '/data/gengshay/code/fusionseg/python/sintel/results/motion/Sintel_%s_%s_%s_%02d_mask.npy' % (passname, seqname1, seqname2, framename)).astype(bool) elif args.method == 'cosnet': bgmask_pred = cv2.resize( cv2.imread( '/data/gengshay/code/COSNet/result/test/davis_iteration_conf/Results/sintel/Sintel_%s_%s_%s_%02d.png' % (passname, seqname1, seqname2, framename), 0), shape[::-1]) < 128 elif args.method == 'matnet': bgmask_pred = cv2.imread( '/data/gengshay/code/MATNet/output/sintel/Sintel_%s_%s_%s_%02d.png' % (passname, seqname1, seqname2, framename), 0) < 128 elif args.method == 'angle': maskp_pred = '%s/%s/pm-Sintel_%s_%s_%s_%02d.pfm' % ( args.path, args.dataset, passname, seqname1, seqname2, framename) mask_pred = readPFM(maskp_pred)[0] bgmask_pred = mask_pred == 0 else: exit() if args.method != 'ours' and args.method != 'detectron2': _, mask_pred = cv2.connectedComponentsWithAlgorithm( (1 - bgmask_pred.astype(np.uint8)), connectivity=8, ltype=cv2.CV_16U, ccltype=cv2.CCL_WU) # bg-iou try: bgiou = np.logical_and( bgmask_gt.astype(bool)[validmask], bgmask_pred[validmask]).sum() / np.logical_or( bgmask_gt.astype(bool)[validmask], bgmask_pred[validmask]).sum() except: pdb.set_trace() # obj-F (penalizes false positive) # defined in https://arxiv.org/abs/1902.03715 Sec.4 # F=2PR/P+R => 2/(1/P+1/R) # since P = sum(|c and g|) / sum(|c|) and R = sum(|c and g|) / sum(|g|), # where c is prediction and g is GT # 1/P+1/R = [sum(|c|) + sum(|g|)] / sum(|c and g|) # therefore F = 2*sum(|c and g|) / [sum(|c|) + sum(|g|)] if '2015' in args.dataset: gtlist = list(set(mask_gt.flatten())) M = len(gtlist) - 1 imatrix = np.zeros((M, mask_pred.max())) fmatrix = np.zeros((M, mask_pred.max())) for i in range(M): # for all bg instances objx_mask_gt = mask_gt == gtlist[i + 1] for j in range(mask_pred.max()): objx_mask_pred = mask_pred == j + 1 imatrix[i, j] = float( np.logical_and(objx_mask_gt, objx_mask_pred)[validmask].sum()) fmatrix[i, j] = imatrix[i, j] / (objx_mask_gt[validmask].sum() + objx_mask_pred[validmask].sum()) * 2 ridx, cidx = scipy.optimize.linear_sum_assignment(1 - fmatrix) objf = imatrix[ridx, cidx].sum() / ((mask_pred > 0)[validmask].sum() + (mask_gt > 0)[validmask].sum()) * 2 else: objf = 0. return bgiou, objf
def default_loader(root, path_imgs, path_flo): imgs = [os.path.join(root, path) for path in path_imgs] flo = os.path.join(root, path_flo) return [imread(img).astype(np.float32) for img in imgs], read_flow(flo)
def main(): model.eval() flowl0 = "/gpfs/gpfs0/y.maximov/kolya/piv/SQG/SQG_00001_flow.flo" iml0 = "/gpfs/gpfs0/y.maximov/kolya/piv/SQG/SQG_00001_img1.tif" iml1 = "/gpfs/gpfs0/y.maximov/kolya/piv/SQG/SQG_00001_img2.tif" iml0 = default_loader(iml0) iml1 = default_loader(iml1) iml1 = np.asarray(iml1) / 255. iml0 = np.asarray(iml0) / 255. iml0 = iml0[:, :, None].copy() # iml0[:,:,::-1].copy() iml1 = iml1[:, :, None].copy() # iml1[:,:,::-1].copy() flowl0 = read_flow(flowl0) # flowl0 = random_incompressible_flow( # 1, # [256, 256], # np.random.choice([30, 40, 50]), # 10. ** (2 * np.random.rand()), # incompressible=False # ) # iml0, iml1 = image_from_flow( # ppp=np.random.uniform(0.008, 0.1), # pip=np.random.uniform(0.95, 1.0), # flow=flowl0, # intensity_bounds=(0.8, 1), # diameter_bounds=(0.35, 6) # ) # iml0 = iml0.transpose(1, 2, 0).copy() # iml1 = iml1.transpose(1, 2, 0).copy() # flowl0 = flowl0[0] # flowl0 = np.concatenate([ # flowl0, # np.ones(flowl0.shape[:-1] + (1,), dtype=flowl0.dtype) # ], axis=-1) flowl0 = np.ascontiguousarray(flowl0, dtype=np.float32) flowl0[np.isnan(flowl0)] = 1e6 # set to max cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "flow-orig"), flow_to_image(flowl0)[:, :, ::-1]) schedule_aug_coeff = 1.0 scl = None # 0.2 * schedule_aug_coeff # if scl > 0: # scl = [0.2 * schedule_aug_coeff, 0., 0.2 * schedule_aug_coeff] # else: # scl = None rot = 0.17 * schedule_aug_coeff if rot > 0: rot = [0.17 * schedule_aug_coeff, 0.0] else: rot = None trans = 0.2 * schedule_aug_coeff if trans > 0: trans = [0.2 * schedule_aug_coeff, 0.0] else: trans = None co_transform = flow_transforms.Compose([ # flow_transforms.Scale(1, order=0), flow_transforms.SpatialAug([256, 256], scale=scl, rot=rot, trans=trans, schedule_coeff=1, order=0, black=True), # flow_transforms.PCAAug(schedule_coeff=schedule_coeff), # flow_transforms.ChromaticAug(schedule_coeff=schedule_coeff, noise=self.noise), ]) augmented, flowl0 = co_transform([iml0, iml1], flowl0) iml0 = augmented[0] iml1 = augmented[1] cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "flow"), flow_to_image(flowl0)[:, :, ::-1]) cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "mask"), 255 * flowl0[:, :, -1]) cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "img1"), 255 * iml0[:, :, ::-1]) cv2.imwrite('%s/%s/%s.png' % (args.outdir, "generated", "img2"), 255 * iml1[:, :, ::-1])
def __getitem__(self, index): ( image1_name, image2_name, reference_video_name, reference_name1, reference_name2, reference_name3, reference_name4, reference_name5, flow_forward_name, flow_backward_name, mask_name, reference_gt1, reference_gt2, reference_gt3, path, ) = self.image_pairs[index] try: I1 = Image.open(os.path.join(path, "input_pad", image1_name)) I2 = Image.open(os.path.join(path, "input_pad", image2_name)) I_reference_video = Image.open( os.path.join(path, "reference_gt", random.choice([reference_gt1, reference_gt2, reference_gt3])) ) I_reference_video_real = Image.open( os.path.join( path, "reference", random.choice( [reference_name1, reference_name2, reference_name3, reference_name4, reference_name5] ), ) ) flow_forward = read_flow(os.path.join(path, "flow", flow_forward_name)) # numpy flow_backward = read_flow(os.path.join(path, "flow", flow_backward_name)) # numpy mask = Image.open(os.path.join(path, "mask", mask_name)) # binary mask mask = np.array(mask) mask[mask < 240] = 0 mask[mask >= 240] = 1 # transform I1 = self.image_transform(I1) I2 = self.image_transform(I2) I_reference_video = self.image_transform(self.CenterPad(I_reference_video)) I_reference_video_real = self.image_transform(self.CenterPad(I_reference_video_real)) flow_forward = self.ToTensor(self.CenterCrop(flow_forward)) flow_backward = self.ToTensor(self.CenterCrop(flow_backward)) mask = self.ToTensor(self.CenterCrop(mask)) if np.random.random() < self.real_reference_probability: I_reference_output = I_reference_video_real placeholder = torch.zeros_like(I1) self_ref_flag = torch.zeros_like(I1) else: I_reference_output = I_reference_video placeholder = I2 if np.random.random() < self.nonzero_placeholder_probability else torch.zeros_like(I1) self_ref_flag = torch.ones_like(I1) outputs = [ I1, I2, I_reference_output, flow_forward, flow_backward, mask, placeholder, self_ref_flag, ] except Exception as e: print("problem in, ", path) print(e) return self.__getitem__(np.random.randint(0, len(self.image_pairs))) return outputs
def main(): model.eval() flow_pairs = [] for inx in range(inx_st, inx_fn + 1): print(test_left_img[inx]) flo = read_flow(test_flow[inx]) imgL_o = np.asarray(Image.open(test_left_img[inx])) imgR_o = np.asarray(Image.open(test_right_img[inx])) # resize maxh = imgL_o.shape[0] * args.testres maxw = imgL_o.shape[1] * args.testres max_h = int(maxh // 64 * 64) max_w = int(maxw // 64 * 64) if max_h < maxh: max_h += 64 if max_w < maxw: max_w += 64 input_size = imgL_o.shape imgL = cv2.resize(imgL_o, (max_w, max_h)) imgR = cv2.resize(imgR_o, (max_w, max_h)) # flip channel, subtract mean imgL = imgL[:, :, None].copy() / 255. - np.asarray(mean_L).mean(0)[ np.newaxis, np.newaxis, :] imgR = imgR[:, :, None].copy() / 255. - np.asarray(mean_R).mean(0)[ np.newaxis, np.newaxis, :] print(imgL.shape) imgL = np.transpose(imgL, [2, 0, 1])[np.newaxis] imgR = np.transpose(imgR, [2, 0, 1])[np.newaxis] # forward imgL = Variable(torch.FloatTensor(imgL).cuda()) imgR = Variable(torch.FloatTensor(imgR).cuda()) with torch.no_grad(): imgLR = torch.cat([imgL, imgR], 0) model.eval() torch.cuda.synchronize() rts = model(imgLR) torch.cuda.synchronize() pred_disp, entropy = rts # upsampling pred_disp = torch.squeeze(pred_disp).data.cpu().numpy() pred_disp = cv2.resize(np.transpose(pred_disp, (1, 2, 0)), (input_size[1], input_size[0])) pred_disp[:, :, 0] *= input_size[1] / max_w pred_disp[:, :, 1] *= input_size[0] / max_h flow = np.ones([pred_disp.shape[0], pred_disp.shape[1], 3]) flow[:, :, :2] = pred_disp flow_pairs.append((flo, flow)) test_compressibility( flow_pairs, '%s/%s/%d-%d-compr.png' % (args.outdir, args.dataset, inx_st, inx_fn)) test_energy_spectrum( flow_pairs, '%s/%s/%d-%d-energy.png' % (args.outdir, args.dataset, inx_st, inx_fn)) test_intermittency_r( flow_pairs, '%s/%s/%d-%d-interm-r.png' % (args.outdir, args.dataset, inx_st, inx_fn)) test_intermittency_n( flow_pairs, '%s/%s/%d-%d-interm-n.png' % (args.outdir, args.dataset, inx_st, inx_fn)) torch.cuda.empty_cache()
def eval_f(fp): import warnings warnings.filterwarnings("ignore") #gts if '2015' in args.dataset: gt_disp0 = disparity_loader(disp0p[fp]) gt_disp1 = disparity_loader(disp1p[fp]) gt_flow = read_flow(flow_paths[fp]).astype(np.float32) ints = load_calib_cam_to_cam(calib[fp]) K0 = ints['K_cam2'] fl = K0[0, 0] bl = ints['b20'] - ints['b30'] elif 'sintel' in args.dataset: gt_disp0 = disparity_read(disp0p[fp]) gt_disp1 = disparity_read(disp1p[fp]) gt_flow = read_flow(flow_paths[fp]).astype(np.float32) K0, _ = cam_read(calib[fp]) fl = K0[0, 0] bl = 0.1 d1mask = gt_disp0 > 0 d2mask = gt_disp1 > 0 flowmask = gt_flow[:, :, -1] == 1 validmask = np.logical_and(np.logical_and(d1mask, d2mask), flowmask) if '2015' in args.dataset: fgmask = cv2.imread(flow_paths[fp].replace('flow_occ', 'obj_map'), 0) > 0 fgmask = np.logical_and(fgmask, validmask) shape = gt_disp0.shape # pred idnum = expansionp[fp].split('/')[-1].split('.')[0] if args.method == 'ours': logdc = disparity_loader('%s/%s/mid-%s.pfm' % (args.path, args.dataset, idnum)) pred_flow = read_flow('%s/%s/flo-%s.pfm' % (args.path, args.dataset, idnum)) try: pred_disp = disparity_loader('%s/%s/%s_disp.pfm' % (args.path, args.dataset, idnum)) except: try: pred_disp = disparity_loader('%s/%s/%s.png' % (args.path, args.dataset, idnum)) except: try: pred_disp = disparity_loader( '%s/%s/disp-%s.pfm' % (args.path, args.dataset, idnum)) except: pred_disp = disparity_loader( '%s/%s/exp-%s.pfm' % (args.path, args.dataset, idnum)) pred_disp[pred_disp == np.inf] = pred_disp[pred_disp != np.inf].max() pred_disp[np.isnan(pred_disp)] = 1e-12 pred_disp[pred_disp < 1e-12] = 1e-12 pred_disp1 = pred_disp / np.exp(logdc) pred_flow = disparity_loader('%s/%s/flo-%s.pfm' % (args.path, args.dataset, idnum)) elif args.method == 'monodepth2': pred_disp = disparity_loader('%s/%s/%s_disp.pfm' % (args.path, args.dataset, idnum)) else: exit() #hom_p = np.stack((pred_disp.flatten(), np.ones(pred_disp.flatten().shape))).T[validmask.flatten()] #xx = np.linalg.inv(np.matmul(hom_p[:,:,np.newaxis],hom_p[:,np.newaxis,:]).sum(0)) #yy = (hom_p[:,:,np.newaxis]*gt_disp0.flatten()[validmask.flatten(),np.newaxis,np.newaxis]).sum(0) #st = xx.dot(yy) #pred_disp = pred_disp*st[0] + st[1] scale_factor = np.median((gt_disp0 / pred_disp)[validmask]) pred_disp = scale_factor * pred_disp pred_disp1 = scale_factor * pred_disp1 # eval d1err = np.abs(pred_disp - gt_disp0) d1err_map = (np.logical_and(d1err >= 3, d1err / gt_disp0 >= 0.05)) d1err = d1err_map[validmask] d2err = np.abs(pred_disp1 - gt_disp1) d2err_map = (np.logical_and(d2err >= 3, d2err / gt_disp1 >= 0.05)) d2err = d2err_map[validmask] flow_epe = np.sqrt(np.power(gt_flow - pred_flow, 2).sum(-1)) gt_flow_mag = np.linalg.norm(gt_flow[:, :, :2], 2, -1) flerr_map = np.logical_and(flow_epe > 3, flow_epe / gt_flow_mag > 0.05) flerr = flerr_map[validmask] flerr_map[~validmask] = False try: d1ferr = d1err_map[fgmask] d2ferr = d2err_map[fgmask] flferr = flerr_map[fgmask] except: d1ferr = np.zeros(1) d2ferr = np.zeros(1) flferr = np.zeros(1) sferr = np.logical_or(np.logical_or(d1err, d2err), flerr) sfferr = np.logical_or(np.logical_or(d1ferr, d2ferr), flferr) img = cv2.imread(test_left_img[fp])[:, :, ::-1] # cv2.imwrite('%s/%s/err-%s.png'%(args.path,args.dataset,idnum),np.vstack((gt_disp0,pred_disp,gt_disp1,pred_disp1,flerr_map.astype(float)*255))) if '2015' in args.dataset: flowvis = cv2.imread(test_left_img[fp].replace( 'image_2', 'viz_flow_occ'))[:, :, ::-1] else: flowvis = flow_to_image(gt_flow) pred_flow[:, :, -1] = 1 cv2.imwrite( '%s/%s/err-%s.png' % (args.path, args.dataset, idnum), np.vstack( (img, flowvis, 255 * visualize_flow(pred_flow, mode='RGB'), np.tile(flerr_map.reshape(shape)[:, :, None], 3).astype(float) * 255))[:, :, ::-1]) return d1err.mean(), d2err.mean(), flerr.mean(), sferr.mean(),\ d1ferr.mean(), d2ferr.mean(), flferr.mean(), sfferr.mean(),gt_flow_mag.mean()
flow_paths = [flow_paths[i] for i, flag in enumerate(split) if flag == 2] #pdb.set_trace() #test_left_img = [i for i in test_left_img if 'clean' in i] #test_right_img = [i for i in test_right_img if 'clean' in i] #flow_paths = [i for i in flow_paths if 'clean' in i] #for i,gtflow_path in enumerate(sorted(flow_paths)): for i, gtflow_path in enumerate(flow_paths): #if not 'Sintel_clean_cave_4_10' in gtflow_path: # continue #if i%10!=1: # continue num = gtflow_path.split('/')[-1].strip().replace('flow.flo', 'img1.png') if not 'test' in args.dataset and not 'clip' in args.dataset: gtflow = read_flow(gtflow_path) num = num.replace('jpg', 'png') flow = read_flow('%s/%s/%s' % (args.path, args.dataset, num)) if args.vis == 'yes': #flowimg = flow_to_image(flow) flowimg = flow_to_image(flow) * np.linalg.norm( flow[:, :, :2], 2, 2)[:, :, np.newaxis] / 100. / 255. #gtflowimg = compute_color(gtflow[:,:,0]/20, gtflow[:,:,1]/20)/255. #flowimg = compute_color(flow[:,:,0]/20, flow[:,:,1]/20)/255. mkdir_p('%s/%s/flowimg' % (args.path, args.dataset)) plt.imsave('%s/%s/flowimg/%s' % (args.path, args.dataset, num), flowimg) if 'test' in args.dataset or 'clip' in args.dataset: continue gtflowimg = flow_to_image(gtflow) mkdir_p('%s/%s/gtimg' % (args.path, args.dataset))