def main(): davis_folder = '/media/iiai/data/VOS/DAVIS2017/JPEGImages/480p' save_dir = '/media/iiai/data/VOS/DAVIS2017/davis2017-hed' videos = os.listdir(davis_folder) print(videos) for idx, video in enumerate(videos): print('process {}[{}/{}]'.format(video, idx, len(videos))) save_dir_video = os.path.join(save_dir, video) if not os.path.exists(save_dir_video): os.makedirs(save_dir_video) imagefiles = sorted(glob.glob(os.path.join(davis_folder, video, '*.jpg'))) for imagefile in imagefiles: tensorInput = torch.FloatTensor(numpy.array(PIL.Image.open(imagefile))[:, :, ::-1].transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0)) tensorOutput = estimate(tensorInput) save_name = os.path.basename(imagefile) save_file = os.path.join(save_dir_video, save_name) PIL.Image.fromarray( (tensorOutput.clamp(0.0, 1.0).numpy().transpose(1, 2, 0)[:, :, 0] * 255.0).astype(numpy.uint8)).save( save_file)
def run(imagefile1, imagefile2, save_file): tensorFirst = torch.FloatTensor( numpy.array(PIL.Image.open(imagefile1))[:, :, ::-1].transpose( 2, 0, 1).astype(numpy.float32) * (1.0 / 255.0)) tensorSecond = torch.FloatTensor( numpy.array(PIL.Image.open(imagefile2))[:, :, ::-1].transpose( 2, 0, 1).astype(numpy.float32) * (1.0 / 255.0)) tensorOutput = estimate(tensorFirst, tensorSecond) flow_color = flow_vis.flow_to_color(tensorOutput.numpy().transpose( 1, 2, 0), convert_to_bgr=True) cv2.imwrite(save_file, flow_color)
def compute_flow(network, inputs): global last_frame current_frame = np.array(inputs["input_image"]) if last_frame is None: output = np.full(current_frame.shape, 255) else: tensorFirst = torch.FloatTensor( np.array(last_frame)[:, :, ::-1].transpose(2, 0, 1).astype( np.float32) * (1.0 / 255.0)) tensorSecond = torch.FloatTensor( np.array(current_frame)[:, :, ::-1].transpose(2, 0, 1).astype( np.float32) * (1.0 / 255.0)) tensorOutput = estimate(network, tensorFirst, tensorSecond) output = tensorOutput.numpy().transpose(1, 2, 0) output = flowiz.convert_from_flow(output) last_frame = current_frame return {"output_image": output}
def main(): count = 20 frames = get_every_nth_frame("data/video.3gp", 3)[:count] models = ["default", "kitti", "sintel"] for model_name in models: arguments_strModel = join(realpath("../"), f"models/network-{model_name}.pytorch") # TODO: launch pycharm from terminal to load env. variables model = Network(arguments_strModel).cuda().eval() if not exists( join(realpath(dirname(__file__)), f"data/flows_{model_name}")): mkdir(join(realpath(dirname(__file__)), f"data/flows_{model_name}")) mse_arr = [] # h, w, c for n, (frame1, frame2) in enumerate(pairwise(frames)): # security through obscurity input_frame1 = tensor( cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB).transpose( (2, 0, 1)).astype(np.float32) / 255) input_frame2 = tensor( cv2.cvtColor(frame2, cv2.COLOR_BGR2RGB).transpose( (2, 0, 1)).astype(np.float32) / 255) flow = estimate(model, input_frame1, input_frame2) warped_image = warp_flow(frame1, flow.cpu().numpy().transpose((1, 2, 0))) # compute mse mse_arr.append(((warped_image - frame2)**2).mean()) print(f"{model_name}: {np.array(mse_arr).mean():.3f}")
def compute_flow(network, inputs): global f global initialize current_frame = np.array(inputs["input_image"]) if initialize: frame = current_frame f.append(current_frame) initialize = False else: f.append(current_frame) prev_frame = f.pop(0) next_frame = f[0] tensorFirst = torch.FloatTensor( np.array(prev_frame)[:, :, ::-1].transpose(2, 0, 1).astype( np.float32) * (1.0 / 255.0)) tensorSecond = torch.FloatTensor( np.array(next_frame)[:, :, ::-1].transpose(2, 0, 1).astype( np.float32) * (1.0 / 255.0)) tensorOutput = estimate(network, tensorFirst, tensorSecond) objectOutput = tempfile.NamedTemporaryFile(suffix=".flo") out = str(Path(objectOutput.name)) np.array([80, 73, 69, 72], np.uint8).tofile(objectOutput) np.array( [tensorOutput.size(2), tensorOutput.size(1)], np.int32).tofile(objectOutput) np.array(tensorOutput.numpy().transpose(1, 2, 0), np.float32).tofile(objectOutput) frame = flowiz.convert_from_file(out) return {"output_image": frame}
npyFirst = npyFirst[540:-540, 1024:-1024, :] npySecond = npySecond[540:-540, 1024:-1024, :] npyReference = npyReference[540:-540, 1024:-1024, :] # end tenFirst = torch.FloatTensor( numpy.ascontiguousarray( npyFirst.transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0))) tenSecond = torch.FloatTensor( numpy.ascontiguousarray( npySecond.transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0))) npyEstimate = (run.estimate(tenFirst, tenSecond).clamp( 0.0, 1.0).numpy().transpose(1, 2, 0) * 255.0).astype( numpy.uint8) fltPsnr.append( skimage.measure.compare_psnr(im_true=npyReference, im_test=npyEstimate, data_range=255)) fltSsim.append( skimage.measure.compare_ssim(X=npyReference, Y=npyEstimate, data_range=255, multichannel=True)) # end # end print('category', strCategory)
for strTruth in sorted(glob.glob('./middlebury/*/frame10i11.png')): tenOne = torch.FloatTensor( numpy.ascontiguousarray( numpy.array( PIL.Image.open(strTruth.replace( 'frame10i11', 'frame10')))[:, :, ::-1].transpose( 2, 0, 1).astype(numpy.float32) * (1.0 / 255.0))) tenTwo = torch.FloatTensor( numpy.ascontiguousarray( numpy.array( PIL.Image.open(strTruth.replace( 'frame10i11', 'frame11')))[:, :, ::-1].transpose( 2, 0, 1).astype(numpy.float32) * (1.0 / 255.0))) npyEstimate = (run.estimate(tenOne, tenTwo).clip( 0.0, 1.0).numpy().transpose(1, 2, 0) * 255.0).astype(numpy.uint8) fltPsnr.append( skimage.metrics.peak_signal_noise_ratio(image_true=numpy.array( PIL.Image.open(strTruth))[:, :, ::-1], image_test=npyEstimate, data_range=255)) fltSsim.append( skimage.metrics.structural_similarity(im1=numpy.array( PIL.Image.open(strTruth))[:, :, ::-1], im2=npyEstimate, data_range=255, multichannel=True)) # end print('computed average psnr', numpy.mean(fltPsnr))
def main(): args = get_args() if args.save_path is not None: if not exists(args.save_path): mkdir(args.save_path) # frames = get_every_nth_frame(args.path, args.dist)[:args.count] paths, frames = load_directory(args.frames) args.count = len(frames) models = ["default"] # models = ["default", "kitti", "sintel"] model_name = "default" arguments_strModel = f"/home/xbakom01/src/pytorch-liteflownet/models/network-{model_name}.pytorch" model = Network(arguments_strModel, 5).cuda().eval() # from custom.model.liteflownet import LiteFlowNet # model = LiteFlowNet().cuda().eval() # h, w, c for n, ((path1, path2), (frame1, frame2)) in enumerate(zip(pairwise(paths), pairwise(frames))): # cv2.imwrite(join(realpath(dirname(__file__) + "/../"), "data", "frames", f"f{n}.png"), frame1) # security through obscurity frame1_t = tensor( cv2.cvtColor(center_crop(frame1), cv2.COLOR_BGR2RGB).transpose( (2, 0, 1)).astype(np.float32) / 255).cuda() frame2_t = tensor( cv2.cvtColor(center_crop(frame2), cv2.COLOR_BGR2RGB).transpose( (2, 0, 1)).astype(np.float32) / 255).cuda() flows = estimate(model, frame1_t, frame2_t) # flow = model(frame1_t.unsqueeze(0), frame2_t.unsqueeze(0))[-1][0].cpu() print(f"{n + 1}/{len(frames) - 1}", flush=True, end="\r") if not args.dataset: flow = flows[-1].cpu() if args.save_path is None: # display flow img = convert_from_flow(flow.numpy().transpose( (1, 2, 0)).astype(np.float16)) cv2.imshow("", img[:, :, ::-1]) cv2.waitKey() else: # save in .flo format f1, f2 = list( map(lambda x: basename(x).split(".")[0], [path1, path2])) with open(join(args.save_path, f"{f1}-{f2}.flowc"), "wb") as f: np.array([80, 73, 69, 72], np.uint8).tofile(f) np.array([flow.size(2), flow.size(1)], np.int32).tofile(f) np.array(flow.numpy().transpose(1, 2, 0), np.float32).tofile(f) # if n == len(frames) - 2: # cv2.imwrite(join(realpath(dirname(__file__)), "data", "frames", f"f{args.count - 1}.png"), frame2) else: # save cascade of flows flow = [flow.cpu().numpy().astype(np.float16) for flow in flows] save_dir = args.save_path if args.save_path is not None else args.frames f1, f2 = list( map(lambda x: basename(x).split(".")[0], [path1, path2])) with open(join(args.save_path, f"{f1}-{f2}.flowc"), "wb") as f: pickle.dump(flow, f)
f'im{center_frame_idx}.png') for idx in range(num_half_frame, 0, -1): neighbor_idx.append(center_frame_idx - idx) neighbor_names.append(f'p{idx}') for idx in range(1, num_half_frame + 1): neighbor_idx.append(center_frame_idx + idx) neighbor_names.append(f'n{idx}') center_frame = read_image(center_frame_path) for idx, name in zip(neighbor_idx, neighbor_names): neighbor_frame_path = os.path.join(data_root, input_folder, key, f'im{idx}.png') output_path = os.path.join(data_root, output_folder, key, f'im{center_frame_idx}_{name}.flo') neighbor_frame = read_image(neighbor_frame_path) flow = estimate(neighbor_frame, center_frame) # import cv2 # import matplotlib.pyplot as plt # # flow = flow.numpy() # mag, ang = cv2.cartToPolar(flow[0], flow[1]) # hsv = np.zeros([flow.shape[1], flow.shape[2], 3], dtype=np.float32) # hsv[..., 2] = 255 # hsv[..., 0] = ang * 180 / np.pi / 2 # hsv[..., 1] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) # hsv = hsv.astype(np.uint8) # rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB) # # plt.imshow(rgb) # plt.show()