def compute_flow(self, kp1=None,kp2=None, return_additional=[], **kwargs ): """ Compute the flow. Parameters ---------- kp1, kp2 : array_like, shape (NUM_KP,2), optional Matrices containing keypoints in image coordinates for first and second frame, respectively. The first column of both matrices contains the x coordinates, the second contains the y coordinates. If kp1 and kp2 are given, no additional feature matching is performed. return_additional: array of strings, optional. If set, return additional data. Possible entries are: 'weights' : Return flow coefficients 'keypoints' : Return matched feature points 'keypoint_labels' : Return assigned layers for keypoints (PCA-Layers only). 'segments' : Return segmentation map (PCA-Layers only) 'segment_flows' : For each layer, return flow. (PCA-Layers only) The additional data is returned as a dict with the same keys. Example: u,v,data = pcaflow.compute_flow(return_additional=['weights',]) weights = data['weights'] Returns ------- u, v : array_like U and V flow fields. data_additional : dict, optional See above for details. The return formats are: 'weights' : array_like, shape (NUM_PC,) 'keypoints' : tuple (array_like, array_like) Each array has shape (NUM_KP,2). 'keypoint_labels' : array_like, shape (NUM_KP,) 'segments' : array_like, shape (WIDTH,HEIGHT) 'segment_flows' : array_like, shape (WIDTH, HEIGHT, 2, NUM_LAYERS) """ # Parse return_additional. return_weights = False return_keypoints = False return_keypoint_labels = False return_segments = False return_segment_flows = False if 'weights' in return_additional: return_weights = True if 'keypoints' in return_additional: return_keypoints = True if 'keypoint_labels' in return_additional: return_keypoint_labels = True if 'segments' in return_additional: return_segments = True if 'segment_flows' in return_additional: return_segment_flows = True if kp1 is not None and kp2 is not None: # We got some initial features. kp1_ = kp1.copy() kp2_ = kp2.copy() else: kp1_,kp2_ = self.feature_matcher.get_features() if len(kp1_) == 0: print('[PCAFlow] Warning: No features found. Setting flow to 0.') u = np.zeros(self.shape_I_orig[:2]) v = np.zeros_like(u) return (u,v) if self.params['remove_homography'] == 1: cprint('[PCAFlow] Removing homography...', self.params) kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_) dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1)) inliers = dists_new < 2 kp1_ = kp1_h kp2_ = kp2_h #kp1[inliers,:] = kp0[inliers,:] I1_warped = cv2.warpPerspective(self.images[1], H, (self.images[1].shape[1],self.images[1].shape[0]), flags=cv2.WARP_INVERSE_MAP+cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE, ) elif self.params['remove_homography'] == 2: cprint('[PCAFlow] Computing homography...', self.params) kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_) dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1)) inliers = dists_new < 2 I1_warped = self.images[1] else: inliers = None I1_warped = self.images[1] H = None kp1_orig = kp1_.copy() kp2_orig = kp2_.copy() if self.reshape_features: h_orig,w_orig = self.shape_I_orig[:2] h_orig_f = float(h_orig) w_orig_f = float(w_orig) scale = [self.pc_w / w_orig_f, self.pc_h / h_orig_f] kp1_ *= scale kp2_ *= scale I0_ = cv2.resize(self.images[0],(self.pc_w,self.pc_h)) I1_ = cv2.resize(I1_warped,(self.pc_w,self.pc_h)) else: I0_ = self.images[0] I1_ = I1_warped cprint('[PCAFLOW] %s features detected...'%kp1_.shape[0], self.params) # Solve if self.params['n_models'] > 1: u_,v_,weights,data_additional_em = self.solver.solve(kp1_,kp2_, I0=I0_, I1=I1_, inliers=inliers, H=H, shape_I_orig=self.shape_I_orig, return_additional=return_additional, **kwargs) else: if return_weights: u_,v_,weights = self.solver.solve(kp1_,kp2_,return_coefficients=True) else: u_,v_ = self.solver.solve(kp1_,kp2_) data_additional_em = {} if self.reshape_features: u = cv2.resize(u_,(w_orig,h_orig)) v = cv2.resize(v_,(w_orig,h_orig)) u *= w_orig_f / self.pc_w v *= h_orig_f / self.pc_h if self.params['remove_homography']==1: cprint('[PCAFlow] Re-applying homography...', self.params) u2,v2 = ht.apply_homography_to_flow(u,v,H) u = u2 v = v2 if len(return_additional) == 0: return u,v else: # Return more additional data data_additional = {} if return_weights: data_additional['weights'] = weights if return_keypoints: data_additional['keypoints'] = (kp1_orig,kp2_orig) # Get additional data from EMSolver for key,value in data_additional_em.items(): data_additional[key] = value return u, v, data_additional
def get_flow_GC(self, kp0, kp1, weights, I0, I1, inliers=None, H=None, shape_I_orig=None): """ Given models, densify using graph cut (i.e., solve labeling problem). """ # Determine ownership of points use_zero_layer = False # At this point, n_models also contains the PCA-Flow model. n_models = self.models.shape[0] point_models = np.argmax(weights, axis=0) # If inliers is not zero, we want to compute a "zero" layer using the # homography if inliers is not None: n_models += 1 use_zero_layer = True use_homography = self.params['remove_homography'] n_coeffs = self.flow_bases_u.shape[0] n_pixels = self.flow_bases_u.shape[1] # Define general cost structures log_unaries = np.zeros((self.pc_h, self.pc_w, n_models), dtype='int32') log_dist = np.zeros_like(log_warp) # Warping takes the images into account. # Thus, we need to rescale them to the size of the principal components. I_ndim = I0.ndim if shape_I_orig is None: Ih, Iw = I0.shape[:2] else: Ih, Iw = shape_I_orig[:2] if I_ndim > 2: I0_ = cv2.resize(cv2.cvtColor(I0, 45), (self.pc_w, self.pc_h)) I1_ = cv2.resize(cv2.cvtColor(I1, 45), (self.pc_w, self.pc_h)) else: I0_ = cv2.resize(I0, (self.pc_w, self.pc_h)) I1_ = cv2.resize(I1, (self.pc_w, self.pc_h)) if I_ndim > 2: I0_bw = I0_[:, :, 0] I1_bw = I1_[:, :, 0] else: I0_bw = I0_ I1_bw = I1_ x, y = np.meshgrid(range(self.pc_w), range(self.pc_h)) # Build basis flow models flow_u_all = np.zeros((n_models, n_pixels)) flow_v_all = np.zeros((n_models, n_pixels)) # Save indices for PCA-Flow and homography models. # If unset, set to invalid indices to catch errors pcaflow_model = n_models + 1 homography_model = n_models + 1 if self.params['remove_homography']: homography_model = n_models - 1 pcaflow_model = n_models - 2 else: pcaflow_model = n_models - 1 # For each model / layer, generate flow fields from coefficients. for m in range(n_models): if m == homography_model: # If we are on the homography layer, generate from from H. # (We generate the flow from H before downscaling it to the # size of the PCs.) ud = np.zeros((Ih, Iw), dtype='float32') vd = np.zeros((Ih, Iw), dtype='float32') if H is None: H = np.eye(3) ud, vd = ht.apply_homography_to_flow(ud, vd, H) u, v = pcautils.scale_u_v(ud, vd, (self.pc_w, self.pc_h)) flow_u_all[m] = u.flatten() flow_v_all[m] = v.flatten() else: # Simply create flow by weighting. flow_u_all[m] = self.models[m, :n_coeffs].dot( self.flow_bases_u) flow_v_all[m] = self.models[m, n_coeffs:].dot(self.flow_bases_v) # Step 1: Color models if self.params['model_gamma_c'] > 0: log_color = self._compute_unaries_color(kp0, kp1, I0_, n_models, pcaflow_model, homography_model, inliers, point_models) log_unaries += log_color if self.params['model_gamma_warp'] > 0: log_warp = self._compute_unaries_warp(I0_bw, I1_bw, n_models, use_homography, homography_model) log_unaries += log_warp if self.params['model_gamma_l'] > 0: log_dist = self._compute_unaries_location(kp0, n_models, homography_model, pcaflow_model, point_models) log_unaries += log_dist cprint('\n', self.params) # # Compute pairwise terms # # This is a simple 0/1 error. All the weighting is done through the # weight variables w_x, w_y. cprint('[GC] Computing edgeweights...', self.params) gamma = self.params['model_gamma'] log_pairwise = (-np.eye(n_models)).astype('int32') # Compute weights according to GrabCut gy, gx = np.gradient(I0_bw.astype('float32')) beta = 1.0 / ((gy**2).mean() + (gx**2).mean()) w_y_gc = np.exp(-beta * gy**2) w_x_gc = np.exp(-beta * gx**2) w_x = (w_x_gc * 100 * gamma).astype('int32') w_y = (w_y_gc * 100 * gamma).astype('int32') cprint('done.\n', self.params) cprint('[GC] Solving...', self.params) try: res_ = pygco.cut_simple_vh(log_unaries, log_pairwise, w_y, w_x) except: cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.') res_ = pygco.cut_simple_vh(log_unaries, log_pairwise, w_y, w_x, algorithm='swap') res = cv2.medianBlur(res_.astype('uint8'), ksize=3).astype('int32') cprint('done.\n', self.params) if self.params['debug'] > 1: self.output_debug2(kp0, point_models, res, flow_u_all, flow_v_all) u_all = flow_u_all[res.ravel(), np.arange(n_pixels)].reshape( (self.pc_h, self.pc_w)) v_all = flow_v_all[res.ravel(), np.arange(n_pixels)].reshape( (self.pc_h, self.pc_w)) return u_all, v_all
def get_flow_GC(self,kp0,kp1,weights,I0,I1,inliers=None,H=None,shape_I_orig=None): """ Given models, densify using graph cut (i.e., solve labeling problem). """ # Determine ownership of points use_zero_layer = False # At this point, n_models also contains the PCA-Flow model. n_models = self.models.shape[0] point_models = np.argmax(weights,axis=0) # If inliers is not zero, we want to compute a "zero" layer using the # homography if inliers is not None: n_models += 1 use_zero_layer = True use_homography = self.params['remove_homography'] n_coeffs = self.flow_bases_u.shape[0] n_pixels = self.flow_bases_u.shape[1] # Define general cost structures log_unaries = np.zeros((self.pc_h,self.pc_w,n_models),dtype='int32') log_dist = np.zeros_like(log_warp) # Warping takes the images into account. # Thus, we need to rescale them to the size of the principal components. I_ndim = I0.ndim if shape_I_orig is None: Ih,Iw = I0.shape[:2] else: Ih,Iw = shape_I_orig[:2] if I_ndim > 2: I0_ = cv2.resize(cv2.cvtColor(I0,45),(self.pc_w,self.pc_h)) I1_ = cv2.resize(cv2.cvtColor(I1,45),(self.pc_w,self.pc_h)) else: I0_ = cv2.resize(I0,(self.pc_w,self.pc_h)) I1_ = cv2.resize(I1,(self.pc_w,self.pc_h)) if I_ndim > 2: I0_bw = I0_[:,:,0] I1_bw = I1_[:,:,0] else: I0_bw = I0_ I1_bw = I1_ x,y = np.meshgrid(range(self.pc_w),range(self.pc_h)) # Build basis flow models flow_u_all = np.zeros((n_models,n_pixels)) flow_v_all = np.zeros((n_models,n_pixels)) # Save indices for PCA-Flow and homography models. # If unset, set to invalid indices to catch errors pcaflow_model = n_models+1 homography_model = n_models+1 if self.params['remove_homography']: homography_model = n_models-1 pcaflow_model = n_models - 2 else: pcaflow_model = n_models - 1 # For each model / layer, generate flow fields from coefficients. for m in range(n_models): if m == homography_model: # If we are on the homography layer, generate from from H. # (We generate the flow from H before downscaling it to the # size of the PCs.) ud = np.zeros((Ih,Iw),dtype='float32') vd = np.zeros((Ih,Iw),dtype='float32') if H is None: H = np.eye(3) ud,vd = ht.apply_homography_to_flow(ud,vd,H) u,v = pcautils.scale_u_v(ud,vd,(self.pc_w,self.pc_h)) flow_u_all[m] = u.flatten() flow_v_all[m] = v.flatten() else: # Simply create flow by weighting. flow_u_all[m] = self.models[m,:n_coeffs].dot(self.flow_bases_u) flow_v_all[m] = self.models[m,n_coeffs:].dot(self.flow_bases_v) # Step 1: Color models if self.params['model_gamma_c'] > 0: log_color = self._compute_unaries_color(kp0, kp1, I0_, n_models, pcaflow_model, homography_model, inliers, point_models) log_unaries += log_color if self.params['model_gamma_warp'] > 0: log_warp = self._compute_unaries_warp(I0_bw, I1_bw, n_models, use_homography, homography_model) log_unaries += log_warp if self.params['model_gamma_l'] > 0: log_dist = self._compute_unaries_location(kp0, n_models, homography_model, pcaflow_model, point_models) log_unaries += log_dist cprint('\n',self.params) # # Compute pairwise terms # # This is a simple 0/1 error. All the weighting is done through the # weight variables w_x, w_y. cprint('[GC] Computing edgeweights...',self.params) gamma = self.params['model_gamma'] log_pairwise = (-np.eye(n_models)).astype('int32') # Compute weights according to GrabCut gy,gx = np.gradient(I0_bw.astype('float32')) beta = 1.0 / ((gy**2).mean() + (gx**2).mean()) w_y_gc = np.exp(- beta * gy**2) w_x_gc = np.exp(- beta * gx**2) w_x = (w_x_gc * 100 * gamma).astype('int32') w_y = (w_y_gc * 100 * gamma).astype('int32') cprint('done.\n',self.params) cprint('[GC] Solving...',self.params) try: res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x) except: cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.') res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x,algorithm='swap') res = cv2.medianBlur(res_.astype('uint8'),ksize=3).astype('int32') cprint('done.\n',self.params) if self.params['debug']>1: self.output_debug2(kp0,point_models,res,flow_u_all,flow_v_all) u_all = flow_u_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w)) v_all = flow_v_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w)) return u_all,v_all