Пример #1
0
    def compute_flow(self,
                       kp1=None,kp2=None,
                       return_additional=[],
                       **kwargs
                      ):
        """
        Compute the flow.

        Parameters
        ----------
        kp1, kp2 : array_like, shape (NUM_KP,2), optional
            Matrices containing keypoints in image coordinates for
            first and second frame, respectively.
            The first column of both matrices contains the x coordinates,
            the second contains the y coordinates.
            If kp1 and kp2 are given, no additional feature matching is
            performed.
        
        return_additional: array of strings, optional.
            If set, return additional data. Possible entries are:
        
                'weights'   : Return flow coefficients
                'keypoints' : Return matched feature points
                'keypoint_labels' : Return assigned layers for keypoints
                                    (PCA-Layers only).
                'segments'  : Return segmentation map
                              (PCA-Layers only)
                'segment_flows' : For each layer, return flow.
                                  (PCA-Layers only)
        
            The additional data is returned as a dict with the same keys.
        
            Example:
                u,v,data = pcaflow.compute_flow(return_additional=['weights',])
                weights = data['weights']



        Returns
        -------
        u, v : array_like
            U and V flow fields.

        data_additional : dict, optional
            See above for details. The return formats are:

                'weights' : array_like, shape (NUM_PC,)
                'keypoints' : tuple (array_like, array_like)
                              Each array has shape (NUM_KP,2).
                'keypoint_labels' : array_like, shape (NUM_KP,)
                'segments' : array_like, shape (WIDTH,HEIGHT)
                'segment_flows' : array_like, shape (WIDTH, HEIGHT, 2, NUM_LAYERS)

        """

        # Parse return_additional.
        return_weights = False
        return_keypoints = False
        return_keypoint_labels = False
        return_segments = False
        return_segment_flows = False
        
        if 'weights' in return_additional:
            return_weights = True
        if 'keypoints' in return_additional:
            return_keypoints = True
        if 'keypoint_labels' in return_additional:
            return_keypoint_labels = True
        if 'segments' in return_additional:
            return_segments = True
        if 'segment_flows' in return_additional:
            return_segment_flows = True
            

        if kp1 is not None and kp2 is not None:
            # We got some initial features.
            kp1_ = kp1.copy()
            kp2_ = kp2.copy()

        else:
            kp1_,kp2_ = self.feature_matcher.get_features()

        if len(kp1_) == 0:
            print('[PCAFlow] Warning: No features found. Setting flow to 0.')
            u = np.zeros(self.shape_I_orig[:2])
            v = np.zeros_like(u)
            return (u,v)

        if self.params['remove_homography'] == 1:
            cprint('[PCAFlow] Removing homography...', self.params)

            kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)

            dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
            inliers = dists_new < 2
            kp1_ = kp1_h
            kp2_ = kp2_h
            #kp1[inliers,:] = kp0[inliers,:]
            I1_warped = cv2.warpPerspective(self.images[1],
                    H,
                    (self.images[1].shape[1],self.images[1].shape[0]),
                    flags=cv2.WARP_INVERSE_MAP+cv2.INTER_LINEAR,
                    borderMode=cv2.BORDER_REPLICATE,
                    )
        elif self.params['remove_homography'] == 2:
            cprint('[PCAFlow] Computing homography...', self.params)

            kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)

            dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
            inliers = dists_new < 2
            I1_warped = self.images[1]

        else:
            inliers = None
            I1_warped = self.images[1]
            H = None

        kp1_orig = kp1_.copy()
        kp2_orig = kp2_.copy()

        if self.reshape_features:
            h_orig,w_orig = self.shape_I_orig[:2]
            h_orig_f = float(h_orig)
            w_orig_f = float(w_orig)
            scale = [self.pc_w / w_orig_f, self.pc_h / h_orig_f]
            kp1_ *= scale
            kp2_ *= scale
            I0_ = cv2.resize(self.images[0],(self.pc_w,self.pc_h))
            I1_ = cv2.resize(I1_warped,(self.pc_w,self.pc_h))
        else:
            I0_ = self.images[0]
            I1_ = I1_warped

        cprint('[PCAFLOW] %s features detected...'%kp1_.shape[0], self.params)

        # Solve
        if self.params['n_models'] > 1:
            u_,v_,weights,data_additional_em = self.solver.solve(kp1_,kp2_,
                    I0=I0_,
                    I1=I1_,
                    inliers=inliers,
                    H=H,
                    shape_I_orig=self.shape_I_orig,
                    return_additional=return_additional,
                    **kwargs)
        else:
            if return_weights:
                u_,v_,weights = self.solver.solve(kp1_,kp2_,return_coefficients=True)
            else:
                u_,v_ = self.solver.solve(kp1_,kp2_)
            data_additional_em = {}

        if self.reshape_features:
            u = cv2.resize(u_,(w_orig,h_orig))
            v = cv2.resize(v_,(w_orig,h_orig))

            u *= w_orig_f / self.pc_w
            v *= h_orig_f / self.pc_h

        if self.params['remove_homography']==1:
            cprint('[PCAFlow] Re-applying homography...', self.params)
            u2,v2 = ht.apply_homography_to_flow(u,v,H)
            u = u2
            v = v2

        if len(return_additional) == 0:
            return u,v

        else:
            # Return more additional data
            data_additional = {}
            if return_weights:
                data_additional['weights'] = weights
            if return_keypoints:
                data_additional['keypoints'] = (kp1_orig,kp2_orig)

            # Get additional data from EMSolver
            for key,value in data_additional_em.items():
                data_additional[key] = value

            return u, v, data_additional    
Пример #2
0
    def get_flow_GC(self,
                    kp0,
                    kp1,
                    weights,
                    I0,
                    I1,
                    inliers=None,
                    H=None,
                    shape_I_orig=None):
        """
        Given models, densify using graph cut (i.e., solve labeling problem).

        """

        # Determine ownership of points
        use_zero_layer = False

        # At this point, n_models also contains the PCA-Flow model.
        n_models = self.models.shape[0]

        point_models = np.argmax(weights, axis=0)

        # If inliers is not zero, we want to compute a "zero" layer using the
        # homography
        if inliers is not None:
            n_models += 1
            use_zero_layer = True

        use_homography = self.params['remove_homography']

        n_coeffs = self.flow_bases_u.shape[0]
        n_pixels = self.flow_bases_u.shape[1]

        # Define general cost structures
        log_unaries = np.zeros((self.pc_h, self.pc_w, n_models), dtype='int32')

        log_dist = np.zeros_like(log_warp)

        # Warping takes the images into account.
        # Thus, we need to rescale them to the size of the principal components.
        I_ndim = I0.ndim
        if shape_I_orig is None:
            Ih, Iw = I0.shape[:2]
        else:
            Ih, Iw = shape_I_orig[:2]

        if I_ndim > 2:
            I0_ = cv2.resize(cv2.cvtColor(I0, 45), (self.pc_w, self.pc_h))
            I1_ = cv2.resize(cv2.cvtColor(I1, 45), (self.pc_w, self.pc_h))
        else:
            I0_ = cv2.resize(I0, (self.pc_w, self.pc_h))
            I1_ = cv2.resize(I1, (self.pc_w, self.pc_h))

        if I_ndim > 2:
            I0_bw = I0_[:, :, 0]
            I1_bw = I1_[:, :, 0]
        else:
            I0_bw = I0_
            I1_bw = I1_

        x, y = np.meshgrid(range(self.pc_w), range(self.pc_h))

        # Build basis flow models
        flow_u_all = np.zeros((n_models, n_pixels))
        flow_v_all = np.zeros((n_models, n_pixels))

        # Save indices for PCA-Flow and homography models.
        # If unset, set to invalid indices to catch errors
        pcaflow_model = n_models + 1
        homography_model = n_models + 1
        if self.params['remove_homography']:
            homography_model = n_models - 1
            pcaflow_model = n_models - 2
        else:
            pcaflow_model = n_models - 1

        # For each model / layer, generate flow fields from coefficients.
        for m in range(n_models):
            if m == homography_model:
                # If we are on the homography layer, generate from from H.
                # (We generate the flow from H before downscaling it to the
                # size of the PCs.)
                ud = np.zeros((Ih, Iw), dtype='float32')
                vd = np.zeros((Ih, Iw), dtype='float32')
                if H is None:
                    H = np.eye(3)
                ud, vd = ht.apply_homography_to_flow(ud, vd, H)
                u, v = pcautils.scale_u_v(ud, vd, (self.pc_w, self.pc_h))
                flow_u_all[m] = u.flatten()
                flow_v_all[m] = v.flatten()
            else:
                # Simply create flow by weighting.
                flow_u_all[m] = self.models[m, :n_coeffs].dot(
                    self.flow_bases_u)
                flow_v_all[m] = self.models[m,
                                            n_coeffs:].dot(self.flow_bases_v)

        # Step 1: Color models

        if self.params['model_gamma_c'] > 0:
            log_color = self._compute_unaries_color(kp0, kp1, I0_, n_models,
                                                    pcaflow_model,
                                                    homography_model, inliers,
                                                    point_models)
            log_unaries += log_color

        if self.params['model_gamma_warp'] > 0:
            log_warp = self._compute_unaries_warp(I0_bw, I1_bw, n_models,
                                                  use_homography,
                                                  homography_model)
            log_unaries += log_warp

        if self.params['model_gamma_l'] > 0:
            log_dist = self._compute_unaries_location(kp0, n_models,
                                                      homography_model,
                                                      pcaflow_model,
                                                      point_models)

            log_unaries += log_dist

        cprint('\n', self.params)

        #
        # Compute pairwise terms
        #

        # This is a simple 0/1 error. All the weighting is done through the
        # weight variables w_x, w_y.
        cprint('[GC] Computing edgeweights...', self.params)

        gamma = self.params['model_gamma']

        log_pairwise = (-np.eye(n_models)).astype('int32')

        # Compute weights according to GrabCut
        gy, gx = np.gradient(I0_bw.astype('float32'))
        beta = 1.0 / ((gy**2).mean() + (gx**2).mean())
        w_y_gc = np.exp(-beta * gy**2)
        w_x_gc = np.exp(-beta * gx**2)
        w_x = (w_x_gc * 100 * gamma).astype('int32')
        w_y = (w_y_gc * 100 * gamma).astype('int32')

        cprint('done.\n', self.params)
        cprint('[GC] Solving...', self.params)
        try:
            res_ = pygco.cut_simple_vh(log_unaries, log_pairwise, w_y, w_x)
        except:
            cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.')
            res_ = pygco.cut_simple_vh(log_unaries,
                                       log_pairwise,
                                       w_y,
                                       w_x,
                                       algorithm='swap')

        res = cv2.medianBlur(res_.astype('uint8'), ksize=3).astype('int32')
        cprint('done.\n', self.params)

        if self.params['debug'] > 1:
            self.output_debug2(kp0, point_models, res, flow_u_all, flow_v_all)

        u_all = flow_u_all[res.ravel(), np.arange(n_pixels)].reshape(
            (self.pc_h, self.pc_w))
        v_all = flow_v_all[res.ravel(), np.arange(n_pixels)].reshape(
            (self.pc_h, self.pc_w))

        return u_all, v_all
Пример #3
0
    def get_flow_GC(self,kp0,kp1,weights,I0,I1,inliers=None,H=None,shape_I_orig=None):
        """
        Given models, densify using graph cut (i.e., solve labeling problem).

        """

        # Determine ownership of points
        use_zero_layer = False
        
        # At this point, n_models also contains the PCA-Flow model.
        n_models = self.models.shape[0]
        
        point_models = np.argmax(weights,axis=0)

        # If inliers is not zero, we want to compute a "zero" layer using the
        # homography
        if inliers is not None:
            n_models += 1
            use_zero_layer = True

        use_homography = self.params['remove_homography']

        n_coeffs = self.flow_bases_u.shape[0]
        n_pixels = self.flow_bases_u.shape[1]


        # Define general cost structures
        log_unaries = np.zeros((self.pc_h,self.pc_w,n_models),dtype='int32')


        log_dist = np.zeros_like(log_warp)


        # Warping takes the images into account.
        # Thus, we need to rescale them to the size of the principal components.
        I_ndim = I0.ndim
        if shape_I_orig is None:
            Ih,Iw = I0.shape[:2]
        else:
            Ih,Iw = shape_I_orig[:2]
            
        if I_ndim > 2:
            I0_ = cv2.resize(cv2.cvtColor(I0,45),(self.pc_w,self.pc_h))
            I1_ = cv2.resize(cv2.cvtColor(I1,45),(self.pc_w,self.pc_h))
        else:
            I0_ = cv2.resize(I0,(self.pc_w,self.pc_h))
            I1_ = cv2.resize(I1,(self.pc_w,self.pc_h))
            
        if I_ndim > 2:
            I0_bw = I0_[:,:,0]
            I1_bw = I1_[:,:,0]
        else:
            I0_bw = I0_
            I1_bw = I1_

        x,y = np.meshgrid(range(self.pc_w),range(self.pc_h))


        # Build basis flow models
        flow_u_all = np.zeros((n_models,n_pixels))
        flow_v_all = np.zeros((n_models,n_pixels))

        # Save indices for PCA-Flow and homography models.
        # If unset, set to invalid indices to catch errors
        pcaflow_model = n_models+1
        homography_model = n_models+1
        if self.params['remove_homography']:
            homography_model = n_models-1
            pcaflow_model = n_models - 2
        else:
            pcaflow_model = n_models - 1


        # For each model / layer, generate flow fields from coefficients.
        for m in range(n_models):
            if m == homography_model:
                # If we are on the homography layer, generate from from H.
                # (We generate the flow from H before downscaling it to the
                # size of the PCs.)
                ud = np.zeros((Ih,Iw),dtype='float32')
                vd = np.zeros((Ih,Iw),dtype='float32')
                if H is None:
                    H = np.eye(3)
                ud,vd = ht.apply_homography_to_flow(ud,vd,H)
                u,v = pcautils.scale_u_v(ud,vd,(self.pc_w,self.pc_h))
                flow_u_all[m] = u.flatten()
                flow_v_all[m] = v.flatten()
            else:
                # Simply create flow by weighting.
                flow_u_all[m] = self.models[m,:n_coeffs].dot(self.flow_bases_u)
                flow_v_all[m] = self.models[m,n_coeffs:].dot(self.flow_bases_v)




        # Step 1: Color models

        if self.params['model_gamma_c'] > 0:
            log_color = self._compute_unaries_color(kp0,
                                                    kp1,
                                                    I0_,
                                                    n_models,
                                                    pcaflow_model,
                                                    homography_model,
                                                    inliers,
                                                    point_models)
            log_unaries += log_color

            

        if self.params['model_gamma_warp'] > 0:
           log_warp = self._compute_unaries_warp(I0_bw,
                                                 I1_bw,
                                                 n_models,
                                                 use_homography,
                                                 homography_model)
           log_unaries += log_warp
           


        if self.params['model_gamma_l'] > 0:
            log_dist = self._compute_unaries_location(kp0,
                                                      n_models,
                                                      homography_model,
                                                      pcaflow_model,
                                                      point_models)

            log_unaries += log_dist

        
        cprint('\n',self.params)


        #
        # Compute pairwise terms
        #

        # This is a simple 0/1 error. All the weighting is done through the
        # weight variables w_x, w_y.
        cprint('[GC] Computing edgeweights...',self.params)
        
        gamma = self.params['model_gamma']        

        log_pairwise = (-np.eye(n_models)).astype('int32')

        # Compute weights according to GrabCut
        gy,gx = np.gradient(I0_bw.astype('float32'))
        beta = 1.0 / ((gy**2).mean() + (gx**2).mean())
        w_y_gc = np.exp(- beta * gy**2)
        w_x_gc = np.exp(- beta * gx**2)
        w_x = (w_x_gc * 100 * gamma).astype('int32')
        w_y = (w_y_gc * 100 * gamma).astype('int32')

        
        cprint('done.\n',self.params)
        cprint('[GC] Solving...',self.params)
        try:
            res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x)
        except:
            cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.')
            res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x,algorithm='swap')

        res = cv2.medianBlur(res_.astype('uint8'),ksize=3).astype('int32')
        cprint('done.\n',self.params)

        if self.params['debug']>1:
            self.output_debug2(kp0,point_models,res,flow_u_all,flow_v_all)

        u_all = flow_u_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w))
        v_all = flow_v_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w))

        return u_all,v_all