def _compute_unaries_warp(self, I0_bw, I1_bw, n_models, use_homography, homography_model, ): """ Compute warping based unaries, i.e. the violation of brightness constancy given a particular flow. """ cprint('[GC] Computing warp models',self.params) l_warp = np.zeros((self.pc_h,self.pc_w, n_models)) dI0dy,dI0dx = np.gradient(I0_bw) dI1dy,dI1dx = np.gradient(I1_bw) dx_weight = 1.0 I1_stacked = np.dstack((I1_bw,dx_weight*dI1dx,dx_weight*dI1dy)) I0_stacked = np.dstack((I0_bw,dx_weight*dI0dx,dx_weight*dI0dy)) I_valid_homography = np.ones_like(I1_bw) for m in range(n_models): if use_homography==1 and m == homography_model: # I1 has the homography already removed. Nothing to do here. I1_warped = I1_stacked I_valid = np.ones(I1_warped.shape[:2],dtype='uint8') else: # Warp back by flow for current model u = flow_u_all[m].reshape((self.pc_h,self.pc_w)) v = flow_v_all[m].reshape((self.pc_h,self.pc_w)) I1_warped,I_valid = pullback_opencv(u,v,I1_stacked) if m == homography_model: I_valid_homography = I_valid df = np.abs(I0_stacked - I1_warped.astype('float32')).mean(axis=2) D = cv2.GaussianBlur(df,ksize=(5,5),sigmaX=-1) if self.params['model_sigma_w'] > 0: D = 1.0 - np.exp(-(D/self.params['model_sigma_w'])**2) l_warp[:,:,m] = D if use_homography == 2: cprint('[CG] Homography mean: {0}'.format(I_valid_homography.astype('float32').mean()), self.params) l_warp[I_valid_homography<1] = 0 log_warp = (100 * self.params['model_gamma_warp'] * (l_warp - l_warp.min(axis=2)[:,:,np.newaxis])).astype('int32') return log_warp
def push_back(self,I): """ Push back frame. When processing a streaming video, this allows to pre-compute features only once per frame. Parameters ---------- I : array_like Image, usually given as H x W x 3 color image. """ cprint('[PCAFLOW] Adding image...', self.params) if not (I.shape[0] == self.pc_h and I.shape[1] == self.pc_w): self.reshape_features = True self.shape_I_orig = I.shape if self.params['image_blur'] > 0: I = cv2.GaussianBlur( I, ksize=(int(self.params['image_blur']),int(self.params['image_blur'])), sigmaX=-1) cprint('[PCAFLOW] Adding image to feature matcher.', self.params) self.feature_matcher.push_back(I) self.images.append(I) cprint('[PCAFLOW] Done adding image.',self.params)
def _compute_unaries_warp( self, I0_bw, I1_bw, n_models, use_homography, homography_model, ): """ Compute warping based unaries, i.e. the violation of brightness constancy given a particular flow. """ cprint('[GC] Computing warp models', self.params) l_warp = np.zeros((self.pc_h, self.pc_w, n_models)) dI0dy, dI0dx = np.gradient(I0_bw) dI1dy, dI1dx = np.gradient(I1_bw) dx_weight = 1.0 I1_stacked = np.dstack((I1_bw, dx_weight * dI1dx, dx_weight * dI1dy)) I0_stacked = np.dstack((I0_bw, dx_weight * dI0dx, dx_weight * dI0dy)) I_valid_homography = np.ones_like(I1_bw) for m in range(n_models): if use_homography == 1 and m == homography_model: # I1 has the homography already removed. Nothing to do here. I1_warped = I1_stacked I_valid = np.ones(I1_warped.shape[:2], dtype='uint8') else: # Warp back by flow for current model u = flow_u_all[m].reshape((self.pc_h, self.pc_w)) v = flow_v_all[m].reshape((self.pc_h, self.pc_w)) I1_warped, I_valid = pullback_opencv(u, v, I1_stacked) if m == homography_model: I_valid_homography = I_valid df = np.abs(I0_stacked - I1_warped.astype('float32')).mean(axis=2) D = cv2.GaussianBlur(df, ksize=(5, 5), sigmaX=-1) if self.params['model_sigma_w'] > 0: D = 1.0 - np.exp(-(D / self.params['model_sigma_w'])**2) l_warp[:, :, m] = D if use_homography == 2: cprint( '[CG] Homography mean: {0}'.format( I_valid_homography.astype('float32').mean()), self.params) l_warp[I_valid_homography < 1] = 0 log_warp = ( 100 * self.params['model_gamma_warp'] * (l_warp - l_warp.min(axis=2)[:, :, np.newaxis])).astype('int32') return log_warp
def _compute_unaries_color(self, kp0, kp1, I0_, n_models, pcaflow_model, homography_model, inliers, point_models): """ Compute unaries based on the color distributions of assigned features. """ # Build color models cprint('[GC] Computing color models', self.params) kp0_ = np.floor(kp0).astype('int') point_surround = 1 if I_ndim > 2: colors_points_ = I0_[kp0_[:, 1], kp0_[:, 0], :].reshape((-1, 3)) else: colors_points_ = I0_[kp0_[:, 1], kp0_[:, 0]].flatten() if I_ndim > 2: color_all_ = I0_.reshape((-1, 3)) else: color_all_ = I0_.flatten() colors_points = colors_points_.astype('float32') color_all = color_all_.astype('float32') # Normalize to mean / std of matched points. color_all -= colors_points.mean(axis=0) color_all /= colors_points.std(axis=0) colors_points -= colors_points.mean(axis=0) colors_points /= colors_points.std(axis=0) scores_colors = np.zeros((self.pc_h, self.pc_w, n_models)) for m in range(n_models): # Compute indices into features at current model if m == homography_model: # For homography layer, use only inliers ind = np.tile(inliers == 1, point_surround) elif m == pcaflow_model: # For PCAFlow layer, use all points ind = np.ones(colors_points.shape[0]) == 1 else: # Otherwise, use current ownerships ind = np.tile(point_models == m, point_surround) # Extract colors of selected features if I_ndim > 2: P = colors_points[ind, :] else: P = colors_points[ind] cprint('[GC] Model {0}: Num points: {1}'.format(m, P.shape[0]), self.params) if P.shape[0] > 1: if P.shape[0] < 10: nc = 1 else: # Currently, this is always one. Mixtures were of no # advantage. nc = self.params['model_color_n_mixtures'] # Fit Gaussian to selected color points, and compute score for # all pixels. G = mixture.GMM(n_components=nc, covariance_type='full').fit(P) score = G.score(color_all) else: # Simple fallback. score = np.ones(color_all.shape[0]) * -100000 S = score.reshape((self.pc_h, self.pc_w)) S = cv2.GaussianBlur(S, ksize=(5, 5), sigmaX=-1) scores_colors[:, :, m] = S log_colors = -(self.params['model_gamma_c'] * 100 * (scores_colors - scores_colors.max( axis=2)[:, :, np.newaxis])).astype('int32') cprint('done\n', self.params) return log_colors
def __init__( self, flow_bases_u, # U basis (n_bases x (width*height)) flow_bases_v, # V basis (n_bases x (widght*height)) cov_matrix, # Covariance matrix pc_size, # (width,height) tuple params, # params cov_matrix_sublayer=None, # Optional different covariance matrix for sublayer solver ): cprint('[EMSolver] Initializing ...', params) t0 = time.time() self.params = dict(params) self.flow_bases_u = flow_bases_u.astype('float32') self.flow_bases_v = flow_bases_v.astype('float32') self.pc_w = pc_size[0] self.pc_h = pc_size[1] self.cov_matrix = cov_matrix.astype('float32').copy() len_bases = len(self.flow_bases_u) n_components_max = self.flow_bases_u.shape[0] self.n_models = params['n_models'] # Define additional solution self.n_models += 1 # Section to add QuadraticSolver as additional solution params_inner = dict(self.params) cprint('[EMSolver] Initializing additional RobustQuadraticSolver...', self.params) self.sub_solver_additional = RobustQuadraticSolver(flow_bases_u, flow_bases_v, cov_matrix, pc_size, params_inner, n_iters=10) self.use_additional = 1 self.models = np.zeros( (params['n_models'] + self.use_additional, 2 * n_components_max), dtype='float32') self.model_medians = np.ones( (params['n_models'] + self.use_additional, 2)) * -1 # If no separate covariance matrix for sublayer is provided, use the full one. if cov_matrix_sublayer is not None: self.cov_sublayer = cov_matrix_sublayer.copy() else: self.cov_sublayer = cov_matrix.copy() params_sublayer = dict(dp.get_sublayer_parameters(self.params)) self.sub_solver = RobustQuadraticSolver(flow_bases_u, flow_bases_v, self.cov_sublayer, pc_size, params_sublayer, n_iters=10) self.debug_path = './output' t1 = time.time() cprint('[EMSolver] Done. Initialization took %2.6f secs' % (t1 - t0), self.params)
def get_flow_GC(self, kp0, kp1, weights, I0, I1, inliers=None, H=None, shape_I_orig=None): """ Given models, densify using graph cut (i.e., solve labeling problem). """ # Determine ownership of points use_zero_layer = False # At this point, n_models also contains the PCA-Flow model. n_models = self.models.shape[0] point_models = np.argmax(weights, axis=0) # If inliers is not zero, we want to compute a "zero" layer using the # homography if inliers is not None: n_models += 1 use_zero_layer = True use_homography = self.params['remove_homography'] n_coeffs = self.flow_bases_u.shape[0] n_pixels = self.flow_bases_u.shape[1] # Define general cost structures log_unaries = np.zeros((self.pc_h, self.pc_w, n_models), dtype='int32') log_dist = np.zeros_like(log_warp) # Warping takes the images into account. # Thus, we need to rescale them to the size of the principal components. I_ndim = I0.ndim if shape_I_orig is None: Ih, Iw = I0.shape[:2] else: Ih, Iw = shape_I_orig[:2] if I_ndim > 2: I0_ = cv2.resize(cv2.cvtColor(I0, 45), (self.pc_w, self.pc_h)) I1_ = cv2.resize(cv2.cvtColor(I1, 45), (self.pc_w, self.pc_h)) else: I0_ = cv2.resize(I0, (self.pc_w, self.pc_h)) I1_ = cv2.resize(I1, (self.pc_w, self.pc_h)) if I_ndim > 2: I0_bw = I0_[:, :, 0] I1_bw = I1_[:, :, 0] else: I0_bw = I0_ I1_bw = I1_ x, y = np.meshgrid(range(self.pc_w), range(self.pc_h)) # Build basis flow models flow_u_all = np.zeros((n_models, n_pixels)) flow_v_all = np.zeros((n_models, n_pixels)) # Save indices for PCA-Flow and homography models. # If unset, set to invalid indices to catch errors pcaflow_model = n_models + 1 homography_model = n_models + 1 if self.params['remove_homography']: homography_model = n_models - 1 pcaflow_model = n_models - 2 else: pcaflow_model = n_models - 1 # For each model / layer, generate flow fields from coefficients. for m in range(n_models): if m == homography_model: # If we are on the homography layer, generate from from H. # (We generate the flow from H before downscaling it to the # size of the PCs.) ud = np.zeros((Ih, Iw), dtype='float32') vd = np.zeros((Ih, Iw), dtype='float32') if H is None: H = np.eye(3) ud, vd = ht.apply_homography_to_flow(ud, vd, H) u, v = pcautils.scale_u_v(ud, vd, (self.pc_w, self.pc_h)) flow_u_all[m] = u.flatten() flow_v_all[m] = v.flatten() else: # Simply create flow by weighting. flow_u_all[m] = self.models[m, :n_coeffs].dot( self.flow_bases_u) flow_v_all[m] = self.models[m, n_coeffs:].dot(self.flow_bases_v) # Step 1: Color models if self.params['model_gamma_c'] > 0: log_color = self._compute_unaries_color(kp0, kp1, I0_, n_models, pcaflow_model, homography_model, inliers, point_models) log_unaries += log_color if self.params['model_gamma_warp'] > 0: log_warp = self._compute_unaries_warp(I0_bw, I1_bw, n_models, use_homography, homography_model) log_unaries += log_warp if self.params['model_gamma_l'] > 0: log_dist = self._compute_unaries_location(kp0, n_models, homography_model, pcaflow_model, point_models) log_unaries += log_dist cprint('\n', self.params) # # Compute pairwise terms # # This is a simple 0/1 error. All the weighting is done through the # weight variables w_x, w_y. cprint('[GC] Computing edgeweights...', self.params) gamma = self.params['model_gamma'] log_pairwise = (-np.eye(n_models)).astype('int32') # Compute weights according to GrabCut gy, gx = np.gradient(I0_bw.astype('float32')) beta = 1.0 / ((gy**2).mean() + (gx**2).mean()) w_y_gc = np.exp(-beta * gy**2) w_x_gc = np.exp(-beta * gx**2) w_x = (w_x_gc * 100 * gamma).astype('int32') w_y = (w_y_gc * 100 * gamma).astype('int32') cprint('done.\n', self.params) cprint('[GC] Solving...', self.params) try: res_ = pygco.cut_simple_vh(log_unaries, log_pairwise, w_y, w_x) except: cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.') res_ = pygco.cut_simple_vh(log_unaries, log_pairwise, w_y, w_x, algorithm='swap') res = cv2.medianBlur(res_.astype('uint8'), ksize=3).astype('int32') cprint('done.\n', self.params) if self.params['debug'] > 1: self.output_debug2(kp0, point_models, res, flow_u_all, flow_v_all) u_all = flow_u_all[res.ravel(), np.arange(n_pixels)].reshape( (self.pc_h, self.pc_w)) v_all = flow_v_all[res.ravel(), np.arange(n_pixels)].reshape( (self.pc_h, self.pc_w)) return u_all, v_all
def solve(self, kp0, kp1, soft=False, I0=None, I1=None, inliers=None, H=None, shape_I_orig=None, **kwargs): """ Solve using EM. This is the main entry function. """ kp0_ = kp0.copy() kp1_ = kp1.copy() # Compute system in order to evaluate models. A, b = self.get_system(kp0_, kp1_) n_points = kp0_.shape[0] n_models = self.params['n_models'] n_components_max = self.flow_bases_u.shape[0] # ownership indicates which keypoint belongs to which model. ownership = np.zeros((n_models, n_points), dtype='bool') ownership_previous = ownership.copy() # Distance of each keypoint to the model dists = np.zeros((n_models, n_points), dtype='float32') weights_all = np.zeros_like(dists) if kwargs.has_key('debug_path'): self.debug_path = kwargs['debug_path'] if self.use_additional: self.models = np.zeros((n_models + 1, 2 * n_components_max), dtype='float32') # Defining a "median" for all points does not make sense (this would # cause center pixels to be more likely to belong to the PCA-Flow # solution self.model_medians = np.ones((n_models, 2)) * -1 # For the additional (=PCAFlow) model, solve using all keypoints. model_additional, weights_features = self.sub_solver_additional.solve( kp0, kp1, return_flow=False, return_coefficients=True, return_weights=True, ) self.models[-1, :] = model_additional else: self.models = np.zeros((n_models, 2 * n_components_max), dtype='float32') self.model_medians = np.ones((n_models, 2)) * -1 ############################## # Initialize ############################## IDs = np.arange(n_points) block_width = self.pc_w / n_models uv = kp1_ - kp0_ data_clustering = np.c_[kp0_, uv].astype('float32') #data_clustering = kp0_.astype('float32') data_clustering -= data_clustering.mean(axis=0) data_clustering /= data_clustering.std(axis=0) # Weigh down the location features data_clustering[:, :2] *= self.params['em_init_loc_weight'] # Use KMeans from scikit-image, since OpenCV's sklearn cannot be # used with a given random seed. L = KMeans(n_clusters=n_models, max_iter=100, tol=0.1, precompute_distances=False, random_state=12345).fit_predict(data_clustering) # For each cluster, extract the points belonging to this cluster, # and recompute the model. for m in range(n_models): weights = self.sub_solver.solve( kp0_[L == m, :], kp1_[L == m, :], return_flow=False, return_coefficients=True, )[0] self.models[m, :] = weights ownership[m, L == m] = True # Robustly compute median of model locations kp0_cur = kp0_[ownership[m, :], :] self.model_medians[m] = np.median(kp0_cur, axis=0) USE_MEDIAN = True MED_FACTOR = self.params['model_factor_dist_to_median'] ############################## # Iterate to get ownership ############################## for iter in range(20): # # M-Step: Determine distances and ownerships # for m in range(n_models): # Compute distance of all points to current model err = (A.dot(self.models[m, :]) - b)**2 dists[m, :] = np.sqrt(err[:n_points] + err[n_points:]) # Add median to distance if USE_MEDIAN and self.model_medians[m, 0] > -1: dists_median = np.sqrt( np.sum((kp0_ - self.model_medians[m])**2, axis=1)) dists[m, :] += dists_median * MED_FACTOR # Set correct ownerships (=binary mask) mn = np.argmin(dists, axis=0) ownership[:] = False ownership[mn, IDs] = True weights_all = np.exp(-dists) weights_all /= np.maximum(1e-9, weights_all.sum(axis=0)) # Check how many entries changed. If no change, exit. n_change = np.sum(np.logical_xor(ownership, ownership_previous)) / 2 cprint('Iter {0}. {1} entries changed...\n'.format(iter, n_change), self.params) if n_change == 0: break # Remove models with < 10 points small_models = ownership.sum(axis=1) < 10 if np.any(small_models): # Prune the empty models m_remove = np.nonzero(small_models)[0] print( '[EMSolver] Removing models {} because they became too small.' .format(m_remove)) weights_all = np.delete(weights_all, m_remove, axis=0) ownership = np.delete(ownership, m_remove, axis=0) ownership_previous = np.delete(ownership_previous, m_remove, axis=0) self.models = np.delete(self.models, m_remove, axis=0) self.model_medians = np.delete(self.model_medians, m_remove, axis=0) dists = np.delete(dists, m_remove, axis=0) n_models -= len(m_remove) continue # # E-Step: Re-compute models # for m in range(n_models): kp0_cur = kp0_[ownership[m, :], :] kp1_cur = kp1_[ownership[m, :], :] self.models[m, :] = self.sub_solver.solve( kp0_cur, kp1_cur, return_flow=False, return_coefficients=True)[0] self.model_medians[m] = np.median(kp0_cur, axis=0) ownership_previous = ownership.copy() # Determine ownerships one last time for m in range(n_models): err = (A.dot(self.models[m, :]) - b)**2 dists[m, :] = np.sqrt(err[:n_points] + err[n_points:]) if USE_MEDIAN and self.model_medians[m, 0] > -1: dists_median = np.sqrt( np.sum((kp0_ - self.model_medians[m])**2, axis=1)) dists[m, :] += dists_median * MED_FACTOR mn = np.argmin(dists, axis=0) ownership[:] = False ownership[mn, IDs] = True weights_all = np.exp(-dists) weights_all /= np.maximum(1e-9, weights_all.sum(axis=0)) if I0 is None or I1 is None: print('[EMSolver] :: ERROR. No images given.') u = None v = None else: if inliers is None: u, v = self.get_flow_GC(kp0_, kp1_, weights_all, I0, I1) else: u, v = self.get_flow_GC(kp0_, kp1_, weights_all, I0, I1, inliers, H, shape_I_orig) return u, v, self.models[0]
def __init__(self,pc_file_u,pc_file_v, covfile, covfile_sublayer=None, pc_size=-1, params={}, preset=None): """ Initialize PCAFlow object. Parameters ---------- pc_file_u, pc_file_v : string Files containing the principal components in horizontal and vertical direction, respectively. These files should be .npy files, in which each row is a flattened principal component (i.e., the total size of these principal component matrices is NUM_PC x (WIDTH*HEIGHT). cov_file : string File containing the covariance matrix of size NUM_PC x NUM_PC for PCA-Flow. covfile_sublayer : string, optional File containing the covariance matrix for the layers (usually biased towards the first PCs). If PCA-Layers is used and this file is not given, use cov_file. pc_size : tuple, optional Size of principal components. Only required if PCs are not of size 512x256 or 1024x436. params : dict, optional Parameters. See parameters.py for documentation of parameters. preset : string Preset with useful parameter values for different datasets. Can be one of 'pcaflow_sintel' 'pcalayers_sintel' 'pcaflow_kitti' 'pcalayers_kitti' """ np.random.seed(1) self.params = defaults.get_parameters(params,preset) cprint('[PCAFlow] Initializing.', self.params) NC = int(self.params['NC']) self.NC = NC pc_u = np.load(pc_file_u) pc_v = np.load(pc_file_v) cov_matrix = np.load(covfile).astype('float32') if covfile_sublayer is not None: cov_matrix_sublayer = np.load(covfile_sublayer).astype('float32') else: cov_matrix_sublayer = None pc_w = 0 pc_h = 0 if pc_size==-1: # Try to guess principal component dimensions if pc_u.shape[1] == 1024*436: cprint('[PCAFLOW] Using PC dimensionality 1024 x 436', self.params) pc_w = 1024 pc_h = 436 elif pc_v.shape[1] == 512*256: cprint('[PCAFLOW] Using PC dimensionality 512 x 256', self.params) pc_w = 512 pc_h = 256 else: print('[PCAFLOW] *** ERROR *** ') print('[PCAFLOW] Could not guess dimensionality of principal components.') print('[PCAFLOW] Please provide as parameter.') sys.exit(1) self.PC = [] # Smooth principal components. self.pc_u = self.filter_pcs(pc_u,(pc_w,pc_h)).astype('float32') self.pc_v = self.filter_pcs(pc_v,(pc_w,pc_h)).astype('float32') self.cov_matrix = cov_matrix self.pc_w = pc_w self.pc_h = pc_h self.reshape_features=True ############################### # Feature matcher ############################### if self.params['features'].lower() == 'libviso' and libviso_available: self.feature_matcher = FeatureMatcherLibviso(self.params) elif self.params['features'].lower() == 'orb': self.feature_matcher = FeatureMatcherORB(self.params) elif self.params['features'].lower() == 'fast': self.feature_matcher = FeatureMatcherFast(self.params) elif self.params['features'].lower() == 'akaze' or not libviso_available: self.feature_matcher = FeatureMatcherAKAZE(self.params) else: print('[PCAFLOW] *** ERROR ***') print('[PCAFLOW] Unknown feature type {}. Please use "libviso" or "fast".'.format(self.params['features'])) sys.exit(1) if self.params['n_models'] <= 1: ############################## # Solver for PCA-Flow ############################## self.solver = RobustQuadraticSolver(self.pc_u, self.pc_v, self.cov_matrix, pc_size=(pc_w,pc_h), params=self.params) else: ############################## # Solver for PCA-Layers ############################## self.solver = EMSolver(self.pc_u, self.pc_v, self.cov_matrix, pc_size = (pc_w,pc_h), params=self.params, cov_matrix_sublayer=cov_matrix_sublayer) self.images = deque(maxlen=2) cprint('[PCAFLOW] Finished initializing.',self.params)
def compute_flow(self, kp1=None,kp2=None, return_additional=[], **kwargs ): """ Compute the flow. Parameters ---------- kp1, kp2 : array_like, shape (NUM_KP,2), optional Matrices containing keypoints in image coordinates for first and second frame, respectively. The first column of both matrices contains the x coordinates, the second contains the y coordinates. If kp1 and kp2 are given, no additional feature matching is performed. return_additional: array of strings, optional. If set, return additional data. Possible entries are: 'weights' : Return flow coefficients 'keypoints' : Return matched feature points 'keypoint_labels' : Return assigned layers for keypoints (PCA-Layers only). 'segments' : Return segmentation map (PCA-Layers only) 'segment_flows' : For each layer, return flow. (PCA-Layers only) The additional data is returned as a dict with the same keys. Example: u,v,data = pcaflow.compute_flow(return_additional=['weights',]) weights = data['weights'] Returns ------- u, v : array_like U and V flow fields. data_additional : dict, optional See above for details. The return formats are: 'weights' : array_like, shape (NUM_PC,) 'keypoints' : tuple (array_like, array_like) Each array has shape (NUM_KP,2). 'keypoint_labels' : array_like, shape (NUM_KP,) 'segments' : array_like, shape (WIDTH,HEIGHT) 'segment_flows' : array_like, shape (WIDTH, HEIGHT, 2, NUM_LAYERS) """ # Parse return_additional. return_weights = False return_keypoints = False return_keypoint_labels = False return_segments = False return_segment_flows = False if 'weights' in return_additional: return_weights = True if 'keypoints' in return_additional: return_keypoints = True if 'keypoint_labels' in return_additional: return_keypoint_labels = True if 'segments' in return_additional: return_segments = True if 'segment_flows' in return_additional: return_segment_flows = True if kp1 is not None and kp2 is not None: # We got some initial features. kp1_ = kp1.copy() kp2_ = kp2.copy() else: kp1_,kp2_ = self.feature_matcher.get_features() if len(kp1_) == 0: print('[PCAFlow] Warning: No features found. Setting flow to 0.') u = np.zeros(self.shape_I_orig[:2]) v = np.zeros_like(u) return (u,v) if self.params['remove_homography'] == 1: cprint('[PCAFlow] Removing homography...', self.params) kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_) dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1)) inliers = dists_new < 2 kp1_ = kp1_h kp2_ = kp2_h #kp1[inliers,:] = kp0[inliers,:] I1_warped = cv2.warpPerspective(self.images[1], H, (self.images[1].shape[1],self.images[1].shape[0]), flags=cv2.WARP_INVERSE_MAP+cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE, ) elif self.params['remove_homography'] == 2: cprint('[PCAFlow] Computing homography...', self.params) kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_) dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1)) inliers = dists_new < 2 I1_warped = self.images[1] else: inliers = None I1_warped = self.images[1] H = None kp1_orig = kp1_.copy() kp2_orig = kp2_.copy() if self.reshape_features: h_orig,w_orig = self.shape_I_orig[:2] h_orig_f = float(h_orig) w_orig_f = float(w_orig) scale = [self.pc_w / w_orig_f, self.pc_h / h_orig_f] kp1_ *= scale kp2_ *= scale I0_ = cv2.resize(self.images[0],(self.pc_w,self.pc_h)) I1_ = cv2.resize(I1_warped,(self.pc_w,self.pc_h)) else: I0_ = self.images[0] I1_ = I1_warped cprint('[PCAFLOW] %s features detected...'%kp1_.shape[0], self.params) # Solve if self.params['n_models'] > 1: u_,v_,weights,data_additional_em = self.solver.solve(kp1_,kp2_, I0=I0_, I1=I1_, inliers=inliers, H=H, shape_I_orig=self.shape_I_orig, return_additional=return_additional, **kwargs) else: if return_weights: u_,v_,weights = self.solver.solve(kp1_,kp2_,return_coefficients=True) else: u_,v_ = self.solver.solve(kp1_,kp2_) data_additional_em = {} if self.reshape_features: u = cv2.resize(u_,(w_orig,h_orig)) v = cv2.resize(v_,(w_orig,h_orig)) u *= w_orig_f / self.pc_w v *= h_orig_f / self.pc_h if self.params['remove_homography']==1: cprint('[PCAFlow] Re-applying homography...', self.params) u2,v2 = ht.apply_homography_to_flow(u,v,H) u = u2 v = v2 if len(return_additional) == 0: return u,v else: # Return more additional data data_additional = {} if return_weights: data_additional['weights'] = weights if return_keypoints: data_additional['keypoints'] = (kp1_orig,kp2_orig) # Get additional data from EMSolver for key,value in data_additional_em.items(): data_additional[key] = value return u, v, data_additional
def _compute_unaries_color(self, kp0, kp1, I0_, n_models, pcaflow_model, homography_model, inliers, point_models): """ Compute unaries based on the color distributions of assigned features. """ # Build color models cprint('[GC] Computing color models',self.params) kp0_ = np.floor(kp0).astype('int') point_surround = 1 if I_ndim > 2: colors_points_ = I0_[kp0_[:,1],kp0_[:,0],:].reshape((-1,3)) else: colors_points_ = I0_[kp0_[:,1],kp0_[:,0]].flatten() if I_ndim > 2: color_all_ = I0_.reshape((-1,3)) else: color_all_ = I0_.flatten() colors_points = colors_points_.astype('float32') color_all = color_all_.astype('float32') # Normalize to mean / std of matched points. color_all -= colors_points.mean(axis=0) color_all /= colors_points.std(axis=0) colors_points -= colors_points.mean(axis=0) colors_points /= colors_points.std(axis=0) scores_colors = np.zeros((self.pc_h,self.pc_w,n_models)) for m in range(n_models): # Compute indices into features at current model if m == homography_model: # For homography layer, use only inliers ind = np.tile(inliers==1,point_surround) elif m == pcaflow_model: # For PCAFlow layer, use all points ind = np.ones(colors_points.shape[0])==1 else: # Otherwise, use current ownerships ind = np.tile(point_models==m,point_surround) # Extract colors of selected features if I_ndim > 2: P = colors_points[ind,:] else: P = colors_points[ind] cprint('[GC] Model {0}: Num points: {1}'.format(m,P.shape[0]),self.params) if P.shape[0] > 1: if P.shape[0] < 10: nc = 1 else: # Currently, this is always one. Mixtures were of no # advantage. nc = self.params['model_color_n_mixtures'] # Fit Gaussian to selected color points, and compute score for # all pixels. G = mixture.GMM(n_components=nc,covariance_type='full').fit(P) score = G.score(color_all) else: # Simple fallback. score = np.ones(color_all.shape[0]) * -100000 S = score.reshape((self.pc_h,self.pc_w)) S = cv2.GaussianBlur(S,ksize=(5,5),sigmaX=-1) scores_colors[:,:,m] = S log_colors = - (self.params['model_gamma_c'] * 100 * (scores_colors - scores_colors.max(axis=2)[:,:,np.newaxis])).astype('int32') cprint('done\n',self.params) return log_colors
def __init__(self, flow_bases_u, # U basis (n_bases x (width*height)) flow_bases_v, # V basis (n_bases x (widght*height)) cov_matrix, # Covariance matrix pc_size, # (width,height) tuple params, # params cov_matrix_sublayer=None, # Optional different covariance matrix for sublayer solver ): cprint('[EMSolver] Initializing ...', params) t0 = time.time() self.params = dict(params) self.flow_bases_u = flow_bases_u.astype('float32') self.flow_bases_v = flow_bases_v.astype('float32') self.pc_w = pc_size[0] self.pc_h = pc_size[1] self.cov_matrix = cov_matrix.astype('float32').copy() len_bases = len(self.flow_bases_u) n_components_max = self.flow_bases_u.shape[0] self.n_models = params['n_models'] # Define additional solution self.n_models += 1 # Section to add QuadraticSolver as additional solution params_inner = dict(self.params) cprint('[EMSolver] Initializing additional RobustQuadraticSolver...', self.params) self.sub_solver_additional = RobustQuadraticSolver( flow_bases_u, flow_bases_v, cov_matrix, pc_size, params_inner, n_iters=10) self.use_additional = 1 self.models = np.zeros((params['n_models']+self.use_additional,2*n_components_max),dtype='float32') self.model_medians = np.ones((params['n_models']+self.use_additional,2)) * -1 # If no separate covariance matrix for sublayer is provided, use the full one. if cov_matrix_sublayer is not None: self.cov_sublayer = cov_matrix_sublayer.copy() else: self.cov_sublayer = cov_matrix.copy() params_sublayer = dict(dp.get_sublayer_parameters(self.params)) self.sub_solver = RobustQuadraticSolver(flow_bases_u, flow_bases_v, self.cov_sublayer, pc_size, params_sublayer, n_iters=10) self.debug_path = './output' t1 = time.time() cprint('[EMSolver] Done. Initialization took %2.6f secs'%(t1-t0), self.params)
def get_flow_GC(self,kp0,kp1,weights,I0,I1,inliers=None,H=None,shape_I_orig=None): """ Given models, densify using graph cut (i.e., solve labeling problem). """ # Determine ownership of points use_zero_layer = False # At this point, n_models also contains the PCA-Flow model. n_models = self.models.shape[0] point_models = np.argmax(weights,axis=0) # If inliers is not zero, we want to compute a "zero" layer using the # homography if inliers is not None: n_models += 1 use_zero_layer = True use_homography = self.params['remove_homography'] n_coeffs = self.flow_bases_u.shape[0] n_pixels = self.flow_bases_u.shape[1] # Define general cost structures log_unaries = np.zeros((self.pc_h,self.pc_w,n_models),dtype='int32') log_dist = np.zeros_like(log_warp) # Warping takes the images into account. # Thus, we need to rescale them to the size of the principal components. I_ndim = I0.ndim if shape_I_orig is None: Ih,Iw = I0.shape[:2] else: Ih,Iw = shape_I_orig[:2] if I_ndim > 2: I0_ = cv2.resize(cv2.cvtColor(I0,45),(self.pc_w,self.pc_h)) I1_ = cv2.resize(cv2.cvtColor(I1,45),(self.pc_w,self.pc_h)) else: I0_ = cv2.resize(I0,(self.pc_w,self.pc_h)) I1_ = cv2.resize(I1,(self.pc_w,self.pc_h)) if I_ndim > 2: I0_bw = I0_[:,:,0] I1_bw = I1_[:,:,0] else: I0_bw = I0_ I1_bw = I1_ x,y = np.meshgrid(range(self.pc_w),range(self.pc_h)) # Build basis flow models flow_u_all = np.zeros((n_models,n_pixels)) flow_v_all = np.zeros((n_models,n_pixels)) # Save indices for PCA-Flow and homography models. # If unset, set to invalid indices to catch errors pcaflow_model = n_models+1 homography_model = n_models+1 if self.params['remove_homography']: homography_model = n_models-1 pcaflow_model = n_models - 2 else: pcaflow_model = n_models - 1 # For each model / layer, generate flow fields from coefficients. for m in range(n_models): if m == homography_model: # If we are on the homography layer, generate from from H. # (We generate the flow from H before downscaling it to the # size of the PCs.) ud = np.zeros((Ih,Iw),dtype='float32') vd = np.zeros((Ih,Iw),dtype='float32') if H is None: H = np.eye(3) ud,vd = ht.apply_homography_to_flow(ud,vd,H) u,v = pcautils.scale_u_v(ud,vd,(self.pc_w,self.pc_h)) flow_u_all[m] = u.flatten() flow_v_all[m] = v.flatten() else: # Simply create flow by weighting. flow_u_all[m] = self.models[m,:n_coeffs].dot(self.flow_bases_u) flow_v_all[m] = self.models[m,n_coeffs:].dot(self.flow_bases_v) # Step 1: Color models if self.params['model_gamma_c'] > 0: log_color = self._compute_unaries_color(kp0, kp1, I0_, n_models, pcaflow_model, homography_model, inliers, point_models) log_unaries += log_color if self.params['model_gamma_warp'] > 0: log_warp = self._compute_unaries_warp(I0_bw, I1_bw, n_models, use_homography, homography_model) log_unaries += log_warp if self.params['model_gamma_l'] > 0: log_dist = self._compute_unaries_location(kp0, n_models, homography_model, pcaflow_model, point_models) log_unaries += log_dist cprint('\n',self.params) # # Compute pairwise terms # # This is a simple 0/1 error. All the weighting is done through the # weight variables w_x, w_y. cprint('[GC] Computing edgeweights...',self.params) gamma = self.params['model_gamma'] log_pairwise = (-np.eye(n_models)).astype('int32') # Compute weights according to GrabCut gy,gx = np.gradient(I0_bw.astype('float32')) beta = 1.0 / ((gy**2).mean() + (gx**2).mean()) w_y_gc = np.exp(- beta * gy**2) w_x_gc = np.exp(- beta * gx**2) w_x = (w_x_gc * 100 * gamma).astype('int32') w_y = (w_y_gc * 100 * gamma).astype('int32') cprint('done.\n',self.params) cprint('[GC] Solving...',self.params) try: res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x) except: cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.') res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x,algorithm='swap') res = cv2.medianBlur(res_.astype('uint8'),ksize=3).astype('int32') cprint('done.\n',self.params) if self.params['debug']>1: self.output_debug2(kp0,point_models,res,flow_u_all,flow_v_all) u_all = flow_u_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w)) v_all = flow_v_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w)) return u_all,v_all
def solve(self,kp0,kp1,soft=False,I0=None,I1=None,inliers=None,H=None,shape_I_orig=None,**kwargs): """ Solve using EM. This is the main entry function. """ kp0_ = kp0.copy() kp1_ = kp1.copy() # Compute system in order to evaluate models. A,b = self.get_system(kp0_,kp1_) n_points = kp0_.shape[0] n_models = self.params['n_models'] n_components_max = self.flow_bases_u.shape[0] # ownership indicates which keypoint belongs to which model. ownership = np.zeros((n_models,n_points),dtype='bool') ownership_previous = ownership.copy() # Distance of each keypoint to the model dists = np.zeros((n_models,n_points),dtype='float32') weights_all = np.zeros_like(dists) if kwargs.has_key('debug_path'): self.debug_path = kwargs['debug_path'] if self.use_additional: self.models = np.zeros((n_models+1,2*n_components_max),dtype='float32') # Defining a "median" for all points does not make sense (this would # cause center pixels to be more likely to belong to the PCA-Flow # solution self.model_medians = np.ones((n_models,2)) * -1 # For the additional (=PCAFlow) model, solve using all keypoints. model_additional,weights_features = self.sub_solver_additional.solve( kp0, kp1, return_flow=False, return_coefficients=True, return_weights=True, ) self.models[-1,:] = model_additional else: self.models = np.zeros((n_models,2*n_components_max),dtype='float32') self.model_medians = np.ones((n_models,2)) * -1 ############################## # Initialize ############################## IDs = np.arange(n_points) block_width = self.pc_w / n_models uv = kp1_-kp0_ data_clustering = np.c_[kp0_,uv].astype('float32') #data_clustering = kp0_.astype('float32') data_clustering -= data_clustering.mean(axis=0) data_clustering /= data_clustering.std(axis=0) # Weigh down the location features data_clustering[:,:2] *= self.params['em_init_loc_weight'] # Use KMeans from scikit-image, since OpenCV's sklearn cannot be # used with a given random seed. L = KMeans(n_clusters=n_models, max_iter=100, tol=0.1, precompute_distances=False, random_state=12345).fit_predict(data_clustering) # For each cluster, extract the points belonging to this cluster, # and recompute the model. for m in range(n_models): weights = self.sub_solver.solve( kp0_[L==m,:], kp1_[L==m,:], return_flow=False, return_coefficients=True, )[0] self.models[m,:] = weights ownership[m,L==m] = True # Robustly compute median of model locations kp0_cur = kp0_[ownership[m,:],:] self.model_medians[m] = np.median(kp0_cur,axis=0) USE_MEDIAN = True MED_FACTOR = self.params['model_factor_dist_to_median'] ############################## # Iterate to get ownership ############################## for iter in range(20): # # M-Step: Determine distances and ownerships # for m in range(n_models): # Compute distance of all points to current model err = (A.dot(self.models[m,:])-b)**2 dists[m,:] = np.sqrt(err[:n_points]+err[n_points:]) # Add median to distance if USE_MEDIAN and self.model_medians[m,0] > -1: dists_median = np.sqrt(np.sum((kp0_ - self.model_medians[m])**2,axis=1)) dists[m,:] += dists_median * MED_FACTOR # Set correct ownerships (=binary mask) mn = np.argmin(dists,axis=0) ownership[:] = False ownership[mn,IDs] = True weights_all = np.exp(-dists) weights_all /= np.maximum(1e-9,weights_all.sum(axis=0)) # Check how many entries changed. If no change, exit. n_change = np.sum(np.logical_xor(ownership,ownership_previous))/2 cprint('Iter {0}. {1} entries changed...\n'.format(iter,n_change),self.params) if n_change == 0: break # Remove models with < 10 points small_models = ownership.sum(axis=1) < 10 if np.any(small_models): # Prune the empty models m_remove = np.nonzero(small_models)[0] print('[EMSolver] Removing models {} because they became too small.'.format(m_remove)) weights_all = np.delete(weights_all,m_remove,axis=0) ownership = np.delete(ownership,m_remove,axis=0) ownership_previous = np.delete(ownership_previous,m_remove,axis=0) self.models = np.delete(self.models,m_remove,axis=0) self.model_medians = np.delete(self.model_medians,m_remove,axis=0) dists = np.delete(dists,m_remove,axis=0) n_models -= len(m_remove) continue # # E-Step: Re-compute models # for m in range(n_models): kp0_cur = kp0_[ownership[m,:],:] kp1_cur = kp1_[ownership[m,:],:] self.models[m,:] = self.sub_solver.solve(kp0_cur,kp1_cur, return_flow=False, return_coefficients=True)[0] self.model_medians[m] = np.median(kp0_cur,axis=0) ownership_previous = ownership.copy() # Determine ownerships one last time for m in range(n_models): err = (A.dot(self.models[m,:])-b)**2 dists[m,:] = np.sqrt(err[:n_points]+err[n_points:]) if USE_MEDIAN and self.model_medians[m,0] > -1: dists_median = np.sqrt(np.sum((kp0_ - self.model_medians[m])**2,axis=1)) dists[m,:] += dists_median * MED_FACTOR mn = np.argmin(dists,axis=0) ownership[:] = False ownership[mn,IDs] = True weights_all = np.exp(-dists) weights_all /= np.maximum(1e-9,weights_all.sum(axis=0)) if I0 is None or I1 is None: print('[EMSolver] :: ERROR. No images given.') u = None v = None else: if inliers is None: u,v = self.get_flow_GC(kp0_,kp1_,weights_all,I0,I1) else: u,v = self.get_flow_GC(kp0_,kp1_,weights_all, I0,I1, inliers,H,shape_I_orig) return u,v,self.models[0]
train_loss += loss.data[0] step_cnt += 1 # backward optimizer.zero_grad() loss.backward() network.clip_gradient(net, 10.) optimizer.step() if step % disp_interval == 0: duration = t.toc(average=False) fps = step_cnt / duration log_text = 'step %d, image: %s, loss: %.4f, fps: %.2f (%.2fs per batch)' % ( step, blobs['im_name'], train_loss / step_cnt, fps, 1. / fps) cprint(log_text, prefix='[.green][.bold]') if _DEBUG: cprint( '\tTP: %.2f%%, TF: %.2f%%, fg/bg=(%d/%d)' % (tp / fg * 100., tf / bg * 100., fg / step_cnt, bg / step_cnt)) cprint( '\trpn_cls: %.4f, rpn_box: %.4f, rcnn_cls: %.4f, rcnn_box: %.4f' % (net.rpn.cross_entropy.data.cpu().numpy()[0], net.rpn.loss_box.data.cpu().numpy()[0], net.cross_entropy.data.cpu().numpy()[0], net.loss_box.data.cpu().numpy()[0])) re_cnt = True if use_tensorboard and step % log_interval == 0: exp.add_scalar_value('train_loss', train_loss / step_cnt, step=step)