def compute_flow(self,
kp1=None,kp2=None,
return_additional=[],
**kwargs
):
"""
Compute the flow.
Parameters
----------
kp1, kp2 : array_like, shape (NUM_KP,2), optional
Matrices containing keypoints in image coordinates for
first and second frame, respectively.
The first column of both matrices contains the x coordinates,
the second contains the y coordinates.
If kp1 and kp2 are given, no additional feature matching is
performed.
return_additional: array of strings, optional.
If set, return additional data. Possible entries are:
'weights' : Return flow coefficients
'keypoints' : Return matched feature points
'keypoint_labels' : Return assigned layers for keypoints
(PCA-Layers only).
'segments' : Return segmentation map
(PCA-Layers only)
'segment_flows' : For each layer, return flow.
(PCA-Layers only)
The additional data is returned as a dict with the same keys.
Example:
u,v,data = pcaflow.compute_flow(return_additional=['weights',])
weights = data['weights']
Returns
-------
u, v : array_like
U and V flow fields.
data_additional : dict, optional
See above for details. The return formats are:
'weights' : array_like, shape (NUM_PC,)
'keypoints' : tuple (array_like, array_like)
Each array has shape (NUM_KP,2).
'keypoint_labels' : array_like, shape (NUM_KP,)
'segments' : array_like, shape (WIDTH,HEIGHT)
'segment_flows' : array_like, shape (WIDTH, HEIGHT, 2, NUM_LAYERS)
"""
# Parse return_additional.
return_weights = False
return_keypoints = False
return_keypoint_labels = False
return_segments = False
return_segment_flows = False
if 'weights' in return_additional:
return_weights = True
if 'keypoints' in return_additional:
return_keypoints = True
if 'keypoint_labels' in return_additional:
return_keypoint_labels = True
if 'segments' in return_additional:
return_segments = True
if 'segment_flows' in return_additional:
return_segment_flows = True
if kp1 is not None and kp2 is not None:
# We got some initial features.
kp1_ = kp1.copy()
kp2_ = kp2.copy()
else:
kp1_,kp2_ = self.feature_matcher.get_features()
if len(kp1_) == 0:
print('[PCAFlow] Warning: No features found. Setting flow to 0.')
u = np.zeros(self.shape_I_orig[:2])
v = np.zeros_like(u)
return (u,v)
if self.params['remove_homography'] == 1:
cprint('[PCAFlow] Removing homography...', self.params)
kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)
dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
inliers = dists_new < 2
kp1_ = kp1_h
kp2_ = kp2_h
#kp1[inliers,:] = kp0[inliers,:]
I1_warped = cv2.warpPerspective(self.images[1],
H,
(self.images[1].shape[1],self.images[1].shape[0]),
flags=cv2.WARP_INVERSE_MAP+cv2.INTER_LINEAR,
borderMode=cv2.BORDER_REPLICATE,
)
elif self.params['remove_homography'] == 2:
cprint('[PCAFlow] Computing homography...', self.params)
kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)
dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
inliers = dists_new < 2
I1_warped = self.images[1]
else:
inliers = None
I1_warped = self.images[1]
H = None
kp1_orig = kp1_.copy()
kp2_orig = kp2_.copy()
if self.reshape_features:
h_orig,w_orig = self.shape_I_orig[:2]
h_orig_f = float(h_orig)
w_orig_f = float(w_orig)
scale = [self.pc_w / w_orig_f, self.pc_h / h_orig_f]
kp1_ *= scale
kp2_ *= scale
I0_ = cv2.resize(self.images[0],(self.pc_w,self.pc_h))
I1_ = cv2.resize(I1_warped,(self.pc_w,self.pc_h))
else:
I0_ = self.images[0]
I1_ = I1_warped
cprint('[PCAFLOW] %s features detected...'%kp1_.shape[0], self.params)
# Solve
if self.params['n_models'] > 1:
u_,v_,weights,data_additional_em = self.solver.solve(kp1_,kp2_,
I0=I0_,
I1=I1_,
inliers=inliers,
H=H,
shape_I_orig=self.shape_I_orig,
return_additional=return_additional,
**kwargs)
else:
if return_weights:
u_,v_,weights = self.solver.solve(kp1_,kp2_,return_coefficients=True)
else:
u_,v_ = self.solver.solve(kp1_,kp2_)
data_additional_em = {}
if self.reshape_features:
u = cv2.resize(u_,(w_orig,h_orig))
v = cv2.resize(v_,(w_orig,h_orig))
u *= w_orig_f / self.pc_w
v *= h_orig_f / self.pc_h
if self.params['remove_homography']==1:
cprint('[PCAFlow] Re-applying homography...', self.params)
u2,v2 = ht.apply_homography_to_flow(u,v,H)
u = u2
v = v2
if len(return_additional) == 0:
return u,v
else:
# Return more additional data
data_additional = {}
if return_weights:
data_additional['weights'] = weights
if return_keypoints:
data_additional['keypoints'] = (kp1_orig,kp2_orig)
# Get additional data from EMSolver
for key,value in data_additional_em.items():
data_additional[key] = value
return u, v, data_additional
def get_flow_GC(self,
kp0,
kp1,
weights,
I0,
I1,
inliers=None,
H=None,
shape_I_orig=None):
"""
Given models, densify using graph cut (i.e., solve labeling problem).
"""
# Determine ownership of points
use_zero_layer = False
# At this point, n_models also contains the PCA-Flow model.
n_models = self.models.shape[0]
point_models = np.argmax(weights, axis=0)
# If inliers is not zero, we want to compute a "zero" layer using the
# homography
if inliers is not None:
n_models += 1
use_zero_layer = True
use_homography = self.params['remove_homography']
n_coeffs = self.flow_bases_u.shape[0]
n_pixels = self.flow_bases_u.shape[1]
# Define general cost structures
log_unaries = np.zeros((self.pc_h, self.pc_w, n_models), dtype='int32')
log_dist = np.zeros_like(log_warp)
# Warping takes the images into account.
# Thus, we need to rescale them to the size of the principal components.
I_ndim = I0.ndim
if shape_I_orig is None:
Ih, Iw = I0.shape[:2]
else:
Ih, Iw = shape_I_orig[:2]
if I_ndim > 2:
I0_ = cv2.resize(cv2.cvtColor(I0, 45), (self.pc_w, self.pc_h))
I1_ = cv2.resize(cv2.cvtColor(I1, 45), (self.pc_w, self.pc_h))
else:
I0_ = cv2.resize(I0, (self.pc_w, self.pc_h))
I1_ = cv2.resize(I1, (self.pc_w, self.pc_h))
if I_ndim > 2:
I0_bw = I0_[:, :, 0]
I1_bw = I1_[:, :, 0]
else:
I0_bw = I0_
I1_bw = I1_
x, y = np.meshgrid(range(self.pc_w), range(self.pc_h))
# Build basis flow models
flow_u_all = np.zeros((n_models, n_pixels))
flow_v_all = np.zeros((n_models, n_pixels))
# Save indices for PCA-Flow and homography models.
# If unset, set to invalid indices to catch errors
pcaflow_model = n_models + 1
homography_model = n_models + 1
if self.params['remove_homography']:
homography_model = n_models - 1
pcaflow_model = n_models - 2
else:
pcaflow_model = n_models - 1
# For each model / layer, generate flow fields from coefficients.
for m in range(n_models):
if m == homography_model:
# If we are on the homography layer, generate from from H.
# (We generate the flow from H before downscaling it to the
# size of the PCs.)
ud = np.zeros((Ih, Iw), dtype='float32')
vd = np.zeros((Ih, Iw), dtype='float32')
if H is None:
H = np.eye(3)
ud, vd = ht.apply_homography_to_flow(ud, vd, H)
u, v = pcautils.scale_u_v(ud, vd, (self.pc_w, self.pc_h))
flow_u_all[m] = u.flatten()
flow_v_all[m] = v.flatten()
else:
# Simply create flow by weighting.
flow_u_all[m] = self.models[m, :n_coeffs].dot(
self.flow_bases_u)
flow_v_all[m] = self.models[m,
n_coeffs:].dot(self.flow_bases_v)
# Step 1: Color models
if self.params['model_gamma_c'] > 0:
log_color = self._compute_unaries_color(kp0, kp1, I0_, n_models,
pcaflow_model,
homography_model, inliers,
point_models)
log_unaries += log_color
if self.params['model_gamma_warp'] > 0:
log_warp = self._compute_unaries_warp(I0_bw, I1_bw, n_models,
use_homography,
homography_model)
log_unaries += log_warp
if self.params['model_gamma_l'] > 0:
log_dist = self._compute_unaries_location(kp0, n_models,
homography_model,
pcaflow_model,
point_models)
log_unaries += log_dist
cprint('\n', self.params)
#
# Compute pairwise terms
#
# This is a simple 0/1 error. All the weighting is done through the
# weight variables w_x, w_y.
cprint('[GC] Computing edgeweights...', self.params)
gamma = self.params['model_gamma']
log_pairwise = (-np.eye(n_models)).astype('int32')
# Compute weights according to GrabCut
gy, gx = np.gradient(I0_bw.astype('float32'))
beta = 1.0 / ((gy**2).mean() + (gx**2).mean())
w_y_gc = np.exp(-beta * gy**2)
w_x_gc = np.exp(-beta * gx**2)
w_x = (w_x_gc * 100 * gamma).astype('int32')
w_y = (w_y_gc * 100 * gamma).astype('int32')
cprint('done.\n', self.params)
cprint('[GC] Solving...', self.params)
try:
res_ = pygco.cut_simple_vh(log_unaries, log_pairwise, w_y, w_x)
except:
cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.')
res_ = pygco.cut_simple_vh(log_unaries,
log_pairwise,
w_y,
w_x,
algorithm='swap')
res = cv2.medianBlur(res_.astype('uint8'), ksize=3).astype('int32')
cprint('done.\n', self.params)
if self.params['debug'] > 1:
self.output_debug2(kp0, point_models, res, flow_u_all, flow_v_all)
u_all = flow_u_all[res.ravel(), np.arange(n_pixels)].reshape(
(self.pc_h, self.pc_w))
v_all = flow_v_all[res.ravel(), np.arange(n_pixels)].reshape(
(self.pc_h, self.pc_w))
return u_all, v_all
def get_flow_GC(self,kp0,kp1,weights,I0,I1,inliers=None,H=None,shape_I_orig=None):
"""
Given models, densify using graph cut (i.e., solve labeling problem).
"""
# Determine ownership of points
use_zero_layer = False
# At this point, n_models also contains the PCA-Flow model.
n_models = self.models.shape[0]
point_models = np.argmax(weights,axis=0)
# If inliers is not zero, we want to compute a "zero" layer using the
# homography
if inliers is not None:
n_models += 1
use_zero_layer = True
use_homography = self.params['remove_homography']
n_coeffs = self.flow_bases_u.shape[0]
n_pixels = self.flow_bases_u.shape[1]
# Define general cost structures
log_unaries = np.zeros((self.pc_h,self.pc_w,n_models),dtype='int32')
log_dist = np.zeros_like(log_warp)
# Warping takes the images into account.
# Thus, we need to rescale them to the size of the principal components.
I_ndim = I0.ndim
if shape_I_orig is None:
Ih,Iw = I0.shape[:2]
else:
Ih,Iw = shape_I_orig[:2]
if I_ndim > 2:
I0_ = cv2.resize(cv2.cvtColor(I0,45),(self.pc_w,self.pc_h))
I1_ = cv2.resize(cv2.cvtColor(I1,45),(self.pc_w,self.pc_h))
else:
I0_ = cv2.resize(I0,(self.pc_w,self.pc_h))
I1_ = cv2.resize(I1,(self.pc_w,self.pc_h))
if I_ndim > 2:
I0_bw = I0_[:,:,0]
I1_bw = I1_[:,:,0]
else:
I0_bw = I0_
I1_bw = I1_
x,y = np.meshgrid(range(self.pc_w),range(self.pc_h))
# Build basis flow models
flow_u_all = np.zeros((n_models,n_pixels))
flow_v_all = np.zeros((n_models,n_pixels))
# Save indices for PCA-Flow and homography models.
# If unset, set to invalid indices to catch errors
pcaflow_model = n_models+1
homography_model = n_models+1
if self.params['remove_homography']:
homography_model = n_models-1
pcaflow_model = n_models - 2
else:
pcaflow_model = n_models - 1
# For each model / layer, generate flow fields from coefficients.
for m in range(n_models):
if m == homography_model:
# If we are on the homography layer, generate from from H.
# (We generate the flow from H before downscaling it to the
# size of the PCs.)
ud = np.zeros((Ih,Iw),dtype='float32')
vd = np.zeros((Ih,Iw),dtype='float32')
if H is None:
H = np.eye(3)
ud,vd = ht.apply_homography_to_flow(ud,vd,H)
u,v = pcautils.scale_u_v(ud,vd,(self.pc_w,self.pc_h))
flow_u_all[m] = u.flatten()
flow_v_all[m] = v.flatten()
else:
# Simply create flow by weighting.
flow_u_all[m] = self.models[m,:n_coeffs].dot(self.flow_bases_u)
flow_v_all[m] = self.models[m,n_coeffs:].dot(self.flow_bases_v)
# Step 1: Color models
if self.params['model_gamma_c'] > 0:
log_color = self._compute_unaries_color(kp0,
kp1,
I0_,
n_models,
pcaflow_model,
homography_model,
inliers,
point_models)
log_unaries += log_color
if self.params['model_gamma_warp'] > 0:
log_warp = self._compute_unaries_warp(I0_bw,
I1_bw,
n_models,
use_homography,
homography_model)
log_unaries += log_warp
if self.params['model_gamma_l'] > 0:
log_dist = self._compute_unaries_location(kp0,
n_models,
homography_model,
pcaflow_model,
point_models)
log_unaries += log_dist
cprint('\n',self.params)
#
# Compute pairwise terms
#
# This is a simple 0/1 error. All the weighting is done through the
# weight variables w_x, w_y.
cprint('[GC] Computing edgeweights...',self.params)
gamma = self.params['model_gamma']
log_pairwise = (-np.eye(n_models)).astype('int32')
# Compute weights according to GrabCut
gy,gx = np.gradient(I0_bw.astype('float32'))
beta = 1.0 / ((gy**2).mean() + (gx**2).mean())
w_y_gc = np.exp(- beta * gy**2)
w_x_gc = np.exp(- beta * gx**2)
w_x = (w_x_gc * 100 * gamma).astype('int32')
w_y = (w_y_gc * 100 * gamma).astype('int32')
cprint('done.\n',self.params)
cprint('[GC] Solving...',self.params)
try:
res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x)
except:
cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.')
res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x,algorithm='swap')
res = cv2.medianBlur(res_.astype('uint8'),ksize=3).astype('int32')
cprint('done.\n',self.params)
if self.params['debug']>1:
self.output_debug2(kp0,point_models,res,flow_u_all,flow_v_all)
u_all = flow_u_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w))
v_all = flow_v_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w))
return u_all,v_all