def __init__(self,pc_file_u,pc_file_v,
covfile,
covfile_sublayer=None,
pc_size=-1,
params={},
preset=None):
"""
Initialize PCAFlow object.
Parameters
----------
pc_file_u, pc_file_v : string
Files containing the principal components in horizontal and
vertical direction, respectively.
These files should be .npy files, in which each row is a flattened
principal component (i.e., the total size of these principal
component matrices is NUM_PC x (WIDTH*HEIGHT).
cov_file : string
File containing the covariance matrix of size NUM_PC x NUM_PC for
PCA-Flow.
covfile_sublayer : string, optional
File containing the covariance matrix for the layers (usually
biased towards the first PCs).
If PCA-Layers is used and this file is not given, use cov_file.
pc_size : tuple, optional
Size of principal components. Only required if PCs are not of size
512x256 or 1024x436.
params : dict, optional
Parameters. See parameters.py for documentation of parameters.
preset : string
Preset with useful parameter values for different datasets.
Can be one of
'pcaflow_sintel'
'pcalayers_sintel'
'pcaflow_kitti'
'pcalayers_kitti'
"""
np.random.seed(1)
self.params = defaults.get_parameters(params,preset)
cprint('[PCAFlow] Initializing.', self.params)
NC = int(self.params['NC'])
self.NC = NC
pc_u = np.load(pc_file_u)
pc_v = np.load(pc_file_v)
cov_matrix = np.load(covfile).astype('float32')
if covfile_sublayer is not None:
cov_matrix_sublayer = np.load(covfile_sublayer).astype('float32')
else:
cov_matrix_sublayer = None
pc_w = 0
pc_h = 0
if pc_size==-1:
# Try to guess principal component dimensions
if pc_u.shape[1] == 1024*436:
cprint('[PCAFLOW] Using PC dimensionality 1024 x 436', self.params)
pc_w = 1024
pc_h = 436
elif pc_v.shape[1] == 512*256:
cprint('[PCAFLOW] Using PC dimensionality 512 x 256', self.params)
pc_w = 512
pc_h = 256
else:
print('[PCAFLOW] *** ERROR *** ')
print('[PCAFLOW] Could not guess dimensionality of principal components.')
print('[PCAFLOW] Please provide as parameter.')
sys.exit(1)
self.PC = []
# Smooth principal components.
self.pc_u = self.filter_pcs(pc_u,(pc_w,pc_h)).astype('float32')
self.pc_v = self.filter_pcs(pc_v,(pc_w,pc_h)).astype('float32')
self.cov_matrix = cov_matrix
self.pc_w = pc_w
self.pc_h = pc_h
self.reshape_features=True
###############################
# Feature matcher
###############################
if self.params['features'].lower() == 'libviso' and libviso_available:
self.feature_matcher = FeatureMatcherLibviso(self.params)
elif self.params['features'].lower() == 'orb':
self.feature_matcher = FeatureMatcherORB(self.params)
elif self.params['features'].lower() == 'fast':
self.feature_matcher = FeatureMatcherFast(self.params)
elif self.params['features'].lower() == 'akaze' or not libviso_available:
self.feature_matcher = FeatureMatcherAKAZE(self.params)
else:
print('[PCAFLOW] *** ERROR ***')
print('[PCAFLOW] Unknown feature type {}. Please use "libviso" or "fast".'.format(self.params['features']))
sys.exit(1)
if self.params['n_models'] <= 1:
##############################
# Solver for PCA-Flow
##############################
self.solver = RobustQuadraticSolver(self.pc_u,
self.pc_v,
self.cov_matrix,
pc_size=(pc_w,pc_h),
params=self.params)
else:
##############################
# Solver for PCA-Layers
##############################
self.solver = EMSolver(self.pc_u, self.pc_v,
self.cov_matrix,
pc_size = (pc_w,pc_h),
params=self.params,
cov_matrix_sublayer=cov_matrix_sublayer)
self.images = deque(maxlen=2)
cprint('[PCAFLOW] Finished initializing.',self.params)
class PCAFlow:
"""
Basic PCAFlow class.
"""
def __init__(self,pc_file_u,pc_file_v,
covfile,
covfile_sublayer=None,
pc_size=-1,
params={},
preset=None):
"""
Initialize PCAFlow object.
Parameters
----------
pc_file_u, pc_file_v : string
Files containing the principal components in horizontal and
vertical direction, respectively.
These files should be .npy files, in which each row is a flattened
principal component (i.e., the total size of these principal
component matrices is NUM_PC x (WIDTH*HEIGHT).
cov_file : string
File containing the covariance matrix of size NUM_PC x NUM_PC for
PCA-Flow.
covfile_sublayer : string, optional
File containing the covariance matrix for the layers (usually
biased towards the first PCs).
If PCA-Layers is used and this file is not given, use cov_file.
pc_size : tuple, optional
Size of principal components. Only required if PCs are not of size
512x256 or 1024x436.
params : dict, optional
Parameters. See parameters.py for documentation of parameters.
preset : string
Preset with useful parameter values for different datasets.
Can be one of
'pcaflow_sintel'
'pcalayers_sintel'
'pcaflow_kitti'
'pcalayers_kitti'
"""
np.random.seed(1)
self.params = defaults.get_parameters(params,preset)
cprint('[PCAFlow] Initializing.', self.params)
NC = int(self.params['NC'])
self.NC = NC
pc_u = np.load(pc_file_u)
pc_v = np.load(pc_file_v)
cov_matrix = np.load(covfile).astype('float32')
if covfile_sublayer is not None:
cov_matrix_sublayer = np.load(covfile_sublayer).astype('float32')
else:
cov_matrix_sublayer = None
pc_w = 0
pc_h = 0
if pc_size==-1:
# Try to guess principal component dimensions
if pc_u.shape[1] == 1024*436:
cprint('[PCAFLOW] Using PC dimensionality 1024 x 436', self.params)
pc_w = 1024
pc_h = 436
elif pc_v.shape[1] == 512*256:
cprint('[PCAFLOW] Using PC dimensionality 512 x 256', self.params)
pc_w = 512
pc_h = 256
else:
print('[PCAFLOW] *** ERROR *** ')
print('[PCAFLOW] Could not guess dimensionality of principal components.')
print('[PCAFLOW] Please provide as parameter.')
sys.exit(1)
self.PC = []
# Smooth principal components.
self.pc_u = self.filter_pcs(pc_u,(pc_w,pc_h)).astype('float32')
self.pc_v = self.filter_pcs(pc_v,(pc_w,pc_h)).astype('float32')
self.cov_matrix = cov_matrix
self.pc_w = pc_w
self.pc_h = pc_h
self.reshape_features=True
###############################
# Feature matcher
###############################
if self.params['features'].lower() == 'libviso' and libviso_available:
self.feature_matcher = FeatureMatcherLibviso(self.params)
elif self.params['features'].lower() == 'orb':
self.feature_matcher = FeatureMatcherORB(self.params)
elif self.params['features'].lower() == 'fast':
self.feature_matcher = FeatureMatcherFast(self.params)
elif self.params['features'].lower() == 'akaze' or not libviso_available:
self.feature_matcher = FeatureMatcherAKAZE(self.params)
else:
print('[PCAFLOW] *** ERROR ***')
print('[PCAFLOW] Unknown feature type {}. Please use "libviso" or "fast".'.format(self.params['features']))
sys.exit(1)
if self.params['n_models'] <= 1:
##############################
# Solver for PCA-Flow
##############################
self.solver = RobustQuadraticSolver(self.pc_u,
self.pc_v,
self.cov_matrix,
pc_size=(pc_w,pc_h),
params=self.params)
else:
##############################
# Solver for PCA-Layers
##############################
self.solver = EMSolver(self.pc_u, self.pc_v,
self.cov_matrix,
pc_size = (pc_w,pc_h),
params=self.params,
cov_matrix_sublayer=cov_matrix_sublayer)
self.images = deque(maxlen=2)
cprint('[PCAFLOW] Finished initializing.',self.params)
def filter_pcs(self,matrix,size):
"""
Apply Gaussian filter to principal components.
This makes them somewhat better behaved.
"""
matrix_out = np.zeros_like(matrix)
#pdb.set_trace()
for i,m in enumerate(matrix):
m_ = m.reshape((size[1],size[0]))
matrix_out[i,:] = cv2.GaussianBlur(m_,
ksize=(0,0),
sigmaX=size[0]/200.0).flatten()
return matrix_out
def push_back(self,I):
"""
Push back frame.
When processing a streaming video, this allows to pre-compute
features only once per frame.
Parameters
----------
I : array_like
Image, usually given as H x W x 3 color image.
"""
cprint('[PCAFLOW] Adding image...', self.params)
if not (I.shape[0] == self.pc_h and I.shape[1] == self.pc_w):
self.reshape_features = True
self.shape_I_orig = I.shape
if self.params['image_blur'] > 0:
I = cv2.GaussianBlur(
I,
ksize=(int(self.params['image_blur']),int(self.params['image_blur'])),
sigmaX=-1)
cprint('[PCAFLOW] Adding image to feature matcher.', self.params)
self.feature_matcher.push_back(I)
self.images.append(I)
cprint('[PCAFLOW] Done adding image.',self.params)
def compute_flow(self,
kp1=None,kp2=None,
return_additional=[],
**kwargs
):
"""
Compute the flow.
Parameters
----------
kp1, kp2 : array_like, shape (NUM_KP,2), optional
Matrices containing keypoints in image coordinates for
first and second frame, respectively.
The first column of both matrices contains the x coordinates,
the second contains the y coordinates.
If kp1 and kp2 are given, no additional feature matching is
performed.
return_additional: array of strings, optional.
If set, return additional data. Possible entries are:
'weights' : Return flow coefficients
'keypoints' : Return matched feature points
'keypoint_labels' : Return assigned layers for keypoints
(PCA-Layers only).
'segments' : Return segmentation map
(PCA-Layers only)
'segment_flows' : For each layer, return flow.
(PCA-Layers only)
The additional data is returned as a dict with the same keys.
Example:
u,v,data = pcaflow.compute_flow(return_additional=['weights',])
weights = data['weights']
Returns
-------
u, v : array_like
U and V flow fields.
data_additional : dict, optional
See above for details. The return formats are:
'weights' : array_like, shape (NUM_PC,)
'keypoints' : tuple (array_like, array_like)
Each array has shape (NUM_KP,2).
'keypoint_labels' : array_like, shape (NUM_KP,)
'segments' : array_like, shape (WIDTH,HEIGHT)
'segment_flows' : array_like, shape (WIDTH, HEIGHT, 2, NUM_LAYERS)
"""
# Parse return_additional.
return_weights = False
return_keypoints = False
return_keypoint_labels = False
return_segments = False
return_segment_flows = False
if 'weights' in return_additional:
return_weights = True
if 'keypoints' in return_additional:
return_keypoints = True
if 'keypoint_labels' in return_additional:
return_keypoint_labels = True
if 'segments' in return_additional:
return_segments = True
if 'segment_flows' in return_additional:
return_segment_flows = True
if kp1 is not None and kp2 is not None:
# We got some initial features.
kp1_ = kp1.copy()
kp2_ = kp2.copy()
else:
kp1_,kp2_ = self.feature_matcher.get_features()
if len(kp1_) == 0:
print('[PCAFlow] Warning: No features found. Setting flow to 0.')
u = np.zeros(self.shape_I_orig[:2])
v = np.zeros_like(u)
return (u,v)
if self.params['remove_homography'] == 1:
cprint('[PCAFlow] Removing homography...', self.params)
kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)
dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
inliers = dists_new < 2
kp1_ = kp1_h
kp2_ = kp2_h
#kp1[inliers,:] = kp0[inliers,:]
I1_warped = cv2.warpPerspective(self.images[1],
H,
(self.images[1].shape[1],self.images[1].shape[0]),
flags=cv2.WARP_INVERSE_MAP+cv2.INTER_LINEAR,
borderMode=cv2.BORDER_REPLICATE,
)
elif self.params['remove_homography'] == 2:
cprint('[PCAFlow] Computing homography...', self.params)
kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)
dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
inliers = dists_new < 2
I1_warped = self.images[1]
else:
inliers = None
I1_warped = self.images[1]
H = None
kp1_orig = kp1_.copy()
kp2_orig = kp2_.copy()
if self.reshape_features:
h_orig,w_orig = self.shape_I_orig[:2]
h_orig_f = float(h_orig)
w_orig_f = float(w_orig)
scale = [self.pc_w / w_orig_f, self.pc_h / h_orig_f]
kp1_ *= scale
kp2_ *= scale
I0_ = cv2.resize(self.images[0],(self.pc_w,self.pc_h))
I1_ = cv2.resize(I1_warped,(self.pc_w,self.pc_h))
else:
I0_ = self.images[0]
I1_ = I1_warped
cprint('[PCAFLOW] %s features detected...'%kp1_.shape[0], self.params)
# Solve
if self.params['n_models'] > 1:
u_,v_,weights,data_additional_em = self.solver.solve(kp1_,kp2_,
I0=I0_,
I1=I1_,
inliers=inliers,
H=H,
shape_I_orig=self.shape_I_orig,
return_additional=return_additional,
**kwargs)
else:
if return_weights:
u_,v_,weights = self.solver.solve(kp1_,kp2_,return_coefficients=True)
else:
u_,v_ = self.solver.solve(kp1_,kp2_)
data_additional_em = {}
if self.reshape_features:
u = cv2.resize(u_,(w_orig,h_orig))
v = cv2.resize(v_,(w_orig,h_orig))
u *= w_orig_f / self.pc_w
v *= h_orig_f / self.pc_h
if self.params['remove_homography']==1:
cprint('[PCAFlow] Re-applying homography...', self.params)
u2,v2 = ht.apply_homography_to_flow(u,v,H)
u = u2
v = v2
if len(return_additional) == 0:
return u,v
else:
# Return more additional data
data_additional = {}
if return_weights:
data_additional['weights'] = weights
if return_keypoints:
data_additional['keypoints'] = (kp1_orig,kp2_orig)
# Get additional data from EMSolver
for key,value in data_additional_em.items():
data_additional[key] = value
return u, v, data_additional