def _compute_unaries_warp(self,
I0_bw,
I1_bw,
n_models,
use_homography,
homography_model,
):
"""
Compute warping based unaries, i.e. the violation of brightness
constancy given a particular flow.
"""
cprint('[GC] Computing warp models',self.params)
l_warp = np.zeros((self.pc_h,self.pc_w, n_models))
dI0dy,dI0dx = np.gradient(I0_bw)
dI1dy,dI1dx = np.gradient(I1_bw)
dx_weight = 1.0
I1_stacked = np.dstack((I1_bw,dx_weight*dI1dx,dx_weight*dI1dy))
I0_stacked = np.dstack((I0_bw,dx_weight*dI0dx,dx_weight*dI0dy))
I_valid_homography = np.ones_like(I1_bw)
for m in range(n_models):
if use_homography==1 and m == homography_model:
# I1 has the homography already removed. Nothing to do here.
I1_warped = I1_stacked
I_valid = np.ones(I1_warped.shape[:2],dtype='uint8')
else:
# Warp back by flow for current model
u = flow_u_all[m].reshape((self.pc_h,self.pc_w))
v = flow_v_all[m].reshape((self.pc_h,self.pc_w))
I1_warped,I_valid = pullback_opencv(u,v,I1_stacked)
if m == homography_model:
I_valid_homography = I_valid
df = np.abs(I0_stacked - I1_warped.astype('float32')).mean(axis=2)
D = cv2.GaussianBlur(df,ksize=(5,5),sigmaX=-1)
if self.params['model_sigma_w'] > 0:
D = 1.0 - np.exp(-(D/self.params['model_sigma_w'])**2)
l_warp[:,:,m] = D
if use_homography == 2:
cprint('[CG] Homography mean: {0}'.format(I_valid_homography.astype('float32').mean()), self.params)
l_warp[I_valid_homography<1] = 0
log_warp = (100 * self.params['model_gamma_warp'] * (l_warp - l_warp.min(axis=2)[:,:,np.newaxis])).astype('int32')
return log_warp
def push_back(self,I):
"""
Push back frame.
When processing a streaming video, this allows to pre-compute
features only once per frame.
Parameters
----------
I : array_like
Image, usually given as H x W x 3 color image.
"""
cprint('[PCAFLOW] Adding image...', self.params)
if not (I.shape[0] == self.pc_h and I.shape[1] == self.pc_w):
self.reshape_features = True
self.shape_I_orig = I.shape
if self.params['image_blur'] > 0:
I = cv2.GaussianBlur(
I,
ksize=(int(self.params['image_blur']),int(self.params['image_blur'])),
sigmaX=-1)
cprint('[PCAFLOW] Adding image to feature matcher.', self.params)
self.feature_matcher.push_back(I)
self.images.append(I)
cprint('[PCAFLOW] Done adding image.',self.params)
def _compute_unaries_warp(
self,
I0_bw,
I1_bw,
n_models,
use_homography,
homography_model,
):
"""
Compute warping based unaries, i.e. the violation of brightness
constancy given a particular flow.
"""
cprint('[GC] Computing warp models', self.params)
l_warp = np.zeros((self.pc_h, self.pc_w, n_models))
dI0dy, dI0dx = np.gradient(I0_bw)
dI1dy, dI1dx = np.gradient(I1_bw)
dx_weight = 1.0
I1_stacked = np.dstack((I1_bw, dx_weight * dI1dx, dx_weight * dI1dy))
I0_stacked = np.dstack((I0_bw, dx_weight * dI0dx, dx_weight * dI0dy))
I_valid_homography = np.ones_like(I1_bw)
for m in range(n_models):
if use_homography == 1 and m == homography_model:
# I1 has the homography already removed. Nothing to do here.
I1_warped = I1_stacked
I_valid = np.ones(I1_warped.shape[:2], dtype='uint8')
else:
# Warp back by flow for current model
u = flow_u_all[m].reshape((self.pc_h, self.pc_w))
v = flow_v_all[m].reshape((self.pc_h, self.pc_w))
I1_warped, I_valid = pullback_opencv(u, v, I1_stacked)
if m == homography_model:
I_valid_homography = I_valid
df = np.abs(I0_stacked - I1_warped.astype('float32')).mean(axis=2)
D = cv2.GaussianBlur(df, ksize=(5, 5), sigmaX=-1)
if self.params['model_sigma_w'] > 0:
D = 1.0 - np.exp(-(D / self.params['model_sigma_w'])**2)
l_warp[:, :, m] = D
if use_homography == 2:
cprint(
'[CG] Homography mean: {0}'.format(
I_valid_homography.astype('float32').mean()),
self.params)
l_warp[I_valid_homography < 1] = 0
log_warp = (
100 * self.params['model_gamma_warp'] *
(l_warp - l_warp.min(axis=2)[:, :, np.newaxis])).astype('int32')
return log_warp
def _compute_unaries_color(self, kp0, kp1, I0_, n_models, pcaflow_model,
homography_model, inliers, point_models):
"""
Compute unaries based on the color distributions of assigned features.
"""
# Build color models
cprint('[GC] Computing color models', self.params)
kp0_ = np.floor(kp0).astype('int')
point_surround = 1
if I_ndim > 2:
colors_points_ = I0_[kp0_[:, 1], kp0_[:, 0], :].reshape((-1, 3))
else:
colors_points_ = I0_[kp0_[:, 1], kp0_[:, 0]].flatten()
if I_ndim > 2:
color_all_ = I0_.reshape((-1, 3))
else:
color_all_ = I0_.flatten()
colors_points = colors_points_.astype('float32')
color_all = color_all_.astype('float32')
# Normalize to mean / std of matched points.
color_all -= colors_points.mean(axis=0)
color_all /= colors_points.std(axis=0)
colors_points -= colors_points.mean(axis=0)
colors_points /= colors_points.std(axis=0)
scores_colors = np.zeros((self.pc_h, self.pc_w, n_models))
for m in range(n_models):
# Compute indices into features at current model
if m == homography_model:
# For homography layer, use only inliers
ind = np.tile(inliers == 1, point_surround)
elif m == pcaflow_model:
# For PCAFlow layer, use all points
ind = np.ones(colors_points.shape[0]) == 1
else:
# Otherwise, use current ownerships
ind = np.tile(point_models == m, point_surround)
# Extract colors of selected features
if I_ndim > 2:
P = colors_points[ind, :]
else:
P = colors_points[ind]
cprint('[GC] Model {0}: Num points: {1}'.format(m, P.shape[0]),
self.params)
if P.shape[0] > 1:
if P.shape[0] < 10:
nc = 1
else:
# Currently, this is always one. Mixtures were of no
# advantage.
nc = self.params['model_color_n_mixtures']
# Fit Gaussian to selected color points, and compute score for
# all pixels.
G = mixture.GMM(n_components=nc, covariance_type='full').fit(P)
score = G.score(color_all)
else:
# Simple fallback.
score = np.ones(color_all.shape[0]) * -100000
S = score.reshape((self.pc_h, self.pc_w))
S = cv2.GaussianBlur(S, ksize=(5, 5), sigmaX=-1)
scores_colors[:, :, m] = S
log_colors = -(self.params['model_gamma_c'] * 100 *
(scores_colors - scores_colors.max(
axis=2)[:, :, np.newaxis])).astype('int32')
cprint('done\n', self.params)
return log_colors
def __init__(
self,
flow_bases_u, # U basis (n_bases x (width*height))
flow_bases_v, # V basis (n_bases x (widght*height))
cov_matrix, # Covariance matrix
pc_size, # (width,height) tuple
params, # params
cov_matrix_sublayer=None, # Optional different covariance matrix for sublayer solver
):
cprint('[EMSolver] Initializing ...', params)
t0 = time.time()
self.params = dict(params)
self.flow_bases_u = flow_bases_u.astype('float32')
self.flow_bases_v = flow_bases_v.astype('float32')
self.pc_w = pc_size[0]
self.pc_h = pc_size[1]
self.cov_matrix = cov_matrix.astype('float32').copy()
len_bases = len(self.flow_bases_u)
n_components_max = self.flow_bases_u.shape[0]
self.n_models = params['n_models']
# Define additional solution
self.n_models += 1
# Section to add QuadraticSolver as additional solution
params_inner = dict(self.params)
cprint('[EMSolver] Initializing additional RobustQuadraticSolver...',
self.params)
self.sub_solver_additional = RobustQuadraticSolver(flow_bases_u,
flow_bases_v,
cov_matrix,
pc_size,
params_inner,
n_iters=10)
self.use_additional = 1
self.models = np.zeros(
(params['n_models'] + self.use_additional, 2 * n_components_max),
dtype='float32')
self.model_medians = np.ones(
(params['n_models'] + self.use_additional, 2)) * -1
# If no separate covariance matrix for sublayer is provided, use the full one.
if cov_matrix_sublayer is not None:
self.cov_sublayer = cov_matrix_sublayer.copy()
else:
self.cov_sublayer = cov_matrix.copy()
params_sublayer = dict(dp.get_sublayer_parameters(self.params))
self.sub_solver = RobustQuadraticSolver(flow_bases_u,
flow_bases_v,
self.cov_sublayer,
pc_size,
params_sublayer,
n_iters=10)
self.debug_path = './output'
t1 = time.time()
cprint('[EMSolver] Done. Initialization took %2.6f secs' % (t1 - t0),
self.params)
def get_flow_GC(self,
kp0,
kp1,
weights,
I0,
I1,
inliers=None,
H=None,
shape_I_orig=None):
"""
Given models, densify using graph cut (i.e., solve labeling problem).
"""
# Determine ownership of points
use_zero_layer = False
# At this point, n_models also contains the PCA-Flow model.
n_models = self.models.shape[0]
point_models = np.argmax(weights, axis=0)
# If inliers is not zero, we want to compute a "zero" layer using the
# homography
if inliers is not None:
n_models += 1
use_zero_layer = True
use_homography = self.params['remove_homography']
n_coeffs = self.flow_bases_u.shape[0]
n_pixels = self.flow_bases_u.shape[1]
# Define general cost structures
log_unaries = np.zeros((self.pc_h, self.pc_w, n_models), dtype='int32')
log_dist = np.zeros_like(log_warp)
# Warping takes the images into account.
# Thus, we need to rescale them to the size of the principal components.
I_ndim = I0.ndim
if shape_I_orig is None:
Ih, Iw = I0.shape[:2]
else:
Ih, Iw = shape_I_orig[:2]
if I_ndim > 2:
I0_ = cv2.resize(cv2.cvtColor(I0, 45), (self.pc_w, self.pc_h))
I1_ = cv2.resize(cv2.cvtColor(I1, 45), (self.pc_w, self.pc_h))
else:
I0_ = cv2.resize(I0, (self.pc_w, self.pc_h))
I1_ = cv2.resize(I1, (self.pc_w, self.pc_h))
if I_ndim > 2:
I0_bw = I0_[:, :, 0]
I1_bw = I1_[:, :, 0]
else:
I0_bw = I0_
I1_bw = I1_
x, y = np.meshgrid(range(self.pc_w), range(self.pc_h))
# Build basis flow models
flow_u_all = np.zeros((n_models, n_pixels))
flow_v_all = np.zeros((n_models, n_pixels))
# Save indices for PCA-Flow and homography models.
# If unset, set to invalid indices to catch errors
pcaflow_model = n_models + 1
homography_model = n_models + 1
if self.params['remove_homography']:
homography_model = n_models - 1
pcaflow_model = n_models - 2
else:
pcaflow_model = n_models - 1
# For each model / layer, generate flow fields from coefficients.
for m in range(n_models):
if m == homography_model:
# If we are on the homography layer, generate from from H.
# (We generate the flow from H before downscaling it to the
# size of the PCs.)
ud = np.zeros((Ih, Iw), dtype='float32')
vd = np.zeros((Ih, Iw), dtype='float32')
if H is None:
H = np.eye(3)
ud, vd = ht.apply_homography_to_flow(ud, vd, H)
u, v = pcautils.scale_u_v(ud, vd, (self.pc_w, self.pc_h))
flow_u_all[m] = u.flatten()
flow_v_all[m] = v.flatten()
else:
# Simply create flow by weighting.
flow_u_all[m] = self.models[m, :n_coeffs].dot(
self.flow_bases_u)
flow_v_all[m] = self.models[m,
n_coeffs:].dot(self.flow_bases_v)
# Step 1: Color models
if self.params['model_gamma_c'] > 0:
log_color = self._compute_unaries_color(kp0, kp1, I0_, n_models,
pcaflow_model,
homography_model, inliers,
point_models)
log_unaries += log_color
if self.params['model_gamma_warp'] > 0:
log_warp = self._compute_unaries_warp(I0_bw, I1_bw, n_models,
use_homography,
homography_model)
log_unaries += log_warp
if self.params['model_gamma_l'] > 0:
log_dist = self._compute_unaries_location(kp0, n_models,
homography_model,
pcaflow_model,
point_models)
log_unaries += log_dist
cprint('\n', self.params)
#
# Compute pairwise terms
#
# This is a simple 0/1 error. All the weighting is done through the
# weight variables w_x, w_y.
cprint('[GC] Computing edgeweights...', self.params)
gamma = self.params['model_gamma']
log_pairwise = (-np.eye(n_models)).astype('int32')
# Compute weights according to GrabCut
gy, gx = np.gradient(I0_bw.astype('float32'))
beta = 1.0 / ((gy**2).mean() + (gx**2).mean())
w_y_gc = np.exp(-beta * gy**2)
w_x_gc = np.exp(-beta * gx**2)
w_x = (w_x_gc * 100 * gamma).astype('int32')
w_y = (w_y_gc * 100 * gamma).astype('int32')
cprint('done.\n', self.params)
cprint('[GC] Solving...', self.params)
try:
res_ = pygco.cut_simple_vh(log_unaries, log_pairwise, w_y, w_x)
except:
cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.')
res_ = pygco.cut_simple_vh(log_unaries,
log_pairwise,
w_y,
w_x,
algorithm='swap')
res = cv2.medianBlur(res_.astype('uint8'), ksize=3).astype('int32')
cprint('done.\n', self.params)
if self.params['debug'] > 1:
self.output_debug2(kp0, point_models, res, flow_u_all, flow_v_all)
u_all = flow_u_all[res.ravel(), np.arange(n_pixels)].reshape(
(self.pc_h, self.pc_w))
v_all = flow_v_all[res.ravel(), np.arange(n_pixels)].reshape(
(self.pc_h, self.pc_w))
return u_all, v_all
def solve(self,
kp0,
kp1,
soft=False,
I0=None,
I1=None,
inliers=None,
H=None,
shape_I_orig=None,
**kwargs):
"""
Solve using EM.
This is the main entry function.
"""
kp0_ = kp0.copy()
kp1_ = kp1.copy()
# Compute system in order to evaluate models.
A, b = self.get_system(kp0_, kp1_)
n_points = kp0_.shape[0]
n_models = self.params['n_models']
n_components_max = self.flow_bases_u.shape[0]
# ownership indicates which keypoint belongs to which model.
ownership = np.zeros((n_models, n_points), dtype='bool')
ownership_previous = ownership.copy()
# Distance of each keypoint to the model
dists = np.zeros((n_models, n_points), dtype='float32')
weights_all = np.zeros_like(dists)
if kwargs.has_key('debug_path'):
self.debug_path = kwargs['debug_path']
if self.use_additional:
self.models = np.zeros((n_models + 1, 2 * n_components_max),
dtype='float32')
# Defining a "median" for all points does not make sense (this would
# cause center pixels to be more likely to belong to the PCA-Flow
# solution
self.model_medians = np.ones((n_models, 2)) * -1
# For the additional (=PCAFlow) model, solve using all keypoints.
model_additional, weights_features = self.sub_solver_additional.solve(
kp0,
kp1,
return_flow=False,
return_coefficients=True,
return_weights=True,
)
self.models[-1, :] = model_additional
else:
self.models = np.zeros((n_models, 2 * n_components_max),
dtype='float32')
self.model_medians = np.ones((n_models, 2)) * -1
##############################
# Initialize
##############################
IDs = np.arange(n_points)
block_width = self.pc_w / n_models
uv = kp1_ - kp0_
data_clustering = np.c_[kp0_, uv].astype('float32')
#data_clustering = kp0_.astype('float32')
data_clustering -= data_clustering.mean(axis=0)
data_clustering /= data_clustering.std(axis=0)
# Weigh down the location features
data_clustering[:, :2] *= self.params['em_init_loc_weight']
# Use KMeans from scikit-image, since OpenCV's sklearn cannot be
# used with a given random seed.
L = KMeans(n_clusters=n_models,
max_iter=100,
tol=0.1,
precompute_distances=False,
random_state=12345).fit_predict(data_clustering)
# For each cluster, extract the points belonging to this cluster,
# and recompute the model.
for m in range(n_models):
weights = self.sub_solver.solve(
kp0_[L == m, :],
kp1_[L == m, :],
return_flow=False,
return_coefficients=True,
)[0]
self.models[m, :] = weights
ownership[m, L == m] = True
# Robustly compute median of model locations
kp0_cur = kp0_[ownership[m, :], :]
self.model_medians[m] = np.median(kp0_cur, axis=0)
USE_MEDIAN = True
MED_FACTOR = self.params['model_factor_dist_to_median']
##############################
# Iterate to get ownership
##############################
for iter in range(20):
#
# M-Step: Determine distances and ownerships
#
for m in range(n_models):
# Compute distance of all points to current model
err = (A.dot(self.models[m, :]) - b)**2
dists[m, :] = np.sqrt(err[:n_points] + err[n_points:])
# Add median to distance
if USE_MEDIAN and self.model_medians[m, 0] > -1:
dists_median = np.sqrt(
np.sum((kp0_ - self.model_medians[m])**2, axis=1))
dists[m, :] += dists_median * MED_FACTOR
# Set correct ownerships (=binary mask)
mn = np.argmin(dists, axis=0)
ownership[:] = False
ownership[mn, IDs] = True
weights_all = np.exp(-dists)
weights_all /= np.maximum(1e-9, weights_all.sum(axis=0))
# Check how many entries changed. If no change, exit.
n_change = np.sum(np.logical_xor(ownership,
ownership_previous)) / 2
cprint('Iter {0}. {1} entries changed...\n'.format(iter, n_change),
self.params)
if n_change == 0:
break
# Remove models with < 10 points
small_models = ownership.sum(axis=1) < 10
if np.any(small_models):
# Prune the empty models
m_remove = np.nonzero(small_models)[0]
print(
'[EMSolver] Removing models {} because they became too small.'
.format(m_remove))
weights_all = np.delete(weights_all, m_remove, axis=0)
ownership = np.delete(ownership, m_remove, axis=0)
ownership_previous = np.delete(ownership_previous,
m_remove,
axis=0)
self.models = np.delete(self.models, m_remove, axis=0)
self.model_medians = np.delete(self.model_medians,
m_remove,
axis=0)
dists = np.delete(dists, m_remove, axis=0)
n_models -= len(m_remove)
continue
#
# E-Step: Re-compute models
#
for m in range(n_models):
kp0_cur = kp0_[ownership[m, :], :]
kp1_cur = kp1_[ownership[m, :], :]
self.models[m, :] = self.sub_solver.solve(
kp0_cur,
kp1_cur,
return_flow=False,
return_coefficients=True)[0]
self.model_medians[m] = np.median(kp0_cur, axis=0)
ownership_previous = ownership.copy()
# Determine ownerships one last time
for m in range(n_models):
err = (A.dot(self.models[m, :]) - b)**2
dists[m, :] = np.sqrt(err[:n_points] + err[n_points:])
if USE_MEDIAN and self.model_medians[m, 0] > -1:
dists_median = np.sqrt(
np.sum((kp0_ - self.model_medians[m])**2, axis=1))
dists[m, :] += dists_median * MED_FACTOR
mn = np.argmin(dists, axis=0)
ownership[:] = False
ownership[mn, IDs] = True
weights_all = np.exp(-dists)
weights_all /= np.maximum(1e-9, weights_all.sum(axis=0))
if I0 is None or I1 is None:
print('[EMSolver] :: ERROR. No images given.')
u = None
v = None
else:
if inliers is None:
u, v = self.get_flow_GC(kp0_, kp1_, weights_all, I0, I1)
else:
u, v = self.get_flow_GC(kp0_, kp1_, weights_all, I0, I1,
inliers, H, shape_I_orig)
return u, v, self.models[0]
def __init__(self,pc_file_u,pc_file_v,
covfile,
covfile_sublayer=None,
pc_size=-1,
params={},
preset=None):
"""
Initialize PCAFlow object.
Parameters
----------
pc_file_u, pc_file_v : string
Files containing the principal components in horizontal and
vertical direction, respectively.
These files should be .npy files, in which each row is a flattened
principal component (i.e., the total size of these principal
component matrices is NUM_PC x (WIDTH*HEIGHT).
cov_file : string
File containing the covariance matrix of size NUM_PC x NUM_PC for
PCA-Flow.
covfile_sublayer : string, optional
File containing the covariance matrix for the layers (usually
biased towards the first PCs).
If PCA-Layers is used and this file is not given, use cov_file.
pc_size : tuple, optional
Size of principal components. Only required if PCs are not of size
512x256 or 1024x436.
params : dict, optional
Parameters. See parameters.py for documentation of parameters.
preset : string
Preset with useful parameter values for different datasets.
Can be one of
'pcaflow_sintel'
'pcalayers_sintel'
'pcaflow_kitti'
'pcalayers_kitti'
"""
np.random.seed(1)
self.params = defaults.get_parameters(params,preset)
cprint('[PCAFlow] Initializing.', self.params)
NC = int(self.params['NC'])
self.NC = NC
pc_u = np.load(pc_file_u)
pc_v = np.load(pc_file_v)
cov_matrix = np.load(covfile).astype('float32')
if covfile_sublayer is not None:
cov_matrix_sublayer = np.load(covfile_sublayer).astype('float32')
else:
cov_matrix_sublayer = None
pc_w = 0
pc_h = 0
if pc_size==-1:
# Try to guess principal component dimensions
if pc_u.shape[1] == 1024*436:
cprint('[PCAFLOW] Using PC dimensionality 1024 x 436', self.params)
pc_w = 1024
pc_h = 436
elif pc_v.shape[1] == 512*256:
cprint('[PCAFLOW] Using PC dimensionality 512 x 256', self.params)
pc_w = 512
pc_h = 256
else:
print('[PCAFLOW] *** ERROR *** ')
print('[PCAFLOW] Could not guess dimensionality of principal components.')
print('[PCAFLOW] Please provide as parameter.')
sys.exit(1)
self.PC = []
# Smooth principal components.
self.pc_u = self.filter_pcs(pc_u,(pc_w,pc_h)).astype('float32')
self.pc_v = self.filter_pcs(pc_v,(pc_w,pc_h)).astype('float32')
self.cov_matrix = cov_matrix
self.pc_w = pc_w
self.pc_h = pc_h
self.reshape_features=True
###############################
# Feature matcher
###############################
if self.params['features'].lower() == 'libviso' and libviso_available:
self.feature_matcher = FeatureMatcherLibviso(self.params)
elif self.params['features'].lower() == 'orb':
self.feature_matcher = FeatureMatcherORB(self.params)
elif self.params['features'].lower() == 'fast':
self.feature_matcher = FeatureMatcherFast(self.params)
elif self.params['features'].lower() == 'akaze' or not libviso_available:
self.feature_matcher = FeatureMatcherAKAZE(self.params)
else:
print('[PCAFLOW] *** ERROR ***')
print('[PCAFLOW] Unknown feature type {}. Please use "libviso" or "fast".'.format(self.params['features']))
sys.exit(1)
if self.params['n_models'] <= 1:
##############################
# Solver for PCA-Flow
##############################
self.solver = RobustQuadraticSolver(self.pc_u,
self.pc_v,
self.cov_matrix,
pc_size=(pc_w,pc_h),
params=self.params)
else:
##############################
# Solver for PCA-Layers
##############################
self.solver = EMSolver(self.pc_u, self.pc_v,
self.cov_matrix,
pc_size = (pc_w,pc_h),
params=self.params,
cov_matrix_sublayer=cov_matrix_sublayer)
self.images = deque(maxlen=2)
cprint('[PCAFLOW] Finished initializing.',self.params)
def compute_flow(self,
kp1=None,kp2=None,
return_additional=[],
**kwargs
):
"""
Compute the flow.
Parameters
----------
kp1, kp2 : array_like, shape (NUM_KP,2), optional
Matrices containing keypoints in image coordinates for
first and second frame, respectively.
The first column of both matrices contains the x coordinates,
the second contains the y coordinates.
If kp1 and kp2 are given, no additional feature matching is
performed.
return_additional: array of strings, optional.
If set, return additional data. Possible entries are:
'weights' : Return flow coefficients
'keypoints' : Return matched feature points
'keypoint_labels' : Return assigned layers for keypoints
(PCA-Layers only).
'segments' : Return segmentation map
(PCA-Layers only)
'segment_flows' : For each layer, return flow.
(PCA-Layers only)
The additional data is returned as a dict with the same keys.
Example:
u,v,data = pcaflow.compute_flow(return_additional=['weights',])
weights = data['weights']
Returns
-------
u, v : array_like
U and V flow fields.
data_additional : dict, optional
See above for details. The return formats are:
'weights' : array_like, shape (NUM_PC,)
'keypoints' : tuple (array_like, array_like)
Each array has shape (NUM_KP,2).
'keypoint_labels' : array_like, shape (NUM_KP,)
'segments' : array_like, shape (WIDTH,HEIGHT)
'segment_flows' : array_like, shape (WIDTH, HEIGHT, 2, NUM_LAYERS)
"""
# Parse return_additional.
return_weights = False
return_keypoints = False
return_keypoint_labels = False
return_segments = False
return_segment_flows = False
if 'weights' in return_additional:
return_weights = True
if 'keypoints' in return_additional:
return_keypoints = True
if 'keypoint_labels' in return_additional:
return_keypoint_labels = True
if 'segments' in return_additional:
return_segments = True
if 'segment_flows' in return_additional:
return_segment_flows = True
if kp1 is not None and kp2 is not None:
# We got some initial features.
kp1_ = kp1.copy()
kp2_ = kp2.copy()
else:
kp1_,kp2_ = self.feature_matcher.get_features()
if len(kp1_) == 0:
print('[PCAFlow] Warning: No features found. Setting flow to 0.')
u = np.zeros(self.shape_I_orig[:2])
v = np.zeros_like(u)
return (u,v)
if self.params['remove_homography'] == 1:
cprint('[PCAFlow] Removing homography...', self.params)
kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)
dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
inliers = dists_new < 2
kp1_ = kp1_h
kp2_ = kp2_h
#kp1[inliers,:] = kp0[inliers,:]
I1_warped = cv2.warpPerspective(self.images[1],
H,
(self.images[1].shape[1],self.images[1].shape[0]),
flags=cv2.WARP_INVERSE_MAP+cv2.INTER_LINEAR,
borderMode=cv2.BORDER_REPLICATE,
)
elif self.params['remove_homography'] == 2:
cprint('[PCAFlow] Computing homography...', self.params)
kp1_h, kp2_h, H, H_inv, inliers_ = ht.remove_homography_from_points(kp1_,kp2_)
dists_new = np.sqrt(np.sum((kp1_h - kp2_h)**2,axis=1))
inliers = dists_new < 2
I1_warped = self.images[1]
else:
inliers = None
I1_warped = self.images[1]
H = None
kp1_orig = kp1_.copy()
kp2_orig = kp2_.copy()
if self.reshape_features:
h_orig,w_orig = self.shape_I_orig[:2]
h_orig_f = float(h_orig)
w_orig_f = float(w_orig)
scale = [self.pc_w / w_orig_f, self.pc_h / h_orig_f]
kp1_ *= scale
kp2_ *= scale
I0_ = cv2.resize(self.images[0],(self.pc_w,self.pc_h))
I1_ = cv2.resize(I1_warped,(self.pc_w,self.pc_h))
else:
I0_ = self.images[0]
I1_ = I1_warped
cprint('[PCAFLOW] %s features detected...'%kp1_.shape[0], self.params)
# Solve
if self.params['n_models'] > 1:
u_,v_,weights,data_additional_em = self.solver.solve(kp1_,kp2_,
I0=I0_,
I1=I1_,
inliers=inliers,
H=H,
shape_I_orig=self.shape_I_orig,
return_additional=return_additional,
**kwargs)
else:
if return_weights:
u_,v_,weights = self.solver.solve(kp1_,kp2_,return_coefficients=True)
else:
u_,v_ = self.solver.solve(kp1_,kp2_)
data_additional_em = {}
if self.reshape_features:
u = cv2.resize(u_,(w_orig,h_orig))
v = cv2.resize(v_,(w_orig,h_orig))
u *= w_orig_f / self.pc_w
v *= h_orig_f / self.pc_h
if self.params['remove_homography']==1:
cprint('[PCAFlow] Re-applying homography...', self.params)
u2,v2 = ht.apply_homography_to_flow(u,v,H)
u = u2
v = v2
if len(return_additional) == 0:
return u,v
else:
# Return more additional data
data_additional = {}
if return_weights:
data_additional['weights'] = weights
if return_keypoints:
data_additional['keypoints'] = (kp1_orig,kp2_orig)
# Get additional data from EMSolver
for key,value in data_additional_em.items():
data_additional[key] = value
return u, v, data_additional
def _compute_unaries_color(self,
kp0,
kp1,
I0_,
n_models,
pcaflow_model,
homography_model,
inliers,
point_models):
"""
Compute unaries based on the color distributions of assigned features.
"""
# Build color models
cprint('[GC] Computing color models',self.params)
kp0_ = np.floor(kp0).astype('int')
point_surround = 1
if I_ndim > 2:
colors_points_ = I0_[kp0_[:,1],kp0_[:,0],:].reshape((-1,3))
else:
colors_points_ = I0_[kp0_[:,1],kp0_[:,0]].flatten()
if I_ndim > 2:
color_all_ = I0_.reshape((-1,3))
else:
color_all_ = I0_.flatten()
colors_points = colors_points_.astype('float32')
color_all = color_all_.astype('float32')
# Normalize to mean / std of matched points.
color_all -= colors_points.mean(axis=0)
color_all /= colors_points.std(axis=0)
colors_points -= colors_points.mean(axis=0)
colors_points /= colors_points.std(axis=0)
scores_colors = np.zeros((self.pc_h,self.pc_w,n_models))
for m in range(n_models):
# Compute indices into features at current model
if m == homography_model:
# For homography layer, use only inliers
ind = np.tile(inliers==1,point_surround)
elif m == pcaflow_model:
# For PCAFlow layer, use all points
ind = np.ones(colors_points.shape[0])==1
else:
# Otherwise, use current ownerships
ind = np.tile(point_models==m,point_surround)
# Extract colors of selected features
if I_ndim > 2:
P = colors_points[ind,:]
else:
P = colors_points[ind]
cprint('[GC] Model {0}: Num points: {1}'.format(m,P.shape[0]),self.params)
if P.shape[0] > 1:
if P.shape[0] < 10:
nc = 1
else:
# Currently, this is always one. Mixtures were of no
# advantage.
nc = self.params['model_color_n_mixtures']
# Fit Gaussian to selected color points, and compute score for
# all pixels.
G = mixture.GMM(n_components=nc,covariance_type='full').fit(P)
score = G.score(color_all)
else:
# Simple fallback.
score = np.ones(color_all.shape[0]) * -100000
S = score.reshape((self.pc_h,self.pc_w))
S = cv2.GaussianBlur(S,ksize=(5,5),sigmaX=-1)
scores_colors[:,:,m] = S
log_colors = - (self.params['model_gamma_c'] * 100 * (scores_colors - scores_colors.max(axis=2)[:,:,np.newaxis])).astype('int32')
cprint('done\n',self.params)
return log_colors
def __init__(self,
flow_bases_u, # U basis (n_bases x (width*height))
flow_bases_v, # V basis (n_bases x (widght*height))
cov_matrix, # Covariance matrix
pc_size, # (width,height) tuple
params, # params
cov_matrix_sublayer=None, # Optional different covariance matrix for sublayer solver
):
cprint('[EMSolver] Initializing ...', params)
t0 = time.time()
self.params = dict(params)
self.flow_bases_u = flow_bases_u.astype('float32')
self.flow_bases_v = flow_bases_v.astype('float32')
self.pc_w = pc_size[0]
self.pc_h = pc_size[1]
self.cov_matrix = cov_matrix.astype('float32').copy()
len_bases = len(self.flow_bases_u)
n_components_max = self.flow_bases_u.shape[0]
self.n_models = params['n_models']
# Define additional solution
self.n_models += 1
# Section to add QuadraticSolver as additional solution
params_inner = dict(self.params)
cprint('[EMSolver] Initializing additional RobustQuadraticSolver...', self.params)
self.sub_solver_additional = RobustQuadraticSolver(
flow_bases_u,
flow_bases_v,
cov_matrix,
pc_size,
params_inner,
n_iters=10)
self.use_additional = 1
self.models = np.zeros((params['n_models']+self.use_additional,2*n_components_max),dtype='float32')
self.model_medians = np.ones((params['n_models']+self.use_additional,2)) * -1
# If no separate covariance matrix for sublayer is provided, use the full one.
if cov_matrix_sublayer is not None:
self.cov_sublayer = cov_matrix_sublayer.copy()
else:
self.cov_sublayer = cov_matrix.copy()
params_sublayer = dict(dp.get_sublayer_parameters(self.params))
self.sub_solver = RobustQuadraticSolver(flow_bases_u,
flow_bases_v,
self.cov_sublayer,
pc_size,
params_sublayer,
n_iters=10)
self.debug_path = './output'
t1 = time.time()
cprint('[EMSolver] Done. Initialization took %2.6f secs'%(t1-t0), self.params)
def get_flow_GC(self,kp0,kp1,weights,I0,I1,inliers=None,H=None,shape_I_orig=None):
"""
Given models, densify using graph cut (i.e., solve labeling problem).
"""
# Determine ownership of points
use_zero_layer = False
# At this point, n_models also contains the PCA-Flow model.
n_models = self.models.shape[0]
point_models = np.argmax(weights,axis=0)
# If inliers is not zero, we want to compute a "zero" layer using the
# homography
if inliers is not None:
n_models += 1
use_zero_layer = True
use_homography = self.params['remove_homography']
n_coeffs = self.flow_bases_u.shape[0]
n_pixels = self.flow_bases_u.shape[1]
# Define general cost structures
log_unaries = np.zeros((self.pc_h,self.pc_w,n_models),dtype='int32')
log_dist = np.zeros_like(log_warp)
# Warping takes the images into account.
# Thus, we need to rescale them to the size of the principal components.
I_ndim = I0.ndim
if shape_I_orig is None:
Ih,Iw = I0.shape[:2]
else:
Ih,Iw = shape_I_orig[:2]
if I_ndim > 2:
I0_ = cv2.resize(cv2.cvtColor(I0,45),(self.pc_w,self.pc_h))
I1_ = cv2.resize(cv2.cvtColor(I1,45),(self.pc_w,self.pc_h))
else:
I0_ = cv2.resize(I0,(self.pc_w,self.pc_h))
I1_ = cv2.resize(I1,(self.pc_w,self.pc_h))
if I_ndim > 2:
I0_bw = I0_[:,:,0]
I1_bw = I1_[:,:,0]
else:
I0_bw = I0_
I1_bw = I1_
x,y = np.meshgrid(range(self.pc_w),range(self.pc_h))
# Build basis flow models
flow_u_all = np.zeros((n_models,n_pixels))
flow_v_all = np.zeros((n_models,n_pixels))
# Save indices for PCA-Flow and homography models.
# If unset, set to invalid indices to catch errors
pcaflow_model = n_models+1
homography_model = n_models+1
if self.params['remove_homography']:
homography_model = n_models-1
pcaflow_model = n_models - 2
else:
pcaflow_model = n_models - 1
# For each model / layer, generate flow fields from coefficients.
for m in range(n_models):
if m == homography_model:
# If we are on the homography layer, generate from from H.
# (We generate the flow from H before downscaling it to the
# size of the PCs.)
ud = np.zeros((Ih,Iw),dtype='float32')
vd = np.zeros((Ih,Iw),dtype='float32')
if H is None:
H = np.eye(3)
ud,vd = ht.apply_homography_to_flow(ud,vd,H)
u,v = pcautils.scale_u_v(ud,vd,(self.pc_w,self.pc_h))
flow_u_all[m] = u.flatten()
flow_v_all[m] = v.flatten()
else:
# Simply create flow by weighting.
flow_u_all[m] = self.models[m,:n_coeffs].dot(self.flow_bases_u)
flow_v_all[m] = self.models[m,n_coeffs:].dot(self.flow_bases_v)
# Step 1: Color models
if self.params['model_gamma_c'] > 0:
log_color = self._compute_unaries_color(kp0,
kp1,
I0_,
n_models,
pcaflow_model,
homography_model,
inliers,
point_models)
log_unaries += log_color
if self.params['model_gamma_warp'] > 0:
log_warp = self._compute_unaries_warp(I0_bw,
I1_bw,
n_models,
use_homography,
homography_model)
log_unaries += log_warp
if self.params['model_gamma_l'] > 0:
log_dist = self._compute_unaries_location(kp0,
n_models,
homography_model,
pcaflow_model,
point_models)
log_unaries += log_dist
cprint('\n',self.params)
#
# Compute pairwise terms
#
# This is a simple 0/1 error. All the weighting is done through the
# weight variables w_x, w_y.
cprint('[GC] Computing edgeweights...',self.params)
gamma = self.params['model_gamma']
log_pairwise = (-np.eye(n_models)).astype('int32')
# Compute weights according to GrabCut
gy,gx = np.gradient(I0_bw.astype('float32'))
beta = 1.0 / ((gy**2).mean() + (gx**2).mean())
w_y_gc = np.exp(- beta * gy**2)
w_x_gc = np.exp(- beta * gx**2)
w_x = (w_x_gc * 100 * gamma).astype('int32')
w_y = (w_y_gc * 100 * gamma).astype('int32')
cprint('done.\n',self.params)
cprint('[GC] Solving...',self.params)
try:
res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x)
except:
cprint('[GC] *** Alpha expansion failed. Using alpha-beta swap.')
res_ = pygco.cut_simple_vh(log_unaries,log_pairwise,w_y,w_x,algorithm='swap')
res = cv2.medianBlur(res_.astype('uint8'),ksize=3).astype('int32')
cprint('done.\n',self.params)
if self.params['debug']>1:
self.output_debug2(kp0,point_models,res,flow_u_all,flow_v_all)
u_all = flow_u_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w))
v_all = flow_v_all[res.ravel(),np.arange(n_pixels)].reshape((self.pc_h,self.pc_w))
return u_all,v_all
def solve(self,kp0,kp1,soft=False,I0=None,I1=None,inliers=None,H=None,shape_I_orig=None,**kwargs):
"""
Solve using EM.
This is the main entry function.
"""
kp0_ = kp0.copy()
kp1_ = kp1.copy()
# Compute system in order to evaluate models.
A,b = self.get_system(kp0_,kp1_)
n_points = kp0_.shape[0]
n_models = self.params['n_models']
n_components_max = self.flow_bases_u.shape[0]
# ownership indicates which keypoint belongs to which model.
ownership = np.zeros((n_models,n_points),dtype='bool')
ownership_previous = ownership.copy()
# Distance of each keypoint to the model
dists = np.zeros((n_models,n_points),dtype='float32')
weights_all = np.zeros_like(dists)
if kwargs.has_key('debug_path'):
self.debug_path = kwargs['debug_path']
if self.use_additional:
self.models = np.zeros((n_models+1,2*n_components_max),dtype='float32')
# Defining a "median" for all points does not make sense (this would
# cause center pixels to be more likely to belong to the PCA-Flow
# solution
self.model_medians = np.ones((n_models,2)) * -1
# For the additional (=PCAFlow) model, solve using all keypoints.
model_additional,weights_features = self.sub_solver_additional.solve(
kp0,
kp1,
return_flow=False,
return_coefficients=True,
return_weights=True,
)
self.models[-1,:] = model_additional
else:
self.models = np.zeros((n_models,2*n_components_max),dtype='float32')
self.model_medians = np.ones((n_models,2)) * -1
##############################
# Initialize
##############################
IDs = np.arange(n_points)
block_width = self.pc_w / n_models
uv = kp1_-kp0_
data_clustering = np.c_[kp0_,uv].astype('float32')
#data_clustering = kp0_.astype('float32')
data_clustering -= data_clustering.mean(axis=0)
data_clustering /= data_clustering.std(axis=0)
# Weigh down the location features
data_clustering[:,:2] *= self.params['em_init_loc_weight']
# Use KMeans from scikit-image, since OpenCV's sklearn cannot be
# used with a given random seed.
L = KMeans(n_clusters=n_models,
max_iter=100,
tol=0.1,
precompute_distances=False,
random_state=12345).fit_predict(data_clustering)
# For each cluster, extract the points belonging to this cluster,
# and recompute the model.
for m in range(n_models):
weights = self.sub_solver.solve(
kp0_[L==m,:],
kp1_[L==m,:],
return_flow=False,
return_coefficients=True,
)[0]
self.models[m,:] = weights
ownership[m,L==m] = True
# Robustly compute median of model locations
kp0_cur = kp0_[ownership[m,:],:]
self.model_medians[m] = np.median(kp0_cur,axis=0)
USE_MEDIAN = True
MED_FACTOR = self.params['model_factor_dist_to_median']
##############################
# Iterate to get ownership
##############################
for iter in range(20):
#
# M-Step: Determine distances and ownerships
#
for m in range(n_models):
# Compute distance of all points to current model
err = (A.dot(self.models[m,:])-b)**2
dists[m,:] = np.sqrt(err[:n_points]+err[n_points:])
# Add median to distance
if USE_MEDIAN and self.model_medians[m,0] > -1:
dists_median = np.sqrt(np.sum((kp0_ - self.model_medians[m])**2,axis=1))
dists[m,:] += dists_median * MED_FACTOR
# Set correct ownerships (=binary mask)
mn = np.argmin(dists,axis=0)
ownership[:] = False
ownership[mn,IDs] = True
weights_all = np.exp(-dists)
weights_all /= np.maximum(1e-9,weights_all.sum(axis=0))
# Check how many entries changed. If no change, exit.
n_change = np.sum(np.logical_xor(ownership,ownership_previous))/2
cprint('Iter {0}. {1} entries changed...\n'.format(iter,n_change),self.params)
if n_change == 0:
break
# Remove models with < 10 points
small_models = ownership.sum(axis=1) < 10
if np.any(small_models):
# Prune the empty models
m_remove = np.nonzero(small_models)[0]
print('[EMSolver] Removing models {} because they became too small.'.format(m_remove))
weights_all = np.delete(weights_all,m_remove,axis=0)
ownership = np.delete(ownership,m_remove,axis=0)
ownership_previous = np.delete(ownership_previous,m_remove,axis=0)
self.models = np.delete(self.models,m_remove,axis=0)
self.model_medians = np.delete(self.model_medians,m_remove,axis=0)
dists = np.delete(dists,m_remove,axis=0)
n_models -= len(m_remove)
continue
#
# E-Step: Re-compute models
#
for m in range(n_models):
kp0_cur = kp0_[ownership[m,:],:]
kp1_cur = kp1_[ownership[m,:],:]
self.models[m,:] = self.sub_solver.solve(kp0_cur,kp1_cur,
return_flow=False,
return_coefficients=True)[0]
self.model_medians[m] = np.median(kp0_cur,axis=0)
ownership_previous = ownership.copy()
# Determine ownerships one last time
for m in range(n_models):
err = (A.dot(self.models[m,:])-b)**2
dists[m,:] = np.sqrt(err[:n_points]+err[n_points:])
if USE_MEDIAN and self.model_medians[m,0] > -1:
dists_median = np.sqrt(np.sum((kp0_ - self.model_medians[m])**2,axis=1))
dists[m,:] += dists_median * MED_FACTOR
mn = np.argmin(dists,axis=0)
ownership[:] = False
ownership[mn,IDs] = True
weights_all = np.exp(-dists)
weights_all /= np.maximum(1e-9,weights_all.sum(axis=0))
if I0 is None or I1 is None:
print('[EMSolver] :: ERROR. No images given.')
u = None
v = None
else:
if inliers is None:
u,v = self.get_flow_GC(kp0_,kp1_,weights_all,I0,I1)
else:
u,v = self.get_flow_GC(kp0_,kp1_,weights_all,
I0,I1,
inliers,H,shape_I_orig)
return u,v,self.models[0]
train_loss += loss.data[0]
step_cnt += 1
# backward
optimizer.zero_grad()
loss.backward()
network.clip_gradient(net, 10.)
optimizer.step()
if step % disp_interval == 0:
duration = t.toc(average=False)
fps = step_cnt / duration
log_text = 'step %d, image: %s, loss: %.4f, fps: %.2f (%.2fs per batch)' % (
step, blobs['im_name'], train_loss / step_cnt, fps, 1. / fps)
cprint(log_text, prefix='[.green][.bold]')
if _DEBUG:
cprint(
'\tTP: %.2f%%, TF: %.2f%%, fg/bg=(%d/%d)' %
(tp / fg * 100., tf / bg * 100., fg / step_cnt, bg / step_cnt))
cprint(
'\trpn_cls: %.4f, rpn_box: %.4f, rcnn_cls: %.4f, rcnn_box: %.4f'
% (net.rpn.cross_entropy.data.cpu().numpy()[0],
net.rpn.loss_box.data.cpu().numpy()[0],
net.cross_entropy.data.cpu().numpy()[0],
net.loss_box.data.cpu().numpy()[0]))
re_cnt = True
if use_tensorboard and step % log_interval == 0:
exp.add_scalar_value('train_loss', train_loss / step_cnt, step=step)
