def test_operator_crop_non_centered(self):
''' Non-centered Crop Operator '''
# Generate Crop Operator
crop_size = (image_size[0] // 2, image_size[1] // 2)
crop_start = (6, 6)
CR = ops.Crop(image_size, crop_size, pad_value=0, dtype=global_dtype, backend=global_backend, crop_start=crop_start)
# Check forward operator
y_1 = yp.changeBackend(CR * self.x, 'numpy')
y_2 = yp.changeBackend(yp.crop(self.x, crop_size, crop_start), 'numpy')
assert yp.sumb(yp.abs(y_1 - y_2)) < eps
# Check Adjoint Operator
pad_size = [int((image_size[i] - crop_size[i]) / 2) for i in range(len(image_size))]
y_3 = yp.pad(yp.crop(self.x, crop_size, crop_start), image_size, crop_start, pad_value=0)
y_4 = yp.reshape(CR.H * CR * self.x, image_size)
assert yp.sumb(yp.abs(y_3 - y_4)) < eps
# Check gradient
CR.gradient_check()
def test_operator_crop(self):
''' Crop Operator '''
# Generate Crop Operator
crop_size = (image_size[0] // 2, image_size[1] // 2)
CR = ops.Crop(image_size, crop_size, pad_value=0, dtype=global_dtype, backend=global_backend)
# Check forward operator
crop_start = tuple(np.asarray(image_size) // 2 - np.asarray(crop_size) // 2)
y_1 = yp.changeBackend(CR * self.x, 'numpy')
y_2 = yp.changeBackend(yp.crop(self.x, crop_size, crop_start), 'numpy')
assert yp.sumb(yp.abs(y_1 - y_2)) < eps
# Check Adjoint Operator
pad_size = [int((image_size[i] - crop_size[i]) / 2) for i in range(len(image_size))]
y_3 = yp.pad(yp.crop(self.x, crop_size, crop_start), image_size, crop_start, pad_value=0)
y_4 = CR.H * CR * self.x
assert yp.sumb(yp.abs(y_3 - y_4)) < eps
# Check gradient
CR.gradient_check()
def VecStack(vector_list, axis=0):
""" This is a helper function to stack vectors """
# Determine output size
single_vector_shape = [
max([shape(vector)[0] for vector in vector_list]),
max([shape(vector)[1] for vector in vector_list])
]
vector_shape = dcopy(single_vector_shape)
vector_shape[axis] *= len(vector_list)
# Allocate memory
vector_full = zeros(vector_shape, getDatatype(vector_list[0]),
getBackend(vector_list[0]))
# Assign vector elements to the output
for index, vector in enumerate(vector_list):
vector_full[index * single_vector_shape[0]:(index + 1) *
single_vector_shape[0], :] = pad(vector,
single_vector_shape)
# Return
return vector_full
def test_pad(self):
pad_size = [self.x_np_randn.shape[i] + 10 for i in range(len(shape))]
assert np.sum(
np.abs(
bops.pad(self.x_np_randn, pad_size, crop_start=(0, 0)) - np.
asarray(bops.pad(self.x_ocl_randn, pad_size, crop_start=(
0, 0))))) < 1e-4
assert np.sum(
np.abs(
bops.pad(self.x_np_randn, pad_size, crop_start=(5, 2)) - np.
asarray(bops.pad(self.x_ocl_randn, pad_size, crop_start=(
5, 2))))) < 1e-4
assert np.sum(
np.abs(
bops.pad(
self.x_np_randn, pad_size, crop_start=(
1, 3), pad_value=4) - np.asarray(
bops.pad(self.x_ocl_randn,
pad_size,
crop_start=(1, 3),
pad_value=4)))) < 1e-4
def registerImage(image0,
image1,
method='xc',
axis=None,
preprocess_methods=['reflect'],
debug=False,
**kwargs):
# Perform preprocessing
if len(preprocess_methods) > 0:
image0, image1 = _preprocessForRegistration(image0, image1,
preprocess_methods,
**kwargs)
# Parameter on whether we can trust our registration
trust_ratio = 1.0
if method in ['xc' or 'cross_correlation']:
# Get energy ratio threshold
trust_threshold = kwargs.get('energy_ratio_threshold', 1.5)
# Pad arrays for optimal speed
pad_size = tuple(
[sp.fftpack.next_fast_len(s) for s in yp.shape(image0)])
# Perform padding
if pad_size is not yp.shape(image0):
image0 = yp.pad(image0, pad_size, pad_value='edge', center=True)
image1 = yp.pad(image1, pad_size, pad_value='edge', center=True)
# Take F.T. of measurements
src_freq, target_freq = yp.Ft(image0, axes=axis), yp.Ft(image1,
axes=axis)
# Whole-pixel shift - Compute cross-correlation by an IFFT
image_product = src_freq * yp.conj(target_freq)
# image_product /= abs(src_freq * yp.conj(target_freq))
cross_correlation = yp.iFt(image_product, center=False, axes=axis)
# Take sum along axis if we're doing 1D
if axis is not None:
axis_to_sum = list(range(yp.ndim(image1)))
del axis_to_sum[axis]
cross_correlation = yp.sum(cross_correlation, axis=axis_to_sum)
# Locate maximum
shape = yp.shape(src_freq)
maxima = yp.argmax(yp.abs(cross_correlation))
midpoints = np.array([np.fix(axis_size / 2) for axis_size in shape])
shifts = np.array(maxima, dtype=np.float64)
shifts[shifts > midpoints] -= np.array(shape)[shifts > midpoints]
# If its only one row or column the shift along that dimension has no
# effect. We set to zero.
for dim in range(yp.ndim(src_freq)):
if shape[dim] == 1:
shifts[dim] = 0
# If energy ratio is too small, set all shifts to zero
trust_metric = yp.scalar(
yp.max(yp.abs(cross_correlation)**2) /
yp.mean(yp.abs(cross_correlation)**2))
# Determine if this registraition can be trusted
trust_ratio = trust_metric / trust_threshold
elif method == 'orb':
# Get user-defined mean_residual_threshold if given
trust_threshold = kwargs.get('mean_residual_threshold', 40.0)
# Get user-defined mean_residual_threshold if given
orb_feature_threshold = kwargs.get('orb_feature_threshold', 25)
match_count = 0
fast_threshold = 0.05
while match_count < orb_feature_threshold:
descriptor_extractor = ORB(n_keypoints=500,
fast_n=9,
harris_k=0.1,
fast_threshold=fast_threshold)
# Extract keypoints from first frame
descriptor_extractor.detect_and_extract(
np.asarray(image0).astype(np.double))
keypoints0 = descriptor_extractor.keypoints
descriptors0 = descriptor_extractor.descriptors
# Extract keypoints from second frame
descriptor_extractor.detect_and_extract(
np.asarray(image1).astype(np.double))
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
# Set match count
match_count = min(len(keypoints0), len(keypoints1))
fast_threshold -= 0.01
if fast_threshold == 0:
raise RuntimeError(
'Could not find any keypoints (even after shrinking fast threshold).'
)
# Match descriptors
matches = match_descriptors(descriptors0,
descriptors1,
cross_check=True)
# Filter descriptors to axes (if provided)
if axis is not None:
matches_filtered = []
for (index_0, index_1) in matches:
point_0 = keypoints0[index_0, :]
point_1 = keypoints1[index_1, :]
unit_vec = point_0 - point_1
unit_vec /= np.linalg.norm(unit_vec)
if yp.abs(unit_vec[axis]) > 0.99:
matches_filtered.append((index_0, index_1))
matches_filtered = np.asarray(matches_filtered)
else:
matches_filtered = matches
# Robustly estimate affine transform model with RANSAC
model_robust, inliers = ransac((keypoints0[matches_filtered[:, 0]],
keypoints1[matches_filtered[:, 1]]),
EuclideanTransform,
min_samples=3,
residual_threshold=2,
max_trials=100)
# Note that model_robust has a translation property, but this doesn't
# seem to be as numerically stable as simply averaging the difference
# between the coordinates along the desired axis.
# Apply match filter
matches_filtered = matches_filtered[inliers, :]
# Process keypoints
if yp.shape(matches_filtered)[0] > 0:
# Compute shifts
difference = keypoints0[matches_filtered[:, 0]] - keypoints1[
matches_filtered[:, 1]]
shifts = (yp.sum(difference, axis=0) / yp.shape(difference)[0])
shifts = np.round(shifts[0])
# Filter to axis mask
if axis is not None:
_shifts = [0, 0]
_shifts[axis] = shifts[axis]
shifts = _shifts
# Calculate residuals
residuals = yp.sqrt(
yp.sum(
yp.abs(keypoints0[matches_filtered[:, 0]] +
np.asarray(shifts) -
keypoints1[matches_filtered[:, 1]])**2))
# Define a trust metric
trust_metric = residuals / yp.shape(
keypoints0[matches_filtered[:, 0]])[0]
# Determine if this registration can be trusted
trust_ratio = 1 / (trust_metric / trust_threshold)
print('===')
print(trust_ratio)
print(trust_threshold)
print(trust_metric)
print(shifts)
else:
trust_metric = 1e10
trust_ratio = 0.0
shifts = np.asarray([0, 0])
elif method == 'optimize':
# Create Operators
L2 = ops.L2Norm(yp.shape(image0), dtype='complex64')
R = ops.PhaseRamp(yp.shape(image0), dtype='complex64')
REAL = ops.RealFilter((2, 1), dtype='complex64')
# Take Fourier Transforms of images
image0_f, image1_f = yp.astype(yp.Ft(image0), 'complex64'), yp.astype(
yp.Ft(image1), 'complex64')
# Diagonalize one of the images
D = ops.Diagonalize(image0_f)
# Form objective
objective = L2 * (D * R * REAL - image1_f)
# Solve objective
solver = ops.solvers.GradientDescent(objective)
shifts = solver.solve(iteration_count=1000, step_size=1e-8)
# Convert to numpy array, take real part, and round.
shifts = yp.round(yp.real(yp.asbackend(shifts, 'numpy')))
# Flip shift axes (x,y to y, x)
shifts = np.fliplr(shifts)
# TODO: Trust metric and trust_threshold
trust_threshold = 1
trust_ratio = 1.0
else:
raise ValueError('Invalid Registration Method %s' % method)
# Mark whether or not this measurement is of good quality
if not trust_ratio > 1:
if debug:
print('Ignoring shift with trust metric %g (threshold is %g)' %
(trust_metric, trust_threshold))
shifts = yp.zeros_like(np.asarray(shifts)).tolist()
# Show debugging figures if requested
if debug:
import matplotlib.pyplot as plt
plt.figure(figsize=(6, 5))
plt.subplot(131)
plt.imshow(yp.abs(image0))
plt.axis('off')
plt.subplot(132)
plt.imshow(yp.abs(image1))
plt.title('Trust ratio: %g' % (trust_ratio))
plt.axis('off')
plt.subplot(133)
if method in ['xc' or 'cross_correlation']:
if axis is not None:
plt.plot(yp.abs(yp.squeeze(cross_correlation)))
else:
plt.imshow(yp.abs(yp.fftshift(cross_correlation)))
else:
plot_matches(plt.gca(), yp.real(image0), yp.real(image1),
keypoints0, keypoints1, matches_filtered)
plt.title(str(shifts))
plt.axis('off')
# Return
return shifts, trust_ratio
def _preprocessForRegistration(image0, image1, methods=['pad'], **kwargs):
# Ensure methods argument is a list
if type(methods) not in (list, tuple):
methods = [methods]
# Perform 'reflection', which pads an object with antisymmetric copies of itself
if 'pad' in methods:
# Get pad factor
pad_factor = kwargs.get('pad_factor', 2)
# Get pad value
pad_type = kwargs.get('pad_type', 'reflect')
# Generate pad operator which pads the object to 2x it's size
pad_size = [
sp.fftpack.next_fast_len(int(pad_factor * s))
for s in yp.shape(image0)
]
# Perform padding
image0 = yp.pad(image0, pad_size, pad_value=pad_type, center=True)
image1 = yp.pad(image1, pad_size, pad_value=pad_type, center=True)
# Normalize to the range [0,1] (Always do if we're filtering)
if 'normalize' in methods:
image0 = filters._normalize(image0)
image1 = filters._normalize(image1)
# Sobel filtering
if 'sobel' in methods:
image0 = filters.sobel(image0)
image1 = filters.sobel(image1)
# Gaussian filtering
if 'gaussian' in methods:
image0 = filters.gaussian(image0, sigma=1)
image1 = filters.gaussian(image1, sigma=1)
# High-pass filtering (using gaussian)
if 'highpass' in methods:
image0 = filters.gaussian(image0, sigma=2) - filters.gaussian(image0,
sigma=4)
image1 = filters.gaussian(image1, sigma=2) - filters.gaussian(image1,
sigma=4)
# Roberts filtering
if 'roberts' in methods:
image0 = filters.roberts(image0)
image1 = filters.roberts(image1)
# Scharr filtering
if 'scharr' in methods:
image0 = filters.scharr(image0)
image1 = filters.scharr(image1)
# Prewitt filtering
if 'prewitt' in methods:
image0 = filters.prewitt(image0)
image1 = filters.prewitt(image1)
# Canny filtering
if 'canny' in methods:
image0 = filters.canny(image0,
sigma=kwargs.get('sigma', 1),
low_threshold=kwargs.get('low_threshold', 0.01),
high_threshold=kwargs.get(
'high_threshold', 0.05))
image1 = filters.canny(image1,
sigma=kwargs.get('sigma', 1),
low_threshold=kwargs.get('low_threshold', 0.01),
high_threshold=kwargs.get(
'high_threshold', 0.05))
return image0, image1