def align_frames(self):
"""
Compute the displacement of all frames relative to the sharpest frame using the alignment
rectangle.
:return: -
"""
if self.x_low_opt is None:
raise WrongOrderingError(
"Method 'align_frames' is called before 'select_alignment_rect'"
)
# Initialize a list which for each frame contains the shifts in y and x directions.
self.frame_shifts = []
# From the sharpest frame cut out the alignment rectangle. The shifts of all other frames
# will be computed relativ to this patch.
self.reference_window = self.frames_mono[self.frame_ranks_max_index][
self.y_low_opt:self.y_high_opt, self.x_low_opt:self.x_high_opt]
self.reference_window_shape = self.reference_window.shape
for idx, frame in enumerate(self.frames_mono):
# For the sharpest frame the displacement is 0 because it is used as the reference.
if idx == self.frame_ranks_max_index:
self.frame_shifts.append([0, 0])
# For all other frames: Cut out the alignment patch and compute its translation
# relative to the reference.
else:
frame_window = self.frames_mono[idx][
self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt]
self.frame_shifts.append(
Miscellaneous.translation(self.reference_window,
frame_window,
self.reference_window_shape))
# Compute the shape of the area contained in all frames in the form [[y_low, y_high],
# [x_low, x_high]]
self.intersection_shape = [[
max(b[0] for b in self.frame_shifts),
min(b[0] for b in self.frame_shifts) + self.shape[0]
],
[
max(b[1] for b in self.frame_shifts),
min(b[1] for b in self.frame_shifts) +
self.shape[1]
]]
self.intersection_number_pixels = (self.intersection_shape[0][1] -
self.intersection_shape[0][0]) * \
(self.intersection_shape[1][1] -
self.intersection_shape[1][0])
def align_frames(self):
"""
Compute the displacement of all frames relative to the sharpest frame using the alignment
rectangle.
:return: -
"""
if self.configuration.align_frames_mode == "Surface":
# For "Surface" mode the alignment rectangle has to be selected first.
if self.x_low_opt is None:
raise WrongOrderingError(
"Method 'align_frames' is called before 'select_alignment_rect'"
)
# From the sharpest frame cut out the alignment rectangle. The shifts of all other frames
# will be computed relativ to this patch.
if self.configuration.align_frames_method == "MultiLevelCorrelation":
# MultiLevelCorrelation uses two reference windows with different resolution. Also,
# please note that the data type is float32 in this case.
reference_frame = self.frames.frames_mono_blurred(
self.frame_ranks_max_index).astype(float32)
self.reference_window = reference_frame[
self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt]
# For the first phase a box with half the resolution is constructed.
self.reference_window_first_phase = self.reference_window[::
2, ::
2]
else:
# For all other methods, the reference window is of type int32.
reference_frame = self.frames.frames_mono_blurred(
self.frame_ranks_max_index).astype(int32)
self.reference_window = reference_frame[
self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt]
self.reference_window_shape = self.reference_window.shape
elif self.configuration.align_frames_mode == "Planet":
# For "Planetary" mode compute the center of gravity for the reference image.
cog_reference_y, cog_reference_x = AlignFrames.center_of_gravity(
self.frames.frames_mono_blurred(self.frame_ranks_max_index))
else:
raise NotSupportedError("Frame alignment mode '" +
self.configuration.align_frames_mode +
"' not supported")
# Initialize a list which for each frame contains the shifts in y and x directions.
self.frame_shifts = [None] * self.frames.number
# Initialize a counter of processed frames for progress bar signalling. It is set to one
# because in the loop below the optimal frame is not counted.
number_processed = 1
# Loop over all frames. Begin with the sharpest (reference) frame
for idx in chain(reversed(range(self.frame_ranks_max_index + 1)),
range(self.frame_ranks_max_index,
self.frames.number)):
if idx == self.frame_ranks_max_index:
# For the sharpest frame the displacement is 0 because it is used as the reference.
self.frame_shifts[idx] = [0, 0]
# Initialize two variables which keep the shift values of the previous step as
# the starting point for the next step. This reduces the search radius if frames are
# drifting.
dy_min_cum = dx_min_cum = 0
# For all other frames: Compute the global shift, using the "blurred" monochrome image.
else:
# After every "signal_step_size"th frame, send a progress signal to the main GUI.
if self.progress_signal is not None and number_processed % self.signal_step_size == 1:
self.progress_signal.emit(
"Align all frames",
int(
round(10 * number_processed / self.frames.number) *
10))
frame = self.frames.frames_mono_blurred(idx)
# In Planetary mode the shift of the "center of gravity" of the image is computed.
# This algorithm cannot fail.
if self.configuration.align_frames_mode == "Planet":
cog_frame = AlignFrames.center_of_gravity(frame)
self.frame_shifts[idx] = [
cog_reference_y - cog_frame[0],
cog_reference_x - cog_frame[1]
]
# In Surface mode various methods can be used to measure the shift from one frame
# to the next. The method "Translation" is special: Using phase correlation it is
# the only method not based on a local search algorithm. It is treated differently
# here because it does not require a re-shifting of the alignment patch.
elif self.configuration.align_frames_method == "Translation":
# The shift is computed with cross-correlation. Cut out the alignment patch and
# compute its translation relative to the reference.
frame_window = self.frames.frames_mono_blurred(
idx)[self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt]
self.frame_shifts[idx] = Miscellaneous.translation(
self.reference_window, frame_window,
self.reference_window_shape)
# Now treat all "Surface" mode cases using local search algorithms. In each case
# the result is the shift vector [dy_min, dx_min]. The search can fail (if within
# the search radius no optimum is found). If that happens for at least one frame,
# an exception is raised. The workflow thread then tries again using another
# alignment patch.
else:
if self.configuration.align_frames_method == "MultiLevelCorrelation":
# The shift is computed in two phases: First on a coarse pixel grid,
# and then on the original grid in a small neighborhood around the optimum
# found in the first phase.
shift_y_local_first_phase, shift_x_local_first_phase, \
success_first_phase, shift_y_local_second_phase, \
shift_x_local_second_phase, success_second_phase = \
Miscellaneous.multilevel_correlation(
self.reference_window_first_phase, frame,
self.configuration.frames_gauss_width,
self.reference_window, self.y_low_opt - dy_min_cum,
self.y_high_opt - dy_min_cum,
self.x_low_opt - dx_min_cum,
self.x_high_opt - dx_min_cum,
self.configuration.align_frames_search_width,
weight_matrix_first_phase=None)
success = success_first_phase and success_second_phase
if success:
[dy_min, dx_min] = [
shift_y_local_first_phase +
shift_y_local_second_phase,
shift_x_local_first_phase +
shift_x_local_second_phase
]
elif self.configuration.align_frames_method == "RadialSearch":
# Spiral out from the shift position of the previous frame and search for the
# local optimum.
[dy_min,
dx_min], dev_r = Miscellaneous.search_local_match(
self.reference_window,
frame,
self.y_low_opt - dy_min_cum,
self.y_high_opt - dy_min_cum,
self.x_low_opt - dx_min_cum,
self.x_high_opt - dx_min_cum,
self.configuration.align_frames_search_width,
self.configuration.align_frames_sampling_stride,
sub_pixel=False)
# The search was not successful if a zero shift was reported after more
# than two search cycles.
success = len(dev_r) <= 2 or dy_min != 0 or dx_min != 0
elif self.configuration.align_frames_method == "SteepestDescent":
# Spiral out from the shift position of the previous frame and search for the
# local optimum.
[dy_min, dx_min
], dev_r = Miscellaneous.search_local_match_gradient(
self.reference_window, frame,
self.y_low_opt - dy_min_cum,
self.y_high_opt - dy_min_cum,
self.x_low_opt - dx_min_cum,
self.x_high_opt - dx_min_cum,
self.configuration.align_frames_search_width,
self.configuration.align_frames_sampling_stride,
self.dev_table)
# The search was not successful if a zero shift was reported after more
# than two search cycles.
success = len(dev_r) <= 2 or dy_min != 0 or dx_min != 0
else:
raise NotSupportedError(
"Frame alignment method " +
configuration.align_frames_method +
" not supported")
# If the local search was unsuccessful, quit the frame loop with an error.
if not success:
raise InternalError("frame " + str(idx))
# Update the cumulative shift values to be used as starting point for the
# next frame.
dy_min_cum += dy_min
dx_min_cum += dx_min
self.frame_shifts[idx] = [dy_min_cum, dx_min_cum]
# If the alignment window gets too close to a frame edge, move it away from
# that edge by half the border width. First check if the reference window still
# fits into the shifted frame.
if self.shape[0] - abs(
dy_min_cum) - 2 * self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width < \
self.reference_window_shape[0] or self.shape[1] - abs(
dx_min_cum) - 2 * self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width < \
self.reference_window_shape[1]:
raise ArgumentError(
"Frame stabilization window does not fit into"
" intersection")
new_reference_window = False
# Start with the lower y edge.
while self.y_low_opt - dy_min_cum < \
self.configuration.align_frames_search_width + \
self.configuration.align_frames_border_width / 2:
self.y_low_opt += ceil(
self.configuration.align_frames_border_width / 2.)
self.y_high_opt += ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# Now the upper y edge.
while self.y_high_opt - dy_min_cum > self.shape[
0] - self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width / 2:
self.y_low_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
self.y_high_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# Now the lower x edge.
while self.x_low_opt - dx_min_cum < \
self.configuration.align_frames_search_width + \
self.configuration.align_frames_border_width / 2:
self.x_low_opt += ceil(
self.configuration.align_frames_border_width / 2.)
self.x_high_opt += ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# Now the upper x edge.
while self.x_high_opt - dx_min_cum > self.shape[
1] - self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width / 2:
self.x_low_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
self.x_high_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# If the window was moved, update the "reference window(s)".
if new_reference_window:
if self.configuration.align_frames_method == "MultiLevelCorrelation":
self.reference_window = reference_frame[
self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt]
# For the first phase a box with half the resolution is constructed.
self.reference_window_first_phase = self.reference_window[::
2, ::
2]
else:
self.reference_window = reference_frame[
self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt]
# This frame is processed, go to next one.
number_processed += 1
if self.progress_signal is not None:
self.progress_signal.emit("Align all frames", 100)
# Compute the shape of the area contained in all frames in the form [[y_low, y_high],
# [x_low, x_high]]
self.intersection_shape = [[
max(b[0] for b in self.frame_shifts),
min(b[0] for b in self.frame_shifts) + self.shape[0]
],
[
max(b[1] for b in self.frame_shifts),
min(b[1] for b in self.frame_shifts) +
self.shape[1]
]]
def compute_shift_alignment_point(self,
frame_mono_blurred,
frame_index,
alignment_point_index,
de_warp=True):
"""
Compute the [y, x] pixel shift vector at a given alignment point relative to the mean frame.
Four different methods can be used to compute the shift values:
- a subpixel algorithm from "skimage.feature"
- a phase correlation algorithm (miscellaneous.translation)
- a local search algorithm (spiralling outwards), see method "search_local_match",
optionally with subpixel accuracy.
- a local search algorithm, based on steepest descent, see method
"search_local_match_gradient". This method is faster than the previous one, but it has no
subpixel option.
Be careful with the sign of the local shift values. For the first two methods, a positive
value means that the current frame has to be shifted in the positive coordinate direction
in order to make objects in the frame match with their counterparts in the reference frame.
In other words: If the shift is positive, an object in the current frame is at lower pixel
coordinates as compared to the reference. This is very counter-intuitive, but to make the
three methods consistent, the same approach was followed in the implementation of the third
method "search_local_match", contained in this module. There, a pixel box around an
alignment point in the current frame is moved until the content of the box matches with the
corresponding box in the reference frame. If at this point the box is shifted towards a
higher coordinate value, this value is returned with a negative sign as the local shift.
:param frame_mono_blurred: Gaussian-blurred version of the frame with index "frame_index"
:param frame_index: Index of the selected frame in the list of frames
:param alignment_point_index: Index of the selected alignment point
:param de_warp: If True, include local warp shift computation. If False, only apply
global frame shift.
:return: Local shift vector [dy, dx]
"""
alignment_point = self.alignment_points[alignment_point_index]
y_low = alignment_point['box_y_low']
y_high = alignment_point['box_y_high']
x_low = alignment_point['box_x_low']
x_high = alignment_point['box_x_high']
reference_box = alignment_point['reference_box']
# The offsets dy and dx are caused by two effects: First, the mean frame is smaller
# than the original frames. It only contains their intersection. And second, because the
# given frame is globally shifted as compared to the mean frame.
dy = self.align_frames.dy[frame_index]
dx = self.align_frames.dx[frame_index]
if de_warp:
# Use subpixel registration from skimage.feature, with accuracy 1/10 pixels.
if self.configuration.alignment_points_method == 'Subpixel':
# Cut out the alignment box from the given frame. Take into account the offsets
# explained above.
box_in_frame = frame_mono_blurred[y_low + dy:y_high + dy,
x_low + dx:x_high + dx]
shift_pixel, error, diffphase = register_translation(
reference_box, box_in_frame, 10, space='real')
# Use a simple phase shift computation (contained in module "miscellaneous").
elif self.configuration.alignment_points_method == 'CrossCorrelation':
# Cut out the alignment box from the given frame. Take into account the offsets
# explained above.
box_in_frame = frame_mono_blurred[y_low + dy:y_high + dy,
x_low + dx:x_high + dx]
shift_pixel = Miscellaneous.translation(
reference_box, box_in_frame, box_in_frame.shape)
# Use a local search (see method "search_local_match" below.
elif self.configuration.alignment_points_method == 'RadialSearch':
shift_pixel, dev_r = Miscellaneous.search_local_match(
reference_box,
frame_mono_blurred,
y_low + dy,
y_high + dy,
x_low + dx,
x_high + dx,
self.configuration.alignment_points_search_width,
self.configuration.alignment_points_sampling_stride,
sub_pixel=self.configuration.
alignment_points_local_search_subpixel)
# Use the steepest descent search method.
elif self.configuration.alignment_points_method == 'SteepestDescent':
shift_pixel, dev_r = Miscellaneous.search_local_match_gradient(
reference_box, frame_mono_blurred, y_low + dy, y_high + dy,
x_low + dx, x_high + dx,
self.configuration.alignment_points_search_width,
self.configuration.alignment_points_sampling_stride,
self.dev_table)
else:
raise NotSupportedError(
"The point shift computation method " +
self.configuration.alignment_points_method +
" is not implemented")
# Return the computed shift vector.
return shift_pixel
else:
# If no de-warping is computed, just return the zero vector.
return [0, 0]
def align_frames(self):
"""
Compute the displacement of all frames relative to the sharpest frame using the alignment
rectangle.
:return: -
"""
if self.configuration.align_frames_mode == "Surface":
# For "Surface" mode the alignment rectangle has to be selected first.
if self.x_low_opt is None:
raise WrongOrderingError(
"Method 'align_frames' is called before 'select_alignment_rect'"
)
# From the sharpest frame cut out the alignment rectangle. The shifts of all other frames
# will be computed relativ to this patch.
self.reference_window = self.frames.frames_mono_blurred(
self.frame_ranks_max_index)[
self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt].astype(int32)
self.reference_window_shape = self.reference_window.shape
elif self.configuration.align_frames_mode == "Planet":
# For "Planetary" mode compute the center of gravity for the reference image.
cog_reference_y, cog_reference_x = AlignFrames.center_of_gravity(
self.frames.frames_mono_blurred(self.frame_ranks_max_index))
else:
raise NotSupportedError("Frame alignment mode '" +
self.configuration.align_frames_mode +
"' not supported")
# Initialize a list which for each frame contains the shifts in y and x directions.
self.frame_shifts = [None] * self.frames.number
# Initialize lists with info on failed frames.
self.dev_r_list = []
self.failed_index_list = []
# Initialize a counter of processed frames for progress bar signalling. It is set to one
# because in the loop below the optimal frame is not counted.
number_processed = 1
# Loop over all frames. Begin with the sharpest (reference) frame
for idx in chain(reversed(range(self.frame_ranks_max_index + 1)),
range(self.frame_ranks_max_index,
self.frames.number)):
if idx == self.frame_ranks_max_index:
# For the sharpest frame the displacement is 0 because it is used as the reference.
self.frame_shifts[idx] = [0, 0]
# Initialize two variables which keep the shift values of the previous step as
# the starting point for the next step. This reduces the search radius if frames are
# drifting.
dy_min_cum = dx_min_cum = 0
# For all other frames: Compute the global shift, using the "blurred" monochrome image.
else:
# After every "signal_step_size"th frame, send a progress signal to the main GUI.
if self.progress_signal is not None and number_processed % self.signal_step_size == 1:
self.progress_signal.emit(
"Align all frames",
int((number_processed / self.frames.number) * 100.))
frame = self.frames.frames_mono_blurred(idx)
if self.configuration.align_frames_mode == "Planet":
# In Planetary mode the shift of the "center of gravity" of the image is
# computed. This algorithm cannot fail.
cog_frame = AlignFrames.center_of_gravity(frame)
self.frame_shifts[idx] = [
cog_reference_y - cog_frame[0],
cog_reference_x - cog_frame[1]
]
number_processed += 1
continue
# In "Surface" mode three alignment algorithms can be chosen from. In each case
# the result is the shift vector [dy_min, dx_min]. The second and third algorithm
# do a local search. It can fail if within the search radius no minimum is found.
# The first algorithm (cross-correlation) can fail as well, but in this case there
# is no indication that this happened.
elif self.configuration.align_frames_method == "Translation":
# The shift is computed with cross-correlation. Cut out the alignment patch and
# compute its translation relative to the reference.
frame_window = self.frames.frames_mono_blurred(
idx)[self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt]
self.frame_shifts[idx] = Miscellaneous.translation(
self.reference_window, frame_window,
self.reference_window_shape)
continue
elif self.configuration.align_frames_method == "RadialSearch":
# Spiral out from the shift position of the previous frame and search for the
# local optimum.
[dy_min, dx_min], dev_r = Miscellaneous.search_local_match(
self.reference_window,
frame,
self.y_low_opt - dy_min_cum,
self.y_high_opt - dy_min_cum,
self.x_low_opt - dx_min_cum,
self.x_high_opt - dx_min_cum,
self.configuration.align_frames_search_width,
self.configuration.align_frames_sampling_stride,
sub_pixel=False)
elif self.configuration.align_frames_method == "SteepestDescent":
# Spiral out from the shift position of the previous frame and search for the
# local optimum.
[dy_min, dx_min
], dev_r = Miscellaneous.search_local_match_gradient(
self.reference_window, frame,
self.y_low_opt - dy_min_cum,
self.y_high_opt - dy_min_cum,
self.x_low_opt - dx_min_cum,
self.x_high_opt - dx_min_cum,
self.configuration.align_frames_search_width,
self.configuration.align_frames_sampling_stride,
self.dev_table)
else:
raise NotSupportedError("Frame alignment method " +
configuration.align_frames_method +
" not supported")
# Update the cumulative shift values to be used as starting point for the
# next frame.
dy_min_cum += dy_min
dx_min_cum += dx_min
self.frame_shifts[idx] = [dy_min_cum, dx_min_cum]
# In "Surface" mode shift computation can fail if no minimum is found within
# the pre-defined search radius.
if len(dev_r) > 2 and dy_min == 0 and dx_min == 0:
self.failed_index_list.append(idx)
self.dev_r_list.append(dev_r)
continue
# If the alignment window gets too close to a frame edge, move it away from
# that edge by half the border width. First check if the reference window still
# fits into the shifted frame.
if self.shape[0] - abs(
dy_min_cum) - 2 * self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width < \
self.reference_window_shape[0] or self.shape[1] - abs(
dx_min_cum) - 2 * self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width < \
self.reference_window_shape[1]:
raise ArgumentError(
"Frame stabilization window does not fit into"
" intersection")
new_reference_window = False
# Start with the lower y edge.
while self.y_low_opt - dy_min_cum < \
self.configuration.align_frames_search_width + \
self.configuration.align_frames_border_width / 2:
self.y_low_opt += ceil(
self.configuration.align_frames_border_width / 2.)
self.y_high_opt += ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# Now the upper y edge.
while self.y_high_opt - dy_min_cum > self.shape[
0] - self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width / 2:
self.y_low_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
self.y_high_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# Now the lower x edge.
while self.x_low_opt - dx_min_cum < \
self.configuration.align_frames_search_width + \
self.configuration.align_frames_border_width / 2:
self.x_low_opt += ceil(
self.configuration.align_frames_border_width / 2.)
self.x_high_opt += ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# Now the upper x edge.
while self.x_high_opt - dx_min_cum > self.shape[
1] - self.configuration.align_frames_search_width - \
self.configuration.align_frames_border_width / 2:
self.x_low_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
self.x_high_opt -= ceil(
self.configuration.align_frames_border_width / 2.)
new_reference_window = True
# If the window was moved, update the "reference_window".
if new_reference_window:
self.reference_window = self.frames.frames_mono_blurred(
self.frame_ranks_max_index)[
self.y_low_opt:self.y_high_opt,
self.x_low_opt:self.x_high_opt].astype(int32)
number_processed += 1
if self.progress_signal is not None:
self.progress_signal.emit("Align all frames", 100)
# Compute the shape of the area contained in all frames in the form [[y_low, y_high],
# [x_low, x_high]]
self.intersection_shape = [[
max(b[0] for b in self.frame_shifts),
min(b[0] for b in self.frame_shifts) + self.shape[0]
],
[
max(b[1] for b in self.frame_shifts),
min(b[1] for b in self.frame_shifts) +
self.shape[1]
]]
if len(self.failed_index_list) > 0:
raise InternalError("No valid shift computed for " +
str(len(self.failed_index_list)) +
" frames: " + str(self.failed_index_list))