def test_differentiate_image(img):
diff_img = differentiate_image(img)
plt.figure()
show_image(img)
plt.figure()
plt.imshow(diff_img[0,0])
plt.figure()
plt.imshow(diff_img[0, 1])
plt.show()
def test_downsacle(img):
plt.figure()
show_image(img)
img2R = downscale(img[0], 0.5)
img2G = downscale(img[1], 0.5)
img2B = downscale(img[1], 0.5)
img2 = np.array([img2R, img2G, img2B])
plt.figure()
show_image(img2)
plt.show()
def compare_flow(computed_flow, real_flow,current_frame,next_frame, plot=True,arrows=True):
"""
:param computed_flow: np.array(float) (YX, height_1,width_1)
:param real_flow: np.array(float) (YX, height_2,width_2)
:param plot: bool
:param current_frame: (ChannelCount, height,width)
:param next_frame: (ChannelCount, height,width)
:return: (min(height_1,height_2),min(width_1,width_2))
"""
height_1 = computed_flow.shape[1]
width_1 = computed_flow.shape[2]
height_2 = real_flow.shape[1]
width_2 = real_flow.shape[2]
width = -1
height = -1
if(height_1>height_2 and width_1>width_2):
computed_flow = down_scale_flow(computed_flow, width_2, height_2)
width = width_2
height = height_2
elif (height_2>height_1 and width_2>width_1):
real_flow = down_scale_flow(real_flow, width_1, height_1)
width = width_1
height = height_1
elif(height_2==height_1 and width_2==width_1):
width = width_1
height=height_1
else:
raise Exception("False dimensions: flow_1: (", height_1, ", ", width_1, ") flow_2: (", height_2, ", ", width_2, ")" )
ang_error = angular_error(computed_flow, real_flow)
abs_error = absolute_endpoint_error(computed_flow, real_flow)
print("Average angular error:")
print(np.mean(ang_error[~np.isnan(ang_error)]))
print("Median angular error:")
print(np.mean(ang_error[~np.isnan(ang_error)]))
print("Absolute endpoint error:")
print(np.mean(abs_error[~np.isnan(abs_error)]))
print("Median endpoint error:")
print(np.median(abs_error[~np.isnan(abs_error)]))
if(plot):
#rescale image
current_frame = scale_image(current_frame,width,height)
next_frame = scale_image(next_frame, width, height)
next_frame_warped = warp_image(next_frame,computed_flow)
plt.figure()
show_image(current_frame)
plt.title("Current frame")
plt.figure()
show_image(next_frame_warped)
plt.title("Next frame warped to current frame")
fig, axs = plt.subplots(1,2)
axs[0].set_title("Current frame - Next frame squared")
show_image((current_frame-next_frame)**2,axes=axs[0])
axs[1].set_title("Current frame - Next frame warped squared")
show_image((current_frame - next_frame_warped) ** 2,axes=axs[1])
if (arrows):
plt.figure()
show_image(current_frame, axes=plt)
show_flow_field_arrow(computed_flow, width, height, axes=plt)
plt.title("Arrows")
fig, axs = plt.subplots(1,2)
show_flow_field(computed_flow,width,height,axes=axs[0])
axs[0].set_title("Computed_flow")
show_flow_field(real_flow, width, height, axes=axs[1])
axs[1].set_title("Ground Truth")
fig, axs = plt.subplots(1,2)
axs[0].imshow(ang_error)
axs[0].set_title("Angular error")
axs[1].imshow(abs_error)
axs[1].set_title("Absolute endpoint error")
plt.show()
def sun_baker_optical_flow(
first_image: np.ndarray,
second_image: np.ndarray,
settings=SolverSettings()) -> np.ndarray:
start_time = time()
factors = settings.scale_factors
gnc_factors = settings.gnc_scale_factors
filter_image, first_gray_image, second_gray_image = preprocessing(
first_image, second_image)
show_image(first_image)
plt.figure()
plt.imshow(first_gray_image)
plt.figure()
plt.imshow(second_gray_image)
plt.show()
pyramid_levels_first_gray_image = create_matrix_pyramid(
first_gray_image, factors)
pyramid_levels_second_gray_image = create_matrix_pyramid(
second_gray_image, factors)
pyramid_levels_filter_image = create_image_pyramid(filter_image, factors)
width = pyramid_levels_filter_image[0].shape[2]
height = pyramid_levels_filter_image[0].shape[1]
gnc_steps = settings.gnc_steps
flow = np.zeros(shape=(2, height, width))
penalty_func_1 = SquaredPenalty()
penalty_func_2 = GeneralizedCharbonnierPenalty()
penalty_func = MixPenalty(penalty_func_1, penalty_func_2, 0)
relaxation_steps = settings.relaxation_steps
max_iter_solve = settings.max_iter_solve
for gnc_iter in range(gnc_steps):
width = pyramid_levels_filter_image[0].shape[2]
height = pyramid_levels_filter_image[0].shape[1]
#downscale
#if (flow.shape[2] > width or flow.shape[1] > height):
if (gnc_iter > 0):
pyramid_levels_first_gray_image = create_matrix_pyramid(
first_gray_image, gnc_factors)
pyramid_levels_second_gray_image = create_matrix_pyramid(
second_gray_image, gnc_factors)
pyramid_levels_filter_image = create_image_pyramid(
filter_image, gnc_factors)
width = pyramid_levels_filter_image[0].shape[2]
height = pyramid_levels_filter_image[0].shape[1]
flow = down_scale_flow(flow, width, height)
for filter_scaled,first_gray_scaled,second_gray_scaled in \
zip(pyramid_levels_filter_image, pyramid_levels_first_gray_image, pyramid_levels_second_gray_image):
width = filter_scaled.shape[2]
height = filter_scaled.shape[1]
flow = upscale_flow(flow, width, height)
print("-------- GNC: ", gnc_iter, " --------")
if (gnc_steps > 1):
mix_factor = gnc_iter / (gnc_steps - 1)
else:
mix_factor = 0
penalty_func.mix_factor = mix_factor
if (mix_factor == 0):
settings.relaxation_steps = 1
settings.max_iter_solve = 100
else:
settings.relaxation_steps = relaxation_steps
settings.max_iter_solve = max_iter_solve
for iter in range(settings.steps_per_level):
flow_next = solve_layer(filter_scaled, first_gray_scaled,
second_gray_scaled, flow, penalty_func,
settings)
if (np.linalg.norm(flow - flow_next) < 1e-03):
flow = flow_next
print("Iter break")
break
flow = flow_next
#plt.title("At level: "+str(width) +","+str(height)+", GNC: "+str(gnc_iter))
#show_flow_field(flow, width, height)
#plt.show()
print("Sun Baker full time: ", time() - start_time)
return flow
plt.show()
def _to_gray(img):
return np.array([(img[0] + img[1] + img[2])], dtype=float) / 3
if __name__ == '__main__':
img1_color = open_image(
r"..\..\..\..\resources\eval-twoframes\Dimetrodon\frame10.png")
img2_color = open_image(
r"..\..\..\..\resources\eval-twoframes\Dimetrodon\frame11.png")
img1_gray = open_image(
r"..\..\..\..\resources\eval-twoframes\Dimetrodon\frame10-gray.png")
img2_gray = open_image(
r"..\..\..\..\resources\eval-twoframes\Dimetrodon\frame11-gray.png")
img1_color = _to_gray(img1_color)
img2_color = _to_gray(img2_color)
show_image(img1_color)
plt.figure()
show_image(img1_gray)
plt.figure()
show_image(img1_gray - img1_color)
plt.title("Diff Image")
plt.figure()
plt.show()
color_vs_gray_compare(img1_color, img2_color, img1_gray, img2_gray)
def test_image_pyramid(img, factors):
pyramid = create_image_pyramid(img, factors)
for level in pyramid:
plt.figure()
show_image(level)
plt.show()
ref_flow = read_flow_field(
r"..\..\..\..\resources\eval-twoframes-groundtruth\RubberWhale\flow10.flo"
)
return img1, img2, ref_flow
def grove2():
img1 = open_image(
r"..\..\..\..\resources\eval-twoframes\Grove2\frame10.png")
img2 = open_image(
r"..\..\..\..\resources\eval-twoframes\Grove2\frame11.png")
ref_flow = read_flow_field(
r"..\..\..\..\resources\eval-twoframes-groundtruth\Grove2\flow10.flo")
return img1, img2, ref_flow
if __name__ == '__main__':
img1, img2, ref_flow = RubberWhale()
#img1 = denoising_chambolle_color_img(img1)
#img2 = denoising_chambolle_color_img(img2)
img1 = scale_np_array_in_range(img1, 0, 1)
img2 = scale_np_array_in_range(img2, 0, 1)
show_image(img1)
plt.figure()
show_image(img2)
plt.show()
computed_flow = test_horn_schunck(img1, img2)
compare_flow(computed_flow, ref_flow, img1, img2, plot=True, arrows=True)