def retrieve_patches_for_queries_and_descr(x_queries,
y_queries,
which_desc,
image_path,
mask,
encoder32,
encoder128,
patch_size=16,
compare_stride=8,
eps=0.0001,
nr_similar_patches=5):
image = imageio.imread(image_path)
image_height = image.shape[0]
image_width = image.shape[1]
# psnr_max_value = 255
image = image / 255.
# psnr_max_value = 1
results_patches_diffs = {}
results_patches_x_coords = {}
results_patches_y_coords = {}
results_patches_positions = {}
counter_query_patches = 0
total_nr_query_patches = len(x_queries)
for query_it in range(total_nr_query_patches):
x_query = x_queries[query_it]
y_query = y_queries[query_it]
sys.stdout.write("\r" + str(counter_query_patches + 1) + "/" +
str(total_nr_query_patches))
# mask = generate_mask()
inverse_mask = np.repeat((1 - mask), 3, axis=1).reshape(
(patch_size, patch_size, 3))
query_patch = image[x_query:x_query + patch_size,
y_query:y_query + patch_size, :]
query_patch = query_patch * inverse_mask
if which_desc == 0:
query_patch_descr = compute_descriptor(query_patch, encoder32)
elif which_desc == 1:
query_patch_descr = compute_descriptor(query_patch, encoder128)
elif which_desc == 2:
query_patch_descr = compute_chen_rgb(query_patch)
else:
query_patch_descr = query_patch
counter_compare_patches = 0
patches_diffs = [1000000000]
patches_x_coords = [-1]
patches_y_coords = [-1]
patches_positions = [-1]
for y_compare in range(0, image_width - patch_size + 1,
compare_stride):
for x_compare in range(0, image_height - patch_size + 1,
compare_stride):
compare_patch = image[x_compare:x_compare + patch_size,
y_compare:y_compare + patch_size, :]
compare_patch = compare_patch * inverse_mask
if which_desc == 0:
compare_patch_descr = compute_descriptor(
compare_patch, encoder32)
elif which_desc == 1:
compare_patch_descr = compute_descriptor(
compare_patch, encoder128)
elif which_desc == 2:
compare_patch_descr = compute_chen_rgb(compare_patch)
else:
compare_patch_descr = compare_patch
diff = calculate_ssd(query_patch_descr, compare_patch_descr)
if diff < eps:
counter_compare_patches += 1
continue
# sorting
for i in range(len(patches_diffs)):
if diff < patches_diffs[i]:
patches_diffs.insert(i, diff)
patches_x_coords.insert(i, x_compare)
patches_y_coords.insert(i, y_compare)
patches_positions.insert(i, counter_compare_patches)
break
counter_compare_patches += 1
results_patches_diffs[
counter_query_patches] = patches_diffs[:nr_similar_patches]
results_patches_x_coords[
counter_query_patches] = patches_x_coords[:nr_similar_patches]
results_patches_y_coords[
counter_query_patches] = patches_y_coords[:nr_similar_patches]
results_patches_positions[
counter_query_patches] = patches_positions[:nr_similar_patches]
counter_query_patches += 1
return results_patches_x_coords, results_patches_y_coords
def calculate_SSDs_for_desc_and_noise_level(which_desc, noise_level):
image_noisy = add_gaussian_noise(image, sigma=noise_level)
query_x_coords = []
query_y_coords = []
results_noisy_descr_patches_diffs = {}
results_noisy_descr_patches_x_coords = {}
results_noisy_descr_patches_y_coords = {}
results_noisy_descr_patches_positions = {}
counter_query_patches = 0
# just for the sake of output
total_nr_query_patches = len(
range(0, image_width - patch_size + 1, query_stride)) * len(
range(0, image_height - patch_size + 1, query_stride))
for y_query in range(0, image_width - patch_size + 1, query_stride):
for x_query in range(0, image_height - patch_size + 1, query_stride):
sys.stdout.write("\r" + str(counter_query_patches + 1) + "/" +
str(total_nr_query_patches))
query_x_coords.append(x_query)
query_y_coords.append(y_query)
query_patch = image_noisy[x_query:x_query + patch_size,
y_query:y_query + patch_size, :]
if which_desc == 0:
query_patch_descr = compute_descriptor(query_patch, encoder32)
elif which_desc == 1:
query_patch_descr = compute_descriptor(query_patch, encoder128)
elif which_desc == 2:
query_patch_descr = compute_chen_rgb(query_patch)
else:
query_patch_descr = query_patch
counter_compare_patches = 0
patches_diffs = [1000000000]
patches_x_coords = [-1]
patches_y_coords = [-1]
patches_positions = [-1]
for y_compare in range(0, image_width - patch_size + 1,
compare_stride):
for x_compare in range(0, image_height - patch_size + 1,
compare_stride):
compare_patch = image_noisy[x_compare:x_compare +
patch_size,
y_compare:y_compare +
patch_size, :]
if which_desc == 0:
compare_patch_descr = compute_descriptor(
compare_patch, encoder32)
elif which_desc == 1:
compare_patch_descr = compute_descriptor(
compare_patch, encoder128)
elif which_desc == 2:
compare_patch_descr = compute_chen_rgb(compare_patch)
else:
compare_patch_descr = compare_patch
diff = calculate_ssd(query_patch_descr,
compare_patch_descr)
if diff < eps:
counter_compare_patches += 1
continue
# sorting
for i in range(len(patches_diffs)):
if diff < patches_diffs[i]:
patches_diffs.insert(i, diff)
patches_x_coords.insert(i, x_compare)
patches_y_coords.insert(i, y_compare)
patches_positions.insert(i,
counter_compare_patches)
break
counter_compare_patches += 1
results_noisy_descr_patches_diffs[
counter_query_patches] = patches_diffs[:nr_similar_patches]
results_noisy_descr_patches_x_coords[
counter_query_patches] = patches_x_coords[:nr_similar_patches]
results_noisy_descr_patches_y_coords[
counter_query_patches] = patches_y_coords[:nr_similar_patches]
results_noisy_descr_patches_positions[
counter_query_patches] = patches_positions[:nr_similar_patches]
counter_query_patches += 1
ssds = []
for q_it in range(total_nr_query_patches):
for c_it in range(nr_similar_patches):
# getting the query patch from the clean image
x_query = query_x_coords[q_it]
y_query = query_y_coords[q_it]
query_patch = image[x_query:x_query + patch_size,
y_query:y_query + patch_size, :]
# getting the compare patch from the clean image
x_compare = results_noisy_descr_patches_x_coords[q_it][c_it]
y_compare = results_noisy_descr_patches_y_coords[q_it][c_it]
compare_patch = image[x_compare:x_compare + patch_size,
y_compare:y_compare + patch_size, :]
# calculating the difference in the clean image
actual_diff = calculate_ssd(query_patch, compare_patch)
ssds.append(actual_diff)
ssds = np.array(ssds)
return ssds
def find_similar_patches_with_Chen():
kernel1D = make_kernel(patch_size)
kernel = np.repeat(kernel1D, 3, axis=1).reshape(
(patch_size, patch_size, 3))
diffs = {}
x_coords = {}
y_coords = {}
total_nr_patches = len(range(
0, image_width - patch_size + 1, stride)) * len(
range(0, image_height - patch_size + 1, stride))
# initialise the dictionaries that store the differences
counter_query_patches = 0
for y_query in range(0, image_width - patch_size + 1, stride):
for x_query in range(0, image_height - patch_size + 1, stride):
diffs[counter_query_patches] = {}
counter_query_patches += 1
counter_query_patches = 0
for y_query in range(0, image_width - patch_size + 1, stride):
for x_query in range(0, image_height - patch_size + 1, stride):
# print(counter_query_patches)
sys.stdout.write("\r" + str(counter_query_patches + 1) + "/" +
str(total_nr_patches))
# query_patch = image_noisy[x_query: x_query + patch_size, y_query: y_query + patch_size, :]
# query_patch = np.multiply(query_patch, kernel)
# query_patch_descr = encoder_patch.predict(np.expand_dims(query_patch, axis=0))[0]
# query_patch_descr = compute_descriptor_from_IR(image_noisy_IR, x_query, y_query, patch_size, encoder_mp)
query_patch = image_noisy[x_query:x_query + patch_size,
y_query:y_query + patch_size, :]
query_patch = np.multiply(query_patch, kernel)
query_patch_descr = compute_chen_rgb(query_patch)
x_coords[counter_query_patches] = x_query
y_coords[counter_query_patches] = y_query
# diffs[counter_query_patches] = {}
counter_compare_patches = 0
for y_compare in range(0, image_width - patch_size + 1, stride):
for x_compare in range(0, image_height - patch_size + 1,
stride):
if counter_query_patches < counter_compare_patches and abs(
x_compare -
x_query) < comparing_window_size and abs(
y_compare - y_query) < comparing_window_size:
# compare_patch = image_noisy[x_compare: x_compare + patch_size, y_compare: y_compare + patch_size, :]
# compare_patch = np.multiply(compare_patch, kernel)
# compare_patch_descr = encoder_patch.predict(np.expand_dims(compare_patch, axis=0))[0]
# compare_patch_descr = compute_descriptor_from_IR(image_noisy_IR, x_compare, y_compare, patch_size, encoder_mp)
compare_patch = image_noisy[x_compare:x_compare +
patch_size,
y_compare:y_compare +
patch_size, :]
compare_patch = np.multiply(compare_patch, kernel)
compare_patch_descr = compute_chen_rgb(compare_patch)
query_compare_diff = calculate_ssd(
query_patch_descr, compare_patch_descr)
diffs[counter_query_patches][
counter_compare_patches] = query_compare_diff
diffs[counter_compare_patches][
counter_query_patches] = query_compare_diff
counter_compare_patches += 1
counter_query_patches += 1
return diffs, x_coords, y_coords
def retrieve_patches_for_queries_and_descr(x_queries,
y_queries,
which_desc,
image_path,
random_seed,
noise_level,
encoder32,
encoder128,
patch_size=16,
compare_stride=8,
eps=0.0001,
nr_similar_patches=5):
np.random.seed(random_seed)
image = imageio.imread(image_path)
image_height = image.shape[0]
image_width = image.shape[1]
image_noisy = add_gaussian_noise(image, sigma=noise_level)
image = image / 255.
image_noisy = image_noisy / 255.
results_patches_diffs = {}
results_patches_x_coords = {}
results_patches_y_coords = {}
results_patches_positions = {}
counter_query_patches = 0
total_nr_query_patches = len(x_queries)
for query_it in range(total_nr_query_patches):
x_query = x_queries[query_it]
y_query = y_queries[query_it]
sys.stdout.write("\r" + str(counter_query_patches + 1) + "/" +
str(total_nr_query_patches))
query_patch = image_noisy[x_query:x_query + patch_size,
y_query:y_query + patch_size, :]
if which_desc == 0:
query_patch_descr = compute_descriptor(query_patch, encoder32)
elif which_desc == 1:
query_patch_descr = compute_descriptor(query_patch, encoder128)
elif which_desc == 2:
query_patch_descr = compute_chen_rgb(query_patch)
else:
query_patch_descr = query_patch
counter_compare_patches = 0
patches_diffs = [1000000000]
patches_x_coords = [-1]
patches_y_coords = [-1]
patches_positions = [-1]
for y_compare in range(0, image_width - patch_size + 1,
compare_stride):
for x_compare in range(0, image_height - patch_size + 1,
compare_stride):
compare_patch = image_noisy[x_compare:x_compare + patch_size,
y_compare:y_compare +
patch_size, :]
if which_desc == 0:
compare_patch_descr = compute_descriptor(
compare_patch, encoder32)
elif which_desc == 1:
compare_patch_descr = compute_descriptor(
compare_patch, encoder128)
elif which_desc == 2:
compare_patch_descr = compute_chen_rgb(compare_patch)
else:
compare_patch_descr = compare_patch
diff = calculate_ssd(query_patch_descr, compare_patch_descr)
if diff < eps:
counter_compare_patches += 1
continue
# sorting
for i in range(len(patches_diffs)):
if diff < patches_diffs[i]:
patches_diffs.insert(i, diff)
patches_x_coords.insert(i, x_compare)
patches_y_coords.insert(i, y_compare)
patches_positions.insert(i, counter_compare_patches)
break
counter_compare_patches += 1
results_patches_diffs[
counter_query_patches] = patches_diffs[:nr_similar_patches]
results_patches_x_coords[
counter_query_patches] = patches_x_coords[:nr_similar_patches]
results_patches_y_coords[
counter_query_patches] = patches_y_coords[:nr_similar_patches]
results_patches_positions[
counter_query_patches] = patches_positions[:nr_similar_patches]
counter_query_patches += 1
return results_patches_x_coords, results_patches_y_coords
def calculate_SSDs_for_desc_and_missing_level(which_desc, mask_perc):
query_x_coords = []
query_y_coords = []
results_noisy_descr_patches_diffs = {}
results_noisy_descr_patches_x_coords = {}
results_noisy_descr_patches_y_coords = {}
results_noisy_descr_patches_positions = {}
counter_query_patches = 0
seed_nr = 0
# just for the sake of output
total_nr_query_patches = len(
range(0, image_width - patch_size + 1, query_stride)) * len(
range(0, image_height - patch_size + 1, query_stride))
for y_query in range(0, image_width - patch_size + 1, query_stride):
for x_query in range(0, image_height - patch_size + 1, query_stride):
sys.stdout.write("\r" + str(counter_query_patches + 1) + "/" +
str(total_nr_query_patches))
query_x_coords.append(x_query)
query_y_coords.append(y_query)
# if there the missing part percentage is 0
if mask_perc == 0:
inverse_mask = np.ones((patch_size, patch_size, 3),
dtype=np.uint8)
else:
# alternate between random corner, random border, and ising mask
if counter_query_patches % 3 == 0:
seed = seeds_for_ising[mask_perc][seed_nr]
seed_nr += 1
if seed_nr == (len(seeds_for_ising[mask_perc])):
seed_nr = 0
np.random.seed(seed * 10)
mask = mask_ising_model(patch_size=patch_size)
if counter_query_patches % 3 == 1:
mask = mask_of_specific_percentage(
mask_perc - 0.05, mask_perc + 0.05,
mask_random_border_rectangle)
if counter_query_patches % 3 == 2:
mask = mask_of_specific_percentage(
mask_perc - 0.05, mask_perc + 0.05,
mask_random_corner_rectangle)
inverse_mask = np.repeat((1 - mask), 3, axis=1).reshape(
(patch_size, patch_size, 3))
query_patch = image[x_query:x_query + patch_size,
y_query:y_query + patch_size, :]
query_patch = query_patch * inverse_mask
if which_desc == 0:
query_patch_descr = compute_descriptor(query_patch, encoder32)
elif which_desc == 1:
query_patch_descr = compute_descriptor(query_patch, encoder128)
elif which_desc == 2:
query_patch_descr = compute_chen_rgb(query_patch)
else:
query_patch_descr = query_patch
counter_compare_patches = 0
compare_patches_scores = {}
patches_diffs = [1000000000]
patches_x_coords = [-1]
patches_y_coords = [-1]
patches_positions = [-1]
for y_compare in range(0, image_width - patch_size + 1,
compare_stride):
for x_compare in range(0, image_height - patch_size + 1,
compare_stride):
compare_patch = image[x_compare:x_compare + patch_size,
y_compare:y_compare + patch_size, :]
compare_patch = compare_patch * inverse_mask
if which_desc == 0:
compare_patch_descr = compute_descriptor(
compare_patch, encoder32)
elif which_desc == 1:
compare_patch_descr = compute_descriptor(
compare_patch, encoder128)
elif which_desc == 2:
compare_patch_descr = compute_chen_rgb(compare_patch)
else:
compare_patch_descr = compare_patch
diff = calculate_ssd(query_patch_descr,
compare_patch_descr)
if diff < eps:
counter_compare_patches += 1
continue
# sorting
for i in range(len(patches_diffs)):
if diff < patches_diffs[i]:
patches_diffs.insert(i, diff)
patches_x_coords.insert(i, x_compare)
patches_y_coords.insert(i, y_compare)
patches_positions.insert(i,
counter_compare_patches)
break
counter_compare_patches += 1
results_noisy_descr_patches_diffs[
counter_query_patches] = patches_diffs[:nr_similar_patches]
results_noisy_descr_patches_x_coords[
counter_query_patches] = patches_x_coords[:nr_similar_patches]
results_noisy_descr_patches_y_coords[
counter_query_patches] = patches_y_coords[:nr_similar_patches]
results_noisy_descr_patches_positions[
counter_query_patches] = patches_positions[:nr_similar_patches]
counter_query_patches += 1
ssds = []
for q_it in range(total_nr_query_patches):
for c_it in range(nr_similar_patches):
# getting the query patch from the clean image
x_query = query_x_coords[q_it]
y_query = query_y_coords[q_it]
query_patch = image[x_query:x_query + patch_size,
y_query:y_query + patch_size, :]
# getting the compare patch from the clean image
x_compare = results_noisy_descr_patches_x_coords[q_it][c_it]
y_compare = results_noisy_descr_patches_y_coords[q_it][c_it]
compare_patch = image[x_compare:x_compare + patch_size,
y_compare:y_compare + patch_size, :]
# calculating the difference in the clean image
actual_diff = calculate_ssd(query_patch, compare_patch)
ssds.append(actual_diff)
ssds = np.array(ssds)
return ssds