def test_extract_pixel_feature():
# Make and test 2D image
sm = 0.5 * np.ones((4, 5))
sm[0, 0] = 0
lg = 0.3 * np.ones((7, 10))
lg[0, 0] = 1
c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(lg)
sm_0 = np.array([[0, 0, 0.5], [0, 0, 0.5], [0.5, 0.5, 0.5]])
lg_0 = np.array([[0.3, 0.3, 0.3, 0.3, 0.3], [0.3, 1, 1, 0.3, 0.3],
[0.3, 1, 1, 0.3, 0.3], [0.3, 0.3, 0.3, 0.3, 0.3],
[0.3, 0.3, 0.3, 0.3, 0.3]])
im_padded = pad_img_pair(sm, lg, c)
# First test full feature
correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()])
feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=True)
assert (np.allclose(feat, correct_feat_0_0))
# Now test half feature
correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()[:c.n_half]])
feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=False)
assert (np.allclose(feat, correct_feat_0_0))
def test_extract_pixel_feature():
# Make and test 2D image
sm = 0.5 * np.ones((4, 5))
sm[0, 0] = 0
lg = 0.3 * np.ones((7, 10))
lg[0, 0] = 1
c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(lg)
sm_0 = np.array([[ 0, 0, 0.5],
[ 0, 0, 0.5],
[0.5, 0.5, 0.5]])
lg_0 = np.array([[0.3, 0.3, 0.3, 0.3, 0.3],
[0.3, 1, 1, 0.3, 0.3],
[0.3, 1, 1, 0.3, 0.3],
[0.3, 0.3, 0.3, 0.3, 0.3],
[0.3, 0.3, 0.3, 0.3, 0.3]])
im_padded = pad_img_pair(sm, lg, c)
# First test full feature
correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()])
feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=True)
assert(np.allclose(feat, correct_feat_0_0))
# Now test half feature
correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()[:c.n_half]])
feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=False)
assert(np.allclose(feat, correct_feat_0_0))
def test_best_coherence_match():
# make A_pd, Ap_pd, BBp_feat, s
A_orig = plt.imread('./test_images/test_best_coherence_match_A.jpg')
Ap_orig = plt.imread('./test_images/test_best_coherence_match_Ap.jpg')
A = convert_to_YIQ( A_orig/255.)[:, :, 0]
Ap = convert_to_YIQ(Ap_orig/255.)[:, :, 0]
A_pyr = compute_gaussian_pyramid( A, min_size=3)
Ap_pyr = compute_gaussian_pyramid(Ap, min_size=3)
imh, imw = A.shape[:2]
c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(A)
c.max_levels = len(A_pyr)
A_pd = pad_img_pair( A_pyr[-2], A_pyr[-1], c)
Ap_pd = pad_img_pair(Ap_pyr[-2], Ap_pyr[-1], c)
flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr, c)
# BBp_feat cases: all corners and middle
indices = [(1, 1),
(1, imw - 1),
(imh - 1, 1),
(imh - 1, imw - 1),
(np.floor(imh/2.).astype(int), np.floor(imw/2.).astype(int))]
for row, col in indices:
num_px = row * imw + col
s_rows = np.random.random_integers(num_px, size=num_px) - 1
s_cols = np.random.random_integers(num_px, size=num_px) - 1
s = [(rr, cc) for rr, cc in zip(s_rows, s_cols)]
s[(row - 1) * imw + col - 1] = (row - 1, col - 1)
Bs_feat = np.hstack([extract_pixel_feature( A_pd, (row, col), c, full_feat=True),
extract_pixel_feature(Ap_pd, (row, col), c, full_feat=False)])
p_coh_orig, r_star_orig = best_coherence_match_orig(A_pd, Ap_pd, Bs_feat, s, (row, col, imw), c)
p_coh_new, r_star_new = best_coherence_match(As[-1], A.shape, Bs_feat, s, (row, col, imw), c)
try:
assert(p_coh_orig == (row, col))
assert(p_coh_new == (row, col))
except:
print('row, col, p_coh_orig, p_coh_new, s', row, col, p_coh_orig, p_coh_new, s)
As_feat = np.hstack([extract_pixel_feature( A_pd, p_coh_orig, p_coh_new, c, full_feat=True),
extract_pixel_feature(Ap_pd, p_coh_orig, p_coh_new, c, full_feat=False)])
print('As_feat', As_feat)
print('Bs_feat', Bs_feat)
assert(False)
def test_best_coherence_match():
# make A_pd, Ap_pd, BBp_feat, s
A_orig = plt.imread('./test_images/test_best_coherence_match_A.jpg')
Ap_orig = plt.imread('./test_images/test_best_coherence_match_Ap.jpg')
A = convert_to_YIQ(A_orig / 255.)[:, :, 0]
Ap = convert_to_YIQ(Ap_orig / 255.)[:, :, 0]
A_pyr = compute_gaussian_pyramid(A, min_size=3)
Ap_pyr = compute_gaussian_pyramid(Ap, min_size=3)
imh, imw = A.shape[:2]
c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(A)
c.max_levels = len(A_pyr)
A_pd = pad_img_pair(A_pyr[-2], A_pyr[-1], c)
Ap_pd = pad_img_pair(Ap_pyr[-2], Ap_pyr[-1], c)
flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr, c)
# BBp_feat cases: all corners and middle
indices = [(1, 1), (1, imw - 1), (imh - 1, 1), (imh - 1, imw - 1),
(np.floor(imh / 2.).astype(int), np.floor(imw / 2.).astype(int))
]
for row, col in indices:
num_px = row * imw + col
s_rows = np.random.random_integers(num_px, size=num_px) - 1
s_cols = np.random.random_integers(num_px, size=num_px) - 1
s = [(rr, cc) for rr, cc in zip(s_rows, s_cols)]
s[(row - 1) * imw + col - 1] = (row - 1, col - 1)
Bs_feat = np.hstack([
extract_pixel_feature(A_pd, (row, col), c, full_feat=True),
extract_pixel_feature(Ap_pd, (row, col), c, full_feat=False)
])
p_coh_orig, r_star_orig = best_coherence_match_orig(
A_pd, Ap_pd, Bs_feat, s, (row, col, imw), c)
p_coh_new, r_star_new = best_coherence_match(As[-1], A.shape, Bs_feat,
s, (row, col, imw), c)
try:
assert (p_coh_orig == (row, col))
assert (p_coh_new == (row, col))
except:
print('row, col, p_coh_orig, p_coh_new, s', row, col, p_coh_orig,
p_coh_new, s)
As_feat = np.hstack([
extract_pixel_feature(A_pd,
p_coh_orig,
p_coh_new,
c,
full_feat=True),
extract_pixel_feature(Ap_pd,
p_coh_orig,
p_coh_new,
c,
full_feat=False)
])
print('As_feat', As_feat)
print('Bs_feat', Bs_feat)
assert (False)
r_star = (np.nan, np.nan)
for r_row in np.arange(row_min, row_max, dtype=int):
col_end = col if r_row == row else col_max
for r_col in np.arange(col_min, col_end, dtype=int):
s_ix = r_row * Bp_w + r_col
# p = s(r) + (q - r)
p_r = np.array(s[s_ix]) + np.array([row, col]) - np.array(
[r_row, r_col])
# check that p_r is inside the bounds of A/Ap lg
A_h, A_w = A_pd[1].shape[:2] - 2 * c.pad_lg
if 0 <= p_r[0] < A_h and 0 <= p_r[1] < A_w:
AAp_feat = np.hstack([
extract_pixel_feature(A_pd, p_r, c, full_feat=True),
extract_pixel_feature(Ap_pd, p_r, c, full_feat=False)
])
assert (AAp_feat.shape == BBp_feat.shape)
new_sum = norm(AAp_feat - BBp_feat, ord=2)**2
if new_sum <= min_sum:
min_sum = new_sum
r_star = np.array([r_row, r_col])
if np.isnan(r_star).any():
return (-1, -1), (0, 0)
# s[r_star] + (q - r_star)
return tuple(s[r_star[0] * Bp_w + r_star[1]] +
def image_analogies_main(A_fname, Ap_fname_list, B_fname, out_path, c, debug=False):
# # This is the setup code
begin_time = time.time()
start_time = time.time()
# Load images
A_pyr, Ap_pyr_list, B_pyr, Bp_pyr, color_pyr_list, c = img_setup(A_fname, Ap_fname_list, B_fname, out_path, c)
# Save parameters for reference
names = ['A_fname', 'Ap_fname_list', 'B_fname', 'c.convert', 'c.remap_lum', 'c.init_rand', 'c.AB_weight', 'c.k']
vars = [ A_fname, Ap_fname_list, B_fname, c.convert, c.remap_lum, c.init_rand, c.AB_weight, c.k ]
save_metadata(out_path, names, vars)
# Pull features from B
B_features = compute_feature_array(B_pyr, c, full_feat=True)
stop_time = time.time()
print 'Environment Setup: %f' % (stop_time - start_time)
# Build Structures for ANN
start_time = time.time()
flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr_list, c)
stop_time = time.time()
ann_time_total = stop_time - start_time
print 'ANN Index Creation: %f' % (ann_time_total)
# ##########################################################################################
# # This is the Algorithm Code
# now we iterate per pixel in each level
for level in range(1, c.max_levels):
start_time = time.time()
ann_time_level = 0
print('Computing level %d of %d' % (level, c.max_levels - 1))
imh, imw = Bp_pyr[level].shape[:2]
color_im_out = np.nan * np.ones((imh, imw, 3))
s = []
im = []
if debug:
# make debugging structures
sa = []
sc = []
rstars = []
p_src = np.nan * np.ones((imh, imw, 3))
img_src = np.zeros((imh, imw))
app_dist = np.zeros((imh, imw))
coh_dist = np.zeros((imh, imw))
app_color = np.array([1, 0, 0])
coh_color = np.array([1, 1, 0])
err_color = np.array([0, 0, 0])
paths = ['%d_psrc.eps' % (level),
'%d_appdist.eps' % (level),
'%d_cohdist.eps' % (level),
'%d_output.eps' % (level),
'%d_imgsrc.eps' % (level)]
vars = [p_src, app_dist, coh_dist, Bp_pyr[level], img_src]
for row in range(imh):
for col in range(imw):
px = np.array([row, col])
# we pad on each iteration so Bp features will be more accurate
Bp_pd = pad_img_pair(Bp_pyr[level - 1], Bp_pyr[level], c)
BBp_feat = np.hstack([B_features[level][px2ix(px, imw), :],
extract_pixel_feature(Bp_pd, px, c, full_feat=False)])
assert(BBp_feat.shape == (As_size[level][1],))
# Find Approx Nearest Neighbor
ann_start_time = time.time()
p_app_ix = best_approximate_match(flann[level], flann_params[level], BBp_feat)
assert(p_app_ix < As_size[level][0])
ann_stop_time = time.time()
ann_time_level = ann_time_level + ann_stop_time - ann_start_time
# translate p_app_ix back to row, col
Ap_imh, Ap_imw = Ap_pyr_list[0][level].shape[:2]
p_app, i_app = Ap_ix2px(p_app_ix, Ap_imh, Ap_imw)
# is this the first iteration for this level?
# then skip coherence step
if len(s) < 1:
p = p_app
i = i_app
# Find Coherence Match and Compare Distances
else:
p_coh, i_coh, r_star = best_coherence_match(As[level], (Ap_imh, Ap_imw), BBp_feat, s, im, px, imw, c)
if np.allclose(p_coh, np.array([-1, -1])):
p = p_app
i = i_app
else:
AAp_feat_app = As[level][p_app_ix]
AAp_feat_coh = As[level][Ap_px2ix(p_coh, i_coh, Ap_imh, Ap_imw)]
d_app = compute_distance(AAp_feat_app, BBp_feat, c.weights)
d_coh = compute_distance(AAp_feat_coh, BBp_feat, c.weights)
if d_coh <= d_app * (1 + (2**(level - c.max_levels)) * c.k):
p = p_coh
i = i_coh
else:
p = p_app
i = i_app
# Update Bp and s
Bp_pyr[level][row, col] = Ap_pyr_list[i][level][tuple(p)]
if not c.convert:
color_im_out[row, col, :] = color_pyr_list[i][level][tuple(p)]
s.append(p)
im.append(i)
if debug:
sa.append(p_app)
if len(s) > 1 and not np.allclose(p_coh, np.array([-1, -1])):
sc.append(p_coh)
rstars.append(r_star)
app_dist[row, col] = d_app
coh_dist[row, col] = d_coh
if np.allclose(p, p_coh):
p_src[row, col] = coh_color
elif np.allclose(p, p_app):
p_src[row, col] = app_color
else:
print('Look, a bug! Squash it!')
raise
else:
sc.append((0, 0))
rstars.append((0, 0))
p_src[row, col] = err_color
ann_time_total = ann_time_total + ann_time_level
if debug:
assert(len(im) == np.product(img_src.shape))
img_src[:, :] = (np.array(im).astype(np.float64)/np.max(im)).reshape(img_src.shape)
# Save debugging structures
for path, var in zip(paths, vars):
fig = plt.imshow(var, interpolation='nearest', cmap='gray')
savefig_noborder(out_path + path, fig)
plt.close()
with open(out_path + '%d_srcs.pickle' % level, 'w') as f:
pickle.dump([sa, sc, rstars, s, im], f)
# Save color output images
if c.convert:
color_im_out = convert_to_RGB(np.dstack([Bp_pyr[level], color_pyr_list[i][level][:, :, 1:]]))
color_im_out = np.clip(color_im_out, 0, 1)
plt.imsave(out_path + 'level_%d_color.jpg' % level, color_im_out)
plt.imsave(out_path + out_path.split('/')[-2] + '.jpg', color_im_out)
stop_time = time.time()
print 'Level %d time: %f' % (level, stop_time - start_time)
print('Level %d ANN time: %f' % (level, ann_time_level))
end_time = time.time()
print 'Total time: %f' % (end_time - begin_time)
print('ANN time: %f' % ann_time_total)
min_sum = float('inf')
r_star = (np.nan, np.nan)
for r_row in np.arange(row_min, row_max, dtype=int):
col_end = col if r_row == row else col_max
for r_col in np.arange(col_min, col_end, dtype=int):
s_ix = r_row * Bp_w + r_col
# p = s(r) + (q - r)
p_r = np.array(s[s_ix]) + np.array([row, col]) - np.array([r_row, r_col])
# check that p_r is inside the bounds of A/Ap lg
A_h, A_w = A_pd[1].shape[:2] - 2 * c.pad_lg
if 0 <= p_r[0] < A_h and 0 <= p_r[1] < A_w:
AAp_feat = np.hstack([extract_pixel_feature( A_pd, p_r, c, full_feat=True),
extract_pixel_feature(Ap_pd, p_r, c, full_feat=False)])
assert(AAp_feat.shape == BBp_feat.shape)
new_sum = norm(AAp_feat - BBp_feat, ord=2)**2
if new_sum <= min_sum:
min_sum = new_sum
r_star = np.array([r_row, r_col])
if np.isnan(r_star).any():
return (-1, -1), (0, 0)
# s[r_star] + (q - r_star)
return tuple(s[r_star[0] * Bp_w + r_star[1]] + (np.array([row, col]) - r_star)), tuple(r_star)
def image_analogies_main(A_fname,
Ap_fname_list,
B_fname,
out_path,
c,
debug=False):
# # This is the setup code
begin_time = time.time()
start_time = time.time()
# Load images
A_pyr, Ap_pyr_list, B_pyr, Bp_pyr, color_pyr_list, c = img_setup(
A_fname, Ap_fname_list, B_fname, out_path, c)
# Save parameters for reference
names = [
'A_fname', 'Ap_fname_list', 'B_fname', 'c.convert', 'c.remap_lum',
'c.init_rand', 'c.AB_weight', 'c.k'
]
vars = [
A_fname, Ap_fname_list, B_fname, c.convert, c.remap_lum, c.init_rand,
c.AB_weight, c.k
]
save_metadata(out_path, names, vars)
# Pull features from B
B_features = compute_feature_array(B_pyr, c, full_feat=True)
stop_time = time.time()
print 'Environment Setup: %f' % (stop_time - start_time)
# Build Structures for ANN
start_time = time.time()
flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr_list, c)
stop_time = time.time()
ann_time_total = stop_time - start_time
print 'ANN Index Creation: %f' % (ann_time_total)
# ##########################################################################################
# # This is the Algorithm Code
# now we iterate per pixel in each level
for level in range(1, c.max_levels):
start_time = time.time()
ann_time_level = 0
print('Computing level %d of %d' % (level, c.max_levels - 1))
imh, imw = Bp_pyr[level].shape[:2]
color_im_out = np.nan * np.ones((imh, imw, 3))
s = []
im = []
if debug:
# make debugging structures
sa = []
sc = []
rstars = []
p_src = np.nan * np.ones((imh, imw, 3))
img_src = np.zeros((imh, imw))
app_dist = np.zeros((imh, imw))
coh_dist = np.zeros((imh, imw))
app_color = np.array([1, 0, 0])
coh_color = np.array([1, 1, 0])
err_color = np.array([0, 0, 0])
paths = [
'%d_psrc.eps' % (level),
'%d_appdist.eps' % (level),
'%d_cohdist.eps' % (level),
'%d_output.eps' % (level),
'%d_imgsrc.eps' % (level)
]
vars = [p_src, app_dist, coh_dist, Bp_pyr[level], img_src]
for row in range(imh):
for col in range(imw):
px = np.array([row, col])
# we pad on each iteration so Bp features will be more accurate
Bp_pd = pad_img_pair(Bp_pyr[level - 1], Bp_pyr[level], c)
BBp_feat = np.hstack([
B_features[level][px2ix(px, imw), :],
extract_pixel_feature(Bp_pd, px, c, full_feat=False)
])
assert (BBp_feat.shape == (As_size[level][1], ))
# Find Approx Nearest Neighbor
ann_start_time = time.time()
p_app_ix = best_approximate_match(flann[level],
flann_params[level],
BBp_feat)
assert (p_app_ix < As_size[level][0])
ann_stop_time = time.time()
ann_time_level = ann_time_level + ann_stop_time - ann_start_time
# translate p_app_ix back to row, col
Ap_imh, Ap_imw = Ap_pyr_list[0][level].shape[:2]
p_app, i_app = Ap_ix2px(p_app_ix, Ap_imh, Ap_imw)
# is this the first iteration for this level?
# then skip coherence step
if len(s) < 1:
p = p_app
i = i_app
# Find Coherence Match and Compare Distances
else:
p_coh, i_coh, r_star = best_coherence_match(
As[level], (Ap_imh, Ap_imw), BBp_feat, s, im, px, imw,
c)
if np.allclose(p_coh, np.array([-1, -1])):
p = p_app
i = i_app
else:
AAp_feat_app = As[level][p_app_ix]
AAp_feat_coh = As[level][Ap_px2ix(
p_coh, i_coh, Ap_imh, Ap_imw)]
d_app = compute_distance(AAp_feat_app, BBp_feat,
c.weights)
d_coh = compute_distance(AAp_feat_coh, BBp_feat,
c.weights)
if d_coh <= d_app * (1 +
(2**
(level - c.max_levels)) * c.k):
p = p_coh
i = i_coh
else:
p = p_app
i = i_app
# Update Bp and s
Bp_pyr[level][row, col] = Ap_pyr_list[i][level][tuple(p)]
if not c.convert:
color_im_out[row,
col, :] = color_pyr_list[i][level][tuple(p)]
s.append(p)
im.append(i)
if debug:
sa.append(p_app)
if len(s) > 1 and not np.allclose(p_coh, np.array([-1, -1
])):
sc.append(p_coh)
rstars.append(r_star)
app_dist[row, col] = d_app
coh_dist[row, col] = d_coh
if np.allclose(p, p_coh):
p_src[row, col] = coh_color
elif np.allclose(p, p_app):
p_src[row, col] = app_color
else:
print('Look, a bug! Squash it!')
raise
else:
sc.append((0, 0))
rstars.append((0, 0))
p_src[row, col] = err_color
ann_time_total = ann_time_total + ann_time_level
if debug:
assert (len(im) == np.product(img_src.shape))
img_src[:, :] = (np.array(im).astype(np.float64) /
np.max(im)).reshape(img_src.shape)
# Save debugging structures
for path, var in zip(paths, vars):
fig = plt.imshow(var, interpolation='nearest', cmap='gray')
savefig_noborder(out_path + path, fig)
plt.close()
with open(out_path + '%d_srcs.pickle' % level, 'w') as f:
pickle.dump([sa, sc, rstars, s, im], f)
# Save color output images
if c.convert:
color_im_out = convert_to_RGB(
np.dstack([Bp_pyr[level], color_pyr_list[i][level][:, :, 1:]]))
color_im_out = np.clip(color_im_out, 0, 1)
plt.imsave(out_path + 'level_%d_color.jpg' % level, color_im_out)
plt.imsave(out_path + out_path.split('/')[-2] + '.jpg', color_im_out)
stop_time = time.time()
print 'Level %d time: %f' % (level, stop_time - start_time)
print('Level %d ANN time: %f' % (level, ann_time_level))
end_time = time.time()
print 'Total time: %f' % (end_time - begin_time)
print('ANN time: %f' % ann_time_total)