def test_extract_pixel_feature():
    # Make and test 2D image
    sm = 0.5 * np.ones((4, 5))
    sm[0, 0] = 0
    lg = 0.3 * np.ones((7, 10))
    lg[0, 0] = 1

    c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(lg)

    sm_0 = np.array([[0, 0, 0.5], [0, 0, 0.5], [0.5, 0.5, 0.5]])

    lg_0 = np.array([[0.3, 0.3, 0.3, 0.3, 0.3], [0.3, 1, 1, 0.3, 0.3],
                     [0.3, 1, 1, 0.3, 0.3], [0.3, 0.3, 0.3, 0.3, 0.3],
                     [0.3, 0.3, 0.3, 0.3, 0.3]])

    im_padded = pad_img_pair(sm, lg, c)

    # First test full feature

    correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()])
    feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=True)
    assert (np.allclose(feat, correct_feat_0_0))

    # Now test half feature

    correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()[:c.n_half]])
    feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=False)
    assert (np.allclose(feat, correct_feat_0_0))
def test_extract_pixel_feature():
    # Make and test 2D image
    sm = 0.5 * np.ones((4, 5))
    sm[0, 0] = 0
    lg = 0.3 * np.ones((7, 10))
    lg[0, 0] = 1

    c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(lg)

    sm_0 = np.array([[  0,   0, 0.5],
                     [  0,   0, 0.5],
                     [0.5, 0.5, 0.5]])

    lg_0 = np.array([[0.3, 0.3, 0.3, 0.3, 0.3],
                     [0.3,   1,   1, 0.3, 0.3],
                     [0.3,   1,   1, 0.3, 0.3],
                     [0.3, 0.3, 0.3, 0.3, 0.3],
                     [0.3, 0.3, 0.3, 0.3, 0.3]])

    im_padded = pad_img_pair(sm, lg, c)

    # First test full feature

    correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()])
    feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=True)
    assert(np.allclose(feat, correct_feat_0_0))

    # Now test half feature

    correct_feat_0_0 = np.hstack([sm_0.flatten(), lg_0.flatten()[:c.n_half]])
    feat = extract_pixel_feature(im_padded, (0, 0), c, full_feat=False)
    assert(np.allclose(feat, correct_feat_0_0))
def test_best_coherence_match():
    # make A_pd, Ap_pd, BBp_feat, s
    A_orig  = plt.imread('./test_images/test_best_coherence_match_A.jpg')
    Ap_orig = plt.imread('./test_images/test_best_coherence_match_Ap.jpg')

    A  = convert_to_YIQ( A_orig/255.)[:, :, 0]
    Ap = convert_to_YIQ(Ap_orig/255.)[:, :, 0]

    A_pyr  = compute_gaussian_pyramid( A, min_size=3)
    Ap_pyr = compute_gaussian_pyramid(Ap, min_size=3)

    imh, imw = A.shape[:2]

    c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(A)
    c.max_levels = len(A_pyr)

    A_pd  = pad_img_pair( A_pyr[-2],  A_pyr[-1], c)
    Ap_pd = pad_img_pair(Ap_pyr[-2], Ap_pyr[-1], c)

    flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr, c)

    # BBp_feat cases: all corners and middle
    indices = [(1, 1),
               (1, imw - 1),
               (imh - 1, 1),
               (imh - 1, imw - 1),
               (np.floor(imh/2.).astype(int), np.floor(imw/2.).astype(int))]

    for row, col in indices:
        num_px = row * imw + col
        s_rows = np.random.random_integers(num_px, size=num_px) - 1
        s_cols = np.random.random_integers(num_px, size=num_px) - 1
        s = [(rr, cc) for rr, cc in zip(s_rows, s_cols)]
        s[(row - 1) * imw + col - 1] = (row - 1, col - 1)

        Bs_feat = np.hstack([extract_pixel_feature( A_pd, (row, col), c, full_feat=True),
                             extract_pixel_feature(Ap_pd, (row, col), c, full_feat=False)])

        p_coh_orig, r_star_orig = best_coherence_match_orig(A_pd, Ap_pd, Bs_feat, s, (row, col, imw), c)
        p_coh_new, r_star_new = best_coherence_match(As[-1], A.shape, Bs_feat, s, (row, col, imw), c)

        try:
            assert(p_coh_orig == (row, col))
            assert(p_coh_new == (row, col))
        except:
            print('row, col, p_coh_orig, p_coh_new, s', row, col, p_coh_orig, p_coh_new, s)
            As_feat = np.hstack([extract_pixel_feature( A_pd, p_coh_orig, p_coh_new, c, full_feat=True),
                                 extract_pixel_feature(Ap_pd, p_coh_orig, p_coh_new, c, full_feat=False)])
            print('As_feat', As_feat)
            print('Bs_feat', Bs_feat)
            assert(False)
def test_best_coherence_match():
    # make A_pd, Ap_pd, BBp_feat, s
    A_orig = plt.imread('./test_images/test_best_coherence_match_A.jpg')
    Ap_orig = plt.imread('./test_images/test_best_coherence_match_Ap.jpg')

    A = convert_to_YIQ(A_orig / 255.)[:, :, 0]
    Ap = convert_to_YIQ(Ap_orig / 255.)[:, :, 0]

    A_pyr = compute_gaussian_pyramid(A, min_size=3)
    Ap_pyr = compute_gaussian_pyramid(Ap, min_size=3)

    imh, imw = A.shape[:2]

    c.num_ch, c.padding_sm, c.padding_lg, c.weights = c.setup_vars(A)
    c.max_levels = len(A_pyr)

    A_pd = pad_img_pair(A_pyr[-2], A_pyr[-1], c)
    Ap_pd = pad_img_pair(Ap_pyr[-2], Ap_pyr[-1], c)

    flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr, c)

    # BBp_feat cases: all corners and middle
    indices = [(1, 1), (1, imw - 1), (imh - 1, 1), (imh - 1, imw - 1),
               (np.floor(imh / 2.).astype(int), np.floor(imw / 2.).astype(int))
               ]

    for row, col in indices:
        num_px = row * imw + col
        s_rows = np.random.random_integers(num_px, size=num_px) - 1
        s_cols = np.random.random_integers(num_px, size=num_px) - 1
        s = [(rr, cc) for rr, cc in zip(s_rows, s_cols)]
        s[(row - 1) * imw + col - 1] = (row - 1, col - 1)

        Bs_feat = np.hstack([
            extract_pixel_feature(A_pd, (row, col), c, full_feat=True),
            extract_pixel_feature(Ap_pd, (row, col), c, full_feat=False)
        ])

        p_coh_orig, r_star_orig = best_coherence_match_orig(
            A_pd, Ap_pd, Bs_feat, s, (row, col, imw), c)
        p_coh_new, r_star_new = best_coherence_match(As[-1], A.shape, Bs_feat,
                                                     s, (row, col, imw), c)

        try:
            assert (p_coh_orig == (row, col))
            assert (p_coh_new == (row, col))
        except:
            print('row, col, p_coh_orig, p_coh_new, s', row, col, p_coh_orig,
                  p_coh_new, s)
            As_feat = np.hstack([
                extract_pixel_feature(A_pd,
                                      p_coh_orig,
                                      p_coh_new,
                                      c,
                                      full_feat=True),
                extract_pixel_feature(Ap_pd,
                                      p_coh_orig,
                                      p_coh_new,
                                      c,
                                      full_feat=False)
            ])
            print('As_feat', As_feat)
            print('Bs_feat', Bs_feat)
            assert (False)
    r_star = (np.nan, np.nan)
    for r_row in np.arange(row_min, row_max, dtype=int):
        col_end = col if r_row == row else col_max
        for r_col in np.arange(col_min, col_end, dtype=int):
            s_ix = r_row * Bp_w + r_col

            # p = s(r) + (q - r)
            p_r = np.array(s[s_ix]) + np.array([row, col]) - np.array(
                [r_row, r_col])

            # check that p_r is inside the bounds of A/Ap lg
            A_h, A_w = A_pd[1].shape[:2] - 2 * c.pad_lg

            if 0 <= p_r[0] < A_h and 0 <= p_r[1] < A_w:
                AAp_feat = np.hstack([
                    extract_pixel_feature(A_pd, p_r, c, full_feat=True),
                    extract_pixel_feature(Ap_pd, p_r, c, full_feat=False)
                ])

                assert (AAp_feat.shape == BBp_feat.shape)

                new_sum = norm(AAp_feat - BBp_feat, ord=2)**2

                if new_sum <= min_sum:
                    min_sum = new_sum
                    r_star = np.array([r_row, r_col])
    if np.isnan(r_star).any():
        return (-1, -1), (0, 0)

    # s[r_star] + (q - r_star)
    return tuple(s[r_star[0] * Bp_w + r_star[1]] +
def image_analogies_main(A_fname, Ap_fname_list, B_fname, out_path, c, debug=False):
    # # This is the setup code
    begin_time = time.time()
    start_time = time.time()

    # Load images
    A_pyr, Ap_pyr_list, B_pyr, Bp_pyr, color_pyr_list, c = img_setup(A_fname, Ap_fname_list, B_fname, out_path, c)

    # Save parameters for reference
    names = ['A_fname', 'Ap_fname_list', 'B_fname', 'c.convert', 'c.remap_lum', 'c.init_rand', 'c.AB_weight', 'c.k']
    vars  = [ A_fname,   Ap_fname_list,   B_fname,   c.convert,   c.remap_lum,   c.init_rand,   c.AB_weight,   c.k ]
    save_metadata(out_path, names, vars)

    # Pull features from B
    B_features = compute_feature_array(B_pyr, c, full_feat=True)

    stop_time = time.time()
    print 'Environment Setup: %f' % (stop_time - start_time)

    # Build Structures for ANN
    start_time = time.time()

    flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr_list, c)

    stop_time = time.time()
    ann_time_total = stop_time - start_time
    print 'ANN Index Creation: %f' % (ann_time_total)

    # ##########################################################################################

    # # This is the Algorithm Code

    # now we iterate per pixel in each level
    for level in range(1, c.max_levels):
        start_time = time.time()
        ann_time_level = 0
        print('Computing level %d of %d' % (level, c.max_levels - 1))

        imh, imw = Bp_pyr[level].shape[:2]
        color_im_out = np.nan * np.ones((imh, imw, 3))

        s = []
        im = []

        if debug:
            # make debugging structures
            sa = []
            sc = []
            rstars = []
            p_src     = np.nan * np.ones((imh, imw, 3))
            img_src   = np.zeros((imh, imw))
            app_dist  = np.zeros((imh, imw))
            coh_dist  = np.zeros((imh, imw))
            app_color = np.array([1, 0, 0])
            coh_color = np.array([1, 1, 0])
            err_color = np.array([0, 0, 0])

            paths = ['%d_psrc.eps'    % (level),
                     '%d_appdist.eps' % (level),
                     '%d_cohdist.eps' % (level),
                     '%d_output.eps'  % (level),
                     '%d_imgsrc.eps'  % (level)]
            vars = [p_src, app_dist, coh_dist, Bp_pyr[level], img_src]

        for row in range(imh):
            for col in range(imw):
                px = np.array([row, col])

                # we pad on each iteration so Bp features will be more accurate
                Bp_pd = pad_img_pair(Bp_pyr[level - 1], Bp_pyr[level], c)
                BBp_feat = np.hstack([B_features[level][px2ix(px, imw), :],
                                      extract_pixel_feature(Bp_pd, px, c, full_feat=False)])

                assert(BBp_feat.shape == (As_size[level][1],))

                # Find Approx Nearest Neighbor
                ann_start_time = time.time()

                p_app_ix = best_approximate_match(flann[level], flann_params[level], BBp_feat)
                assert(p_app_ix < As_size[level][0])

                ann_stop_time = time.time()
                ann_time_level = ann_time_level + ann_stop_time - ann_start_time

                # translate p_app_ix back to row, col
                Ap_imh, Ap_imw = Ap_pyr_list[0][level].shape[:2]
                p_app, i_app = Ap_ix2px(p_app_ix, Ap_imh, Ap_imw)

                # is this the first iteration for this level?
                # then skip coherence step
                if len(s) < 1:
                    p = p_app
                    i = i_app

                # Find Coherence Match and Compare Distances
                else:
                    p_coh, i_coh, r_star = best_coherence_match(As[level], (Ap_imh, Ap_imw), BBp_feat, s, im, px, imw, c)

                    if np.allclose(p_coh, np.array([-1, -1])):
                        p = p_app
                        i = i_app

                    else:
                        AAp_feat_app = As[level][p_app_ix]
                        AAp_feat_coh = As[level][Ap_px2ix(p_coh, i_coh, Ap_imh, Ap_imw)]

                        d_app = compute_distance(AAp_feat_app, BBp_feat, c.weights)
                        d_coh = compute_distance(AAp_feat_coh, BBp_feat, c.weights)

                        if d_coh <= d_app * (1 + (2**(level - c.max_levels)) * c.k):
                            p = p_coh
                            i = i_coh
                        else:
                            p = p_app
                            i = i_app

                # Update Bp and s
                Bp_pyr[level][row, col] = Ap_pyr_list[i][level][tuple(p)]

                if not c.convert:
                    color_im_out[row, col, :] = color_pyr_list[i][level][tuple(p)]

                s.append(p)
                im.append(i)

                if debug:
                    sa.append(p_app)
                    if len(s) > 1 and not np.allclose(p_coh, np.array([-1, -1])):
                        sc.append(p_coh)
                        rstars.append(r_star)
                        app_dist[row, col] = d_app
                        coh_dist[row, col] = d_coh
                        if np.allclose(p, p_coh):
                            p_src[row, col] = coh_color
                        elif np.allclose(p, p_app):
                            p_src[row, col] = app_color
                        else:
                            print('Look, a bug! Squash it!')
                            raise
                    else:
                        sc.append((0, 0))
                        rstars.append((0, 0))
                        p_src[row, col] = err_color

        ann_time_total = ann_time_total + ann_time_level

        if debug:
            assert(len(im) == np.product(img_src.shape))
            img_src[:, :] = (np.array(im).astype(np.float64)/np.max(im)).reshape(img_src.shape)
            # Save debugging structures
            for path, var in zip(paths, vars):
                fig = plt.imshow(var, interpolation='nearest', cmap='gray')
                savefig_noborder(out_path + path, fig)
                plt.close()

            with open(out_path + '%d_srcs.pickle' % level, 'w') as f:
                pickle.dump([sa, sc, rstars, s, im], f)

        # Save color output images
        if c.convert:
            color_im_out = convert_to_RGB(np.dstack([Bp_pyr[level], color_pyr_list[i][level][:, :, 1:]]))
            color_im_out = np.clip(color_im_out, 0, 1)
        plt.imsave(out_path + 'level_%d_color.jpg' % level, color_im_out)
        plt.imsave(out_path + out_path.split('/')[-2] + '.jpg', color_im_out)

        stop_time = time.time()
        print 'Level %d time: %f' % (level, stop_time - start_time)
        print('Level %d ANN time: %f' % (level, ann_time_level))

    end_time = time.time()
    print 'Total time: %f' % (end_time - begin_time)
    print('ANN time: %f' % ann_time_total)
    min_sum = float('inf')
    r_star = (np.nan, np.nan)
    for r_row in np.arange(row_min, row_max, dtype=int):
        col_end = col if r_row == row else col_max
        for r_col in np.arange(col_min, col_end, dtype=int):
            s_ix = r_row * Bp_w + r_col

            # p = s(r) + (q - r)
            p_r = np.array(s[s_ix]) + np.array([row, col]) - np.array([r_row, r_col])

            # check that p_r is inside the bounds of A/Ap lg
            A_h, A_w = A_pd[1].shape[:2] - 2 * c.pad_lg

            if 0 <= p_r[0] < A_h and 0 <= p_r[1] < A_w:
                AAp_feat = np.hstack([extract_pixel_feature( A_pd, p_r, c, full_feat=True),
                                      extract_pixel_feature(Ap_pd, p_r, c, full_feat=False)])

                assert(AAp_feat.shape == BBp_feat.shape)

                new_sum = norm(AAp_feat - BBp_feat, ord=2)**2

                if new_sum <= min_sum:
                    min_sum = new_sum
                    r_star = np.array([r_row, r_col])
    if np.isnan(r_star).any():
        return (-1, -1), (0, 0)

    # s[r_star] + (q - r_star)
    return tuple(s[r_star[0] * Bp_w + r_star[1]] + (np.array([row, col]) - r_star)), tuple(r_star)
Esempio n. 8
0
def image_analogies_main(A_fname,
                         Ap_fname_list,
                         B_fname,
                         out_path,
                         c,
                         debug=False):
    # # This is the setup code
    begin_time = time.time()
    start_time = time.time()

    # Load images
    A_pyr, Ap_pyr_list, B_pyr, Bp_pyr, color_pyr_list, c = img_setup(
        A_fname, Ap_fname_list, B_fname, out_path, c)

    # Save parameters for reference
    names = [
        'A_fname', 'Ap_fname_list', 'B_fname', 'c.convert', 'c.remap_lum',
        'c.init_rand', 'c.AB_weight', 'c.k'
    ]
    vars = [
        A_fname, Ap_fname_list, B_fname, c.convert, c.remap_lum, c.init_rand,
        c.AB_weight, c.k
    ]
    save_metadata(out_path, names, vars)

    # Pull features from B
    B_features = compute_feature_array(B_pyr, c, full_feat=True)

    stop_time = time.time()
    print 'Environment Setup: %f' % (stop_time - start_time)

    # Build Structures for ANN
    start_time = time.time()

    flann, flann_params, As, As_size = create_index(A_pyr, Ap_pyr_list, c)

    stop_time = time.time()
    ann_time_total = stop_time - start_time
    print 'ANN Index Creation: %f' % (ann_time_total)

    # ##########################################################################################

    # # This is the Algorithm Code

    # now we iterate per pixel in each level
    for level in range(1, c.max_levels):
        start_time = time.time()
        ann_time_level = 0
        print('Computing level %d of %d' % (level, c.max_levels - 1))

        imh, imw = Bp_pyr[level].shape[:2]
        color_im_out = np.nan * np.ones((imh, imw, 3))

        s = []
        im = []

        if debug:
            # make debugging structures
            sa = []
            sc = []
            rstars = []
            p_src = np.nan * np.ones((imh, imw, 3))
            img_src = np.zeros((imh, imw))
            app_dist = np.zeros((imh, imw))
            coh_dist = np.zeros((imh, imw))
            app_color = np.array([1, 0, 0])
            coh_color = np.array([1, 1, 0])
            err_color = np.array([0, 0, 0])

            paths = [
                '%d_psrc.eps' % (level),
                '%d_appdist.eps' % (level),
                '%d_cohdist.eps' % (level),
                '%d_output.eps' % (level),
                '%d_imgsrc.eps' % (level)
            ]
            vars = [p_src, app_dist, coh_dist, Bp_pyr[level], img_src]

        for row in range(imh):
            for col in range(imw):
                px = np.array([row, col])

                # we pad on each iteration so Bp features will be more accurate
                Bp_pd = pad_img_pair(Bp_pyr[level - 1], Bp_pyr[level], c)
                BBp_feat = np.hstack([
                    B_features[level][px2ix(px, imw), :],
                    extract_pixel_feature(Bp_pd, px, c, full_feat=False)
                ])

                assert (BBp_feat.shape == (As_size[level][1], ))

                # Find Approx Nearest Neighbor
                ann_start_time = time.time()

                p_app_ix = best_approximate_match(flann[level],
                                                  flann_params[level],
                                                  BBp_feat)
                assert (p_app_ix < As_size[level][0])

                ann_stop_time = time.time()
                ann_time_level = ann_time_level + ann_stop_time - ann_start_time

                # translate p_app_ix back to row, col
                Ap_imh, Ap_imw = Ap_pyr_list[0][level].shape[:2]
                p_app, i_app = Ap_ix2px(p_app_ix, Ap_imh, Ap_imw)

                # is this the first iteration for this level?
                # then skip coherence step
                if len(s) < 1:
                    p = p_app
                    i = i_app

                # Find Coherence Match and Compare Distances
                else:
                    p_coh, i_coh, r_star = best_coherence_match(
                        As[level], (Ap_imh, Ap_imw), BBp_feat, s, im, px, imw,
                        c)

                    if np.allclose(p_coh, np.array([-1, -1])):
                        p = p_app
                        i = i_app

                    else:
                        AAp_feat_app = As[level][p_app_ix]
                        AAp_feat_coh = As[level][Ap_px2ix(
                            p_coh, i_coh, Ap_imh, Ap_imw)]

                        d_app = compute_distance(AAp_feat_app, BBp_feat,
                                                 c.weights)
                        d_coh = compute_distance(AAp_feat_coh, BBp_feat,
                                                 c.weights)

                        if d_coh <= d_app * (1 +
                                             (2**
                                              (level - c.max_levels)) * c.k):
                            p = p_coh
                            i = i_coh
                        else:
                            p = p_app
                            i = i_app

                # Update Bp and s
                Bp_pyr[level][row, col] = Ap_pyr_list[i][level][tuple(p)]

                if not c.convert:
                    color_im_out[row,
                                 col, :] = color_pyr_list[i][level][tuple(p)]

                s.append(p)
                im.append(i)

                if debug:
                    sa.append(p_app)
                    if len(s) > 1 and not np.allclose(p_coh, np.array([-1, -1
                                                                       ])):
                        sc.append(p_coh)
                        rstars.append(r_star)
                        app_dist[row, col] = d_app
                        coh_dist[row, col] = d_coh
                        if np.allclose(p, p_coh):
                            p_src[row, col] = coh_color
                        elif np.allclose(p, p_app):
                            p_src[row, col] = app_color
                        else:
                            print('Look, a bug! Squash it!')
                            raise
                    else:
                        sc.append((0, 0))
                        rstars.append((0, 0))
                        p_src[row, col] = err_color

        ann_time_total = ann_time_total + ann_time_level

        if debug:
            assert (len(im) == np.product(img_src.shape))
            img_src[:, :] = (np.array(im).astype(np.float64) /
                             np.max(im)).reshape(img_src.shape)
            # Save debugging structures
            for path, var in zip(paths, vars):
                fig = plt.imshow(var, interpolation='nearest', cmap='gray')
                savefig_noborder(out_path + path, fig)
                plt.close()

            with open(out_path + '%d_srcs.pickle' % level, 'w') as f:
                pickle.dump([sa, sc, rstars, s, im], f)

        # Save color output images
        if c.convert:
            color_im_out = convert_to_RGB(
                np.dstack([Bp_pyr[level], color_pyr_list[i][level][:, :, 1:]]))
            color_im_out = np.clip(color_im_out, 0, 1)
        plt.imsave(out_path + 'level_%d_color.jpg' % level, color_im_out)
        plt.imsave(out_path + out_path.split('/')[-2] + '.jpg', color_im_out)

        stop_time = time.time()
        print 'Level %d time: %f' % (level, stop_time - start_time)
        print('Level %d ANN time: %f' % (level, ann_time_level))

    end_time = time.time()
    print 'Total time: %f' % (end_time - begin_time)
    print('ANN time: %f' % ann_time_total)