Пример #1
0
def augment_data(imgs, embbed_imgs, thetas, aug_num_per_img, std=1):

    aug_imgs = np.repeat(imgs, aug_num_per_img, axis=0)
    aug_thetas = np.repeat(thetas, aug_num_per_img, axis=0)
    aug_img_embbed = np.repeat(embbed_imgs, aug_num_per_img, axis=0)

    for j, (img, embbed_img, theta) in enumerate(zip(imgs, embbed_imgs, thetas)):
        for i in range(aug_num_per_img):
            T = random_affine_T(std=std)
            aug_imgs[j*aug_num_per_img+i], _ = warp_image(img, T)
            aug_img_embbed[j*aug_num_per_img+i], _ = warp_image(embbed_img, T)
            aug_thetas[j*aug_num_per_img+i] = compose_transform(T, theta)

    return aug_imgs, aug_img_embbed, aug_thetas
Пример #2
0
    def prepare_image_transformations(self):
        tic = time.time()
        print('\nPrepare all transformed images..')
        for i, I in enumerate(self.imgs_big_embd):
            I_warped, _ = warp_image(I, self.trans[i], cv2.INTER_CUBIC)
            I_warped = self.embed_to_normal_sz_image(I_warped)
            I_warped = np.abs(I_warped / np.nanmax(I_warped))
            self.imgs_trans_all[i] = I_warped

        panoramic_img = np.nanmean(self.imgs_trans_all, axis=0)  # nanmean
        np.save(self.data_path + 'panoramic_img.npy', panoramic_img)

        fig4 = open_figure(4, 'Panoramic Image', (3, 2))
        PlotImages(4,
                   1,
                   1,
                   1, [panoramic_img], [''],
                   'gray',
                   axis=False,
                   colorbar=False)
        plt.show()
        fig4.savefig(self.mypath + '2_learning/BG/alignment_mean/panorama.png',
                     dpi=1000)

        toc = time.time()
        print('done images transformations: ', toc - tic)
Пример #3
0
 def unwarp_original_image(self, x_warped):
     x_warped_ = x_warped
     height, width, _ = x_warped_.shape
     x_unwarped_tmp = cv2.warpPerspective(x_warped_, inv(self.x_theta_refine), (width, height), flags=cv2.INTER_CUBIC)
     x_unwarped, _ = warp_image(x_unwarped_tmp, self.x_theta_inv, cv2.INTER_CUBIC)
     x_unwarped = x_unwarped[self.st_idx_y:self.st_idx_y + self.img_sz[0], self.st_idx_x:self.st_idx_x + self.img_sz[1], :]
     x_unwarped = np.abs(x_unwarped / np.nanmax(x_unwarped))
     return x_unwarped
Пример #4
0
 def unwarp_bg(self,bg_emb):
     bg_emb_ = bg_emb
     height, width, _ = bg_emb_.shape
     bg_tmp = cv2.warpPerspective(bg_emb_, inv(self.x_theta_refine), (width, height), flags=cv2.INTER_CUBIC)
     bg, _ = warp_image(bg_tmp, self.x_theta_inv, cv2.INTER_CUBIC)
     bg = bg[self.st_idx_y:self.st_idx_y + self.img_sz[0], self.st_idx_x:self.st_idx_x + self.img_sz[1], :]
     bg = np.abs(bg / np.nanmax(bg))
     return bg
Пример #5
0
def plot_results(batch_x, x_theta_test, theta_exp_test, img_sz, num_channels, batch_size, loss_list, loss_align_list, loss_regul_list, data_orig, mypath):
    plt.clf()
    plt.plot(loss_list)
    plt.plot(loss_align_list)
    plt.plot(loss_regul_list)
    plt.legend(['Total loss', 'Alignment loss','Regulator loss'], loc='best')
    plt.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/loss.png')

    imgs_trans_all = []
    imgs_trans_all2 = []
    imgs_notrans_all = []
    for i in range(len(batch_x)):
        I_orig = np.reshape(data_orig[i, ...], (img_sz[0], img_sz[1], num_channels))
        I_orig = prepare_nan_img(np.abs(I_orig))
        imgs_notrans_all.append(I_orig)

        I = np.reshape(batch_x[i,...], (img_sz[0], img_sz[1], num_channels))
        I = prepare_nan_img(np.abs(I))
        T = theta_exp_test[i, ...]
        T = np.reshape(T, (2, 3))

        I_t, d = warp_image(I, T, cv2.INTER_CUBIC)
        I_t = np.abs(I_t/np.nanmax(I_t))
        imgs_trans_all.append(I_t)

        # take transform image from the special transformer
        I_t2 = np.reshape(x_theta_test[i, ...], (img_sz[0], img_sz[1], num_channels))
        I_t2 = prepare_nan_img(np.abs(I_t2))
        I_t2 = np.abs(I_t2 / np.nanmax(I_t2))
        imgs_trans_all2.append(I_t2)

    # --------- build panoramic image of original images:------------
    panoramic_img_notrans = np.nanmedian(imgs_notrans_all, axis=0)  # nanmean
    fig3 = open_figure(3, 'Panoramic Image', (3, 2))
    PlotImages(3, 1, 1, 1, [panoramic_img_notrans], [''], 'gray', axis=False, colorbar=False)

    # --------- build panoramic image of transformed images using my warping:--------
    panoramic_img = np.nanmedian(imgs_trans_all, axis=0)  # nanmean
    fig4 = open_figure(4, 'Panoramic Image', (3, 2))
    PlotImages(4, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False)

    # --------- build panoramic image of transformed images using spacial transformer:----------
    #panoramic_img2 = np.nanmedian(imgs_trans_all2, axis=0)
    #fig5 = open_figure(5,'Panoramic Image',(3,2))
    #PlotImages(5,1,1,1,[panoramic_img2],[''],'gray',axis=False,colorbar=False)

    plt.show()
    fig3.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/STN_Panorama_initial.png', dpi=1000)
    fig4.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/STN_Panorama_transformed.png', dpi=1000)
Пример #6
0
def get_global_AFFINE():

    mypath = config.paths['my_path']
    imgs_big_embd = np.load(mypath + 'data/imgs_big_embd.npy')
    SE_trans = np.load(mypath + 'data/SE_trans.npy')
    AFFINE_residual = np.load(mypath + 'data/AFFINE_residual.npy')
    img_embd_sz_arr = np.load(mypath + 'data/img_embd_sz.npy')
    img_big_embd_sz_arr = np.load(mypath + 'data/img_big_emb_sz.npy')

    #print('imgs_big_embd: ', imgs_big_embd.shape)
    #print('SE_trans: ', SE_trans.shape)
    #print('AFFINE_residual: ', AFFINE_residual.shape)

    img_big_emb_sz = (int(img_big_embd_sz_arr[0]), int(img_big_embd_sz_arr[1]),
                      3)
    img_emb_sz = (int(img_embd_sz_arr[0]), int(img_embd_sz_arr[1]), 3)

    # Prepare final transformations:
    final_trans = []
    imgs_trans_all = []
    for i, I in enumerate(imgs_big_embd):
        T_Final = concat_trans(SE_trans[i],
                               np.reshape(AFFINE_residual[i], (2, 3)))
        I_warped, d = warp_image(I, T_Final, cv2.INTER_CUBIC)
        I_warped = embed_to_normal_sz_image(I_warped, img_emb_sz,
                                            img_big_emb_sz)
        I_warped = np.abs(I_warped / np.nanmax(I_warped))
        imgs_trans_all.append(I_warped)
        final_trans.append(T_Final)

    # build panoramic image with final transformations:
    # print('\nBuild panorama...')
    # panoramic_img = np.nanmedian(imgs_trans_all, axis=0)  # nanmean
    # fig1 = open_figure(1, 'Panoramic Image', (3, 2))
    # PlotImages(1, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False)
    # fig1.savefig('AFFINE/affine_alignment_results/Panorama_AFFINE_final.png', dpi=1000)
    # plt.show()

    # Save final transformations:
    final_trans = np.array(final_trans)
    np.save(mypath + 'data/final_AFFINE_trans.npy', final_trans)
Пример #7
0
    def warp_image_by_T_and_get_H(self, I, T, panoramic_img, ref_image, gap_refine):

        height,width,_ = I.shape
        x_warped,_ = warp_image(I, T, cv2.INTER_CUBIC)
        #test_imgs_warped_before_refine.append(x_warped)

        # ------ Refine theta: use SIFT to warp x_warped towards X_mean_warped:
        I_warped_tmp = self.embed_to_normal_sz_image(x_warped.copy())

        start_x,start_y,end_x,end_y = self.get_enclosing_rectangle(I_warped_tmp.copy())
        x0,y0,x1,y1 = self.add_refine_gap_to_enclosing_rectangle(start_x,start_y,end_x,end_y,gap_refine)
        ref_image.fill(np.nan)
        ref_image[y0:y1,x0:x1,:] = panoramic_img[y0:y1,x0:x1,:]

        # refine the transformed test image:
        if self.is_global_model:
            H = np.eye(3)
        else:
            H = self.get_relative_trans(I_warped_tmp,ref_image)

        return H, I_warped_tmp
Пример #8
0
    def transform_images_globally(self, n_frames, nparray_path):
        print('\nTransform Images globally...')
        imgs_trans_all = []
        img_sz = self.data.img_sz

        # The transformations received from SE-Sync are: from the original image location (center) -> to the global location in the panorama.

        # find the middle frame to create a minimum-size panoramic domain:
        trans_x = []
        for v in self.V:
            if v in range(0, n_frames):
                trans_x.append(self.V[v].ravel())

        trans_x = np.array(trans_x)
        middle_frame = np.argsort(trans_x[:,2])[trans_x.shape[0] // 2]
        print('middle_frame: ', middle_frame)

        # prepare transformations from center:
        trans = []
        T0_inv = invert_T(self.V[middle_frame])
        for v in self.V:
            if v in range(0, n_frames):
                T = revert_to_T0(self.V[v], T0_inv)
                if np.isfinite(np.linalg.cond(T)):
                    T_SE = invert_T(T)
                    if T_SE is not None:
                        trans.append(T_SE.ravel())
                    else:
                        trans.append(np.zeros(2,3).ravel())
                else:
                    trans.append(np.zeros(2,3).ravel())

        trans = np.array(trans)

        # get extreme translations to get estimation for img_embd_sz:
        x_sz = (2 * (int(np.max(np.abs(trans[:,2]))))) + np.max(img_sz) + 50
        y_sz = (2 * (int(np.max(np.abs(trans[:,5]))))) + np.max(img_sz) + 50
        img_emb_sz = (y_sz, x_sz, 3)
        img_big_emb_sz = (y_sz+400, x_sz+400, 3)

        print('img_emb_sz: ', img_emb_sz)
        print('img_big_emb_sz: ', img_big_emb_sz)
        np.save(nparray_path + "img_embd_sz.npy", img_emb_sz)
        np.save(nparray_path + "img_big_emb_sz.npy", img_big_emb_sz)

        # transform images in embeded frames:
        for v in self.V:
            if v in range(0, n_frames):
                T_SE = np.reshape(trans[v], (2,3))

                # embed the image:
                I = embed_to_big_image(self.data.imgs[v], img_emb_sz, img_big_emb_sz)

                if np.sum(T_SE) != 0: # if the transformation is valid, use it.
                    I_Rt, d = warp_image(I, T_SE, cv2.INTER_CUBIC)
                    I_Rt = np.abs(I_Rt)
                    I_Rt = embed_to_normal_sz_image(I_Rt, img_emb_sz, img_big_emb_sz)
                    imgs_trans_all.append(I_Rt)
                    self.data.imgs_trans.append(I_Rt)
                    self.data.trans.append(T_SE)
                    self.data.imgs_big_embd.append(I)
                    self.data.imgs_relevant.append(np.abs(self.data.imgs[v] / np.nanmax(self.data.imgs[v])))

        # build panoramic image (Sesync results):
        print('\nBuild panorama...')
        panoramic_img = np.nanmedian(imgs_trans_all, axis=0) # nanmean
        fig1 = open_figure(1, 'Panoramic Image', (3, 2))
        PlotImages(1, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False)
        fig1.savefig(config.paths['my_path'] + '1_joint_alignment/SE/se_alignment_results/SE_Panorama.png', dpi=1000)
        plt.show()