def augment_data(imgs, embbed_imgs, thetas, aug_num_per_img, std=1): aug_imgs = np.repeat(imgs, aug_num_per_img, axis=0) aug_thetas = np.repeat(thetas, aug_num_per_img, axis=0) aug_img_embbed = np.repeat(embbed_imgs, aug_num_per_img, axis=0) for j, (img, embbed_img, theta) in enumerate(zip(imgs, embbed_imgs, thetas)): for i in range(aug_num_per_img): T = random_affine_T(std=std) aug_imgs[j*aug_num_per_img+i], _ = warp_image(img, T) aug_img_embbed[j*aug_num_per_img+i], _ = warp_image(embbed_img, T) aug_thetas[j*aug_num_per_img+i] = compose_transform(T, theta) return aug_imgs, aug_img_embbed, aug_thetas
def prepare_image_transformations(self): tic = time.time() print('\nPrepare all transformed images..') for i, I in enumerate(self.imgs_big_embd): I_warped, _ = warp_image(I, self.trans[i], cv2.INTER_CUBIC) I_warped = self.embed_to_normal_sz_image(I_warped) I_warped = np.abs(I_warped / np.nanmax(I_warped)) self.imgs_trans_all[i] = I_warped panoramic_img = np.nanmean(self.imgs_trans_all, axis=0) # nanmean np.save(self.data_path + 'panoramic_img.npy', panoramic_img) fig4 = open_figure(4, 'Panoramic Image', (3, 2)) PlotImages(4, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False) plt.show() fig4.savefig(self.mypath + '2_learning/BG/alignment_mean/panorama.png', dpi=1000) toc = time.time() print('done images transformations: ', toc - tic)
def unwarp_original_image(self, x_warped): x_warped_ = x_warped height, width, _ = x_warped_.shape x_unwarped_tmp = cv2.warpPerspective(x_warped_, inv(self.x_theta_refine), (width, height), flags=cv2.INTER_CUBIC) x_unwarped, _ = warp_image(x_unwarped_tmp, self.x_theta_inv, cv2.INTER_CUBIC) x_unwarped = x_unwarped[self.st_idx_y:self.st_idx_y + self.img_sz[0], self.st_idx_x:self.st_idx_x + self.img_sz[1], :] x_unwarped = np.abs(x_unwarped / np.nanmax(x_unwarped)) return x_unwarped
def unwarp_bg(self,bg_emb): bg_emb_ = bg_emb height, width, _ = bg_emb_.shape bg_tmp = cv2.warpPerspective(bg_emb_, inv(self.x_theta_refine), (width, height), flags=cv2.INTER_CUBIC) bg, _ = warp_image(bg_tmp, self.x_theta_inv, cv2.INTER_CUBIC) bg = bg[self.st_idx_y:self.st_idx_y + self.img_sz[0], self.st_idx_x:self.st_idx_x + self.img_sz[1], :] bg = np.abs(bg / np.nanmax(bg)) return bg
def plot_results(batch_x, x_theta_test, theta_exp_test, img_sz, num_channels, batch_size, loss_list, loss_align_list, loss_regul_list, data_orig, mypath): plt.clf() plt.plot(loss_list) plt.plot(loss_align_list) plt.plot(loss_regul_list) plt.legend(['Total loss', 'Alignment loss','Regulator loss'], loc='best') plt.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/loss.png') imgs_trans_all = [] imgs_trans_all2 = [] imgs_notrans_all = [] for i in range(len(batch_x)): I_orig = np.reshape(data_orig[i, ...], (img_sz[0], img_sz[1], num_channels)) I_orig = prepare_nan_img(np.abs(I_orig)) imgs_notrans_all.append(I_orig) I = np.reshape(batch_x[i,...], (img_sz[0], img_sz[1], num_channels)) I = prepare_nan_img(np.abs(I)) T = theta_exp_test[i, ...] T = np.reshape(T, (2, 3)) I_t, d = warp_image(I, T, cv2.INTER_CUBIC) I_t = np.abs(I_t/np.nanmax(I_t)) imgs_trans_all.append(I_t) # take transform image from the special transformer I_t2 = np.reshape(x_theta_test[i, ...], (img_sz[0], img_sz[1], num_channels)) I_t2 = prepare_nan_img(np.abs(I_t2)) I_t2 = np.abs(I_t2 / np.nanmax(I_t2)) imgs_trans_all2.append(I_t2) # --------- build panoramic image of original images:------------ panoramic_img_notrans = np.nanmedian(imgs_notrans_all, axis=0) # nanmean fig3 = open_figure(3, 'Panoramic Image', (3, 2)) PlotImages(3, 1, 1, 1, [panoramic_img_notrans], [''], 'gray', axis=False, colorbar=False) # --------- build panoramic image of transformed images using my warping:-------- panoramic_img = np.nanmedian(imgs_trans_all, axis=0) # nanmean fig4 = open_figure(4, 'Panoramic Image', (3, 2)) PlotImages(4, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False) # --------- build panoramic image of transformed images using spacial transformer:---------- #panoramic_img2 = np.nanmedian(imgs_trans_all2, axis=0) #fig5 = open_figure(5,'Panoramic Image',(3,2)) #PlotImages(5,1,1,1,[panoramic_img2],[''],'gray',axis=False,colorbar=False) plt.show() fig3.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/STN_Panorama_initial.png', dpi=1000) fig4.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/STN_Panorama_transformed.png', dpi=1000)
def get_global_AFFINE(): mypath = config.paths['my_path'] imgs_big_embd = np.load(mypath + 'data/imgs_big_embd.npy') SE_trans = np.load(mypath + 'data/SE_trans.npy') AFFINE_residual = np.load(mypath + 'data/AFFINE_residual.npy') img_embd_sz_arr = np.load(mypath + 'data/img_embd_sz.npy') img_big_embd_sz_arr = np.load(mypath + 'data/img_big_emb_sz.npy') #print('imgs_big_embd: ', imgs_big_embd.shape) #print('SE_trans: ', SE_trans.shape) #print('AFFINE_residual: ', AFFINE_residual.shape) img_big_emb_sz = (int(img_big_embd_sz_arr[0]), int(img_big_embd_sz_arr[1]), 3) img_emb_sz = (int(img_embd_sz_arr[0]), int(img_embd_sz_arr[1]), 3) # Prepare final transformations: final_trans = [] imgs_trans_all = [] for i, I in enumerate(imgs_big_embd): T_Final = concat_trans(SE_trans[i], np.reshape(AFFINE_residual[i], (2, 3))) I_warped, d = warp_image(I, T_Final, cv2.INTER_CUBIC) I_warped = embed_to_normal_sz_image(I_warped, img_emb_sz, img_big_emb_sz) I_warped = np.abs(I_warped / np.nanmax(I_warped)) imgs_trans_all.append(I_warped) final_trans.append(T_Final) # build panoramic image with final transformations: # print('\nBuild panorama...') # panoramic_img = np.nanmedian(imgs_trans_all, axis=0) # nanmean # fig1 = open_figure(1, 'Panoramic Image', (3, 2)) # PlotImages(1, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False) # fig1.savefig('AFFINE/affine_alignment_results/Panorama_AFFINE_final.png', dpi=1000) # plt.show() # Save final transformations: final_trans = np.array(final_trans) np.save(mypath + 'data/final_AFFINE_trans.npy', final_trans)
def warp_image_by_T_and_get_H(self, I, T, panoramic_img, ref_image, gap_refine): height,width,_ = I.shape x_warped,_ = warp_image(I, T, cv2.INTER_CUBIC) #test_imgs_warped_before_refine.append(x_warped) # ------ Refine theta: use SIFT to warp x_warped towards X_mean_warped: I_warped_tmp = self.embed_to_normal_sz_image(x_warped.copy()) start_x,start_y,end_x,end_y = self.get_enclosing_rectangle(I_warped_tmp.copy()) x0,y0,x1,y1 = self.add_refine_gap_to_enclosing_rectangle(start_x,start_y,end_x,end_y,gap_refine) ref_image.fill(np.nan) ref_image[y0:y1,x0:x1,:] = panoramic_img[y0:y1,x0:x1,:] # refine the transformed test image: if self.is_global_model: H = np.eye(3) else: H = self.get_relative_trans(I_warped_tmp,ref_image) return H, I_warped_tmp
def transform_images_globally(self, n_frames, nparray_path): print('\nTransform Images globally...') imgs_trans_all = [] img_sz = self.data.img_sz # The transformations received from SE-Sync are: from the original image location (center) -> to the global location in the panorama. # find the middle frame to create a minimum-size panoramic domain: trans_x = [] for v in self.V: if v in range(0, n_frames): trans_x.append(self.V[v].ravel()) trans_x = np.array(trans_x) middle_frame = np.argsort(trans_x[:,2])[trans_x.shape[0] // 2] print('middle_frame: ', middle_frame) # prepare transformations from center: trans = [] T0_inv = invert_T(self.V[middle_frame]) for v in self.V: if v in range(0, n_frames): T = revert_to_T0(self.V[v], T0_inv) if np.isfinite(np.linalg.cond(T)): T_SE = invert_T(T) if T_SE is not None: trans.append(T_SE.ravel()) else: trans.append(np.zeros(2,3).ravel()) else: trans.append(np.zeros(2,3).ravel()) trans = np.array(trans) # get extreme translations to get estimation for img_embd_sz: x_sz = (2 * (int(np.max(np.abs(trans[:,2]))))) + np.max(img_sz) + 50 y_sz = (2 * (int(np.max(np.abs(trans[:,5]))))) + np.max(img_sz) + 50 img_emb_sz = (y_sz, x_sz, 3) img_big_emb_sz = (y_sz+400, x_sz+400, 3) print('img_emb_sz: ', img_emb_sz) print('img_big_emb_sz: ', img_big_emb_sz) np.save(nparray_path + "img_embd_sz.npy", img_emb_sz) np.save(nparray_path + "img_big_emb_sz.npy", img_big_emb_sz) # transform images in embeded frames: for v in self.V: if v in range(0, n_frames): T_SE = np.reshape(trans[v], (2,3)) # embed the image: I = embed_to_big_image(self.data.imgs[v], img_emb_sz, img_big_emb_sz) if np.sum(T_SE) != 0: # if the transformation is valid, use it. I_Rt, d = warp_image(I, T_SE, cv2.INTER_CUBIC) I_Rt = np.abs(I_Rt) I_Rt = embed_to_normal_sz_image(I_Rt, img_emb_sz, img_big_emb_sz) imgs_trans_all.append(I_Rt) self.data.imgs_trans.append(I_Rt) self.data.trans.append(T_SE) self.data.imgs_big_embd.append(I) self.data.imgs_relevant.append(np.abs(self.data.imgs[v] / np.nanmax(self.data.imgs[v]))) # build panoramic image (Sesync results): print('\nBuild panorama...') panoramic_img = np.nanmedian(imgs_trans_all, axis=0) # nanmean fig1 = open_figure(1, 'Panoramic Image', (3, 2)) PlotImages(1, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False) fig1.savefig(config.paths['my_path'] + '1_joint_alignment/SE/se_alignment_results/SE_Panorama.png', dpi=1000) plt.show()