def augment_data(imgs, embbed_imgs, thetas, aug_num_per_img, std=1):
aug_imgs = np.repeat(imgs, aug_num_per_img, axis=0)
aug_thetas = np.repeat(thetas, aug_num_per_img, axis=0)
aug_img_embbed = np.repeat(embbed_imgs, aug_num_per_img, axis=0)
for j, (img, embbed_img, theta) in enumerate(zip(imgs, embbed_imgs, thetas)):
for i in range(aug_num_per_img):
T = random_affine_T(std=std)
aug_imgs[j*aug_num_per_img+i], _ = warp_image(img, T)
aug_img_embbed[j*aug_num_per_img+i], _ = warp_image(embbed_img, T)
aug_thetas[j*aug_num_per_img+i] = compose_transform(T, theta)
return aug_imgs, aug_img_embbed, aug_thetas
def prepare_image_transformations(self):
tic = time.time()
print('\nPrepare all transformed images..')
for i, I in enumerate(self.imgs_big_embd):
I_warped, _ = warp_image(I, self.trans[i], cv2.INTER_CUBIC)
I_warped = self.embed_to_normal_sz_image(I_warped)
I_warped = np.abs(I_warped / np.nanmax(I_warped))
self.imgs_trans_all[i] = I_warped
panoramic_img = np.nanmean(self.imgs_trans_all, axis=0) # nanmean
np.save(self.data_path + 'panoramic_img.npy', panoramic_img)
fig4 = open_figure(4, 'Panoramic Image', (3, 2))
PlotImages(4,
1,
1,
1, [panoramic_img], [''],
'gray',
axis=False,
colorbar=False)
plt.show()
fig4.savefig(self.mypath + '2_learning/BG/alignment_mean/panorama.png',
dpi=1000)
toc = time.time()
print('done images transformations: ', toc - tic)
def unwarp_original_image(self, x_warped):
x_warped_ = x_warped
height, width, _ = x_warped_.shape
x_unwarped_tmp = cv2.warpPerspective(x_warped_, inv(self.x_theta_refine), (width, height), flags=cv2.INTER_CUBIC)
x_unwarped, _ = warp_image(x_unwarped_tmp, self.x_theta_inv, cv2.INTER_CUBIC)
x_unwarped = x_unwarped[self.st_idx_y:self.st_idx_y + self.img_sz[0], self.st_idx_x:self.st_idx_x + self.img_sz[1], :]
x_unwarped = np.abs(x_unwarped / np.nanmax(x_unwarped))
return x_unwarped
def unwarp_bg(self,bg_emb):
bg_emb_ = bg_emb
height, width, _ = bg_emb_.shape
bg_tmp = cv2.warpPerspective(bg_emb_, inv(self.x_theta_refine), (width, height), flags=cv2.INTER_CUBIC)
bg, _ = warp_image(bg_tmp, self.x_theta_inv, cv2.INTER_CUBIC)
bg = bg[self.st_idx_y:self.st_idx_y + self.img_sz[0], self.st_idx_x:self.st_idx_x + self.img_sz[1], :]
bg = np.abs(bg / np.nanmax(bg))
return bg
def plot_results(batch_x, x_theta_test, theta_exp_test, img_sz, num_channels, batch_size, loss_list, loss_align_list, loss_regul_list, data_orig, mypath):
plt.clf()
plt.plot(loss_list)
plt.plot(loss_align_list)
plt.plot(loss_regul_list)
plt.legend(['Total loss', 'Alignment loss','Regulator loss'], loc='best')
plt.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/loss.png')
imgs_trans_all = []
imgs_trans_all2 = []
imgs_notrans_all = []
for i in range(len(batch_x)):
I_orig = np.reshape(data_orig[i, ...], (img_sz[0], img_sz[1], num_channels))
I_orig = prepare_nan_img(np.abs(I_orig))
imgs_notrans_all.append(I_orig)
I = np.reshape(batch_x[i,...], (img_sz[0], img_sz[1], num_channels))
I = prepare_nan_img(np.abs(I))
T = theta_exp_test[i, ...]
T = np.reshape(T, (2, 3))
I_t, d = warp_image(I, T, cv2.INTER_CUBIC)
I_t = np.abs(I_t/np.nanmax(I_t))
imgs_trans_all.append(I_t)
# take transform image from the special transformer
I_t2 = np.reshape(x_theta_test[i, ...], (img_sz[0], img_sz[1], num_channels))
I_t2 = prepare_nan_img(np.abs(I_t2))
I_t2 = np.abs(I_t2 / np.nanmax(I_t2))
imgs_trans_all2.append(I_t2)
# --------- build panoramic image of original images:------------
panoramic_img_notrans = np.nanmedian(imgs_notrans_all, axis=0) # nanmean
fig3 = open_figure(3, 'Panoramic Image', (3, 2))
PlotImages(3, 1, 1, 1, [panoramic_img_notrans], [''], 'gray', axis=False, colorbar=False)
# --------- build panoramic image of transformed images using my warping:--------
panoramic_img = np.nanmedian(imgs_trans_all, axis=0) # nanmean
fig4 = open_figure(4, 'Panoramic Image', (3, 2))
PlotImages(4, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False)
# --------- build panoramic image of transformed images using spacial transformer:----------
#panoramic_img2 = np.nanmedian(imgs_trans_all2, axis=0)
#fig5 = open_figure(5,'Panoramic Image',(3,2))
#PlotImages(5,1,1,1,[panoramic_img2],[''],'gray',axis=False,colorbar=False)
plt.show()
fig3.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/STN_Panorama_initial.png', dpi=1000)
fig4.savefig(mypath + '1_joint_alignment/STN/stn_alignment_results/STN_Panorama_transformed.png', dpi=1000)
def get_global_AFFINE():
mypath = config.paths['my_path']
imgs_big_embd = np.load(mypath + 'data/imgs_big_embd.npy')
SE_trans = np.load(mypath + 'data/SE_trans.npy')
AFFINE_residual = np.load(mypath + 'data/AFFINE_residual.npy')
img_embd_sz_arr = np.load(mypath + 'data/img_embd_sz.npy')
img_big_embd_sz_arr = np.load(mypath + 'data/img_big_emb_sz.npy')
#print('imgs_big_embd: ', imgs_big_embd.shape)
#print('SE_trans: ', SE_trans.shape)
#print('AFFINE_residual: ', AFFINE_residual.shape)
img_big_emb_sz = (int(img_big_embd_sz_arr[0]), int(img_big_embd_sz_arr[1]),
3)
img_emb_sz = (int(img_embd_sz_arr[0]), int(img_embd_sz_arr[1]), 3)
# Prepare final transformations:
final_trans = []
imgs_trans_all = []
for i, I in enumerate(imgs_big_embd):
T_Final = concat_trans(SE_trans[i],
np.reshape(AFFINE_residual[i], (2, 3)))
I_warped, d = warp_image(I, T_Final, cv2.INTER_CUBIC)
I_warped = embed_to_normal_sz_image(I_warped, img_emb_sz,
img_big_emb_sz)
I_warped = np.abs(I_warped / np.nanmax(I_warped))
imgs_trans_all.append(I_warped)
final_trans.append(T_Final)
# build panoramic image with final transformations:
# print('\nBuild panorama...')
# panoramic_img = np.nanmedian(imgs_trans_all, axis=0) # nanmean
# fig1 = open_figure(1, 'Panoramic Image', (3, 2))
# PlotImages(1, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False)
# fig1.savefig('AFFINE/affine_alignment_results/Panorama_AFFINE_final.png', dpi=1000)
# plt.show()
# Save final transformations:
final_trans = np.array(final_trans)
np.save(mypath + 'data/final_AFFINE_trans.npy', final_trans)
def warp_image_by_T_and_get_H(self, I, T, panoramic_img, ref_image, gap_refine):
height,width,_ = I.shape
x_warped,_ = warp_image(I, T, cv2.INTER_CUBIC)
#test_imgs_warped_before_refine.append(x_warped)
# ------ Refine theta: use SIFT to warp x_warped towards X_mean_warped:
I_warped_tmp = self.embed_to_normal_sz_image(x_warped.copy())
start_x,start_y,end_x,end_y = self.get_enclosing_rectangle(I_warped_tmp.copy())
x0,y0,x1,y1 = self.add_refine_gap_to_enclosing_rectangle(start_x,start_y,end_x,end_y,gap_refine)
ref_image.fill(np.nan)
ref_image[y0:y1,x0:x1,:] = panoramic_img[y0:y1,x0:x1,:]
# refine the transformed test image:
if self.is_global_model:
H = np.eye(3)
else:
H = self.get_relative_trans(I_warped_tmp,ref_image)
return H, I_warped_tmp
def transform_images_globally(self, n_frames, nparray_path):
print('\nTransform Images globally...')
imgs_trans_all = []
img_sz = self.data.img_sz
# The transformations received from SE-Sync are: from the original image location (center) -> to the global location in the panorama.
# find the middle frame to create a minimum-size panoramic domain:
trans_x = []
for v in self.V:
if v in range(0, n_frames):
trans_x.append(self.V[v].ravel())
trans_x = np.array(trans_x)
middle_frame = np.argsort(trans_x[:,2])[trans_x.shape[0] // 2]
print('middle_frame: ', middle_frame)
# prepare transformations from center:
trans = []
T0_inv = invert_T(self.V[middle_frame])
for v in self.V:
if v in range(0, n_frames):
T = revert_to_T0(self.V[v], T0_inv)
if np.isfinite(np.linalg.cond(T)):
T_SE = invert_T(T)
if T_SE is not None:
trans.append(T_SE.ravel())
else:
trans.append(np.zeros(2,3).ravel())
else:
trans.append(np.zeros(2,3).ravel())
trans = np.array(trans)
# get extreme translations to get estimation for img_embd_sz:
x_sz = (2 * (int(np.max(np.abs(trans[:,2]))))) + np.max(img_sz) + 50
y_sz = (2 * (int(np.max(np.abs(trans[:,5]))))) + np.max(img_sz) + 50
img_emb_sz = (y_sz, x_sz, 3)
img_big_emb_sz = (y_sz+400, x_sz+400, 3)
print('img_emb_sz: ', img_emb_sz)
print('img_big_emb_sz: ', img_big_emb_sz)
np.save(nparray_path + "img_embd_sz.npy", img_emb_sz)
np.save(nparray_path + "img_big_emb_sz.npy", img_big_emb_sz)
# transform images in embeded frames:
for v in self.V:
if v in range(0, n_frames):
T_SE = np.reshape(trans[v], (2,3))
# embed the image:
I = embed_to_big_image(self.data.imgs[v], img_emb_sz, img_big_emb_sz)
if np.sum(T_SE) != 0: # if the transformation is valid, use it.
I_Rt, d = warp_image(I, T_SE, cv2.INTER_CUBIC)
I_Rt = np.abs(I_Rt)
I_Rt = embed_to_normal_sz_image(I_Rt, img_emb_sz, img_big_emb_sz)
imgs_trans_all.append(I_Rt)
self.data.imgs_trans.append(I_Rt)
self.data.trans.append(T_SE)
self.data.imgs_big_embd.append(I)
self.data.imgs_relevant.append(np.abs(self.data.imgs[v] / np.nanmax(self.data.imgs[v])))
# build panoramic image (Sesync results):
print('\nBuild panorama...')
panoramic_img = np.nanmedian(imgs_trans_all, axis=0) # nanmean
fig1 = open_figure(1, 'Panoramic Image', (3, 2))
PlotImages(1, 1, 1, 1, [panoramic_img], [''], 'gray', axis=False, colorbar=False)
fig1.savefig(config.paths['my_path'] + '1_joint_alignment/SE/se_alignment_results/SE_Panorama.png', dpi=1000)
plt.show()