def poisson_blend_old(input, output, mask): """ * inputs: - input (torch.Tensor, required) Input tensor of Completion Network. - output (torch.Tensor, required) Output tensor of Completion Network. - mask (torch.Tensor, required) Input mask tensor of Completion Network. * returns: Image tensor inpainted using poisson image editing method. """ num_samples = input.shape[0] ret = [] # convert torch array to numpy array followed by # converting 'channel first' format to 'channel last' format. input_np = np.transpose(np.copy(input.cpu().numpy()), axes=(0, 2, 3, 1)) output_np = np.transpose(np.copy(output.cpu().numpy()), axes=(0, 2, 3, 1)) mask_np = np.transpose(np.copy(mask.cpu().numpy()), axes=(0, 2, 3, 1)) # apply poisson image editing method for each input/output image and mask. for i in range(num_samples): inpainted_np = blend(input_np[i], output_np[i], mask_np[i]) inpainted = torch.from_numpy(np.transpose(inpainted_np, axes=(2, 0, 1))) inpainted = torch.unsqueeze(inpainted, dim=0) ret.append(inpainted) ret = torch.cat(ret, dim=0) return ret
def poisson_blend(self, imgs1, imgs2, mask): out = np.zeros(imgs1.shape) for i in range(0, len(imgs1)): img1 = (imgs1[i] + 1.) / 2.0 img2 = (imgs2[i] + 1.) / 2.0 out[i] = np.clip((poissonblending.blend(img1, img2, 1 - mask) - 0.5) * 2, -1.0, 1.0) # print (np.max(out[i]), np.min(out[i])) return out.astype(np.float32)
def blend(self, g_out): # Conduct Poission Blending to the output out_im = (np.array(g_out) + 1) / 2 in_im = (np.array(self.input_images) + 1) / 2 for i in range(len(g_out)): out_im[i] = poissonblending.blend(in_im[i], out_im[i], 1 - self.binary_mask) return out_im
def poisson_blend2(self, imgs1, imgs2, mask): # call this while performing consistency experiment out = np.zeros(imgs1.shape) for i in range(0, len(imgs1)): img1 = (imgs1[i] + 1.) / 2.0 img2 = (imgs2[i] + 1.) / 2.0 out[i] = np.clip( (poissonblending.blend(img1, img2, 1 - mask[i]) - 0.5) * 2, -1.0, 1.0) return out.astype(np.float32)
def postprocess(self, g_out): images_out = (np.array(g_out) + 1) / 2.0 images_in = (np.array(self.images_data) + 1) / 2.0 for i in range(len(g_out)): images_out[i] = poissonblending.blend(images_in[i], images_out[i], 1 - self.bin_mask) else: images_out = np.multiply(images_out, 1 - self.masks_data) + np.multiply( images_in, self.masks_data) return images_out
def leftshift(self): a = self.left_list[0] for b in self.left_list[1:]: H = self.matcher_obj.match(a, b, 'left') xh = np.linalg.inv(H) rt = np.dot(xh, np.array([a.shape[1], 0, 1])) # right top rt = rt/rt[-1] lb = np.dot(xh, np.array([0, a.shape[0], 1])) # left bottom lb = lb/lb[-1] ds = np.dot(xh, np.array([a.shape[1], a.shape[0], 1])) # right bottom ds = ds/ds[-1] f1 = np.dot(xh, np.array([0,0,1])) # left top f1 = f1/f1[-1] offsetx = min(lb[0], f1[0], 0) offsety = min(rt[1], f1[1], 0) if(offsetx < 0): xh[0][-1] += abs(offsetx) offsetx = math.ceil(abs(offsetx)) if(offsety < 0): xh[1][-1] += abs(offsety) offsety = math.ceil(abs(offsety)) d_y = max(b.shape[0], math.ceil(lb[1]), math.ceil(ds[1])) dsize = (offsetx + b.shape[1], offsety + d_y) # tmp = cv2.warpPerspective(a, xh, dsize, flags = cv2.INTER_CUBIC) tmp = gt.perspectivetrans(a, xh, dsize) # Poisson-blending mask = cv2.warpPerspective(np.ones(a.shape, dtype=np.uint8)*255, xh, dsize, flags = cv2.INTER_CUBIC) mask[offsety:b.shape[0]+offsety, offsetx+10:b.shape[1]+offsetx] = np.zeros((b.shape[0], b.shape[1]-10, 3), dtype=np.uint8) # cv2.imwrite('mask.jpg', mask) blend_img = np.zeros(tmp.shape, dtype=np.uint8) blend_img[offsety:b.shape[0]+offsety, offsetx:b.shape[1]+offsetx] = b black_mask = np.zeros(tmp.shape, dtype=np.uint8) for i in range(0, tmp.shape[0]): for j in range(0, tmp.shape[1]): if (not np.array_equal(tmp[i, j], np.array([0, 0, 0]))) or (not np.array_equal(blend_img[i, j], np.array([0, 0, 0]))): black_mask[i, j] = (1, 1, 1); tmp = p_b.blend(blend_img, tmp, mask) for i in range(0, tmp.shape[0]): for j in range(0, tmp.shape[1]): if np.array_equal(black_mask[i, j], np.array([0, 0, 0])): tmp[i, j] = (0, 0, 0); # directly stitch # tmp[offsety:b.shape[0]+offsety, offsetx:b.shape[1]+offsetx] = b a = tmp self.leftImage = tmp
def poisson_blend_old(input, output, mask): """ * inputs: - input (torch.Tensor, required) Input tensor of Completion Network. [N, C, W, H] - output (torch.Tensor, required) Output tensor of Completion Network. - mask (torch.Tensor, required) Input mask tensor of Completion Network. * returns: Image tensor inpainted using poisson image editing method. """ num_samples = input.shape[0] ret = [] # convert torch array to numpy array followed by # converting 'channel first' format to 'channel last' format.(N, C, W, H) input_np = np.transpose(np.copy(input.cpu().numpy()), axes=(0, 2, 3, 1)) # (N, W, H, C) output_np = np.transpose(np.copy(output.cpu().numpy()), axes=(0, 2, 3, 1)) mask_np = np.transpose(np.copy(mask.cpu().numpy()), axes=(0, 2, 3, 1)) # apply poisson image editing method for each input/output image and mask. for i in range(num_samples): inpainted_np = blend(input_np[i], output_np[i], mask_np[i]) # (W, H, C) inpainted = torch.from_numpy(np.transpose(inpainted_np, axes=(2, 0, 1))) # (C, W, H) inpainted = torch.unsqueeze(inpainted, dim=0) # (1, C, W, H) ret.append(inpainted) ret = torch.cat(ret, dim=0) return ret # if __name__ == '__main__': # x = torch.ones([100, 3, 10, 8]) # N=100, C=3, W=10, H=8 # y = np.transpose(np.copy(x.cpu().numpy()), axes=(0, 2, 3, 1)) # print(y.shape) # (N,W,H,C) # print(y[0].shape) # z = torch.from_numpy(np.transpose(y[0], axes=(2, 0, 1))) # print(z.shape) # print(type(z)) # wc = torch.unsqueeze(z, dim=0) # print(wc.shape)
def rightshift(self): for each in self.right_list: H = self.matcher_obj.match(self.leftImage, each, 'right') rt = np.dot(H, np.array([each.shape[1], 0, 1])) # right top rt = rt/rt[-1] lb = np.dot(H, np.array([0, each.shape[0], 1])) # left bottom lb = lb/lb[-1] f1 = np.dot(H, np.array([0,0,1])) # left top f1 = f1/f1[-1] ds = np.dot(H, np.array([each.shape[1], each.shape[0], 1])) # right bottom ds = ds/ds[-1] dsize = (int(max(rt[0], ds[0])), int(max(lb[1], ds[1], self.leftImage.shape[0]))) # tmp = cv2.warpPerspective(each, H, dsize, flags = cv2.INTER_CUBIC) tmp = gt.perspectivetrans(each, H, dsize) # Poisson-blending c = self.leftImage.shape[1]-1 for i in range(0, self.leftImage.shape[0]-1): if not np.array_equal(self.leftImage[i, c], np.array([0, 0, 0])): if np.array_equal(self.leftImage[i-1, c], np.array([0, 0, 0])): r1 = i if not np.array_equal(self.leftImage[i, c], np.array([0, 0, 0])): if np.array_equal(self.leftImage[i+1, c], np.array([0, 0, 0])): r2 = i+1 mask = cv2.warpPerspective(np.ones(each.shape, dtype=np.uint8)*255, H, dsize, flags = cv2.INTER_CUBIC) mask[r1:r2, :self.leftImage.shape[1]] = np.zeros((r2-r1, self.leftImage.shape[1], 3), dtype=np.uint8) blend_img = np.zeros(tmp.shape, dtype=np.uint8) blend_img[:self.leftImage.shape[0], :self.leftImage.shape[1]] = self.leftImage black_mask = np.zeros(tmp.shape, dtype=np.uint8) for i in range(0, tmp.shape[0]): for j in range(0, tmp.shape[1]): if (not np.array_equal(tmp[i, j], np.array([0, 0, 0]))) or (not np.array_equal(blend_img[i, j], np.array([0, 0, 0]))): black_mask[i, j] = (1, 1, 1); tmp = p_b.blend(blend_img, tmp, mask) for i in range(0, tmp.shape[0]): for j in range(0, tmp.shape[1]): if np.array_equal(black_mask[i, j], np.array([0, 0, 0])): tmp[i, j] = (0, 0, 0); # directly stitch # tmp = self.mix_and_match(self.leftImage, tmp) self.leftImage = tmp
def postprocess(self, ori_imgs, gen_imgs, is_blend=True): outputs = np.zeros_like(ori_imgs) tar_imgs = np.asarray([ utils.inverse_transform(img) for img in ori_imgs ]) # from (-1, 1) to (0, 1) sour_imgs = np.asarray([ utils.inverse_transform(img) for img in gen_imgs ]) # from (-1, 1) to (0, 1) if is_blend is True: for idx in range(tar_imgs.shape[0]): outputs[idx] = np.clip( poisson.blend(tar_imgs[idx], sour_imgs[idx], ((1. - self.model.masks[idx]) * 255.).astype( np.uint8)), 0, 1) else: outputs = np.multiply(tar_imgs, self.model.masks) + np.multiply( sour_imgs, 1. - self.model.masks) return outputs
def blend(self): src = np.asarray(self.src_img_manager.image_src) src.flags.writeable = True dst = np.asarray(self.dst_img_manager.image) dst.flags.writeable = True mask = np.asarray(self.src_img_manager.image_mask) mask.flags.writeable = True # poissonblending.blend takes (y, x) as offset, # whereas gui has (x, y) as offset values so reverse these values. reversed_offset = self.dst_img_manager.offset[::-1] blended_image = poissonblending.blend(dst, src, mask, reversed_offset) self.image_result = PIL.Image.fromarray(np.uint8(blended_image)) self.image_tk_result = PIL.ImageTk.PhotoImage(self.image_result) result_window = Tkinter.Toplevel() label = Tkinter.Label(result_window, image=self.image_tk_result) label.image = self.image_tk_result # for holding reference counter label.pack() save_button = Tkinter.Button(result_window, text='Save', command=self.save_result(result_window)) save_button.pack() result_window.title("Blended Result")
# Display the resultinfname_maskg frame cv2.imshow('frame', frame_mask) # When everything done, release the capture cap.release() cv2.destroyAllWindows() I = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if doit: nbsample = 500 off = (0, 0) img_ret4 = blend(img_target, I, img_mask, reg=5, nbsubsample=nbsample, offset=off, adapt=adapt_list[idimg], verbose=True) #%% fs = 30 f, axarr = pl.subplots(1, 3, figsize=(30, 10)) newax = f.add_axes([0.15, 0, 0.32, 0.32], anchor='NW', zorder=1) newax.imshow(img_mask) newax.axis('off') newax.set_title('mask') axarr[0].imshow(I) axarr[0].set_title('Source', fontsize=fs) axarr[0].axis('off') axarr[1].imshow(img_target)
print('using GPU...') model.cuda() input = input.cuda() # evaluate res = model.forward(input)[0].cpu() # make out for i in range(3): I[i, :, :] = I[i, :, :] + datamean[i] out = res.float() * M_3ch.float() + I.float() * (M_3ch * (-1) + 1).float() # post-processing if opt.postproc: print('post-postprocessing...') target = input_img # background source = tensor2cvimg(out.numpy()) # foreground mask = input_mask out = blend(target, source, mask, offset=(0, 0)) out = torch.from_numpy(cvimg2tensor(out)) # save images print('save images...') vutils.save_image(out, 'out.png', normalize=True) # vutils.save_image(Im, 'masked_input.png', normalize=True) # vutils.save_image(M_3ch, 'mask.png', normalize=True) # vutils.save_image(res, 'res.png', normalize=True) print('Done')
return 100 return 20 * math.log10(255.0 / math.sqrt(mse)) # In[]: for k, v in dic.items(): img = cv2.cvtColor(cv2.imread(train_path_dataset + k), cv2.COLOR_BGR2RGB) img_gt = cv2.cvtColor(cv2.imread(train_gt_path_dataset + k), cv2.COLOR_BGR2RGB) img_pred = cv2.cvtColor(cv2.imread(preds_path + k), cv2.COLOR_BGR2RGB) mask = np.zeros((600, 500, 3), dtype=np.uint8) for bbox in v: mask[bbox[0]:bbox[2], bbox[1]:bbox[3]] = 255 out = blend(img.copy(), img_pred.copy(), mask.copy(), offset=(0, 0)) print("BEFORE: {}, AFTER: {}".format(psnr(img_gt, img_pred), psnr(img_gt, out))) #plt.imshow(img) #psnr(img_pred, img_gt) # In[]: # In[]: # In[]: # In[]: # In[]:
import numpy as np import PIL.Image import pylab as pl from poissonblending import blend img_mask = np.asarray(PIL.Image.open('./data/me_mask.png')) img_mask = img_mask[:, :, :3] # remove alpha img_source = np.asarray(PIL.Image.open('./data/me.png')) img_source = img_source[:, :, :3] # remove alpha img_target = np.asarray(PIL.Image.open('./data/target.png')) nbsample = 500 off = (35, -15) img_ret1 = blend(img_target, img_source, img_mask, offset=off) img_ret3 = blend(img_target, img_source, img_mask, reg=5, eta=1, nbsubsample=nbsample, offset=off, adapt='linear') img_ret4 = blend(img_target, img_source, img_mask, reg=5, eta=1, nbsubsample=nbsample, offset=off,
def poissonblending(image1, image2, mask): """ We are using code from poissonblending. Need to give the original code credits. """ return blending.blend(image1, image2, 1 - mask)
#frame_mask2= cv2.cvtColor((img_mask2>0)*frame, cv2.COLOR_BGR2RGB) I = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # #pl.figure(1) #pl.subplot(211) #pl.imshow(I) #pl.subplot(212) #pl.imshow(frame_mask2) # #pl.show() nbsample = 500 off = (0, 0) img_ret4 = blend(img_target, I, img_mask, reg=5, nbsubsample=nbsample, offset=off, adapt='kernel', verbose=True) #%% fs = 30 f, axarr = pl.subplots(1, 3, figsize=(30, 10)) newax = f.add_axes([0.15, 0, 0.32, 0.32], anchor='NW', zorder=1) newax.imshow(img_mask) newax.axis('off') newax.set_title('mask') axarr[0].imshow(I) axarr[0].set_title('Source', fontsize=fs) axarr[0].axis('off') axarr[1].imshow(img_target)
def gl_inpaint(input_img, mask, datamean, model, postproc, device): # load data # print('image.shape: {}'.format(image.shape)) # input_img = cv2.imread(image) I = torch.from_numpy(cvimg2tensor(input_img)).float().to(device) if mask.shape[2] == 3: input_mask = mask M = torch.from_numpy( cv2.cvtColor(input_mask, cv2.COLOR_BGR2GRAY) / 255).float().to(device) # print('M.shape: {}'.format(M.shape)) M[M <= 0.2] = 0.0 M[M > 0.2] = 1.0 M = M.view(1, M.size(0), M.size(1)) assert I.size(1) == M.size(1) and I.size(2) == M.size(2) else: print('[ERROR] Mask image is invalid') for i in range(3): I[i, :, :] = I[i, :, :] - datamean[i] # make mask_3ch M_3ch = torch.cat((M, M, M), 0) Im = I * (M_3ch * (-1) + 1) # set up input input = torch.cat((Im, M), 0) input = input.view(1, input.size(0), input.size(1), input.size(2)).float() model.to(device) input = input.to(device) # evaluate res = model.forward(input) # make out for i in range(3): I[i, :, :] = I[i, :, :] + datamean[i] out = res.float() * M_3ch.float() + I.float() * (M_3ch * (-1) + 1).float() out = out[0] out = np.array(out.cpu().detach()).transpose(1, 2, 0) out = out[:, :, [2, 1, 0]] # cv2.imshow('out_before', out) # cv2.waitKey(0) # post-processing if postproc: print('[INFO] Post Processing...') target = input_img # background mask = input_mask out = blend(target, out, mask, offset=(0, 0)) out = out / 255 # cv2.imshow('out_after', out) # cv2.waitKey(0) # print(out) # print(out.shape) return out
d = 20 #prepare(r'Images/Carb.nhdr', r'Images/Carb2.nhdr') img_source, image_header = nrrd.read(r'source.nhdr') img_target, image_header = nrrd.read(r'target.nhdr') offset = int(img_target.shape[0] / 2 - 3 * d / 4) sls = [(slice(None), 0, slice(None)), (slice(None), -1, slice(None)), (slice(None), slice(None), 0), (slice(None), slice(None), -1)] for sl in sls: img_boundary = img_target[sl] img_boundary_source = img_source[sl] img_mask = np.zeros(img_boundary_source.shape) img_mask[1:-1, 1:-1] = 255 boundary_res = poissonblending.blend(img_boundary, img_boundary_source, img_mask, offset=(offset, 0)) #boundary_res[boundary_res>=0.5] = 1 #boundary_res[boundary_res<0.5] = 0 img_ret = PIL.Image.fromarray(np.uint8(boundary_res)) img_ret.save('./testimages/bound' + str(sls.index(sl)) + '.png') img_target[sl] = boundary_res img_mask = np.zeros(img_source.shape) img_mask[1:-1, 1:-1, 1:-1] = 255 data = poisson3d.blend(img_target, img_source, img_mask, offset=(offset, 0, 0)) options = {'encoding': 'raw'} nrrd.write(r'result.nhdr', data, options=options)