def Return_Invalid_Margin_Size_in_LR(self, filter,
max_allowed_perturbation):
TEST_IM_SIZE = 100
assert filter in ['ds_kernel', 'inv_hTh']
if filter == 'ds_kernel':
output_im = imresize(np.ones(
[self.ds_factor * TEST_IM_SIZE,
self.ds_factor * TEST_IM_SIZE]), [1 / self.ds_factor],
use_zero_padding=True)
elif filter == 'inv_hTh':
output_im = conv2(np.ones([TEST_IM_SIZE, TEST_IM_SIZE]),
self.inv_hTh,
mode='same')
output_im /= output_im[int(TEST_IM_SIZE / 2), int(TEST_IM_SIZE / 2)]
output_im[
output_im <=
0] = max_allowed_perturbation / 2 # Negative output_im are hella invalid... (and would not be identified as such without this line since I'm taking their log).
invalidity_mask = np.exp(
-np.abs(np.log(output_im))) < max_allowed_perturbation
# Finding invalid shoulder size, by searching for the index of the deepest invalid pixel, to accomodate cases of non-conitinous invalidity:
margin_sizes = [
np.argwhere(invalidity_mask[:int(TEST_IM_SIZE / 2),
int(TEST_IM_SIZE / 2)])[-1][0] + 1,
np.argwhere(invalidity_mask[int(TEST_IM_SIZE /
2), :int(TEST_IM_SIZE / 2)])[-1][0]
+ 1
]
margin_sizes = np.max(margin_sizes) * np.ones([2]).astype(
margin_sizes[0].dtype)
return np.max(margin_sizes)
def DT_Satisfying_Upscale(self, LR_image):
margin_size = 2 * self.inv_hTh_invalidity_half_size + self.ds_kernel_invalidity_half_size_LR
LR_image = Pad_Image(LR_image, margin_size)
HR_image = imresize(np.stack([
conv2(LR_image[:, :, channel_num], self.inv_hTh, mode='same')
for channel_num in range(LR_image.shape[-1])
], -1),
scale_factor=[self.ds_factor])
return Unpad_Image(HR_image, self.ds_factor * margin_size)
def __getitem__(self, index):
HR_path, LR_path = None, None
scale = self.opt['scale']
HR_size = self.opt['patch_size']
# get HR image
HR_path = self.paths_HR[index]
img_HR = util.read_img(self.HR_env, HR_path)
# modcrop in the validation / test phase
if self.opt['phase'] != 'train':
img_HR = util.modcrop(img_HR, scale)
# change color space if necessary
if self.opt['color']:
img_HR = util.channel_convert(img_HR.shape[2], self.opt['color'],
[img_HR])[0]
# get LR image
if self.paths_LR:
LR_path = self.paths_LR[index]
img_LR = util.read_img(self.LR_env, LR_path)
else: # down-sampling on-the-fly
# randomly scale during training
if self.opt['phase'] == 'train':
random_scale = random.choice(self.random_scale_list)
H_s, W_s, _ = img_HR.shape
def _mod(n, random_scale, scale, thres):
rlt = int(n * random_scale)
rlt = (rlt // scale) * scale
return thres if rlt < thres else rlt
H_s = _mod(H_s, random_scale, scale, HR_size)
W_s = _mod(W_s, random_scale, scale, HR_size)
img_HR = cv2.resize(np.copy(img_HR), (W_s, H_s),
interpolation=cv2.INTER_LINEAR)
# force to 3 channels
if img_HR.ndim == 2:
img_HR = cv2.cvtColor(img_HR, cv2.COLOR_GRAY2BGR)
H, W, _ = img_HR.shape
# using CEM imresize:
img_LR = imresize(img_HR,
scale_factor=[1 / float(scale)],
kernel=self.kernel)
# # using matlab imresize
# img_LR = util.imresize_np(img_HR, 1 / scale, True)
if img_LR.ndim == 2:
img_LR = np.expand_dims(img_LR, axis=2)
if self.opt['phase'] == 'train':
# if the image size is too small
H, W, _ = img_HR.shape
if H < HR_size or W < HR_size:
img_HR = cv2.resize(np.copy(img_HR), (HR_size, HR_size),
interpolation=cv2.INTER_LINEAR)
# using matlab imresize
img_LR = util.imresize_np(img_HR, 1 / scale, True)
if img_LR.ndim == 2:
img_LR = np.expand_dims(img_LR, axis=2)
H, W, C = img_LR.shape
LR_size = HR_size // scale
# randomly crop
rnd_h = random.randint(0, max(0, H - LR_size))
rnd_w = random.randint(0, max(0, W - LR_size))
img_LR = img_LR[rnd_h:rnd_h + LR_size, rnd_w:rnd_w + LR_size, :]
rnd_h_HR, rnd_w_HR = int(rnd_h * scale), int(rnd_w * scale)
img_HR = img_HR[rnd_h_HR:rnd_h_HR + HR_size,
rnd_w_HR:rnd_w_HR + HR_size, :]
# augmentation - flip, rotate
img_LR, img_HR = util.augment([img_LR, img_HR], self.opt['use_flip'], \
self.opt['use_rot'])
# change color space if necessary
if self.opt['color']:
img_LR = util.channel_convert(
C, self.opt['color'],
[img_LR])[0] # TODO during val no definetion
# BGR to RGB, HWC to CHW, numpy to tensor
if img_HR.shape[2] == 3:
img_HR = img_HR[:, :, [2, 1, 0]]
img_LR = img_LR[:, :, [2, 1, 0]]
img_HR = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_HR, (2, 0, 1)))).float()
img_LR = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_LR, (2, 0, 1)))).float()
if LR_path is None:
LR_path = HR_path
return {
'LR': img_LR,
'HR': img_HR,
'LR_path': LR_path,
'HR_path': HR_path
}
horiz_pixel_borders = np.concatenate([
np.zeros([1, 1]).astype(np.int), 1 + np.argwhere(
np.mean(
np.mean(np.diff(LR_im, axis=1) != 0, 2) > MAJORITY_THRESHOLD,
0) > MAJORITY_THRESHOLD).reshape([1, -1])
], 1)
LR_im = LR_im[vert_pixel_borders, horiz_pixel_borders, :]
plt.imsave('Depixelized_LR.png', LR_im)
else:
LR_im = util.read_img(None, LR_IM_PATH)[:, :, [2, 1, 0]]
# Adding 1 row at the bottom (edge-padding) to make the size 32x32:
LR_im = np.concatenate([LR_im, LR_im[-1:, :, :]], 0)
# plt.imshow(LR_im)
# Loading PULSE SR output:
for SR_path in sr_image_paths:
print('Processing image %s...' % (SR_path))
SR_im = util.read_img(None, SR_path)[:, :, [2, 1, 0]]
downsampled_SR = imresize(SR_im, 1 / SCALE_FACTOR)
plt.imsave(SR_path[:-4] + '_DS.png', np.clip(downsampled_SR, 0, 1))
consistent_im = CEM_net.Enforce_DT_on_Image_Pair(LR_im, SR_im)
plt.imsave(SR_path[:-4] + '_consistent.png', np.clip(consistent_im, 0, 1))
downsampled_consistent = imresize(consistent_im, 1 / SCALE_FACTOR)
plt.imsave(SR_path[:-4] + '_consistent_DS.png',
np.clip(downsampled_consistent, 0, 1))
# LR_im_padded = np.pad(LR_im,((CEM_net.invalidity_margins_LR,CEM_net.invalidity_margins_LR),(CEM_net.invalidity_margins_LR,CEM_net.invalidity_margins_LR),(0,0)),'edge')
# SR_im_padded = np.pad(SR_im,((CEM_net.invalidity_margins_HR,CEM_net.invalidity_margins_HR),(CEM_net.invalidity_margins_HR,CEM_net.invalidity_margins_HR),(0,0)),'edge')
# consistent_im_padded = CEM_net.Enforce_DT_on_Image_Pair(LR_im_padded,SR_im_padded)[CEM_net.invalidity_margins_HR:-CEM_net.invalidity_margins_HR,CEM_net.invalidity_margins_HR:-CEM_net.invalidity_margins_HR,:]
# plt.imsave('consistent_SR_padded.png',np.clip(consistent_im_padded,0,1))
save_HR_folder), 'Folder [{:s}] already exists. Exit...'.format(
save_HR_folder)
if save_LR_folder is not None:
os.makedirs(save_LR_folder)
print('mkdir [{:s}] ...'.format(save_LR_folder))
if upscale_kernel is not None:
np.save(os.path.join(save_LR_folder, 'upscale_kernel.npy'),
upscale_kernel)
if save_HR_folder is not None:
os.makedirs(save_HR_folder)
print('mkdir [{:s}] ...'.format(save_HR_folder))
# img_list = []
# for file_name in os.listdir(input_folder):
# img_list.append(os.path.join(input_folder,file_name))
progress_bar = tqdm(os.listdir(input_folder))
for file_name in progress_bar:
cur_im = mod_img(cv2.imread(os.path.join(input_folder, file_name)),
mod_scale)
cv2.imwrite(
os.path.join(save_HR_folder, file_name.replace('.jpg', '.png')),
cur_im)
if save_LR_folder is not None:
# cur_im = cv2.resize(cur_im,dsize=(0,0),fx=1/up_scale,fy=1/up_scale,interpolation = cv2.INTER_CUBIC)
cur_im = imresize(cur_im,
scale_factor=[1 / float(up_scale)],
kernel=upscale_kernel)
cv2.imwrite(
os.path.join(save_LR_folder, file_name.replace('.jpg', '.png')),
cur_im)
def Return_kernel(ds_factor, upscale_kernel=None):
return np.rot90(
imresize(None, [ds_factor, ds_factor],
return_upscale_kernel=True,
kernel=upscale_kernel), 2).astype(np.float32) / (ds_factor**2)
def Project_2_kernel_subspace(self, HR_input):
# Return the projection of the given High Resolution image onto the affine subspace defined by the downsampling kernel, by downsampling and then upsampling it back.
return self.DT_Satisfying_Upscale(
imresize(HR_input, scale_factor=[1 / self.ds_factor]))
def Project_2_ortho_2_NS(self,HR_input):
downscaled_input = imresize(HR_input,scale_factor=[1/self.ds_factor])
if downscaled_input.ndim<HR_input.ndim:#In case input was of size self.ds_factor in at least one of its axes:
downscaled_input = np.reshape(downscaled_input,list(HR_input.shape[:2]//self.ds_factor)+([HR_input.shape[2]] if HR_input.ndim>2 else []))
return self.DT_Satisfying_Upscale(downscaled_input)