def plot_point_cloud(self,
rgb_img=None,
down_scale: int = 4,
view_angles: (int, int) = (50, 70),
s: float = 0.5,
ax: Axes3D = None) -> Axes3D:
# use valid central view if rgb image is missing
rgb_img = self.central_view if rgb_img is None and self.central_view is not None else rgb_img
# downsample via interpolation
if down_scale >= 1:
rgb_img = img_resize(rgb_img.copy(), x_scale=1. /
down_scale) if rgb_img is not None else None
dpt_map = img_resize(self.depth_map.copy(),
x_scale=1. / down_scale)
else:
warnings.warn(
'Downscale parameter has to be a positive integer greater than zero.'
)
dpt_map = self.depth_map.copy()
if self.depth_map is not None:
ax = plot_point_cloud(dpt_map,
rgb_img=rgb_img,
down_scale=1,
view_angles=view_angles,
s=s,
ax=ax)
else:
self.sta.status_msg(msg='Depth map variable is empty')
ax = Axes3D(fig=plt.figure())
return ax
def export_vp_stack(self, type='png', downscale=None):
# print status
self.sta.status_msg('Write viewpoint image stack',
self.cfg.params[self.cfg.opt_prnt])
self.sta.progress(None, self.cfg.params[self.cfg.opt_prnt])
# downscale image
downscale = True if downscale is None else downscale
views_stacked_img = misc.img_resize(self.views_stacked_img.copy(), 1 / self._M) \
if downscale else self.views_stacked_img.copy()
# normalization
p_lo = np.percentile(misc.rgb2gray(self.central_view), 0.05)
p_hi = np.percentile(misc.rgb2gray(self.central_view), 99.995)
views_stacked_img = misc.Normalizer(views_stacked_img,
min=p_lo,
max=p_hi).uint8_norm()
# export all viewpoints in single image
views_stacked_path = os.path.join(
self.cfg.exp_path, 'views_stacked_img_' + str(self._M) + 'px')
misc.save_img_file(views_stacked_img,
file_path=views_stacked_path,
file_type=type)
self.sta.progress(100, self.cfg.params[self.cfg.opt_prnt])
return True
def export_vp_stack(self, type='tiff'):
# print status
self.sta.status_msg('Write viewpoint image stack', self.cfg.params[self.cfg.opt_prnt])
self.sta.progress(None, self.cfg.params[self.cfg.opt_prnt])
# export all viewpoints in single image
views_stacked_path = os.path.join(self.cfg.exp_path, 'views_stacked_img_'+str(self._M)+'px')
misc.save_img_file(misc.img_resize(self.views_stacked_img, 1/self._M),
file_path=views_stacked_path, file_type=type)
self.sta.progress(100, self.cfg.params[self.cfg.opt_prnt])
return True
coords_lists_pcam[2, :, :2] += [40, -100]
s_list = list()
# loop over directories
for set in [(fp_lytro, coords_lists_lytro), (fp_ours, coords_lists_pcam)]:
fp, coords_lists = set
img_tiles, slice_fns = crop_imgs(fp, coords_lists)
# loop over filenames in directory
for img_tile, slice_fn in zip(img_tiles, slice_fns):
# leave out alpha channel if present
img_tile = img_tile[..., :3]
# rescale tile for fair comparison
img_tile = misc.img_resize(img_tile, scale_comp_x, scale_comp_y) if fp.__contains__('refo_lytro') else img_tile
img_tile = np.asarray(img_tile, dtype='uint16')
# store results
s_list.append((slice_fn, blur_metric(img_tile), michelson_contrast(img_tile), brisque_metric(img_tile)))
for s in s_list:
print(s)
s_arr = np.asarray(s_list)
np.save('refocus_metrics', s_list)
def refo_from_vp(self):
""" computational refocusing based on viewpoint shift and integration """
# print status
self.sta.progress(0, self.cfg.params[self.cfg.opt_prnt])
# initialize variables
self._refo_stack = list()
patch_len = self.cfg.params[self.cfg.ptc_leng]
factor = self.cfg.params[self.cfg.ptc_leng] if self.cfg.params[
self.cfg.opt_refi] else 1
# divide intensity to prevent clipping in shift and sum process
self._vp_img_arr /= patch_len
# iterate through refocusing parameter a
ran_refo = self.cfg.params[self.cfg.ran_refo]
a_list = tuple([factor * a for a in ran_refo
]) if self.cfg.params[self.cfg.opt_refi] else ran_refo
for a in range(*a_list):
overlap = abs(a) * (patch_len - 1)
img_slice = np.zeros(
np.append(
np.array(self._vp_img_arr.shape[2:-1]) * factor +
[overlap, overlap], 3))
for j in range(patch_len):
for i in range(patch_len):
# perform sub-pixel refinement if required
vp_img = misc.img_resize(
self._vp_img_arr[j, i],
factor) if factor > 1 else self._vp_img_arr[j, i]
# get viewpoint padding for each border
tb = (abs(a) * j, abs(a) * (patch_len - 1 - j)
) # top, bottom
lr = (abs(a) * i, abs(a) * (patch_len - 1 - i)
) # left, right
# flip padding for each axis if a is negative
pad_width = (tb, lr, (0, 0)) if a >= 0 else (tb[::-1],
lr[::-1], (0,
0))
# shift viewpoint image and add its values to refocused image slice
img_slice = np.add(img_slice,
np.pad(vp_img, pad_width, 'edge'))
# check interrupt status
if self.sta.interrupt:
return False
# print status
percentage = ((j * patch_len + i + 1) / patch_len**2 + a -
a_list[0]) / (a_list[-1] - a_list[0]) * 100
self.sta.progress(percentage,
self.cfg.params[self.cfg.opt_prnt])
# crop refocused image for consistent image dimensions
crop = int(overlap / 2)
final_img = img_slice[crop:-crop,
crop:-crop, :] if (a != 0) else img_slice
# write upscaled version to hard drive
if self.cfg.params[self.cfg.opt_refi]:
upscale_img = LfpContrast().auto_hist_align(final_img.copy(),
ref_img=final_img,
opt=True)
upscale_img = GammaConverter().srgb_conv(img=upscale_img)
fname = np.round(a / patch_len, 2)
LfpExporter(cfg=self.cfg,
sta=self.sta).save_refo_slice(fname,
upscale_img,
string='upscale_')
del upscale_img
# spatially downscale image to original resolution (less memory required)
final_img = misc.img_resize(final_img, 1. /
factor) if factor > 1 else final_img
self._refo_stack.append(final_img)
return True
def refo_from_scratch_upsample(self): # pragma: no cover
# print status
self.sta.progress(0, self.cfg.params[self.cfg.opt_prnt])
# initialize variables
self._refo_stack = []
patch_len = self.cfg.params[self.cfg.ptc_leng]
m, n, P = self.lfp_img.shape if len(
self.lfp_img.shape
) == 3 else self.lfp_img.shape[0], self.lfp_img.shape[1], 1
factor = self.cfg.params[self.cfg.ptc_leng] if self.cfg.params[
self.cfg.opt_refi] else 1
img = self.lfp_img / patch_len # divide to prevent clipping
a_range = np.arange(*self.cfg.params[self.cfg.ran_refo])
for a in a_range:
# initialization
overlap = abs(a) * (patch_len - 1)
hor_refo = np.zeros([m, n + overlap, P])
ver_refo = np.zeros([m + overlap, n + overlap, P])
# consider negative shift
negative_a = overlap if a < 0 else 0
# horizontal shift and integration
for y in range(m):
for x in range(n):
# prevent from taking adjacent pixel being beyond image border
adj_idx = x + patch_len if x + patch_len < n else x
# interpolate values (spatial 1-D) of 2 adjacent pixels for all color channels
fraction_vec = np.array([
np.ogrid[img[y, x, 0]:img[y, adj_idx,
0]:complex(patch_len)],
np.ogrid[img[y, x, 1]:img[y, adj_idx,
1]:complex(patch_len)],
np.ogrid[img[y, x, 2]:img[y, adj_idx,
2]:complex(patch_len)]
]).T
# calculate horizontal shift coordinate
new_x = x + np.mod(x, patch_len) * (a - 1) + negative_a
# add fractional values to refocused image row at shifted coordinate
hor_refo[y, new_x:new_x + patch_len, :] += fraction_vec
# vertical shift and integration
for y in range(m):
# prevent from taking adjacent pixel being beyond image border
adj_idx = y + patch_len if y + patch_len < m else y
frac_vec = np.zeros([patch_len, n + overlap, P])
for x in range(n + overlap):
# interpolate values (spatial 2-D) of 2 adjacent rows for all color channels
frac_vec[:, x] = np.array([
np.ogrid[hor_refo[y, x,
0]:hor_refo[adj_idx, x,
0]:complex(patch_len)],
np.ogrid[hor_refo[y, x,
1]:hor_refo[adj_idx, x,
1]:complex(patch_len)],
np.ogrid[hor_refo[y, x,
2]:hor_refo[adj_idx, x,
2]:complex(patch_len)]
]).T
# calculate vertical shift coordinate
new_y = y + np.mod(y, patch_len) * (a - 1) + negative_a
# add fractional values to refocused image at shifted coordinate
ver_refo[new_y:new_y + patch_len, :, :] += frac_vec
# crop refocused image for consistent image dimensions
crop = int(overlap / 2)
final_img = ver_refo[crop:-crop,
crop:-crop, :] if (a != 0) else ver_refo
# write upscaled version to hard drive
LfpExporter(cfg=self.cfg,
sta=self.sta).save_refo_slice(a=a, refo_img=final_img)
# spatially downscale image to original resolution (for less memory usage)
final_img = misc.img_resize(final_img, 1. /
factor) if factor > 1 else final_img
# append refocused image to stack
self._refo_stack.append(final_img)
# check interrupt status
if self.sta.interrupt:
return False
# print progress status
self.sta.progress((a - a_range[0] + 1) / len(a_range) * 100,
self.cfg.params[self.cfg.opt_prnt])
return True
def refo_from_vp(self):
''' computational refocusing based on viewpoint shift and integration (see Eq. ) '''
# print status
self.sta.progress(0, self.cfg.params[self.cfg.opt_prnt])
# initialize variables
self._refo_stack = list()
patch_len = self.cfg.params[self.cfg.ptc_leng]
factor = self.cfg.params[self.cfg.ptc_leng] if self.cfg.params[
self.cfg.opt_refi] else 1
# divide intensity to prevent clipping in shift and sum process
self._vp_img_arr /= patch_len
# iterate through
a_list = self.cfg.params[self.cfg.ran_refo]
for a in range(*a_list):
overlap = abs(a) * (patch_len - 1)
img_slice = np.zeros(
np.append(
np.array(self._vp_img_arr.shape[2:-1]) * factor +
[overlap, overlap], 3))
for j in range(patch_len):
for i in range(patch_len):
# perform sub-pixel refinement if required
vp_img = misc.img_resize(
self._vp_img_arr[j, i],
factor) if factor > 1 else self._vp_img_arr[j, i]
# get viewpoint pad widths for each border
tb = (abs(a) * j, abs(a) * (patch_len - 1 - j)
) # top, bottom
lr = (abs(a) * i, abs(a) * (patch_len - 1 - i)
) # left, right
# flip pad widths for each axis if a is negative
pad_width = (tb, lr, (0, 0)) if a >= 0 else (tb[::-1],
lr[::-1], (0,
0))
# shift viewpoint image and add its values to refocused image slice
img_slice = np.add(img_slice,
np.pad(vp_img, pad_width, 'edge'))
# check interrupt status
if self.sta.interrupt:
return False
# print status
percentage = ((j * patch_len + i + 1) / patch_len**2 + a -
a_list[0]) / (a_list[-1] - a_list[0]) * 100
self.sta.progress(percentage,
self.cfg.params[self.cfg.opt_prnt])
# crop refocused image for consistent image dimensions
crop = int(overlap / 2)
final_img = img_slice[crop:-crop,
crop:-crop, :] if (a != 0) else img_slice
# spatially downscale image to original resolution (for less memory usage)
final_img = misc.img_resize(final_img, 1. /
factor) if factor > 1 else final_img
self._refo_stack.append(final_img)
return True
for set in [(fp_lytro, coords_lists_lytro), (fp_ours, coords_lists_pcam),
(fp_ours_native, coords_lists_native)]:
fp, coords_lists = set
img_tiles, slice_fns = crop_imgs(fp, coords_lists)
i = 0
# iterate through file names in directory
for img_tile, slice_fn in zip(img_tiles, slice_fns):
# leave out alpha channel if present
img_tile = img_tile[..., :3]
# rescale tile for fair comparison
if fp.__contains__('refo_lytro') or fp.__contains__('refo_lpt'):
img_tile = misc.img_resize(img_tile, scale_comp_x,
scale_comp_y)
elif not fp.__contains__('upscale'):
img_tile = misc.img_resize(img_tile, 7)
# normalize intensities
img_tile = misc.Normalizer(img_tile).uint8_norm()
# store results
s_list.append(
(slice_fn, blur_metric(img_tile), michelson_contrast(img_tile),
brisque_metric(img_tile), img_tile.shape))
for s in s_list:
print(s)
s_arr = np.asarray(s_list)
elif platform.system() == 'Darwin':
fp_lytro = '/Volumes/SD CARD 1/IEEEtran/img/refo_lytro'
fp_ours = '/Volumes/SD CARD 1/IEEEtran/img/refo_upscale_7px'
else:
fp_lytro = ''
fp_ours = ''
# loop over directories
for fp in [fp_lytro, fp_ours]:
slice_fns = [f for f in os.listdir(fp) if f.__contains__('crop')]
s_list = list()
# loop over filenames in directory
for slice_fn in slice_fns:
# load cropped slice
img_tile = misc.load_img_file(os.path.join(fp, slice_fn))
# remove alpha channel if present
img_tile = img_tile[..., :3]
# rescale tile for fair comparison
img_tile = misc.img_resize(img_tile, 2.28122448) if fp.__contains__('refo_lytro') else img_tile
# store results
s_list.append((slice_fn, blur_metric(img_tile), michelson_contrast(img_tile)))
for s in s_list:
print(s)
