def convert2YCBCR(input, stream, idx):
streamtmp = input[stream][idx]
streamtmp = rgb2YCbCr(np.clip(streamtmp, 0.0, 1.0))
input[stream][idx] = streamtmp
return (input)
def convert2YCBCR(input, stream, idx):
streamtmp = input[stream][idx]
streamtmp = rgb2YCbCr(streamtmp)
input[stream][idx] = streamtmp
return (input)
def encode_decode_lightfield(data,
LF_LR,
LF_HR,
inputs,
outputs,
ColorSpace,
decoder_path='cv'):
# light field size
H = LF_LR.shape[2]
W = LF_LR.shape[3]
H_HR = LF_HR.shape[2]
W_HR = LF_HR.shape[3]
dc = cdf.data_config
# patch step sizes
bs_y = dc['SY']
bs_x = dc['SX']
bs_y_HR = dc['SY_HR']
bs_x_HR = dc['SX_HR']
# patch height/width
ps_y = dc['H']
ps_x = dc['W']
ps_y_HR = dc['H_HR']
ps_x_HR = dc['W_HR']
ps_v = dc['D']
# patches per row/column
by = np.int16((H - ps_y) / bs_y) + 1
bx = np.int16((W - ps_x) / bs_x) + 1
# one complete row per batch
cv_interp = np.zeros([H_HR, W_HR, 3], np.float32)
cv_raw = np.zeros([H_HR, W_HR, 3], np.float32)
mask_sum = np.zeros([H_HR, W_HR], dtype=np.float32)
print('starting LF encoding/decoding [', end='', flush=True)
start = timer()
results_received = 0
for py in range(by):
print('.', end='', flush=True)
stacks_h = np.zeros([bx, ps_v, ps_y, ps_x, 3], np.float32)
stacks_v = np.zeros([bx, ps_v, ps_y, ps_x, 3], np.float32)
cv_in = np.zeros([bx, ps_y_HR, ps_x_HR, 3], np.float32)
for px in range(bx):
# get single patch
patch = cdf.get_patch(LF_LR, py, px)
stacks_v[px, :, :, :, :] = patch['stack_v']
stacks_h[px, :, :, :, :] = patch['stack_h']
cv_in[px, :, :, :] = patch['cv']
if ColorSpace == 'YUV':
stacks_v[px, :, :, :, :] = rgb2YUV(stacks_v[px, :, :, :, :])
stacks_h[px, :, :, :, :] = rgb2YUV(stacks_h[px, :, :, :, :])
cv_in[px, :, :, :] = rgb2YUV(cv_in[px, :, :, :])
elif ColorSpace == 'YCBCR':
stacks_v[px, :, :, :, :] = rgb2YCbCr(stacks_v[px, :, :, :, :])
stacks_h[px, :, :, :, :] = rgb2YCbCr(stacks_h[px, :, :, :, :])
cv_in[px, :, :, :] = rgb2YCbCr(cv_in[px, :, :, :])
elif ColorSpace == 'LAB':
stacks_v[px, :, :, :, :] = rgb2lab(stacks_v[px, :, :, :, :])
stacks_h[px, :, :, :, :] = rgb2lab(stacks_h[px, :, :, :, :])
cv_in[px, :, :, :] = rgb2lab(cv_in[px, :, :, :])
# push complete batch to encoder/decoder pipeline
batch = dict()
batch['stacks_h'] = stacks_h
batch['stacks_v'] = stacks_v
batch['cv'] = cv_in
batch['py'] = py
batch['decoder_path'] = decoder_path
inputs.put(batch)
#
if not outputs.empty():
result = outputs.get()
add_result_to_cv(data, result, cv_interp, cv_raw, mask_sum,
bs_x_HR, bs_y_HR, bx, dc)
results_received += 1
outputs.task_done()
# catch remaining results
while results_received < by:
result = outputs.get()
add_result_to_cv(data, result, cv_interp, cv_raw, mask_sum, bs_x_HR,
bs_y_HR, bx, dc)
results_received += 1
outputs.task_done()
# elapsed time since start of dmap computation
end = timer()
total_time = end - start
print('] done, total time %g seconds.' % total_time)
# evaluate result
mse = 0.0
# compute stats and return result
print('total time ', end - start)
# print('MSE : ', mse)
# code.interact( local=locals() )
return (cv_interp, total_time, mse, mask_sum, cv_raw)
LF_HR,
inputs,
outputs,
color_space,
decoder_path=decoder_path,
)
cv_out2 = result_cv[0]
mask = result_cv[3]
cv_out2 = scale_back(cv_out2, mask)
cv_out_orig = np.concatenate((cv_out1, cv_out2), -1)
cv_out_rgb = yuv2rgb(cv_out_orig)
cv_out_rgb = cv_out_rgb.astype(np.float32)
elif color_space == 'YCBCR':
cv_gt_orig = rgb2YCbCr(cv_gt)
cv_LR_orig = rgb2YCbCr(cv_LR)
decoder_path = 'Y'
result_cv = encode_decode_lightfield(
data,
LF_LR,
LF_HR,
inputs,
outputs,
color_space,
decoder_path=decoder_path,
)
cv_out1 = result_cv[0].astype(np.float64)
mask = result_cv[3].astype(np.float64)
# mask = result_lf_cr[1][scales[i]]
# lf_out2_cr[scales[i]] = scale_back(lf_out2_cr[scales[i]], mask)
#
# lf_out3 = dict()
# for i in range(0, len(scales)):
# lf_out3[scales[i]] = np.clip(np.concatenate((lf_out2[scales[i]],lf_out2_cb[scales[i]],lf_out2_cr[scales[i]]), axis=-1), 16/255, 240/255)
lf_YCBCR = dict()
sh_cross = LF_net_in.shape
LF_net_in_cross = np.zeros((2, sh_cross[1], sh_cross[2], sh_cross[3], sh_cross[4]), np.float32)
LF_net_in_cross[0, :, :, :, :] = LF_net_in[:, 4, :, :, :]
LF_net_in_cross[1, :, :, :, :] = LF_net_in[4, :, :, :, :]
for i in range(0, len(scales)):
sh1 = lf_out2[scales[i]].shape
lf_YCBCR[scales[i]] = np.zeros((2, 9, sh1[2], sh1[3], 3), np.float32)
lf_YCBCR[scales[i]][0, ...] = np.stack([cv.resize(rgb2YCbCr(LF_net_in_cross[0, j, :, :, :]),
(sh1[3], sh1[2]), interpolation=cv.INTER_CUBIC) for j
in range(0, 9)])
lf_YCBCR[scales[i]][1, ...] = np.stack([cv.resize(rgb2YCbCr(LF_net_in_cross[1, j, :, :, :]),
(sh1[3], sh1[2]), interpolation=cv.INTER_CUBIC) for j
in range(0, 9)])
lf_out3 = dict()
for i in range(0, len(scales)):
lf_out3[scales[i]] = np.clip(np.concatenate((lf_out2[scales[i]], lf_YCBCR[scales[i]][:, :, :, :, 1:]), axis=-1), 16/255, 240/255)
else:
lf_out3 = lf_out2
for i in range(0, len(scales)):
tmp = np.zeros(lf_out3[scales[i]].shape, np.float32)
tmp[0, ...] = np.stack(
def encode_decode_lightfield(data,
LF,
inputs,
outputs,
ColorSpace,
lf_scale,
decoder_path,
scales,
target_channel=0):
# light field size
H = LF.shape[2]
W = LF.shape[3]
# if lf_scale =='s2':
# # patch step sizes
# bs_y = hp.sy_s2
# bs_x = hp.sx_s2
# # patch height/width
# ps_y = hp.H_s2
# ps_x = hp.W_s2
# ps_v = hp.D
#
# if lf_scale =='s4':
# # patch step sizes
# bs_y = hp.sy_HR
# bs_x = hp.sx_HR
# # patch height/width
# ps_y = hp.H_s4
# ps_x = hp.W_s4
# ps_v = hp.D
# patch step sizes
bs_y = hp.sy
bs_x = hp.sx
# patch height/width
ps_y = hp.H
ps_x = hp.W
ps_v = hp.D
# patches per row/column
by = np.int16((H - ps_y) / bs_y) + 1
bx = np.int16((W - ps_x) / bs_x) + 1
num_channels = hp.decoders_3D[0]['channels']
print('starting LF encoding/decoding [', end='', flush=True)
start = timer()
# one complete row per batch
mask_sum = dict()
LF_crosshair = dict()
if 's2' in scales:
mask_sum['s2'] = np.zeros([np.int(2 * H), np.int(2 * W)],
dtype=np.float32)
LF_crosshair['s2'] = np.zeros(
[2, 9, np.int(2 * H),
np.int(2 * W), num_channels],
dtype=np.float32)
if 's4' in scales:
mask_sum['s4'] = np.zeros([np.int(4 * H), np.int(4 * W)],
dtype=np.float32)
LF_crosshair['s4'] = np.zeros(
[2, 9, np.int(4 * H),
np.int(4 * W), num_channels],
dtype=np.float32)
results_received = 0
for py in range(by):
print('.', end='', flush=True)
batch = dict()
batch['stacks_h'] = np.zeros([bx, ps_v, ps_y, ps_x, hp.C], np.float32)
batch['stacks_v'] = np.zeros([bx, ps_v, ps_y, ps_x, hp.C], np.float32)
batch['stacks_bicubic_v'] = np.zeros([bx] + [9, 48, 48, 3], np.float32)
batch['stacks_bicubic_h'] = np.zeros([bx] + [9, 48, 48, 3], np.float32)
for px in range(bx):
# get single patch
patch = cdf.get_patch(LF, py, px, lf_scale)
if ColorSpace == 'YCBCR':
patch['stack_v'] = rgb2YCbCr(
np.clip(patch['stack_v'], 0.0, 1.0))
patch['stack_h'] = rgb2YCbCr(
np.clip(patch['stack_h'], 0.0, 1.0))
# patch['stack_v'] = np.clip(patch[ 'stack_v' ] , 0.0, 1.0)
# patch['stack_h'] = np.clip(patch[ 'stack_h' ] , 0.0, 1.0)
patch['stack_v'][..., 0:1] = patch['stack_v'][
..., target_channel:target_channel + 1]
patch['stack_h'][..., 0:1] = patch['stack_h'][
..., target_channel:target_channel + 1]
batch['stacks_v'][px, ...] = patch['stack_v']
batch['stacks_h'][px, ...] = patch['stack_h']
# biclinear stuff
[tq, vq, uq] = np.meshgrid(range(9), range(48), range(48))
tq = tq.transpose(1, 0, 2)
vq = vq.transpose(1, 0, 2)
uq = uq.transpose(1, 0, 2)
points = [[0, 4, 8], np.arange(48), np.arange(48)]
xi = (tq.ravel(), vq.ravel(), uq.ravel())
for ch in range(0, 3):
batch['stacks_bicubic_v'][px, :, :, :, ch] = np.clip(
interpn(points, batch['stacks_v'][px, 0:9:4, :, :, ch],
xi).reshape((9, 48, 48)), 16.0 / 255.0,
240 / 255)
batch['stacks_bicubic_h'][px, :, :, :, ch] = np.clip(
interpn(points, batch['stacks_h'][px, 0:9:4, :, :, ch],
xi).reshape((9, 48, 48)), 16.0 / 255.0,
240 / 255)
# batch['stacks_bicubic_v'][px, :, :, :, ch] = np.clip(
# interpn(points, batch['stacks_v'][px, 0:9:4, :, :, ch], xi).reshape((9, 48, 48)), 0.0,
# 1.0)
# batch['stacks_bicubic_h'][px, :, :, :, ch] = np.clip(
# interpn(points, batch['stacks_h'][px, 0:9:4, :, :, ch],
# xi).reshape((9, 48, 48)), 0.0, 1.0)
# push complete batch to encoder/decoder pipeline
batch['py'] = py
batch['decoder_path'] = decoder_path
inputs.put(batch)
#
if not outputs.empty():
result = outputs.get()
if 's2' in scales:
add_result_to_cv(data, result, LF_crosshair['s2'],
mask_sum['s2'], hp.sx_s2, hp.sy_s2, bx, 's2',
ColorSpace)
if 's4' in scales:
add_result_to_cv(data, result, LF_crosshair['s4'],
mask_sum['s4'], hp.sx_HR, hp.sy_HR, bx, 's4',
ColorSpace)
results_received += 1
outputs.task_done()
# catch remaining results
while results_received < by:
result = outputs.get()
if 's2' in scales:
add_result_to_cv(data, result, LF_crosshair['s2'], mask_sum['s2'],
hp.sx_s2, hp.sy_s2, bx, 's2', ColorSpace)
if 's4' in scales:
add_result_to_cv(data, result, LF_crosshair['s4'], mask_sum['s4'],
hp.sx_HR, hp.sy_HR, bx, 's4', ColorSpace)
results_received += 1
outputs.task_done()
# elapsed time since start of dmap computation
end = timer()
total_time = end - start
print('] done, total time %g seconds.' % total_time)
# evaluate result
# mse = 0.0
# compute stats and return result
print('total time ', end - start)
# print('MSE : ', mse)
# code.interact( local=locals() )
return (total_time, mask_sum, LF_crosshair)
decoder_path=decoder_path,
scales=scales)
lf_out2 = dict()
for i in range(0, len(scales)):
lf_out2[scales[i]] = result_lf[2][scales[i]]
mask = result_lf[1][scales[i]]
lf_out2[scales[i]] = scale_back(lf_out2[scales[i]], mask)
if grey:
lf_YCBCR = dict()
for i in range(0, len(scales)):
sh1 = lf_out2[scales[i]].shape
lf_YCBCR[scales[i]] = np.zeros((2, 9, sh1[2], sh1[3], 3),
np.float32)
lf_YCBCR[scales[i]][0, ...] = np.stack([
cv.resize(rgb2YCbCr(LF_crosshair['input'][0, j, :, :, :]),
(sh1[3], sh1[2]),
interpolation=cv.INTER_CUBIC)
for j in range(0, 9)
])
lf_YCBCR[scales[i]][1, ...] = np.stack([
cv.resize(rgb2YCbCr(LF_crosshair['input'][1, j, :, :, :]),
(sh1[3], sh1[2]),
interpolation=cv.INTER_CUBIC)
for j in range(0, 9)
])
lf_out3 = dict()
for i in range(0, len(scales)):
lf_out3[scales[i]] = np.concatenate(
(lf_out2[scales[i]], lf_YCBCR[scales[i]][:, :, :, :, 1:]),