def main(_):
tl.files.exists_or_mkdir(FLAGS.save_dir)
img_paths = get_img_paths(FLAGS.dataset_dir)
with tf.device('/cpu:0'):
x = tf.placeholder(dtype=tf.float32,
shape=[1, None, None, 3],
name='input_tensor')
o_Y, o_UV, net_out = a2net(x, is_train=False, reuse=False)
out_tensor = net_out.outputs
config = tf.ConfigProto()
config.allow_soft_placement = True
sess = tf.InteractiveSession(config=config)
saver = tf.train.Saver()
saver.restore(sess, FLAGS.weights_path)
ssim_list = []
psnr_list = []
for r, g in img_paths:
rain_img = load_and_process_image(r)
gt_img = load_and_process_image(g)
out = sess.run(out_tensor, feed_dict={x: np.expand_dims(rain_img, 0)})
out_img = out[0]
rain_img = yuv2rgb(rain_img)
out_img = yuv2rgb(out_img)
gt_img = yuv2rgb(gt_img)
ssim = compare_ssim(rgb2gray(gt_img), rgb2gray(out_img))
psnr = compare_psnr(rgb2gray(gt_img), rgb2gray(out_img))
ssim_list.append(ssim)
psnr_list.append(psnr)
print('SSIM: {:.5f}'.format(ssim))
print('PSNR: {:.5f}'.format(psnr))
gen_img = np.array([rain_img, out_img, gt_img])
num = r.split('/')[-1].split('_')[0]
tl.visualize.save_images(
gen_img, [1, 3], path.join(FLAGS.save_dir,
'out_{}.png'.format(num)))
sess.close()
print('SSIM: {:.5f}'.format(np.mean(ssim_list)))
print('PSNR: {:.5f}'.format(np.mean(psnr_list)))
def test_yuv_roundtrip(self):
img_rgb = img_as_float(self.img_rgb)[::16, ::16]
assert_array_almost_equal(yuv2rgb(rgb2yuv(img_rgb)), img_rgb)
assert_array_almost_equal(yiq2rgb(rgb2yiq(img_rgb)), img_rgb)
assert_array_almost_equal(ypbpr2rgb(rgb2ypbpr(img_rgb)), img_rgb)
assert_array_almost_equal(ycbcr2rgb(rgb2ycbcr(img_rgb)), img_rgb)
assert_array_almost_equal(ydbdr2rgb(rgb2ydbdr(img_rgb)), img_rgb)
def test_yuv_roundtrip(self):
img_rgb = img_as_float(self.img_rgb)[::16, ::16]
assert_array_almost_equal(yuv2rgb(rgb2yuv(img_rgb)), img_rgb)
assert_array_almost_equal(yiq2rgb(rgb2yiq(img_rgb)), img_rgb)
assert_array_almost_equal(ypbpr2rgb(rgb2ypbpr(img_rgb)), img_rgb)
assert_array_almost_equal(ycbcr2rgb(rgb2ycbcr(img_rgb)), img_rgb)
assert_array_almost_equal(ydbdr2rgb(rgb2ydbdr(img_rgb)), img_rgb)
def gen_pencil_drawing(img, kernel_size, stroke_width=0, num_of_directions=8, smooth_kernel="gauss",
gradient_method=0, rgb=False, w_group=0, pencil_texture_path="", stroke_darkness=1, tone_darkness=1):
if not rgb:
# Grayscale image:
im = img
else:
# RGB image:
yuv_img = color.rgb2yuv(img)
im = yuv_img[:,:,0]
# Generate the Stroke Map:
S = gen_stroke_map(im, kernel_size, stroke_width=stroke_width, num_of_directions=num_of_directions,
smooth_kernel=smooth_kernel, gradient_method=gradient_method)
S = np.power(S, stroke_darkness)
# Generate the Tone Map:
J = gen_tone_map(im, w_group=w_group)
# Read the pencil texture:
if not pencil_texture_path:
pencil_texture = io.imread('./pencils/pencil0.jpg', as_gray=True)
else:
pencil_texture = io.imread(pencil_texture_path, as_gray=True)
# Generate the Pencil Texture Map:
T = gen_pencil_texture(im, pencil_texture, J)
T = np.power(T, tone_darkness)
# The final Y channel:
R = np.multiply(S, T)
if not rgb:
return R
else:
yuv_img[:,:,0] = R
return exposure.rescale_intensity(color.yuv2rgb(yuv_img), in_range=(0, 1))
def process_frame(self, n, filename, frame):
with warnings.catch_warnings():
warnings.simplefilter('ignore')
frame = img_as_float(
resize(frame, (self.height, self.width), mode='constant'))
frame = img_as_float(color.rgb2gray(color.yuv2rgb(frame)))
misc.imsave('{}_{}_{}_0.jpg'.format(filename, n, self.hash_type()),
frame)
return frame
def show_yuv(yuv_original, yuv_pred):
rgb_original = np.clip(color.yuv2rgb(yuv_original), 0, 1)
rgb_pred = np.clip(np.abs(color.yuv2rgb(yuv_pred)), 0, 1)
grey = color.rgb2grey(yuv_original)
fig = plt.figure()
fig.add_subplot(1, 3, 1).set_title('greyscale')
plt.axis('off')
plt.imshow(grey, cmap='gray')
fig.add_subplot(1, 3, 2).set_title('original')
plt.axis('off')
plt.imshow(rgb_original)
fig.add_subplot(1, 3, 3).set_title('gan')
plt.axis('off')
plt.imshow(rgb_pred)
plt.show()
def read(self, offset_frame=None):
if not offset_frame == None:
self.fp.seek(offset_frame * self.frame_length, 0)
Y = np.fromfile(self.fp, np.uint8, count=self.Y_length)
U = np.fromfile(self.fp, np.uint8, count=self.Uv_length)
V = np.fromfile(self.fp, np.uint8, count=self.Uv_length)
if Y.size < self.Y_length or \
U.size < self.Uv_length or \
V.size < self.Uv_length:
return None, False
Y = np.reshape(Y, [self.w, self.h], order='F')
Y = np.transpose(Y)
U = np.reshape(U, [int(self.w / 2), int(self.h / 2)], order='F')
U = np.transpose(U)
V = np.reshape(V, [int(self.w / 2), int(self.h / 2)], order='F')
V = np.transpose(V)
U = imresize(U, [self.h, self.w], interp='nearest')
V = imresize(V, [self.h, self.w], interp='nearest')
# plt.figure(3)
# plt.title("GT")
# plt.imshow(np.stack((Y,Y,Y),axis=-1))
# plt.show()
#
# plt.figure(3)
# plt.title("GT")
# plt.imshow(np.stack((U,U,U),axis=-1))
# plt.show()
# plt.figure(3)
# plt.title("GT")
# plt.imshow(np.stack((V,V,V),axis=-1))
# plt.show()
if self.toRGB:
Y = Y / 255.0
U = U / 255.0 - 0.5
V = V / 255.0 - 0.5
self.YUV = np.stack((Y, U, V), axis=-1)
self.RGB = (255.0 *
np.clip(yuv2rgb(self.YUV), 0.0, 1.0)).astype('uint8')
# plt.figure(3)
# plt.title("GT")
# plt.imshow(self.RGB)
# plt.show()
self.YUV = None
return self.RGB, True
else:
self.YUV = np.stack((Y, U, V), axis=-1)
return self.YUV, True
def pencil_draw_color(I, texture):
if len(I.shape) == 2:
return pencil_draw(I, texture)
else:
I_yuv = color.rgb2yuv(I)
Y_gray = pencil_draw(I_yuv[:, :, 0], texture)
I_yuv[:, :, 0] = Y_gray
I_after = color.yuv2rgb(I_yuv)
I_after = np.maximum(I_after, 0)
I_after = np.minimum(I_after, 1)
#I_ruv[:, :, 0] = 0.5
return I_after
def process_frame(self, n, filename, frame):
frame = resize(frame, (self.height, self.width), mode='constant')
misc.imsave('{}_{}_{}_0.jpg'.format(filename, n, self.hash_type()),
frame)
frame = gaussian(frame, sigma=5.0, multichannel=True)
misc.imsave('{}_{}_{}_1.jpg'.format(filename, n, self.hash_type()),
frame)
with warnings.catch_warnings():
warnings.simplefilter('ignore')
frame = img_as_float(color.rgb2gray(color.yuv2rgb(frame)))
misc.imsave('{}_{}_{}_2.jpg'.format(filename, n, self.hash_type()),
frame)
return frame
def save_images_from_prediction(input, output, N):
outdir = 'outputs'
image = np.zeros((N, PIXEL_SIZE, PIXEL_SIZE, 3))
image[:, :, :, 0] = input.eval().reshape((N, PIXEL_SIZE, PIXEL_SIZE))
image[:, :, :, 1:] = output
image = color.yuv2rgb(image) # returns floats between [0,1]
for i in range(N):
# print(image)
io.imsave(outdir + "/out" + str(i) + ".jpg",
image[i],
check_contrast=False)
def alterYUV(img):
img = color.rgb2yuv(img)
params = []
#Y
img[:, :, 0] += randUnifC(-0.05, 0.05, params=params)
#U
img[:, :, 1] += randUnifC(-0.02, 0.02, params=params)
#V
img[:, :, 2] += randUnifC(-0.02, 0.02, params=params)
# U & V channels can have negative values; clip only Y
img[:, :, 0] = np.clip(img[:, :, 0], 0, 1.0)
img = color.yuv2rgb(img)
img = np.clip(img, 0, 1.0)
return img
def inference(G,in_path,out_path):
p=Image.open(in_path).convert('RGB')
img_yuv = rgb2yuv(p)
H,W,_ = img_yuv.shape
infimg = np.expand_dims(np.expand_dims(img_yuv[...,0], axis=0), axis=0)
img_variable = Variable(torch.Tensor(infimg-0.5))
if args.gpu>=0:
img_variable=img_variable.cuda(args.gpu)
res = G(img_variable)
uv=res.cpu().detach().numpy()
uv[:,0,:,:] *= 0.436
uv[:,1,:,:] *= 0.615
(_,_,H1,W1) = uv.shape
uv = zoom(uv,(1,1,H/H1,W/W1))
yuv = np.concatenate([infimg,uv],axis=1)[0]
rgb=yuv2rgb(yuv.transpose(1,2,0))
cv2.imwrite(out_path,(rgb.clip(min=0,max=1)*256)[:,:,[2,1,0]])
def convolver_rgb(image, kernel, iterations=1):
img_yuv = rgb2yuv(image)
img_yuv[:, :, 0] = multi_convolver(img_yuv[:, :, 0], kernel, iterations)
final_image = yuv2rgb(img_yuv)
fig, ax = plt.subplots(1, 2, figsize=(17, 10))
ax[0].imshow(image)
ax[0].set_title(f'Original', fontsize=20)
ax[1].imshow(final_image)
ax[1].set_title(f'YUV Adjusted, Iterations = {iterations}', fontsize=20)
[axi.set_axis_off() for axi in ax.ravel()]
fig.tight_layout()
return final_image
def test_yuv_roundtrip(self, channel_axis):
img_rgb = img_as_float(self.img_rgb)[::16, ::16]
img_rgb = np.moveaxis(img_rgb, source=-1, destination=channel_axis)
assert_array_almost_equal(
yuv2rgb(rgb2yuv(img_rgb, channel_axis=channel_axis),
channel_axis=channel_axis), img_rgb)
assert_array_almost_equal(
yiq2rgb(rgb2yiq(img_rgb, channel_axis=channel_axis),
channel_axis=channel_axis), img_rgb)
assert_array_almost_equal(
ypbpr2rgb(rgb2ypbpr(img_rgb, channel_axis=channel_axis),
channel_axis=channel_axis), img_rgb)
assert_array_almost_equal(
ycbcr2rgb(rgb2ycbcr(img_rgb, channel_axis=channel_axis),
channel_axis=channel_axis), img_rgb)
assert_array_almost_equal(
ydbdr2rgb(rgb2ydbdr(img_rgb, channel_axis=channel_axis),
channel_axis=channel_axis), img_rgb)
def generate_layers_simple(config, back_list, objects_all, num_objects_list):
max_objects = max(num_objects_list)
num_objects = np.random.choice(num_objects_list)
idx_back = np.random.randint(0, len(back_list))
back = back_list[idx_back]
image_shape = back.shape
obj_layers = np.zeros((max_objects, *image_shape), dtype=np.uint8)
occ_masks = np.zeros((num_objects, *image_shape[:-1]), dtype=np.float)
indices = np.random.randint(len(objects_all), size=num_objects)
scales = np.random.uniform(config['range_scale'][0],
config['range_scale'][1],
size=num_objects)
obj_images = [
rescale_image(objects_all[idx][0], scale)
for idx, scale in zip(indices, scales)
]
if 'range_intensity' in config:
y_scales = np.random.uniform(config['range_intensity'][0],
config['range_intensity'][1],
size=num_objects)
yuv_images = [rgb2yuv(image[..., :-1]) for image in obj_images]
for image, y_scale in zip(yuv_images, y_scales):
image[..., 0] *= y_scale
rgb_images = [(np.clip(yuv2rgb(image), 0, 1) * 255).astype(np.uint8)
for image in yuv_images]
obj_images = [
np.concatenate([rgb, obj[..., -1:]], axis=-1)
for rgb, obj in zip(rgb_images, obj_images)
]
obj_classes = np.array([objects_all[idx][1] for idx in indices])
for idx, image in enumerate(obj_images):
row1 = np.random.randint(image_shape[0] - image.shape[0] + 1)
row2 = row1 + image.shape[0]
col1 = np.random.randint(image_shape[1] - image.shape[1] + 1)
col2 = col1 + image.shape[1]
obj_layers[idx, row1:row2, col1:col2] = image
occ_masks[idx, row1:row2,
col1:col2] = 1 if config['overlap_bbox'] else image[...,
-1] / 255
layers = np.concatenate([back[None], obj_layers])
layers = np.rollaxis(layers, -1, -3)
classes = np.full(max_objects + 1, -1, dtype=np.int8)
classes[1:obj_classes.shape[0] + 1] = obj_classes
return layers, classes, occ_masks
def getcolor(input_image):
p = np.repeat(input_image, 3, axis=2)
if torch.cuda.is_available():
g_available = 1
else:
g_available = -1
args = Namespace(model=baseLoc + 'neural-colorization/model.pth',
gpu=g_available)
G = generator()
if torch.cuda.is_available():
G = G.cuda()
G.load_state_dict(torch.load(args.model))
else:
G.load_state_dict(
torch.load(args.model, map_location=torch.device('cpu')))
img_yuv = rgb2yuv(p)
H, W, _ = img_yuv.shape
infimg = np.expand_dims(np.expand_dims(img_yuv[..., 0], axis=0), axis=0)
img_variable = Variable(torch.Tensor(infimg - 0.5))
if args.gpu >= 0:
img_variable = img_variable.cuda(args.gpu)
res = G(img_variable)
uv = res.cpu().detach().numpy()
uv[:, 0, :, :] *= 0.436
uv[:, 1, :, :] *= 0.615
(_, _, H1, W1) = uv.shape
uv = zoom(uv, (1, 1, float(H) / H1, float(W) / W1))
yuv = np.concatenate([infimg, uv], axis=1)[0]
rgb = yuv2rgb(yuv.transpose(1, 2, 0))
out = (rgb.clip(min=0, max=1) * 255)[:, :, [0, 1, 2]]
out = out.astype(np.uint8)
return out
def next_frame(self):
hdr = self.fd.readline()
if hdr == b"":
return []
if hdr[0:5] != b"FRAME":
raise Exception("Invalid Y4M Frame")
raw_y = self.fd.read(self.width * self.height)
raw_u = self.fd.read(self.halfwidth * self.halfheight)
raw_v = self.fd.read(self.halfwidth * self.halfheight)
y = convert(
np.frombuffer(raw_y,
dtype='uint8').reshape(self.height, self.width),
np.float32)
u = resize(
convert(
np.frombuffer(raw_u,
dtype='uint8').reshape(self.halfheight,
self.halfwidth),
np.float32), (self.height, self.width), 0, 'reflect')
v = resize(
convert(
np.frombuffer(raw_v,
dtype='uint8').reshape(self.halfheight,
self.halfwidth),
np.float32), (self.height, self.width), 0, 'reflect')
yuv = np.array([y, u, v])
from skimage.color import yuv2rgb
rgb = yuv2rgb(yuv.transpose(1, 2, 0)).transpose(2, 0, 1)
#transform = np.array([[1, 0, 1.139883], [1, -0.39464233, -0.580621850], [1, 2.03206, 0]], dtype=np.float32)
#rgb = yuv.transpose(1, 2, 0).dot(transform).transpose(2, 0, 1)
return rgb
plt.figure(figsize=(18, 6))
plt.subplot(1, 3, 1)
plt.title('Raw')
plt.imshow(img)
plt.subplot(1, 3, 2)
plt.title('adapthist')
plt.imshow(img1)
plt.subplot(1, 3, 3)
plt.title('hist')
plt.imshow(img2)
# %%
img3 = color.rgb2yuv(img)
img4 = img3.copy()
img4[:, :, 0] = exposure.equalize_adapthist(img3[:, :, 0])
img4 = color.yuv2rgb(img4)
img5 = img3.copy()
img5[:, :, 0] = exposure.equalize_hist(img3[:, :, 0])
img5 = color.yuv2rgb(img5)
plt.figure(figsize=(18, 6))
plt.subplot(1, 3, 1)
plt.title('Raw')
plt.imshow(img)
plt.subplot(1, 3, 2)
plt.title('adapthist')
plt.imshow(img4)
plt.subplot(1, 3, 3)
plt.title('hist')
plt.imshow(img5)
def network_prediction_to_rgb(self, prediction, inputs):
yuv = np.concatenate((inputs, prediction), axis=2)
yuv[:, :, 1:] /= 2.
return yuv2rgb(yuv)
def main():
'''
main function, start point
'''
# 引数関連
parser = argparse.ArgumentParser()
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='GPU ID (negative value indicates CPU)')
args = parser.parse_args()
# データの読み込み
data_path = path.join(ROOT_PATH, 'dataset', 'demo_videos',
'09_Streets_of_India_960x540_s001_image')
image_list = os.listdir(data_path)
# データサイズ取得
img = io.imread(path.join(data_path, image_list[0]))
h, w, _ = img.shape
# gray_images = np.zeros((len(image_list),1, h, w), dtype=np.float32)
# cbcr_images = np.zeros((len(image_list),2, h, w), dtype=np.float32)
# prepare model
# model = GenEvaluator(N.SRCNN(2, [9, 5, 5]))
model = GenEvaluator(N.DRLSR())
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
serializers.load_npz(path.join(ROOT_PATH, 'models', 'drlsr'), model)
for idx, image_name in tqdm(enumerate(image_list)):
img = io.imread(path.join(data_path, image_name))
img = transform.rescale(transform.rescale(img, 0.5), 2)
io.imsave('/media/shimo/HDD_storage/BICUBIC/bic_' + image_name, img)
img = color.rgb2yuv(img).astype(np.float32)
img_gray = img[:, :, 0].reshape(1, 1, h, w)
img_cbcr = img[:, :, 1:].transpose(2, 0, 1).reshape(1, 2, h, w)
data = img_gray
data = cp.array(data)
y_data = chainer.backends.cuda.to_cpu(model.generator(data).data)
img_gray = np.clip(y_data, 0., 1.)
img = np.concatenate((img_gray, img_cbcr), axis=1)
img = img.transpose(0, 2, 3, 1).reshape(h, w, 3)
# print(img.dtype, img.shape, img.max(), img.min())
# print(img[:,:,0].max(), img[:,:,0].min())
# print(img[:,:,1].max(), img[:,:,1].min())
# print(img[:,:,2].max(), img[:,:,2].min())
img = np.clip(color.yuv2rgb(img) * 255, 0, 255).astype(np.uint8)
# print(img.dtype, img.shape, img.max(), img.min())
# print(img[:,:,0].max(), img[:,:,0].min())
# print(img[:,:,1].max(), img[:,:,1].min())
# print(img[:,:,2].max(), img[:,:,2].min())
io.imsave('/media/shimo/HDD_storage/DRLSR_demo/drlsr_' + image_name,
img.astype(np.uint8))
def shape_color(shape, color):
y = rgb2yuv(shape)[:, :, :1]
uv = rgb2yuv(color)[:, :, 1:]
return np.clip(yuv2rgb(np.concatenate((y, uv), axis=2)) * 255, 0, 255)
def check_colorization(batch, labels, col_space, Col_Net, classifier, alex, T=1):
gray = torch.tensor([0.2989 ,0.5870, 0.1140])[:,None,None].float()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
batch_size = batch.shape[0]
rgb, yuv, lab = [False]*3
if col_space == 'rgb':
rgb =True
elif col_space == 'lab':
lab = True
elif col_space == 'yuv':
yuv = True
if yuv or lab:
Y = batch[:,:1,:,:]
#build gray image
if lab or yuv:
X=torch.unsqueeze(batch[:,0,:,:],1).to(device) #set X to the Lightness of the image
batch=batch[:,1:,:,:].to(device) #image is a and b channel
else:
#using the matlab formula: 0.2989 * R + 0.5870 * G + 0.1140 * B and load data to gpu
X=(batch.clone()*gray).sum(1).to(device).view(-1,1,96,96)
batch=batch.float().to(device)
#if rgb:
# normalize(X,(0,),(1,),True)
if yuv:
normalize(X,(.5,),(.5,),True)
elif lab:
normalize(X,(50,),(1,),True)
#do colorization
col_batch = Col_Net(X).detach().cpu()
classes = col_batch.shape[1]
#construct rgb image form network output
if classes != 3:
#for lab/yuw and GAN net
if classes == 2:
if yuv or lab:
col_batch = torch.cat((Y, col_batch), 1).numpy().transpose((0,2,3,1))
#for -c
elif classes > 3:
if yuv:
col_batch = UV_from_distr(col_batch, T, Y)
elif lab:
col_batch = ab_from_distr(col_batch, T, Y)
if yuv:
rgb_batch = (color.yuv2rgb(col_batch))
elif lab: #lab2rgb doesn't support batches
rgb_batch=np.zeros_like(col_batch)
for k in range(len(col_batch)):
rgb_batch[k] = color.lab2rgb(col_batch[k])
rgb_batch = torch.tensor(np.array(rgb_batch).transpose((0,3,1,2))).float()
else:
rgb_batch = col_batch
rgb_batch = F.interpolate(rgb_batch, size = (224,224))
rgb_batch = normalize(rgb_batch,[0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
with torch.no_grad():
class_out = alex(rgb_batch.to(device))
pred = classifier(class_out)
correct_pred = (pred.argmax(1)==labels.to(device)).sum().cpu().item()
return correct_pred, batch_size
def yuv_combine(y, u, v):
yuv = cv2.merge((y, u, v))
rgb = yuv2rgb(yuv)
return rgb
im = io.imread('ressources/images/poissons.jpg')
plt.imshow(im, cmap=plt.cm.gray, vmin=30, vmax=200)
plt.show()
inverted_img = util.invert(im)
plt.imshow(inverted_img, cmap=plt.cm.gray, vmin=30, vmax=200)
plt.show()
hsv_img = rgb2hsv(im)
hue_img = hsv_img[:, :, 0]
value_img = hsv_img[:, :, 2]
fig, (ax0, ax1, ax2) = plt.subplots(ncols=3, figsize=(8, 2))
ax0.imshow(im)
ax0.set_title("RGB image")
ax0.axis('off')
ax1.imshow(hue_img, cmap='hsv')
ax1.set_title("Hue channel")
ax1.axis('off')
ax2.imshow(value_img)
ax2.set_title("Value channel")
ax2.axis('off')
fig.tight_layout()
yuv_img = yuv2rgb(im)
value_img = yuv_img[:, :, 2]
plt.imshow(value_img)
U = LU.solve(b1)
V = LU.solve(b2)
sol = np.zeros_like(img)
sol[:, :, 0] = Y
sol[:, :, 1] = U.reshape((H, W))
sol[:, :, 2] = V.reshape((H, W))
return sol
# img_path = fn("samples/flag.bmp")
# mark_img_path = fn("samples/flag_marked.bmp")
# out_img_path = "output/flag_out_"
img_path = fn("samples/baby.bmp")
mark_img_path = fn("samples/baby_marked.bmp")
out_img_path = "output/baby_out_"
# img_path = fn("samples/smiley.bmp")
# mark_img_path = fn("samples/smiley_marked.bmp")
# out_img_path = "output/smiley_out_"
img = np.float32(imread(img_path))
marked_img = np.float32(imread(mark_img_path))
out = colorization(img, marked_img, 5)
imsave(fn(out_img_path + "kernel_5.png"), yuv2rgb(out))
result_cv = encode_decode_lightfield(
data,
LF_LR,
LF_HR,
inputs,
outputs,
color_space,
decoder_path=decoder_path,
)
cv_out = result_cv[0]
mask = result_cv[3]
cv_out = scale_back(cv_out, mask)
if color_space == 'YUV':
cv_out = yuv2rgb(cv_out.astype(np.float64))
cv_out = cv_out.astype(np.float32)
elif color_space == 'YCBCR':
cv_out = YCbCr2rgb(cv_out)
cv_out = cv_out.astype(np.float32)
elif color_space == 'LAB':
cv_out = lab2rgb(cv_out.astype(np.float64))
cv_out = cv_out.astype(np.float32)
PSNR_out = measure.compare_psnr(cv_gt,
cv_out,
data_range=1,
dynamic_range=None)
SSIM_out = measure.compare_ssim(cv_gt,
cv_out,
data_range=1,
def preprocess(img):
choices = np.zeros(shape=[25])
#ABOUT TRAINING: slowly increase nums here
nums = np.random.random_integers(0, 10)
picked = np.random.random_integers(0, 24, nums)
choices[picked] = 1
#print(picked)
#Color reduction
if (choices[0] == 1):
scales = [
np.asscalar(np.random.random_integers(8, 200)) for x in range(3)
]
multi_channel = np.random.choice(2) == 0
params = [multi_channel] + [s / 200.0 for s in scales]
if multi_channel:
img = np.round(img * scales[0]) / scales[0]
else:
for i in range(3):
img[:, :, i] = np.round(img[:, :, i] * scales[i]) / scales[i]
#JPEG Noise
if (choices[1] == 1):
quality = np.asscalar(np.random.random_integers(55, 95))
params = [quality / 100.0]
pil_image = Image.fromarray((img * 255.0).astype(np.uint8))
f = BytesIO()
pil_image.save(f, format='jpeg', quality=quality)
jpeg_image = np.asarray(Image.open(f), ).astype(np.float32) / 255.0
img = jpeg_image
#Swirl
if (choices[2] == 1):
strength = (2.0 - 0.01) * np.random.random(1)[0] + 0.01
c_x = np.random.random_integers(1, 256)
c_y = np.random.random_integers(1, 256)
radius = np.random.random_integers(10, 200)
params = [strength / 2.0, c_x / 256.0, c_y / 256.0, radius / 200.0]
img = transform.swirl(img,
rotation=0,
strength=strength,
radius=radius,
center=(c_x, c_y))
#Noise Injection
if (choices[3] == 1):
params = []
options = ['gaussian', 'poisson', 'salt', 'pepper', 's&p', 'speckle']
noise_type = np.random.choice(options, 1)[0]
params.append(options.index(noise_type) / 6.0)
per_channel = np.random.choice(2) == 0
params.append(per_channel)
if per_channel:
for i in range(3):
img[:, :, i] = skimage.util.random_noise(img[:, :, i],
mode=noise_type)
else:
img = skimage.util.random_noise(img, mode=noise_type)
# FFT Perturbation
if (choices[4] == 1):
r, c, _ = img.shape
point_factor = (1.02 - 0.98) * np.random.random((r, c)) + 0.98
randomized_mask = [np.random.choice(2) == 0 for x in range(3)]
keep_fraction = [(0.95 - 0.0) * np.random.random(1)[0] + 0.0
for x in range(3)]
params = randomized_mask + keep_fraction
for i in range(3):
im_fft = fftpack.fft2(img[:, :, i])
r, c = im_fft.shape
if randomized_mask[i]:
mask = np.ones(im_fft.shape[:2]) > 0
im_fft[int(r * keep_fraction[i]):int(r *
(1 - keep_fraction[i]))] = 0
im_fft[:,
int(c * keep_fraction[i]):int(c *
(1 - keep_fraction[i]))] = 0
mask = ~mask
mask = mask * ~(np.random.uniform(size=im_fft.shape[:2]) <
keep_fraction[i])
mask = ~mask
im_fft = np.multiply(im_fft, mask)
else:
im_fft[int(r * keep_fraction[i]):int(r *
(1 - keep_fraction[i]))] = 0
im_fft[:,
int(c * keep_fraction[i]):int(c *
(1 - keep_fraction[i]))] = 0
im_fft = np.multiply(im_fft, point_factor)
im_new = fftpack.ifft2(im_fft).real
im_new = np.clip(im_new, 0, 1)
img[:, :, i] = im_new
#Zooms
if (choices[5] == 1):
h, w, _ = img.shape
i_s = np.random.random_integers(10, 50)
i_e = np.random.random_integers(10, 50)
j_s = np.random.random_integers(10, 50)
j_e = np.random.random_integers(10, 50)
params = [i_s / 50, i_e / 50, j_s / 50, j_e / 50]
i_e = h - i_e
j_e = w - j_e
# Crop the image...
img = img[i_s:i_e, j_s:j_e, :]
# ...now scale it back up
img = skimage.transform.resize(img, (h, w, 3))
if (choices[11] == 1):
#DO NOTHING HERE BECAUSE SEAM_CARVE REMOVED
"""
h, w, _ = img.shape
both_axis = np.random.choice(2) == 0
toRemove_1 = np.random.random_integers(10, 50)
toRemove_2 = np.random.random_integers(10, 50)
params = [both_axis, toRemove_1 / 50, toRemove_2 / 50]
if both_axis:
# First remove from vertical
eimg = skimage.filters.sobel(skimage.color.rgb2gray(img) )
img = transform.seam_carve(img, eimg, 'vertical', toRemove_1)
# Now from horizontal
eimg = skimage.filters.sobel(skimage.color.rgb2gray(img) )
img = transform.seam_carve(img, eimg,'horizontal', toRemove_2)
else:
eimg = skimage.filters.sobel(skimage.color.rgb2gray(img) )
direction = 'horizontal'
if toRemove_2 < 30:
direction = 'vertical'
img = transform.seam_carve(img, eimg,direction, toRemove_1)
# Now scale it back up
img = skimage.transform.resize(img, (h, w, 3))
"""
#Color space processes
if (choices[6] == 1):
img = color.rgb2hsv(img)
params = []
# Hue
img[:, :, 0] += randUnifC(-0.05, 0.05, params=params)
# Saturation
img[:, :, 1] += randUnifC(-0.25, 0.25, params=params)
# Value
img[:, :, 2] += randUnifC(-0.25, 0.25, params=params)
img = np.clip(img, 0, 1.0)
img = color.hsv2rgb(img)
img = (img * 2.0) - 1.0
img = np.clip(img, -1.0, 1.0)
if (choices[8] == 1):
img = (img + 1.0) * 0.5
img = color.rgb2xyz(img)
params = []
# X
img[:, :, 0] += randUnifC(-0.05, 0.05, params=params)
# Y
img[:, :, 1] += randUnifC(-0.05, 0.05, params=params)
# Z
img[:, :, 2] += randUnifC(-0.05, 0.05, params=params)
img = np.clip(img, 0, 1.0)
img = color.xyz2rgb(img)
img = (img * 2.0) - 1.0
img = np.clip(img, -1.0, 1.0)
if (choices[9] == 1):
img = (img + 1.0) * 0.5
img = color.rgb2lab(img)
params = []
# L
img[:, :, 0] += randUnifC(-5.0, 5.0, params=params)
# a
img[:, :, 1] += randUnifC(-2.0, 2.0, params=params)
# b
img[:, :, 2] += randUnifC(-2.0, 2.0, params=params)
img[:, :, 0] = np.clip(img[:, :, 0], 0, 100.0)
img = color.lab2rgb(img)
img = (img * 2.0) - 1.0
img = np.clip(img, -1.0, 1.0)
if (choices[10] == 1):
img = (img + 1.0) * 0.5
img = color.rgb2yuv(img)
params = []
# Y
img[:, :, 0] += randUnifC(-0.05, 0.05, params=params)
# U
img[:, :, 1] += randUnifC(-0.02, 0.02, params=params)
# V
img[:, :, 2] += randUnifC(-0.02, 0.02, params=params)
# U & V channels can have negative values; clip only Y
img[:, :, 0] = np.clip(img[:, :, 0], 0, 1.0)
img = color.yuv2rgb(img)
img = (img * 2.0) - 1.0
img = np.clip(img, -1.0, 1.0)
if (choices[7] == 1):
nbins = np.random.random_integers(40, 256)
params = [nbins / 256.0]
for i in range(3):
img[:, :, i] = skimage.exposure.equalize_hist(img[:, :, i],
nbins=nbins)
img = (img * 2.0) - 1.0
img = np.clip(img, -1.0, 1.0)
#if(choices[12]==1):
if (choices[13] == 1):
per_channel = np.random.choice(2) == 0
params = [per_channel]
low_precentile = [
randUnifC(0.01, 0.04, params=params) for x in range(3)
]
hi_precentile = [
randUnifC(0.96, 0.99, params=params) for x in range(3)
]
if per_channel:
for i in range(3):
p2, p98 = np.percentile(
img[:, :,
i], (low_precentile[i] * 100, hi_precentile[i] * 100))
img[:, :,
i] = skimage.exposure.rescale_intensity(img[:, :, i],
in_range=(p2, p98))
else:
p2, p98 = np.percentile(
img, (low_precentile[0] * 100, hi_precentile[0] * 100))
img = skimage.exposure.rescale_intensity(img, in_range=(p2, p98))
img = (img * 2.0) - 1.0
img = np.clip(img, -1.0, 1.0)
if (choices[14] == 1):
ratios = np.random.rand(3)
ratios /= ratios.sum()
params = [x for x in ratios]
img_g = img[:, :, 0] * ratios[0] + img[:, :, 1] * ratios[
1] + img[:, :, 2] * ratios[2]
for i in range(3):
img[:, :, i] = img_g
img = np.clip(img, -1.0, 1.0)
if (choices[15] == 1):
ratios = np.random.rand(3)
ratios /= ratios.sum()
prop_ratios = np.random.rand(3)
params = [x for x in ratios] + [x for x in prop_ratios]
img_g = img[:, :, 0] * ratios[0] + img[:, :, 1] * ratios[
1] + img[:, :, 2] * ratios[2]
for i in range(3):
p = max(prop_ratios[i], 0.2)
img[:, :, i] = img[:, :, i] * p + img_g * (1.0 - p)
img = np.clip(img, -1.0, 1.0)
if (choices[16] == 1):
params = []
channels = [0, 1, 2]
remove_channel = np.random.choice(3)
channels.remove(remove_channel)
params.append(remove_channel)
ratios = np.random.rand(2)
ratios /= ratios.sum()
params.append(ratios[0])
img_g = img[:, :,
channels[0]] * ratios[0] + img[:, :,
channels[1]] * ratios[1]
for i in channels:
img[:, :, i] = img_g
img = np.clip(img, -1.0, 1.0)
if (choices[17] == 1):
params = []
channels = [0, 1, 2]
to_alter = np.random.choice(3)
channels.remove(to_alter)
params.append(to_alter)
ratios = np.random.rand(2)
ratios /= ratios.sum()
params.append(ratios[0])
img_g = img[:, :,
channels[0]] * ratios[0] + img[:, :,
channels[1]] * ratios[1]
# Lets mix it back in with the original channel
p = (0.9 - 0.1) * np.random.random(1)[0] + 0.1
params.append(p)
img[:, :, to_alter] = img_g * p + img[:, :, to_alter] * (1.0 - p)
img = np.clip(img, -1.0, 1.0)
if (choices[18] == 1):
if randUnifC(0, 1) > 0.5:
sigma = [randUnifC(0.1, 3)] * 3
else:
sigma = [randUnifC(0.1, 3), randUnifC(0.1, 3), randUnifC(0.1, 3)]
img[:, :, 0] = skimage.filters.gaussian(img[:, :, 0], sigma=sigma[0])
img[:, :, 1] = skimage.filters.gaussian(img[:, :, 1], sigma=sigma[1])
img[:, :, 2] = skimage.filters.gaussian(img[:, :, 2], sigma=sigma[2])
img = np.clip(img, -1.0, 1.0)
if (choices[19] == 1):
if randUnifC(0, 1) > 0.5:
radius = [randUnifI(2, 5)] * 3
else:
radius = [randUnifI(2, 5), randUnifI(2, 5), randUnifI(2, 5)]
# median blur - different sigma for each channel
for i in range(3):
mask = skimage.morphology.disk(radius[i])
img[:, :, i] = skimage.filters.rank.median(
np.clip(img[:, :, i], -1.0, 1.0), mask) / 255.0
img = np.clip(img, -1.0, 1.0)
if (choices[20] == 1):
if randUnifC(0, 1) > 0.5:
radius = [randUnifI(2, 3)] * 3
else:
radius = [randUnifI(2, 3), randUnifI(2, 3), randUnifI(2, 3)]
# mean blur w/ different sigma for each channel
for i in range(3):
mask = skimage.morphology.disk(radius[i])
img[:, :, i] = skimage.filters.rank.mean(
np.clip(img[:, :, i], -1.0, 1.0), mask) / 255.0
img = np.clip(img, -1.0, 1.0)
if (choices[21] == 1):
params = []
radius = []
ss = []
for i in range(3):
radius.append(randUnifI(2, 20, params=params))
ss.append(randUnifI(5, 20, params=params))
ss.append(randUnifI(5, 20, params=params))
for i in range(3):
mask = skimage.morphology.disk(radius[i])
img[:, :, i] = skimage.filters.rank.mean_bilateral(
np.clip(img[:, :, i], -1.0,
1.0), mask, s0=ss[i], s1=ss[3 + i]) / 255.0
img = np.clip(img, -1.0, 1.0)
if (choices[22] == 1):
params = []
weight = (0.25 - 0.05) * np.random.random(1)[0] + 0.05
params.append(weight)
multi_channel = np.random.choice(2) == 0
params.append(multi_channel)
img = skimage.restoration.denoise_tv_chambolle(
img, weight=weight, multichannel=multi_channel)
img = np.clip(img, -1.0, 1.0)
if (choices[23] == 1):
wavelets = ['db1', 'db2', 'haar', 'sym9']
convert2ycbcr = np.random.choice(2) == 0
wavelet = np.random.choice(wavelets)
mode_ = np.random.choice(["soft", "hard"])
denoise_kwargs = dict(multichannel=True,
convert2ycbcr=convert2ycbcr,
wavelet=wavelet,
mode=mode_)
max_shifts = np.random.choice([0, 1])
params = [
convert2ycbcr,
wavelets.index(wavelet) / float(len(wavelets)), max_shifts / 5.0,
(mode_ == "soft")
]
img = skimage.restoration.cycle_spin(
img,
func=skimage.restoration.denoise_wavelet,
max_shifts=max_shifts,
func_kw=denoise_kwargs)
img = np.clip(img, -1.0, 1.0)
if (choices[24] == 1):
wavelets = ['db1', 'db2', 'haar', 'sym9']
convert2ycbcr = np.random.choice(2) == 0
wavelet = np.random.choice(wavelets)
mode_ = np.random.choice(["soft", "hard"])
denoise_kwargs = dict(multichannel=True,
convert2ycbcr=convert2ycbcr,
wavelet=wavelet,
mode=mode_)
max_shifts = np.random.choice([0, 1])
params = [
convert2ycbcr,
wavelets.index(wavelet) / float(len(wavelets)), max_shifts / 5.0,
(mode_ == "soft")
]
img = skimage.restoration.cycle_spin(
img,
func=skimage.restoration.denoise_wavelet,
max_shifts=max_shifts,
func_kw=denoise_kwargs)
img = np.clip(img, -1.0, 1.0)
return img
test_yuv = rgb2yuv(test_img)
test_inf = test_yuv[..., 0].reshape(1, 1, 256, 256)
test_var = Variable(torch.Tensor(test_inf - 0.5)).cuda(args.gpu)
if args.d_init is not None:
D.load_state_dict(torch.load(args.d_init, map_location='cuda:0'))
if args.g_init is not None:
G.load_state_dict(torch.load(args.g_init, map_location='cuda:0'))
# save test image for beginning
if args.test_image is not None:
test_res = G(test_var)
uv = test_res.cpu().detach().numpy()
uv[:, 0, :, :] *= 0.436
uv[:, 1, :, :] *= 0.615
test_yuv = np.concatenate([test_inf, uv], axis=1).reshape(3, 256, 256)
test_rgb = yuv2rgb(test_yuv.transpose(1, 2, 0))
cv2.imwrite(os.path.join(args.checkpoint_location, 'test_init.jpg'),
(test_rgb.clip(min=0, max=1) * 256)[:, :, [2, 1, 0]])
i = 0
adversarial_loss = torch.nn.BCELoss()
optimizer_G = torch.optim.Adam(G.parameters(),
lr=args.g_lr,
betas=(0.5, 0.999))
optimizer_D = torch.optim.Adam(D.parameters(),
lr=args.d_lr,
betas=(0.5, 0.999))
for epoch in range(args.epoch):
for y, uv in training_generator:
# Adversarial ground truths
valid = Variable(torch.Tensor(y.size(0), 1).fill_(1.0),
def visualize_yuv(image):
"""
Visualize YUV image in RGB scale
"""
return plt.imshow(tf.clip_by_value(yuv2rgb(image),0,1))
def test_yuv2rgb_dtype(self):
img = rgb2yuv(self.colbars_array).astype('float64')
img32 = img.astype('float32')
assert yuv2rgb(img).dtype == img.dtype
assert yuv2rgb(img32).dtype == img32.dtype
result_cv = encode_decode_lightfield(
data,
LF_LR,
LF_HR,
inputs,
outputs,
color_space,
decoder_path=decoder_path,
)
cv_out2 = result_cv[0]
mask = result_cv[3]
cv_out2 = scale_back(cv_out2, mask)
cv_out_orig = np.concatenate((cv_out1, cv_out2), -1)
cv_out_rgb = yuv2rgb(cv_out_orig)
cv_out_rgb = cv_out_rgb.astype(np.float32)
elif color_space == 'YCBCR':
cv_gt_orig = rgb2YCbCr(cv_gt)
cv_LR_orig = rgb2YCbCr(cv_LR)
decoder_path = 'Y'
result_cv = encode_decode_lightfield(
data,
LF_LR,
LF_HR,
inputs,
outputs,
color_space,