def plot_batch(color_model, q_ab, X_batch_black, X_batch_color, batch_size, h, w, nb_q, epoch):
# Format X_colorized
X_colorized = color_model.predict(X_batch_black / 100.)[:, :, :, :-1]
X_colorized = X_colorized.reshape((batch_size * h * w, nb_q))
X_colorized = q_ab[np.argmax(X_colorized, 1)]
X_a = X_colorized[:, 0].reshape((batch_size, 1, h, w))
X_b = X_colorized[:, 1].reshape((batch_size, 1, h, w))
X_colorized = np.concatenate((X_batch_black, X_a, X_b), axis=1).transpose(0, 2, 3, 1)
X_colorized = [np.expand_dims(color.lab2rgb(im), 0) for im in X_colorized]
X_colorized = np.concatenate(X_colorized, 0).transpose(0, 3, 1, 2)
X_batch_color = [np.expand_dims(color.lab2rgb(im.transpose(1, 2, 0)), 0) for im in X_batch_color]
X_batch_color = np.concatenate(X_batch_color, 0).transpose(0, 3, 1, 2)
list_img = []
for i, img in enumerate(X_colorized[:min(32, batch_size)]):
arr = np.concatenate([X_batch_color[i], np.repeat(X_batch_black[i] / 100., 3, axis=0), img], axis=2)
list_img.append(arr)
plt.figure(figsize=(20,20))
list_img = [np.concatenate(list_img[4 * i: 4 * (i + 1)], axis=2) for i in range(len(list_img) / 4)]
arr = np.concatenate(list_img, axis=1)
plt.imshow(arr.transpose(1,2,0))
ax = plt.gca()
ax.get_xaxis().set_ticks([])
ax.get_yaxis().set_ticks([])
plt.tight_layout()
plt.savefig("../../figures/fig_epoch%s.png" % epoch)
plt.clf()
plt.close()
def imshow_rand(im, labrandom=True):
"""Show a segmentation using a random colormap.
Parameters
----------
im : np.ndarray of int, shape (M, N)
The segmentation to be displayed.
labrandom : bool, optional
Use random points in the Lab colorspace instead of RGB.
Returns
-------
fig : plt.Figure
The image shown.
"""
rand_colors = np.random.rand(ceil(im.max()), 3)
if labrandom:
rand_colors[:, 0] = rand_colors[:, 0] * 60 + 20
rand_colors[:, 1] = rand_colors[:, 1] * 185 - 85
rand_colors[:, 2] = rand_colors[:, 2] * 198 - 106
rand_colors = color.lab2rgb(rand_colors[np.newaxis, ...])[0]
rand_colors[rand_colors < 0] = 0
rand_colors[rand_colors > 1] = 1
rcmap = matplotlib.colors.ListedColormap(np.concatenate(
(np.zeros((1,3)), rand_colors)
))
return plt.imshow(im, cmap=rcmap, interpolation='nearest')
def sync_buffers_from_lab(self):
lab = self.lab.value
rgb = color.lab2rgb(lab[:,:,:3])
rgb *= 255
rgb_ = np.zeros((rgb.shape[0], rgb.shape[1], 4), dtype=np.uint8)
rgb_[:,:,:3] = rgb[:,:,:3]
self.rgb.value = rgb_
def colorizeTest(test_img_lum, trainData): n = (sampling_side-1)/2 x_min = n x_max = test_img_lum.shape[0] - n - 1 y_min = n y_max = test_img_lum.shape[1] - n - 1 # classify output = np.zeros(test_img_lum.shape + (3,)) count = 0 total_pixels = (x_max - x_min + 1)*(y_max - y_min + 1) / 100.0 for (x, y), lum in np.ndenumerate(test_img_lum): # make sure we don't look at pixels without enough neighbors if x < x_min or x > x_max or y < y_min or y > y_max : continue count += 1 if count%100 == 0: print 1.0 * count / total_pixels output[x,y,0] = lum stddev = computeStdDevLuminance(test_img_lum, x, y) closestIndex = getClosestTraining([lum, stddev], trainData[:,[0,3]]) output[x, y, 1:3] = trainData[closestIndex, 1:3] return color.lab2rgb(output[x_min : x_max + 1, y_min : y_max + 1])
def dominant_colors(image, num_colors, mask=None):
"""Reduce image colors to a representative set of a given size.
Args:
image (ndarray): BGR image of shape n x m x 3.
num_colors (int): Number of colors to reduce to.
mask (array_like, optional): Foreground mask. Defaults to None.
Returns:
list: The list of Color objects representing the most dominant colors in the image.
"""
image = rgb2lab(image / 255.0)
if mask is not None:
data = image[mask > 250]
else:
data = np.reshape(image, (-1, 3))
# kmeans algorithm has inherent randomness - result will not be exactly the same
# every time. Fairly consistent with >= 30 iterations
centroids, labels = kmeans2(data, num_colors, iter=30)
counts = np.histogram(labels, bins=range(0, num_colors + 1), normed=True)[0]
centroids_RGB = lab2rgb(centroids.reshape(-1, 1, 3))[:, 0, :] * 255.0
colors = [Color(centroid, count) for centroid, count in zip(centroids_RGB, counts)]
colors.sort(key=lambda color: np.mean(color.BGR))
return colors
def sshow(im, labrandom=True):
"""Show a segmentation (or cross-section) using a random colormap.
Parameters
----------
im : np.ndarray of int
The segmentation to be displayed.
labrandom : bool, optional
Use random points in the Lab colorspace instead of RGB.
Returns
-------
ax : matplotlib AxesImage object
The figure axes.
"""
if im.ndim > 2:
mid = im.shape[0] // 2
ax = sshow(im[mid], labrandom)
else:
rand_colors = np.random.rand(np.ceil(im.max()), 3)
if labrandom:
rand_colors[:, 0] = rand_colors[:, 0] * 60 + 20
rand_colors[:, 1] = rand_colors[:, 1] * 185 - 85
rand_colors[:, 2] = rand_colors[:, 2] * 198 - 106
rand_colors = color.lab2rgb(rand_colors[np.newaxis, ...])[0]
rand_colors[rand_colors < 0] = 0
rand_colors[rand_colors > 1] = 1
rcmap = colors.ListedColormap(np.concatenate((np.zeros((1, 3)),
rand_colors)))
ax = plt.imshow(im, cmap=rcmap, interpolation='nearest')
return ax
def save_img(img_path, lab):
lab = clamp_lab_img(lab)
rgb = color.lab2rgb(np.float64(lab))
print 'rgb min max', np.min(rgb[:, :, 0]), np.max(rgb[:, :, 0]), \
np.min(rgb[:, :, 1]), np.max(rgb[:, :, 1]), np.min(rgb[:, :, 2]), np.max(rgb[:, :, 2])
scipy.misc.imsave(img_path, rgb)
def check_HDF5(size=64):
"""
Plot images with landmarks to check the processing
"""
# Get hdf5 file
hdf5_file = os.path.join(data_dir, "CelebA_%s_data.h5" % size)
with h5py.File(hdf5_file, "r") as hf:
data_color = hf["training_color_data"]
data_lab = hf["training_lab_data"]
data_black = hf["training_black_data"]
for i in range(data_color.shape[0]):
fig = plt.figure()
gs = gridspec.GridSpec(3, 1)
for k in range(3):
ax = plt.subplot(gs[k])
if k == 0:
img = data_color[i, :, :, :].transpose(1,2,0)
ax.imshow(img)
elif k == 1:
img = data_lab[i, :, :, :].transpose(1,2,0)
img = color.lab2rgb(img)
ax.imshow(img)
elif k == 2:
img = data_black[i, 0, :, :] / 255.
ax.imshow(img, cmap="gray")
gs.tight_layout(fig)
plt.show()
plt.clf()
plt.close()
def imshow_rand(im, axis=None, labrandom=True):
"""Show a segmentation using a random colormap.
Parameters
----------
im : np.ndarray of int, shape (M, N)
The segmentation to be displayed.
labrandom : bool, optional
Use random points in the Lab colorspace instead of RGB.
Returns
-------
fig : plt.Figure
The image shown.
"""
if axis is None:
fig, axis = plt.subplots()
rand_colors = np.random.random(size=(ceil(np.max(im)), 3))
if labrandom:
rand_colors[:, 0] = rand_colors[:, 0] * 81 + 39
rand_colors[:, 1] = rand_colors[:, 1] * 185 - 86
rand_colors[:, 2] = rand_colors[:, 2] * 198 - 108
rand_colors = color.lab2rgb(rand_colors[np.newaxis, ...])[0]
rand_colors[rand_colors < 0] = 0
rand_colors[rand_colors > 1] = 1
rcmap = cm.colors.ListedColormap(np.concatenate((np.zeros((1, 3)),
rand_colors)))
return axis.imshow(im, cmap=rcmap)
def JPEG_decompression(data, channels=3): #Meta Data height = data[-1] width = data[-2] quality = data[-3] d_height = data[-4] d_width = data[-5] #Remove Meta Data data = data[:-5] #Unzigzag data_z = zigzag_decode(data, d_height, d_width, channels) #Unquantize im_q = unquantize(data_z,quality) #IDCT im_idct = idct_2d(im_q) #Unblock and Unpad im = unblock_image(im_idct,d_height,d_width) #Upsample #Undo offset and return to RGB im[:,:,[1,2]] -= 128 im = lab2rgb(im) * 255 # lab2rgb converts to float64 #Undo Padding im = im[:d_height,:d_width] #Upsample im = utils.upsample(im,(height,width)) return im.astype(np.uint8)
def test_lab_rgb_outlier(self):
lab_array = np.ones((3, 1, 3))
lab_array[0] = [50, -12, 85]
lab_array[1] = [50, 12, -85]
lab_array[2] = [90, -4, -47]
rgb_array = np.array([[[0.501, 0.481, 0]], [[0, 0.482, 1.0]], [[0.578, 0.914, 1.0]]])
assert_almost_equal(lab2rgb(lab_array), rgb_array, decimal=3)
def run_color(image, image_out):
caffe.set_mode_cpu()
net = caffe.Net('colorization_deploy_v0.prototxt', 'colorization_release_v0.caffemodel', caffe.TEST)
(H_in,W_in) = net.blobs['data_l'].data.shape[2:] # get input shape
(H_out,W_out) = net.blobs['class8_ab'].data.shape[2:] # get output shape
net.blobs['Trecip'].data[...] = 6/np.log(10) # 1/T, set annealing temperature
img_rgb = caffe.io.load_image(image)
img_lab = color.rgb2lab(img_rgb) # convert image to lab color space
img_l = img_lab[:,:,0] # pull out L channel
(H_orig,W_orig) = img_rgb.shape[:2] # original image size
# resize image to network input size
img_rs = caffe.io.resize_image(img_rgb,(H_in,W_in)) # resize image to network input size
img_lab_rs = color.rgb2lab(img_rs)
img_l_rs = img_lab_rs[:,:,0]
net.blobs['data_l'].data[0,0,:,:] = img_l_rs-50 # subtract 50 for mean-centering
net.forward() # run network
ab_dec = net.blobs['class8_ab'].data[0,:,:,:].transpose((1,2,0)) # this is our result
ab_dec_us = sni.zoom(ab_dec,(1.*H_orig/H_out,1.*W_orig/W_out,1)) # upsample to match size of original image L
img_lab_out = np.concatenate((img_l[:,:,np.newaxis],ab_dec_us),axis=2) # concatenate with original image L
img_rgb_out = np.clip(color.lab2rgb(img_lab_out),0,1) # convert back to rgb
scipy.misc.imsave(image_out, img_rgb_out)
def test_rgb_lch_roundtrip(self):
rgb = img_as_float(self.img_rgb)
lab = rgb2lab(rgb)
lch = lab2lch(lab)
lab2 = lch2lab(lch)
rgb2 = lab2rgb(lab2)
assert_array_almost_equal(rgb, rgb2)
def snap_ab(input_l, input_rgb, return_type='rgb'):
''' given an input lightness and rgb, snap the color into a region where l,a,b is in-gamut
'''
T = 20
warnings.filterwarnings("ignore")
input_lab = rgb2lab_1d(np.array(input_rgb)) # convert input to lab
conv_lab = input_lab.copy() # keep ab from input
for t in range(T):
conv_lab[0] = input_l # overwrite input l with input ab
old_lab = conv_lab
tmp_rgb = color.lab2rgb(conv_lab[np.newaxis, np.newaxis, :]).flatten()
tmp_rgb = np.clip(tmp_rgb, 0, 1)
conv_lab = color.rgb2lab(tmp_rgb[np.newaxis, np.newaxis, :]).flatten()
dif_lab = np.sum(np.abs(conv_lab-old_lab))
if dif_lab < 1:
break
# print(conv_lab)
conv_rgb_ingamut = lab2rgb_1d(conv_lab, clip=True, dtype='uint8')
if (return_type == 'rgb'):
return conv_rgb_ingamut
elif(return_type == 'lab'):
conv_lab_ingamut = rgb2lab_1d(conv_rgb_ingamut)
return conv_lab_ingamut
def LAB(img, k, filename):
# print 'lab'
# restructure image pixel values into range from 0 to 1 - needed for library
img = img * 1.0 / MAX_COLOR_VAL
# convert rgb to LAB
pixels_lab = color.rgb2lab(img)
# remove the L channel
L = pixels_lab[:, :, 0]
# reshape, cluster, and retrieve quantized values
pixels_l = np.reshape(L, (L.shape[0] * L.shape[1], 1))
clustered = cluster_pixels(pixels_l, k, (L.shape[0], L.shape[1]))
pixels_lab[:, :, 0] = clustered[:, :, 0]
# convert result to 255 RGB space
quanted_img = color.lab2rgb(pixels_lab) * MAX_COLOR_VAL
quanted_img = quanted_img.astype('uint8')
fig = plt.figure(1)
plt.imshow(quanted_img)
plt.title("LAB quantization where k is " + str(k))
plt.savefig('Q2/' + filename + '_LAB.png')
plt.close(fig)
return quanted_img
def save_image_LAB_into_sRGB(img_path, lab):
lab = np.float64(lab)
srgb = color.lab2rgb(lab)
# clamping to [0,1]
idx1, idx2 = np.nonzero(srgb < 0), np.nonzero(srgb > 1)
srgb[idx1[0], idx1[1], idx1[2]] = 0
srgb[idx2[0], idx2[1], idx2[2]] = 1
scipy.misc.imsave(img_path, srgb)
def lab2rgb_1d(in_lab, clip=True, dtype='uint8'):
warnings.filterwarnings("ignore")
tmp_rgb = color.lab2rgb(in_lab[np.newaxis, np.newaxis, :]).flatten()
if clip:
tmp_rgb = np.clip(tmp_rgb, 0, 1)
if dtype == 'uint8':
tmp_rgb = np.round(tmp_rgb * 255).astype('uint8')
return tmp_rgb
def equalize_adapthist(image, ntiles_x=8, ntiles_y=8, clip_limit=0.01,
nbins=256):
"""Contrast Limited Adaptive Histogram Equalization.
Parameters
----------
image : array-like
Input image.
ntiles_x : int, optional
Number of tile regions in the X direction. Ranges between 2 and 16.
ntiles_y : int, optional
Number of tile regions in the Y direction. Ranges between 2 and 16.
clip_limit : float: optional
Clipping limit, normalized between 0 and 1 (higher values give more
contrast).
nbins : int, optional
Number of gray bins for histogram ("dynamic range").
Returns
-------
out : ndarray
Equalized image.
Notes
-----
* The algorithm relies on an image whose rows and columns are even
multiples of the number of tiles, so the extra rows and columns are left
at their original values, thus preserving the input image shape.
* For color images, the following steps are performed:
- The image is converted to LAB color space
- The CLAHE algorithm is run on the L channel
- The image is converted back to RGB space and returned
* For RGBA images, the original alpha channel is removed.
References
----------
.. [1] http://tog.acm.org/resources/GraphicsGems/gems.html#gemsvi
.. [2] https://en.wikipedia.org/wiki/CLAHE#CLAHE
"""
args = [None, ntiles_x, ntiles_y, clip_limit * nbins, nbins]
if image.ndim > 2:
lab_img = color.rgb2lab(skimage.img_as_float(image))
l_chan = lab_img[:, :, 0]
l_chan /= np.max(np.abs(l_chan))
l_chan = skimage.img_as_uint(l_chan)
args[0] = rescale_intensity(l_chan, out_range=(0, NR_OF_GREY - 1))
new_l = _clahe(*args).astype(float)
new_l = rescale_intensity(new_l, out_range=(0, 100))
lab_img[:new_l.shape[0], :new_l.shape[1], 0] = new_l
image = color.lab2rgb(lab_img)
image = rescale_intensity(image, out_range=(0, 1))
else:
image = skimage.img_as_uint(image)
args[0] = rescale_intensity(image, out_range=(0, NR_OF_GREY - 1))
out = _clahe(*args)
image[:out.shape[0], :out.shape[1]] = out
image = rescale_intensity(image)
return image
def labs2rgbs():
# N.B. we ignore the luminant, as we just need a bare idea of how the color looks like...
from skimage.color import lab2rgb
colors_df = read_colors_df()
# lab2rgb expects an image of floats...
labs = colors_df[['photoshop_l', 'photoshop_a', 'photoshop_b']].values[None, :].astype(float)
# we will just save tuples
colors_df['rgb'] = map(tuple, lab2rgb(labs)[0])
return colors_df
def lab2rgb_transpose(img_l, img_ab):
''' INPUTS
img_l 1xXxX [0,100]
img_ab 2xXxX [-100,100]
OUTPUTS
returned value is XxXx3 '''
pred_lab = np.concatenate((img_l, img_ab), axis=0).transpose((1, 2, 0))
pred_rgb = (np.clip(color.lab2rgb(pred_lab), 0, 1)*255).astype('uint8')
return pred_rgb
def fuckwith(image, p, colors=None):
if isinstance(image, Image.Image):
pil = True
#image = np.array(image.convert('RGB'))
else:
pil = False
a, b, t = p
h, w = image.size
lab = color.rgb2lab(image)
for y in range(h):
lmax = np.max(lab[y,:,0])
lavg = np.mean(lab[y,:,0])
saw = ( (y + 25*t) % 25 )*0.15
shift = int( (lavg*0.1 + lmax*0.15)*a + rand()*5 + saw )
lab[y,:,:1] = np.roll(lab[y,:,:1], shift, 0)
lab[y,:,2] = np.roll(lab[y,:,2], -shift//2, 0)
lab[y,:,0] += lmax * ( (1.2 ** a) - 1 )
lab[y,:,0] *= ((a * 0.8) + 1.0)
lab[y,:,0] += (a*5) - 1
lab[y,:,1] += lmax * 0.2 * a
lab[y,:,2] += lmax * 0.5 * a
for x in range(w):
lmax = np.max(lab[:,x,0])
#shift = int(lmax*0.5*b + rand()*2)
#lab[:,x] = np.roll(lab[:,x], shift, 0)
lab[:,x,0] -= lmax * ( (1.2 ** b) - 1 )
lab[:,x,1] -= lmax * 0.2 * b
lab[:,x,2] -= lmax * 0.3 * b
rgb = color.lab2rgb(lab)
rgb = np.clip(rgb, 0, 1)
if colors:
rgb_scaled = (rgb*255).astype('uint8')
mod = Image.fromarray(rgb_scaled)
mod = mod.convert('P', palette=Image.ADAPTIVE, colors=colors)
if pil == True:
return mod
else:
mod = mod.convert('RGB')
pix = np.array(mod)
return pix
elif pil == True:
rgb_scaled = (rgb*255).astype('uint8')
mod = Image.fromarray(rgb_scaled)
return mod
else:
return rgb
def applyNailPolish(x , y , r = Rg, g = Gg, b = Bg): val = color.rgb2lab((im[x, y]/255.).reshape(len(x), 1, 3)).reshape(len(x), 3) L, A, B = mean(val[:,0]), mean(val[:,1]), mean(val[:,2]) L1, A1, B1 = color.rgb2lab(np.array((r/255., g/255., b/255.)).reshape(1, 1, 3)).reshape(3,) ll, aa, bb = L1 - L, A1 - A, B1 - B val[:, 0] = np.clip(val[:, 0] + ll, 0, 100) val[:, 1] = np.clip(val[:, 1] + aa, -127, 128) val[:, 2] = np.clip(val[:, 2] + bb, -127, 128) im[x, y] = color.lab2rgb(val.reshape(len(x), 1, 3)).reshape(len(x), 3)*255
def ab_selected(self, extents):
x0, x1, y0, y1 = extents
lab_masked = self.lab_image.copy()
L, a, b = lab_masked.T
mask = ((a > y0) & (a < y1)) & ((b > x0) & (b < x1))
lab_masked[..., 1:][~mask.T] = 0
self.image_viewer.image = color.lab2rgb(lab_masked)
def getBackgroundHueHist(imgLab,semMap,semFgBg):
imgHsv=color.rgb2hsv(color.lab2rgb(imgLab))
assert imgLab.shape[:2] == semMap.shape
fgBgMap=semFgBg[semMap]
bgIdx=np.nonzero(fgBgMap==0)
bgHue=imgHsv[bgIdx[0],bgIdx[1],0]
hist,binEdges=np.histogram(bgHue, bins=30, range=(0,1))
hist=np.float32(hist)
hist/=np.sum(hist)
return hist
def lch2rgb(lch):
"""Convert LCH to RGB colorspace (via LAB)
Input and output are in (bands, cols, rows) order
"""
# reshape for skimage (bands, cols, rows) -> (cols, rows, bands)
slch = np.swapaxes(lch, 0, 2)
# convert colorspace
rgb = lab2rgb(lch2lab(slch))
# return in (bands, cols, rows) order
return np.swapaxes(rgb, 2, 0)
def applyBlushColor(r = Rg, g = Gg, b = Bg): global im val = color.rgb2lab((im/255.)).reshape(width*height, 3) L, A, B = mean(val[:,0]), mean(val[:,1]), mean(val[:,2]) L1, A1, B1 = color.rgb2lab(np.array((r/255., g/255., b/255.)).reshape(1, 1, 3)).reshape(3,) ll, aa, bb = (L1 - L)*intensity, (A1 - A)*intensity, (B1 - B)*intensity val[:, 0] = np.clip(val[:, 0] + ll, 0, 100) val[:, 1] = np.clip(val[:, 1] + aa, -127, 128) val[:, 2] = np.clip(val[:, 2] + bb, -127, 128) im = color.lab2rgb(val.reshape(height, width, 3))*255
def merge_images(self, img_a, img_b):
i_a = skic.rgb2lab(img_a)
i_b = skic.rgb2lab(img_b)
norm_lum = np.max(np.asarray([i_a[..., 0], i_b[..., 0]]), axis=0)
res_img = i_a.copy()
res_img[..., 0] = norm_lum
return skic.lab2rgb(res_img)
def apply_texture(x, y):
xmin, ymin = amin(x), amin(y)
X = (x - xmin).astype(int)
Y = (y - ymin).astype(int)
val1 = color.rgb2lab((text[X, Y] / 255.).reshape(len(X), 1, 3)).reshape(len(X), 3)
val2 = color.rgb2lab((im[x, y] / 255.).reshape(len(x), 1, 3)).reshape(len(x), 3)
L, A, B = mean(val2[:, 0]), mean(val2[:, 1]), mean(val2[:, 2])
val2[:, 0] = np.clip(val2[:, 0] - L + val1[:, 0], 0, 100)
val2[:, 1] = np.clip(val2[:, 1] - A + val1[:, 1], -127, 128)
val2[:, 2] = np.clip(val2[:, 2] - B + val1[:, 2], -127, 128)
im[x, y] = color.lab2rgb(val2.reshape(len(x), 1, 3)).reshape(len(x), 3) * 255
def itf( X, npx, mode='RGB' ):
X = ( X.reshape( -1, nc, npx, npx ).transpose( 0, 2, 3, 1 ) + 1. ) / 2.
if mode == 'LAB':
X[:,:,:,0] *= 100
X[:,:,:,1] *= 255
X[:,:,:,2] *= 255
X[:,:,:,1] -= 128
X[:,:,:,2] -= 128
for i in range(X.shape[0]):
X[i,:,:,:] = color.lab2rgb(X[i,:,:,:].astype('int8'))
return X.astype('float32')
def suggest_color(self, h, w, K=5):
if self.dist_model is not None and self.image_loaded:
ab, conf = self.dist_model.get_ab_reccs(h=h, w=w, K=K, N=25000, return_conf=True)
L = np.tile(self.im_lab[h, w, 0], (K, 1))
colors_lab = np.concatenate((L, ab), axis=1)
colors_lab3 = colors_lab[:, np.newaxis, :]
colors_rgb = np.clip(np.squeeze(color.lab2rgb(colors_lab3)), 0, 1)
colors_rgb_withcurr = np.concatenate((self.model.get_img_forward()[h, w, np.newaxis, :] / 255., colors_rgb), axis=0)
return colors_rgb_withcurr
else:
return None
def equalize_adapthist(image,
ntiles_x=8,
ntiles_y=8,
clip_limit=0.01,
nbins=256):
"""Contrast Limited Adaptive Histogram Equalization.
Parameters
----------
image : array-like
Input image.
ntiles_x : int, optional
Number of tile regions in the X direction. Ranges between 2 and 16.
ntiles_y : int, optional
Number of tile regions in the Y direction. Ranges between 2 and 16.
clip_limit : float: optional
Clipping limit, normalized between 0 and 1 (higher values give more
contrast).
nbins : int, optional
Number of gray bins for histogram ("dynamic range").
Returns
-------
out : ndarray
Equalized image.
Notes
-----
* The algorithm relies on an image whose rows and columns are even
multiples of the number of tiles, so the extra rows and columns are left
at their original values, thus preserving the input image shape.
* For color images, the following steps are performed:
- The image is converted to LAB color space
- The CLAHE algorithm is run on the L channel
- The image is converted back to RGB space and returned
* For RGBA images, the original alpha channel is removed.
References
----------
.. [1] http://tog.acm.org/resources/GraphicsGems/gems.html#gemsvi
.. [2] https://en.wikipedia.org/wiki/CLAHE#CLAHE
"""
args = [None, ntiles_x, ntiles_y, clip_limit * nbins, nbins]
if image.ndim > 2:
lab_img = color.rgb2lab(skimage.img_as_float(image))
l_chan = lab_img[:, :, 0]
l_chan /= np.max(np.abs(l_chan))
l_chan = skimage.img_as_uint(l_chan)
args[0] = rescale_intensity(l_chan, out_range=(0, NR_OF_GREY - 1))
new_l = _clahe(*args).astype(float)
new_l = rescale_intensity(new_l, out_range=(0, 100))
lab_img[:new_l.shape[0], :new_l.shape[1], 0] = new_l
image = color.lab2rgb(lab_img)
image = rescale_intensity(image, out_range=(0, 1))
else:
image = skimage.img_as_uint(image)
args[0] = rescale_intensity(image, out_range=(0, NR_OF_GREY - 1))
out = _clahe(*args)
image[:out.shape[0], :out.shape[1]] = out
image = rescale_intensity(image)
return image
def train_model(device,model_name ,classification_model,colorization_model, dataloaders, criterion, optimizer, save_dir = None, save_all_epochs=False, num_epochs=25):
'''
model: The NN to train
dataloaders: A dictionary containing at least the keys
'train','val' that maps to Pytorch data loaders for the dataset
criterion: The Loss function
optimizer: The algorithm to update weights
(Variations on gradient descent)
num_epochs: How many epochs to train for
save_dir: Where to save the best model weights that are found,
as they are found. Will save to save_dir/weights_best.pt
Using None will not write anything to disk
save_all_epochs: Whether to save weights for ALL epochs, not just the best
validation error epoch. Will save to save_dir/weights_e{#}.pt
'''
since = time.time()
val_acc_history = []
best_model_wts = copy.deepcopy(colorization_model.state_dict())
best_loss = float('inf')
total_loss = []
total_acc = []
start_time = time.perf_counter()
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
colorization_model.train() # Set model to training mode
else:
colorization_model.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
# TQDM has nice progress bars
num_iter = 0
for inputs, _ in tqdm(dataloaders[phase]):
if num_iter > MAX_ITER:
break
num_iter += 1
inputs = inputs.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
# Get model outputs and calculate loss
outputs = forwards(inputs,classification_model,colorization_model)
loss = post_processing(inputs,outputs,criterion)
# torch.max outputs the maximum value, and its index
# Since the input is batched, we take the max along axis 1
# (the meaningful outputs)
_, preds = torch.max(outputs, 1)
# backprop + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
#running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / len(dataloaders[phase].dataset)
#epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)
total_loss.append(epoch_loss)
print("epoch_loss",epoch_loss)
#val_loss = evaluate(classification_model,colorization_model, dataloaders['val'], criterion, is_labelled = True, generate_labels = generate_validation_labels, k = 5)
if phase == "train":
writer.add_scalar('Loss/train', epoch_loss, epoch)
# deep copy the model
if phase == 'val' and epoch_loss < best_loss:
best_loss = epoch_loss
best_model_wts = copy.deepcopy(colorization_model.state_dict())
torch.save(best_model_wts, os.path.join(save_dir, model_name + '_best.pt'))
if phase == 'val':
pass
if save_all_epochs:
torch.save(colorization_model.state_dict(), os.path.join(save_dir, f'weights_{epoch}.pt'))
print()
for inputs, _ in tqdm(dataloaders['example']):
gray = transforms.Grayscale(num_output_channels=3)(inputs)
outputs = forwards(inputs, classification_model, colorization_model)
inputs = np.swapaxes(inputs,1,3)
inputs = np.swapaxes(inputs,1,2)
outputs = np.swapaxes(outputs,1,3)
outputs = np.swapaxes(outputs,1,2)
f, ax = plt.subplots(inputs.shape[0],3)
for i in range(inputs.shape[0]):
# use the created array to output your multiple images. In this case I have stacked 4 images vertically
ax[i][0].imshow(inputs[i])
#ax[i][1].imshow(outputs[i].detach().numpy())
ax[i][1].imshow(gray)
lab = color.rgb2lab(inputs[i])
full_output = np.stack([lab[:,:,0],outputs[i,:,:,0].detach().numpy(),outputs[i,:,:,1].detach().numpy()],axis=2)
ax[i][2].imshow(color.lab2rgb(full_output))
plt.savefig('outputs/'+save_file + '/figure epoch number ' + str(epoch) +'.png')
break
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_loss))
# save and load best model weights
torch.save(best_model_wts, os.path.join(save_dir, model_name + '_best.pt'))
model.load_state_dict(best_model_wts)
return colorization_model, val_acc_history
count = 0
frames = int(videoCap.get(cv2.CAP_PROP_FRAME_COUNT))
#frame_size = int(videoCap.get(cv2.CAP_PROP_FRAME_WIDTH))
#print(frame_size)
#fourcc = cv2.VideoWriter_fourcc(*'XVID')
#out_file = cv2.VideoWriter('coloured_video.avi',-1,20.0, frame_size)
#bar = progressbar.ProgressBar(redirect_stdout=True)
for i in range(frames):
try:
X = rgb2lab(1.0 / 255 * image)[:, :, 0]
#Y = rgb2lab(1.0/255*image)[:,:,1:]
X = X.reshape(1, len(image), len(image[0]), 1)
#Y = Y.reshape(1, len(image), len(image[0]), 2)
output = model.predict(X)
output *= 128
cur = np.zeros((len(image), len(image[0]), 3))
cur[:, :, 0] = X[0][:, :, 0]
cur[:, :, 1:] = output[0]
imsave("video_output/img_" + str(count) + ".png", lab2rgb(cur))
count += 1
#for j in range(30):
ret, image = videoCap.read()
#bar.update(i/frames*100)
#output_image = lab2rgb(cur)
#out_file.write(output_image)
except:
pass
#out_file.release()
K = 3
centers = np.array([X.mean(0) + (np.random.randn(3) / 10) for _ in range(K)])
y_kmeans = cluster_assignments(X, centers)
cluster_changes = True
# repeat estimation a number of times (could do something smarter, like comparing if clusters change)
while cluster_changes:
# for i in range(2):
former_centers = centers.tolist()
# assign each point to the closest center
y_kmeans = cluster_assignments(X, centers)
# move the centers to the mean of their assigned points (if any)
for i, c in enumerate(centers):
points = X[y_kmeans == i]
print(("Cluster: {} , Points: {}").format(i, len(points)))
if len(points):
centers[i] = points.mean(0)
if former_centers == centers.tolist():
break
# print("Centers: {}".format(centers.tolist()))
# print("Former_centers {}".format(former_centers))
plot_with_centers(X, y_kmeans, centers)
plt.figure()
plt.imshow(lab2rgb(centers[y_kmeans, :].reshape(shape[0], shape[1], 3)))
plt.axis('off')
plt.show()
#%%
if __name__ == "__main__":
from skimage.io import imread
from skimage.color import rgb2lab, lab2rgb
from skimage import img_as_float
import matplotlib.pyplot as plt
import numpy as np
filepath = "./img/flower.png"
image = imread(filepath)
lab = rgb2lab(image)
luma = lab[:, :, 0]
print(np.max(luma), np.min(luma))
res = wls_filter(luma=luma, lambda_=0.125, alpha=1.2)
image_out = lab2rgb(
lab=np.asarray([res, lab[..., 1], lab[..., 2]]).transpose(1, 2, 0))
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(1, 2, 1)
ax.imshow(image)
ax.axis("off")
ax = fig.add_subplot(1, 2, 2)
ax.imshow(image_out)
ax.axis("off")
plt.show()
#%%
bmean = mean(rgbmean[:, 2])
# print rmean, gmean, bmean
L, A, bB = color.rgb2lab(
np.array(
(rmean / 255., gmean / 255., bmean / 255.)).reshape(1, 1,
3)).reshape(3, )
L1, A1, B1 = color.rgb2lab(
np.array((R / 255., G / 255., B / 255.)).reshape(1, 1, 3)).reshape(3, )
val[:, 0] += (L1 - L) * inten
val[:, 1] += (A1 - A) * inten
val[:, 2] += (B1 - bB) * inten
image_blank = imread('Input.jpg')
image_blank *= 0
image_blank[x, y] = color.lab2rgb(val.reshape(len(x), 1, 3)).reshape(
len(x), 3) * 255
original = color.rgb2lab((im[x, y] * 0 / 255.).reshape(len(x), 1,
3)).reshape(len(x), 3)
tobeadded = color.rgb2lab(
(image_blank[x, y] / 255.).reshape(len(x), 1, 3)).reshape(len(x), 3)
original += tobeadded
im[x,
y] = color.lab2rgb(original.reshape(len(x), 1, 3)).reshape(len(x), 3) * 255
# Blur Filter
filter = np.zeros((height, width))
cv2.fillConvexPoly(filter, np.array(c_[yleft, xleft], dtype='int32'), 1)
cv2.fillConvexPoly(filter, np.array(c_[yright, xright], dtype='int32'), 1)
plt.imshow(filter)
def lab_to_rgb(labIm):
return color.lab2rgb(labIm) * 255
'D:\\CS520\\project4\\imgs\\sunset01.jpg')
out_array = take_divide(out_arr)
nn_3 = NN(input_array=in_arr,
output_array=out_arr[:, :, 0],
kernel_size=3,
method='SAME',
activation_function='tanh',
lr=1e-11,
model_path='nn_3.txt')
nn_4 = NN(input_array=in_arr,
output_array=out_arr[:, :, 1],
kernel_size=3,
method='SAME',
activation_function='tanh',
lr=1e-11,
model_path='nn_4.txt')
nn_3.train(50)
nn_4.train(50)
nn_3.save_model('nn_3.txt')
nn_4.save_model('nn_4.txt')
y1, y2 = nn_3.predict(in_arr, out_arr[:, :, 0]) * 64, nn_3.predict(
in_arr, out_arr[:, :, 1]) * 64
tmp = np.zeros(in_shape)
tmp[:, :, 0] = in_arr
tmp[:, :, 1] = y1
tmp[:, :, 2] = y2
io.imsave("D:\\CS520\\project4\\imgs\\test_image_result_1.png",
img_as_ubyte(color.lab2rgb(tmp)))
# io.imsave("test_image_gray.png", color.rgb2gray(lab2rgb(tmp)))
# load the original image
img_rgb = skimage.io.imread('img/NotreDame.png')
if len(img_rgb.shape) == 2:
img_rgb = np.stack((img_rgb, ) * 3, -1)
elif img_rgb.shape[2] != 3:
img_rgb = np.stack((img_rgb, ) * 3, -1)
img_lab = color.rgb2lab(img_rgb) # convert image to lab color space
img_l = img_lab[:, :, 0] # pull out L channel
(H_orig, W_orig) = img_rgb.shape[:2] # original image size
# create grayscale version of image (just for displaying)
img_lab_bw = img_lab.copy()
img_lab_bw[:, :, 1:] = 0
img_rgb_bw = color.lab2rgb(img_lab_bw)
# resize image to network input size
img_rs = resize(img_rgb, (H_in, W_in)) # resize image to network input size
img_lab_rs = color.rgb2lab(img_rs)
img_l_rs = img_lab_rs[:, :, 0]
# In[10]:
# show original image, along with grayscale input to the network
plt.figure(figsize=(20, 10))
img_pad = np.ones((H_orig, int(W_orig / 10), 3))
plt.imshow(np.hstack((img_rgb, img_pad, img_rgb_bw)))
plt.title('(Left) Loaded image / (Right) Grayscale input to network')
plt.axis('off')
i = resize(i, (299, 299, 3), mode='constant')
i = lab2rgb(i)
grayscaled_rgb_resized.append(i)
grayscaled_rgb_resized = np.array(grayscaled_rgb_resized)
grayscaled_rgb_resized = preprocess_input(grayscaled_rgb_resized)
with inception.graph.as_default():
embed = inception.predict(grayscaled_rgb_resized)
return embed
color_me = []
for filename in os.listdir('Test/People/'):
img = cv2.imread('Test/People/' + filename)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (256, 256))
color_me.append(img)
color_me = np.array(color_me, dtype=float)
color_me = rgb2lab(1.0 / 255 * color_me)[:, :, :, 0]
color_me = color_me.reshape(color_me.shape + (1, ))
# Test model
output = model.predict([color_me, create_inception_embedding(color_me)])
output *= 128
# Output colorizations
for i in range(len(output)):
cur = np.zeros((256, 256, 3))
cur[:, :, 0] = color_me[i][:, :, 0]
cur[:, :, 1:] = output[i]
imsave("Result/img_" + str(i) + ".png", lab2rgb(cur))
def lch2rgb(lch):
lab = color.lch2lab(lch)
return color.lab2rgb(lab)
color_me = []
for filename in os.listdir('frontendWorks/dist/imageUsers/'):
if filename.endswith('.jpg') or filename.endswith('.jpeg'):
img = image.img_to_array(
image.load_img('frontendWorks/dist/imageUsers/' + filename))
color_me.append(img)
color_me = np.array(color_me, dtype=float)
print(color_me.shape)
color_me = color_me / 255.0
color_me = color.gray2rgb(color.rgb2gray(color_me))
color_me_embed = create_inception_embedding(color_me)
color_me = color.rgb2lab(color_me)[:, :, :, 0]
color_me = color_me.reshape(color_me.shape + (1, ))
output = model.predict([color_me, color_me_embed])
output = output * 128
# Output colorizations
for i in range(len(output)):
cur = np.zeros((256, 256, 3))
cur[:, :, 0] = color_me[i][:, :, 0]
cur[:, :, 1:] = output[i]
scipy.misc.imsave("frontendWorks/dist/image_Output/img_" + str(i) + ".png",
color.lab2rgb(cur))
#
print("done")
sys.stdout.flush()
def lab2image(image_array):
image_array = image_array.astype(dtype=numpy.float64)
rgb = (lab2rgb(image_array) * 255).astype(numpy.uint8)
image = Image.fromarray(rgb)
return image
r = np.random.randint(2)
if x == 0:
dx = r
elif x == self.M:
dx = -r
else:
dx = 2*r - 1
dE = np.abs(dx) if x0 == x else (dx if x0 < x else -dx)
self.dx = dx
self.dE = dE
self.Eν = self.E + dE
def accept(self):
self.I[self.i] += self.dx
self.E = self.Eν
# ### CIE L\*a\*b\* Colors
import numpy as np
from skimage import color
color.rgb2lab(np.array([[np.random.rand(3)]]))
color.rgb2lab(np.array([[[0,.0,1.0]]]))
color.lab2rgb(np.array([[[50.0, -100, -100]]]))
def save_image(self, path, img_lab):
img_rgb = color.lab2rgb(img_lab)
io.imsave(path, img_rgb)
def lab2rgb(lab):
return color.lab2rgb(lab)
def array_to_image(
color_images_array=None,
gray_images_array=None,
mode='RGB',
color_normalize=False,
linedrawing=None,
):
# type: (any,any,any,any,any) -> typing.List[Image.Image]
"""
:param color_images_array: shape is [number of image, channel(3), width, height]
:param gray_images_array: used when mode=='ab' or 'gray'
:param mode: mode of input images array (RGB, Lab, ab, gray)
:param color_normalize: normalize rgb color to [min(rgb_images_array) max(rgb_images_array)]
"""
if color_images_array is not None:
color_images_array = chainer.cuda.to_cpu(color_images_array)
if gray_images_array is not None:
gray_images_array = chainer.cuda.to_cpu(gray_images_array)
if mode == 'gray':
color_images_array = numpy.concatenate([gray_images_array] * 3, axis=1)
mode = 'RGB'
if mode == 'ab':
# concat gray image(luminance) and ab(chromaticity)
color_images_array = chainer.cuda.to_cpu(color_images_array)
color_images_array = numpy.concatenate(
(gray_images_array, color_images_array), axis=1)
mode = 'Lab'
color_images_array = color_images_array.transpose(0, 2, 3, 1)
if mode == 'Lab':
color_images_array = color_images_array.astype(dtype=numpy.float64)
image_array_list = [
lab2rgb(image_array) * 255 for image_array in color_images_array
]
color_images_array = numpy.concatenate([
numpy.expand_dims(image_array, axis=0)
for image_array in image_array_list
])
mode = 'RGB'
if mode == 'RGB':
rgb_images_array = color_images_array
else:
raise ValueError('{} mode is not supported'.format(mode))
# to uint8
if color_normalize:
minmax = (rgb_images_array.min(), rgb_images_array.max())
else:
if linedrawing is not None:
minmax = (0, 1)
else:
minmax = (0, 255)
def clip_image(x):
x = (x - minmax[0]) / (minmax[1] -
minmax[0]) * 255 # normalize to 0~255
return numpy.float32(0 if x < 0 else (255 if x > 255 else x))
rgb_images_array = numpy.vectorize(clip_image)(rgb_images_array)
rgb_images_array = rgb_images_array.astype(numpy.uint8)
return [Image.fromarray(image_array) for image_array in rgb_images_array]
def colorize(paths):
gpu_id = 1
caffe.set_mode_gpu()
caffe.set_device(gpu_id)
for iter in range(
450000, 455000,
1000): # change stop number when more snapshots have been saved
# Select desired model
net = caffe.Net(
'/mnt/6TB-WD-Black/cs450/repos/colorization/models/colorization_deploy_v2.prototxt',
'/mnt/6TB-WD-Black/cs450/repos/colorization/train/models/colornet_iter_'
+ str(iter) + '.caffemodel', caffe.TEST)
(H_in, W_in) = net.blobs['data_l'].data.shape[2:] # get input shape
(H_out,
W_out) = net.blobs['class8_ab'].data.shape[2:] # get output shape
print 'Input dimensions: (%i,%i)' % (H_in, W_in)
print 'Output dimensions: (%i,%i)' % (H_out, W_out)
pts_in_hull = np.load(
'/mnt/6TB-WD-Black/cs450/repos/colorization/resources/pts_in_hull.npy'
) # load cluster centers
net.params['class8_ab'][0].data[:, :, 0, 0] = pts_in_hull.transpose(
(1, 0)) # populate cluster centers as 1x1 convolution kernel
print 'Annealed-Mean Parameters populated'
for path in paths:
print('image: ' + path)
# load the original image
img_rgb = caffe.io.load_image(path)
# first_height, first_width = img_rgb.shape[:2]
# if first_height > INPUT_SIZE or first_width > INPUT_SIZE:
# img_rgb = caffe.io.resize_image(img_rgb, (INPUT_SIZE, INPUT_SIZE)) # in case they are too big
img_lab = color.rgb2lab(
img_rgb) # convert image to lab color space
img_l = img_lab[:, :, 0] # pull out L channel
(H_orig, W_orig) = img_rgb.shape[:2] # original image size
# create grayscale version of image (just for displaying)
#img_lab_bw = img_lab.copy()
#img_lab_bw[:, :, 1:] = 0
#img_rgb_bw = color.lab2rgb(img_lab_bw)
# resize image to network input size
img_rs = caffe.io.resize_image(
img_rgb, (H_in, W_in)) # resize image to network input size
img_lab_rs = color.rgb2lab(img_rs)
img_l_rs = img_lab_rs[:, :, 0]
# show original image, along with grayscale input to the network
# img_pad = np.ones((H_orig, W_orig / 10, 3))
# plt.imshow(np.hstack((img_rgb, img_pad, img_rgb_bw)))
# plt.title('(Left) Loaded image / (Right) Grayscale input to network')
# plt.axis('off');
net.blobs['data_l'].data[
0, 0, :, :] = img_l_rs - 50 # subtract 50 for mean-centering
net.forward() # run network
ab_dec = net.blobs['class8_ab'].data[0, :, :, :].transpose(
(1, 2, 0)) # this is our result
ab_dec_us = sni.zoom(
ab_dec, (1. * H_orig / H_out, 1. * W_orig / W_out,
1)) # upsample to match size of original image L
img_lab_out = np.concatenate(
(img_l[:, :, np.newaxis], ab_dec_us),
axis=2) # concatenate with original image L
img_rgb_out = np.clip(color.lab2rgb(img_lab_out), 0,
1) # convert back to rgb
# plt.imshow(img_rgb_out)
# plt.axis('off')
# img_rgb_out = caffe.io.resize_image(img_rgb_out, (first_height, first_width))
file_name = os.path.basename(path)
save_path = '/mnt/6TB-WD-Black/cs450/automatic-colorization/SPECIAL_OUT/' + str(
iter) + '/' + file_name
print('save path: ' + save_path)
imsave(save_path, img_rgb_out)
#!/usr/bin/env python
import sys, warnings
from skimage import io, color, img_as_ubyte
if len(sys.argv) != 4:
print 'Usage: %s <cfam.png> <fam.png> <outfile.png>' % sys.argv[0]
sys.exit(1)
fam = io.imread(sys.argv[2])
fam_norm = fam.astype(float) / 255
cfam = io.imread(sys.argv[1])[:, :, 0:3]
cfam_lab = color.rgb2lab(cfam)
cfam_lightness = cfam_lab[:, :, 0].astype(float) / 100
lcfam_lab = cfam_lab
lcfam_lightness = fam_norm * 100
lcfam_lab[:, :, 0] = lcfam_lightness
lcfam = color.lab2rgb(lcfam_lab)
warnings.simplefilter("ignore")
io.imsave(sys.argv[3], lcfam)
def test_lab2rgb_dtype(self):
img = self.lab_array.astype('float64')
img32 = img.astype('float32')
assert lab2rgb(img).dtype == img.dtype
assert lab2rgb(img32).dtype == img32.dtype
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name="input_image")
annotation = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 2],
name="annotation")
z = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z")
# mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask")
# mask2 = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask2")
z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_new")
istrain = tf.placeholder(tf.bool)
#z_lip = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip")
#z_lip_inv = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip_inv")
e = tf.placeholder(tf.float32, shape=[None, 4, 4, 522], name="e")
e_p = tf.placeholder(tf.float32, shape=[None, 1, 1, 8202], name="e_p")
save_itr = 0
# pred_annotation, logits = inference(image, keep_probability,z)
# tf.summary.image("input_image", image, max_outputs=2)
# tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
# mask_ = tf.ones([FLAGS.batch_size,32,64,3])
# mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]])
# mask2__ = tf.ones([FLAGS.batch_size,78,78,3])
# mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]])
# mask2 = mask2_ - mask
zero = tf.zeros([FLAGS.batch_size, 1, 1, 8202])
logits, h = inference(image, keep_probability, z, 0.0, istrain)
logits_e, h_e = inference(image, keep_probability, z, e, istrain)
#logits_lip,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip,istrain )
#logits_lip_inv,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip_inv,istrain )
z_pred = predictor(h, z, zero, istrain)
z_pred_e = predictor(h, z, e_p, istrain)
# z_pred_lip = predictor(h,z_lip,istrain)
# z_pred_lip_inv = predictor(h,z_lip_inv,istrain)
# logits = inference(image, keep_probability,z,istrain)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3]))
# lossz = 0.1 * tf.reduce_mean(tf.abs(z))
# loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3])))
# loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1))
# loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3])))
g_k = gaussian_kernel(2, 0.0, 1.0)
gauss_kernel = tf.tile(g_k[:, :, tf.newaxis, tf.newaxis], [1, 1, 2, 2])
# Convolve.
logits_smooth = tf.nn.conv2d(tf.abs(logits),
gauss_kernel,
strides=[1, 1, 1, 1],
padding="SAME")
mean, var = tf.nn.moments(logits_smooth, axes=[1, 2])
logits_smooth_std = tf.reduce_sum(-tf.log(tf.sqrt(var)))
# logits_smooth = tf.maximum(logits_smooth_, 0.0001)
# logits_smooth_norm = tf.contrib.layers.flatten(logits_smooth)/tf.reduce_sum(logits_smooth,axis=[1,2,3], keep_dims = True)
# ones = tf.ones([FLAGS.batch_size,IMAGE_SIZE,IMAGE_SIZE,2])
# zeros = tf.zeros([FLAGS.batch_size,IMAGE_SIZE,IMAGE_SIZE,2])
# normal_dist_d = tf.distributions.Uniform(low = zeros, high = ones)
# normal_dist_ = normal_dist_d.sample()
# normal_dist = tf.maximum(normal_dist_, 0.0001)
# normal_dist_norm = tf.contrib.layers.flatten(normal_dist)/tf.reduce_sum(normal_dist,axis=[1,2,3], keep_dims = True)
# X = tf.distributions.Categorical(probs=logits_smooth_norm)
# Y = tf.distributions.Categorical(probs=normal_dist_norm)
# kl_dist = tf.reduce_mean(tf.distributions.kl_divergence(X, Y))
# kl_dist = tf.reduce_sum(logits_smooth_norm * tf.log(logits_smooth_norm/normal_dist_norm))
# kl_dist = tf.contrib.distributions.kl_divergence(logits_smooth_norm,normal_dist_norm)
# logits_std = tf.reduce_std(logits_smooth, axis =[1,2],keep_dims=True )
# logits_mean = tf.reduce_mean(logits_smooth, axis =[1,2], keep_dims=True)
# logits_normalized = (logits_smooth - logits_mean)/logits_std
# annotation_weights_norm = tf.reduce_sum(tf.exp(tf.abs(annotation))/tf.exp(1.0))
# annotation_weights = (tf.exp(tf.abs(annotation))/tf.exp(1.0))
loss_ = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(
(annotation - logits))), 1)) + 0.1 * logits_smooth_std
# loss_ = 0.4*loss_mask + loss_mask2
# loss = tf.reduce_mean(tf.squared_difference(logits ,annotation ))
loss_summary = tf.summary.scalar("entropy", loss_)
# zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1))
zloss_ = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))), 1))
# zloss_lip = tf.reduce_mean(tf.reduce_sum(tf.con`trib.layers.flatten(tf.abs((z_pred - z_pred_lip))),1))
# zloss_lip_inv = -tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip_inv))),1))
# z_loss = zloss_ + 0.1* zloss_lip# + zloss_lip_inv
lip_loss_dec = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((logits - logits_e))),
1))
loss = loss_ + 0.1 * lip_loss_dec
lip_loss_pred = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_e))),
1))
zloss = zloss_ + 0.1 * lip_loss_pred
grads = train_z(loss_, z)
trainable_var = tf.trainable_variables()
trainable_z_pred_var = tf.trainable_variables(scope="predictor")
trainable_d_pred_var = tf.trainable_variables(scope="decoder")
print(trainable_z_pred_var)
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
train_pred = train_predictor(zloss, trainable_z_pred_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
saved = True
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
print("$$$$$$$$$$$$$$$$$$$$")
print(train_images.shape)
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 10))
# train_images[train_images < 0.] = -1.
# train_annotations[train_annotations < 0.] = -1.
# train_images[train_images >= 0.] = 1.0
# train_annotations[train_annotations >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
cond = random.randint(0, 10)
# saved = True
if False:
saved = False
train_images_m, train_annotations_m = train_dataset_reader.get_random_batch(
FLAGS.batch_size)
train_images_m[train_images_m < 0.] = -1.
train_annotations_m[train_annotations_m < 0.] = -1.
train_images_m[train_images_m >= 0.] = 1.0
train_annotations_m[train_annotations_m >= 0.] = 1.0
train_images = (train_images + 1.) / 2.0 * 255.0
train_annotations = (train_annotations + 1.) / 2.0 * 255.0
train_images_m = (train_images_m + 1.) / 2.0 * 255.0
train_annotations_m = (train_annotations_m + 1.) / 2.0 * 255.0
train_images_m[:, 32:, :, :] = 0
train_annotations_m[:, 32:, :, :] = 0
train_images = np.clip((train_images + train_images_m), 0.0,
255.0)
train_annotations = np.clip(
(train_annotations + train_annotations_m), 0.0, 255.0)
'''
train_images[train_images < 0.] = -1.
train_annotations[train_annotations < 0.] = -1.
train_images[train_images >= 0.] = 1.0
train_annotations[train_annotations >= 0.] = 1.0
'''
train_annotations_ = np.squeeze(train_annotations, axis=3)
train_images_ = train_images
train_images = train_images / 127.5 - 1.0
train_annotations = train_annotations / 127.5 - 1.0
# for itr_ in range(FLAGS.batch_size):
# utils.save_image(train_images_[itr_].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr_) )
# utils.save_image(train_annotations_[itr_].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr_) )
# train_images[:,x1:w1,y1:h1,:] = 0
# print(train_images)
r_m, r_m2 = random_mask(64)
#feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,mask:r_m, istrain:True }
#train_images[:,50:100,50:100,:] =0
v = 0
# print(train_images)
error_dec = np.random.normal(0.0, 0.001,
(FLAGS.batch_size, 4, 4, 522))
error_dec_ = np.random.normal(0.0, 0.001,
(FLAGS.batch_size, 1, 1, 8202))
# z_l_inv = z_ + np.random.normal(0.0,0.1)
# feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_, e:error_dec, mask:r_m, istrain:True }
# z_l = z_ + np.random.normal(0.0,0.001)
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# z_l = z_ + np.random.normal(0.0,0.001)
# print("Step: %d, lip_loss:%g" % (itr,lloss))
for p in range(20):
z_ol = np.copy(z_)
# z_l = z_ol + np.random.normal(0.0,0.001)
# print("666666666666666666666666666666666666666")
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
z: z_,
e: error_dec,
istrain: True
}
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# print("Step: %d, z_step: %d, lip_loss:%g" % (itr,p,lloss))
z_loss, summ = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
#z_ = np.clip(z_, -10.0, 10.0)
'''
m = interp1d([-10.0,10.0],[-1.0,1.0])
print(np.max(z_))
print(np.min(z_))
z_ol_interp = m(z_ol)
z_interp = m(z_)
_,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol_interp,z_new:z_interp,e_p:error_dec_,istrain:True,keep_probability: 0.85})
print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
'''
# _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol,z_new:z_,istrain:True,keep_probability: 0.85})
# print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
# z_ = np.clip(z_, -1.0, 1.0)
# print(v.shape)
# print(z_.shape)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
e: error_dec,
z: z_,
istrain: True
}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
# valid_annotations[valid_annotations < 0.] = -1.
# valid_images[valid_images < 0.] = -1.
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
valid_loss, summary_sva = sess.run(
[loss, loss_summary],
feed_dict={
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_,
e: error_dec,
istrain: False
})
print("%s ---> Validation_loss: %g" %
(datetime.datetime.now(), valid_loss))
# add validation loss to TensorBoard
validation_writer.add_summary(summary_sva, itr)
if itr % 3000 == 0:
save_itr = save_itr + 3000
saver.save(sess, FLAGS.logs_dir + "model_fuse.ckpt", save_itr)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = validation_dataset_reader.get_random_batch(
FLAGS.batch_size)
# valid_annotations[valid_annotations < 0.] = -1.0
# valid_images[valid_images < 0.] = -1.0
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
r_m, r_m2 = random_mask(64)
# z_ = np.zeros(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10))
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 10))
feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 0.85,
z: z_,
istrain: False
}
v = 0
m__ = interp1d([-10.0, 10.0], [-1.0, 1.0])
z_ = m__(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
for p in range(20):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
# print(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
# z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict)
# print("z_step: %d, Train_loss:%g" % (p,z_loss))
# z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m})
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ = z_ol + 0.001 * v_prev + (1 + 0.001) * v
# z_ = z_ol + 0.001 * v_prev + (1+0.001)*v
# print("z_____________")
# print(z__)
# print("z_")
# print(z_)
# m__ = interp1d([-10.0,10.0],[-1.0,1.0])
# z_ol = m__(z_ol)
# z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z:z_ol, istrain:False,mask:r_m})
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
# z_ = np.clip(z_, -1.0, 1.0)
# print(z_pred_loss)
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
pred = sess.run(logits,
feed_dict={
image: valid_images,
annotation: valid_annotations,
z: z_,
istrain: False,
keep_probability: 0.85
})
print(
sess.run(var,
feed_dict={
image: valid_images,
annotation: valid_annotations,
z: z_,
istrain: False,
keep_probability: 0.85
}))
print("#######################")
# print(sess.run(normal_dist_norm))
valid_images = (valid_images + 1.) / 2.0 * 100.0
# predicted_patch = sess.run(mask) * pred
# pred = valid_images_masked + predicted_patch
pred_ = pred * 128.0
# pred = pred + 1./2.0*255
print(np.max(pred_))
print(np.min(pred_))
pred = np.reshape(np.concatenate((valid_images, pred_), axis=3),
(-1, 64, 64, 3))
valid_annotations_ = valid_annotations * 128.0
valid_annotations = np.reshape(
np.concatenate((valid_images, valid_annotations_), axis=3),
(-1, 64, 64, 3))
valid_images_gray = np.squeeze(valid_images)
# for itr in range(FLAGS.batch_size):
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr))
# utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr))
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr))
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) )
# utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) )
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) )
# print("Saved image: %d" % itr)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images_gray[itr],
FLAGS.logs_dir,
name="inp_" + str(5 + itr))
utils.save_image(color.lab2rgb(pred[itr]),
FLAGS.logs_dir,
name="predz_" + str(5 + itr))
utils.save_image(color.lab2rgb(valid_annotations[itr]),
FLAGS.logs_dir,
name="gt_" + str(5 + itr))
def test_lab_rgb_roundtrip(self):
img_rgb = img_as_float(self.img_rgb)
assert_array_almost_equal(lab2rgb(rgb2lab(img_rgb)), img_rgb)
def slic(image,
n_segments=100,
compactness=10.,
max_iter=10,
sigma=0,
spacing=None,
multichannel=True,
convert2lab=None,
enforce_connectivity=True,
min_size_factor=0.5,
max_size_factor=3,
slic_zero=False):
"""Segments image using k-means clustering in Color-(x,y,z) space.
Parameters
----------
image : 2D, 3D or 4D ndarray
Input image, which can be 2D or 3D, and grayscale or multichannel
(see `multichannel` parameter).
n_segments : int, optional
The (approximate) number of labels in the segmented output image.
compactness : float, optional
Balances color proximity and space proximity. Higher values give
more weight to space proximity, making superpixel shapes more
square/cubic. In SLICO mode, this is the initial compactness.
This parameter depends strongly on image contrast and on the
shapes of objects in the image. We recommend exploring possible
values on a log scale, e.g., 0.01, 0.1, 1, 10, 100, before
refining around a chosen value.
max_iter : int, optional
Maximum number of iterations of k-means.
sigma : float or (3,) array-like of floats, optional
Width of Gaussian smoothing kernel for pre-processing for each
dimension of the image. The same sigma is applied to each dimension in
case of a scalar value. Zero means no smoothing.
Note, that `sigma` is automatically scaled if it is scalar and a
manual voxel spacing is provided (see Notes section).
spacing : (3,) array-like of floats, optional
The voxel spacing along each image dimension. By default, `slic`
assumes uniform spacing (same voxel resolution along z, y and x).
This parameter controls the weights of the distances along z, y,
and x during k-means clustering.
multichannel : bool, optional
Whether the last axis of the image is to be interpreted as multiple
channels or another spatial dimension.
convert2lab : bool, optional
Whether the input should be converted to Lab colorspace prior to
segmentation. The input image *must* be RGB. Highly recommended.
This option defaults to ``True`` when ``multichannel=True`` *and*
``image.shape[-1] == 3``.
enforce_connectivity: bool, optional
Whether the generated segments are connected or not
min_size_factor: float, optional
Proportion of the minimum segment size to be removed with respect
to the supposed segment size ```depth*width*height/n_segments```
max_size_factor: float, optional
Proportion of the maximum connected segment size. A value of 3 works
in most of the cases.
slic_zero: bool, optional
Run SLIC-zero, the zero-parameter mode of SLIC. [2]_
Returns
-------
labels : 2D or 3D array
Integer mask indicating segment labels.
Raises
------
ValueError
If ``convert2lab`` is set to ``True`` but the last array
dimension is not of length 3.
Notes
-----
* If `sigma > 0`, the image is smoothed using a Gaussian kernel prior to
segmentation.
* If `sigma` is scalar and `spacing` is provided, the kernel width is
divided along each dimension by the spacing. For example, if ``sigma=1``
and ``spacing=[5, 1, 1]``, the effective `sigma` is ``[0.2, 1, 1]``. This
ensures sensible smoothing for anisotropic images.
* The image is rescaled to be in [0, 1] prior to processing.
* Images of shape (M, N, 3) are interpreted as 2D RGB images by default. To
interpret them as 3D with the last dimension having length 3, use
`multichannel=False`.
References
----------
.. [1] Radhakrishna Achanta, Appu Shaji, Kevin Smith, Aurelien Lucchi,
Pascal Fua, and Sabine Süsstrunk, SLIC Superpixels Compared to
State-of-the-art Superpixel Methods, TPAMI, May 2012.
.. [2] http://ivrg.epfl.ch/research/superpixels#SLICO
Examples
--------
>>> from skimage.segmentation import slic
>>> from skimage.data import astronaut
>>> img = astronaut()
>>> segments = slic(img, n_segments=100, compactness=10)
Increasing the compactness parameter yields more square regions:
>>> segments = slic(img, n_segments=100, compactness=20)
"""
image = img_as_float(image)
is_2d = False
##添加维度
if image.ndim == 2:
# 2D grayscale image
image = image[np.newaxis, ..., np.newaxis]
is_2d = True
elif image.ndim == 3 and multichannel:
# Make 2D multichannel image 3D with depth = 1
image = image[np.newaxis, ...]
is_2d = True
elif image.ndim == 3 and not multichannel:
# Add channel as single last dimension
image = image[..., np.newaxis]
if spacing is None:
spacing = np.ones(3)
elif isinstance(spacing, (list, tuple)):
spacing = np.array(spacing, dtype=np.double)
if not isinstance(sigma, coll.Iterable):
sigma = np.array([sigma, sigma, sigma], dtype=np.double)
sigma /= spacing.astype(np.double)
elif isinstance(sigma, (list, tuple)):
sigma = np.array(sigma, dtype=np.double)
if (sigma > 0).any():
# add zero smoothing for multichannel dimension
sigma = list(sigma) + [0]
image = ndi.gaussian_filter(image, sigma)
if multichannel and (convert2lab or convert2lab is None):
if image.shape[-1] != 3 and convert2lab:
raise ValueError("Lab colorspace conversion requires a RGB image.")
elif image.shape[-1] == 3:
image = rgb2lab(image)
depth, height, width = image.shape[:3]
# initialize cluster centroids for desired number of segments
grid_z, grid_y, grid_x = np.mgrid[:depth, :height, :width]
slices = regular_grid(image.shape[:3], n_segments)
step_z, step_y, step_x = [
int(s.step if s.step is not None else 1) for s in slices
]
segments_z = grid_z[slices]
segments_y = grid_y[slices]
segments_x = grid_x[slices]
segments_color = np.zeros(segments_z.shape + (image.shape[3], ))
segments = np.concatenate([
segments_z[..., np.newaxis], segments_y[..., np.newaxis],
segments_x[..., np.newaxis], segments_color
],
axis=-1).reshape(-1, 3 + image.shape[3])
segments = np.ascontiguousarray(segments)
# we do the scaling of ratio in the same way as in the SLIC paper
# so the values have the same meaning
step = float(max((step_z, step_y, step_x)))
ratio = 1.0 / compactness
image = np.ascontiguousarray(image * ratio)
labels, color_center, edge = _slic_cython(image, segments, step, max_iter,
spacing, slic_zero)
if enforce_connectivity:
segment_size = depth * height * width / n_segments
min_size = int(min_size_factor * segment_size)
max_size = int(max_size_factor * segment_size)
labels, color_center, edge = _enforce_label_connectivity_cython(
labels, color_center, min_size, max_size)
if is_2d:
labels = labels[0]
color_center = color_center[:, 3:]
color_center = color_center * 10
color_center = color_center[np.newaxis, ...]
color_center = lab2rgb(color_center)
color_center = color_center * 255
color_center = color_center[0]
edgeList = [[] for i in range(len(edge))]
for i in range(len(edge)):
for j in range(len(edge[i])):
if edge[i, j] != -1 and edge[i, j] not in edgeList[i]:
edgeList[i].append(edge[i, j])
return labels, color_center[:(np.amax(labels) + 1)], edgeList
def save_imgs(self, tensor, filename):
for index, im in enumerate(tensor):
# print(im.shape)
# im =np.clip(im.numpy().transpose(1,2,0), -1, 1)
img_rgb_out = (255 * np.clip(lab2rgb(im), 0, 1)).astype('uint8')
io.imsave(filename + 'rgb' + str(index) + '.png', img_rgb_out)
def test_text2colors(self):
# Load model.
if self.args.resume_epoch:
self.load_model(self.args.mode, self.args.resume_epoch)
print('Start testing...')
for batch_idx, (txt_embeddings, real_palettes,
images) in enumerate(self.test_loader):
if txt_embeddings.size(0) != self.args.batch_size:
break
# Compute text input size (without zero padding).
batch_size = txt_embeddings.size(0)
nonzero_indices = list(torch.nonzero(txt_embeddings)[:, 0])
each_input_size = [
nonzero_indices.count(j) for j in range(batch_size)
]
# Prepare test data.
txt_embeddings = txt_embeddings.to(self.device)
real_palettes = real_palettes.to(self.device).float()
for num_gen in range(5):
# Prepare input and output variables.
palette = torch.FloatTensor(batch_size,
3).zero_().to(self.device)
fake_palettes = torch.FloatTensor(batch_size,
15).zero_().to(self.device)
# ============================== Text-to-Palette ==============================#
# Condition for the generator.
encoder_hidden = self.encoder.init_hidden(batch_size).to(
self.device)
encoder_outputs, decoder_hidden, mu, logvar = self.encoder(
txt_embeddings, encoder_hidden)
# Generate color palette.
for i in range(5):
palette, decoder_context, decoder_hidden, _ = self.G_TPN(
palette, decoder_hidden.squeeze(0), encoder_outputs,
each_input_size, i)
fake_palettes[:, 3 * i:3 * (i + 1)] = palette
# ========================= Palette-to-Colorization ============================#
# Prepare testing data.
inputs, labels = process_image(images, batch_size, imsize)
inputs, labels = inputs.to(self.device), labels.to(self.device)
fake_palettes_ = fake_palettes.view(-1, 5,
3).cpu().data.numpy()
for_global = process_palette_lab(fake_palettes_, batch_size)
side_inputs = process_global_sampling_lab(
for_global, batch_size, self.imsize, 0.03, 0.13)
fake_images = self.G_PCN(inputs, side_inputs)
# ================================ Save Results ================================#
colored_img = torch.cat([inputs, fake_images],
1).data.cpu().numpy().transpose(
(0, 2, 3, 1))
gt_img = images.cpu().numpy().transpose((0, 2, 3, 1))
for x in range(batch_size):
# Input text.
input_text = ''
for idx in txt_embeddings[x]:
if idx.item() == 0: break
input_text += self.input_dict.index2word[
idx.item()] + " "
## Save palette generation results.
fig1, axs1 = plt.subplots(nrows=2, ncols=5)
axs1[0][0].set_title(input_text +
'fake {}'.format(num_gen + 1))
for k in range(5):
lab = np.array([
fake_palettes.data[x][3 * k],
fake_palettes.data[x][3 * k + 1],
fake_palettes.data[x][3 * k + 2]
],
dtype='float64')
rgb = lab2rgb_1d(lab)
axs1[0][k].imshow([[rgb]])
axs1[0][k].axis('off')
axs1[1][0].set_title(input_text + 'real')
for k in range(5):
lab = np.array([
real_palettes.data[x][3 * k],
real_palettes.data[x][3 * k + 1],
real_palettes.data[x][3 * k + 2]
],
dtype='float64')
rgb = lab2rgb_1d(lab)
axs1[1][k].imshow([[rgb]])
axs1[1][k].axis('off')
fig1.savefig(
os.path.join(
self.args.test_sample_dir, self.args.mode,
'{}_palette{}.jpg'.format(
self.args.batch_size * batch_idx + x + 1,
num_gen + 1)))
## Save colorization results.
fig2, axs2 = plt.subplots(ncols=2)
# Make images back to RGB.
colored_img[k][:, :, 0] = colored_img[k][:, :, 0] * 100
colored_img[k][:, :,
1] = (colored_img[k][:, :, 1] * 185) - 88
colored_img[k][:, :,
2] = (colored_img[k][:, :, 2] * 212) - 127
colored_img[k] = lab2rgb(colored_img[k].astype(np.float64),
illuminant='D50')
gt_img[k][:, :, 0] = gt_img[k][:, :, 0] * 100
gt_img[k][:, :, 1] = (gt_img[k][:, :, 1] * 185) - 88
gt_img[k][:, :, 2] = (gt_img[k][:, :, 2] * 212) - 127
gt_img[k] = lab2rgb(gt_img[k].astype(np.float64),
illuminant='D50')
axs2[0].set_title(input_text + '/ Prediction')
axs2[0].imshow(colored_img[x])
axs2[0].axis('off')
axs2[1].set_title(input_text + '/ Ground Truth')
axs2[1].imshow(gt_img[x])
axs2[1].axis('off')
fig2.savefig(
os.path.join(
self.args.test_sample_dir, self.args.mode,
'{}_color{}.jpg'.format(
self.args.batch_size * batch_idx + x + 1,
num_gen + 1)))
print('Saved data [{}], input text [{}], test sample [{}]'.
format(self.args.batch_size * batch_idx + x + 1,
input_text, num_gen + 1))
import pandas as pd
import SimpleITK as sitk
from PIL import Image
from skimage.color import (rgb2gray, gray2rgb, rgb2hsv, hsv2rgb, rgb2lab,
lab2rgb, lch2lab, lab2lch)
#: landmarks coordinates, loading from CSV file
LANDMARK_COORDS = ['X', 'Y']
# PIL.Image.DecompressionBombError: could be decompression bomb DOS attack.
# SEE: https://gitlab.mister-muffin.de/josch/img2pdf/issues/42
Image.MAX_IMAGE_PIXELS = None
CONVERT_RGB = {
'hsv': (rgb2hsv, hsv2rgb),
'lab': (rgb2lab, lab2rgb),
'lch':
(lambda img: lab2lch(rgb2lab(img)), lambda img: lab2rgb(lch2lab(img))),
}
def create_folder(path_folder, ok_existing=True):
""" create a folder if it not exists
:param str path_folder: path to creating folder
:param bool ok_existing: suppress warning for missing
:return str|None: path to created folder
>>> p_dir = create_folder('./sample-folder', ok_existing=True)
>>> create_folder('./sample-folder', ok_existing=False)
False
>>> os.rmdir(p_dir)
"""
def save_each_image(colored_layers, BW_layer, cycle, nr, path, ending):
cur = np.zeros((128, 128, 3))
cur[:, :, 0] = BW_layer[:, :, 0] * 100
cur[:, :, 1:] = colored_layers * 128
imsave(os.path.join(path, cycle + nr + ending), lab2rgb(cur))
padding='same')(decoder_output)
decoder_output = UpSampling2D((2, 2))(decoder_output)
decoder_output = Conv2D(2, (3, 3), activation='tanh',
padding='same')(decoder_output)
decoder_output = UpSampling2D((2, 2))(decoder_output)
model = Model(inputs=encoder_input, outputs=decoder_output)
model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
model.fit(vggfeatures, Y, validation_split=0.2, epochs=1000)
testpath = 'C://Users/Asus/Desktop/dataset/data/'
files = os.listdir(testpath)
for idx, file in enumerate(files):
test = img_to_array(load_img(testpath + file))
test = resize(test, (224, 224), anti_aliasing=True)
test *= 1.0 / 255
lab = rgb2lab(test)
l = lab[:, :, 0]
L = gray2rgb(l)
L = L.reshape((1, 224, 224, 3))
print(L.shape)
vggpred = newmodel.predict(L)
vggpred = vggpred.reshape((1, 7, 7, 512))
ab = model.predict(vggpred)
print(ab.shape)
ab = ab * 128
cur = np.zeros((224, 224, 3))
cur[:, :, 0] = l
cur[:, :, 1:] = ab
imsave('C://Users/Asus/Desktop/dataset/' + str(idx) + ".jpg", lab2rgb(cur))
def main(args):
data_dir = args.test_images_dir
dirs = os.listdir(data_dir)
print(dirs)
# Model instance whose architecture was configured in the 'model.py' file
color_model = Color_model().cuda().eval()
# Loading a pretrained model weights to the model instance archtecture
color_model.load_state_dict(torch.load(args.model_file))
tensor_to_PIL = transforms.ToPILImage(
) # Create a transform that convert a tensor to a PIL Object
# Annealed Mean
T = 0.38 # Temperature
soft = nn.Softmax(dim=1) # Instance of softmax function
Q_bins = np.load(args.Q_bins_file) # 313 bins of quantized colors
#print(Q_bins.shape)
#print(Q_bins)
for file in dirs:
# The 'load_image()' function loads the test dataset images to be submitted to the trained model.
# This function returns the following data:
# - img_l_resized -> L channel image with shape (224x224) that will be the model input
# - img_l_original -> This image will be combined to the predicted (a,b) channels. Both 'img_l_original' and predicted (a,b) channels are in original image shape
# - ori_size -> Shape of original image that will be used to resized the final colored image.
img_l_resized, img_l_original, ori_size = load_image(
data_dir + '/' + file, scale_transform)
img_l_resized = img_l_resized.unsqueeze(0).float().cuda()
# Submitting L channel image with shape (224x224) as input to the model. The model output is a tensor with shape (313x56x56)
img_ab_313 = color_model(img_l_resized)
#img_ab_313 = abs(img_ab_313)
img_ab_313 = img_ab_313.cpu()
# Resize the tensor with shape (313x56x56) to the original image shape
img_ab_313 = F.upsample(img_ab_313, size=ori_size, mode='nearest')
# The Annealed Mean is applied to the output tensor with shape (313x[Original Width]x[Original Height])
img_ab_313_log_t = (torch.log10(img_ab_313 + 0.001)) / T
# Apply softmax function over the 313 layers of the tensor. This will generate a probability distribution across the 313 layers for each pixel
soft_image_log_t = soft(img_ab_313_log_t)
# - bins_matrix_values -> Matrix containing the highest probability value of the distribution for each pixel
# - bins_matrix_indexes -> Matrix containing the index of the probability distribution over the 313 layers that has the highest value of the probability distribution
bins_matrix_values, bins_matrix_indexes = torch.max(soft_image_log_t,
dim=1,
keepdim=True)
bins_matrix_indexes_flatten = torch.flatten(bins_matrix_indexes)
# Use the indexes of highest probabilities values of the distribution for each pixel to select the bins(a and b values) of the 313 quantized colors stored in 'Q_bins'
predicted_ab_channels = Q_bins[bins_matrix_indexes[:, :]]
predicted_ab_channels = predicted_ab_channels.squeeze(0).squeeze(
0).transpose(2, 0, 1)
predicted_ab_channels = predicted_ab_channels.transpose(1, 2, 0)
img_l_original = img_l_original.numpy()
# Create the final image matrix structure to receive the L channel and the (a,b) predicted channels.
# The shape of this final image is ([Original Width]x[Original Height]x3) and this structure is filled initialy with zeros.
img_lab_final = np.zeros((ori_size[0], ori_size[1], 3))
# Assign the L channel image values to the first channel of the 'img_lab_final' image and assign the (a,b) predicted channels to the second and third channels of the 'img_lab_final' image
img_lab_final[:, :, 0] = img_l_original
img_lab_final[:, :, 1:] = predicted_ab_channels
# Converting the final colorized imagem colorspace from CIE Lab to RGB
img_rgb_final = lab2rgb(img_lab_final)
# Saving the colorized image
file = file.replace("jpg", "png")
#imageio.imwrite(args.output_images_dir + file, img_as_ubyte(abs(img_rgb_final)))
imageio.imwrite(args.output_images_dir + file, img_rgb_final)
model = cluster.KMeans(n_clusters=nColors)
labels = model.fit_predict(a_b)
palette = model.cluster_centers_
array_clustered = np.hstack([reshaped_raster[:,:1],palette[labels]])
quantized_raster = np.reshape(
array_clustered, (width, height, palette.shape[1]+1)
)
return quantized_raster, palette
# Quantize test image into 16 bins->palettes
quantize_image_on_lab, pal = quantize(lab_image_train, 16)
# Convert to RGB for plotting the quantized image
rgb_raster = color.lab2rgb(quantize_image_on_lab)
rgb_raster = (rgb_raster*255).astype('uint8')
# Plot to see the quantized space and before quantize
# fig = plt.figure()
# ax1 = fig.add_subplot(2,2,1)
# ax1.imshow(rgb_image_train)
# ax2 = fig.add_subplot(2,2,2)
# ax2.imshow(rgb_raster)
# plt.show()
# ******* SLIC ALGORITHM FOR IMAGE SEGMENTATION *******
segments_slic_train = slic(rgb_image_train, n_segments=250, compactness=10, sigma=1)
