def image_histogram_mapping_segmentation(*x, uv=False, normalize=False, reg_head=True, weighted=False, multioutput=False, clustering=False, correct=True):
mask = x[1]
img = x[0]
gt = tf.cast(x[-1], dtype=tf.float32)
if gt != None and correct:
gt1 = tf.reshape(gt, (-1, 3))
img = img / gt1[0]
gt = gt1[1] / gt1[0]
gt = tf.reshape(gt, (3,))
h = tf.image.rgb_to_hsv(img)[..., 0]
img_uv, Iy = histogram.to_uv(img)
Iy = tf.where(Iy < 0.01, 0., Iy)
if normalize:
Iy = tf.image.per_image_standardization(Iy[:, :, tf.newaxis])
Iy = Iy[..., 0]
img_hy = tf.stack([Iy, h, Iy], axis=-1)
img_uv = tf.stack([Iy, img_uv[..., 0], img_uv[..., 1]], axis=-1)
img = img_uv if uv else img
if multioutput:
features = tf.concat((img, img_hy), axis=-1)
if weighted:
return features, mask, gt, tf.where(Iy == 0., 0., 1.), tf.ones((1,))
return features, mask, gt
elif clustering:
gt1, gt2 = gt[:3], gt[3:]
gt1 = gt1 / (tf.linalg.norm(gt1, axis=-1, ord=2, keepdims=True) + 1e-7)
gt2 = gt2 / (tf.linalg.norm(gt2, axis=-1, ord=2, keepdims=True) + 1e-7)
gt1 = visualizer.create_mask(gt1, sz =img.shape[-3:-1])
gt2 = visualizer.create_mask(gt2, sz =img.shape[-3:-1])
features = tf.concat((img, img_hy, gt1, gt2), axis=-1)
# gt_mask = mask[..., :3]
# gt_mask = gt_mask / (tf.linalg.norm(gt_mask, axis=-1, ord=2, keepdims=True) + 1e-7)
# mask = mask[..., -1:]
if weighted:
return features, mask, tf.where(Iy == 0., 0., 1.)
return features, mask
elif reg_head:
features = tf.concat((img, img_hy), axis=-1)
gt_uv, gty = histogram.to_uv(tf.reshape(gt, (-1, 3)))
gt_uv = tf.reshape(gt_uv, (2,))
gt = gt_uv if uv else gt
gt = [mask, gt] if multioutput else tf.concat([mask, tf.broadcast_to(gt, (*mask.shape[:-1], *gt.shape))], axis=-1)
else:
features = tf.concat((img, img_hy), axis=-1)
gt = x[1]
if weighted:
return features, gt, tf.where(Iy == 0., 0., 1.)
return features, gt
def on_epoch_end(self, epoch, logs=None):
if epoch % 4 != 0:
return
for data in self.images:
img, mask = data[0], data[1]
pred_mask = model.predict(img)
gt1, gt2 = None, None
if type(mask) == tuple:
gt1 = mask[-2]
gt2 = mask[-1]
mask = mask[-3]
pgt1 = pred_mask[-2][..., :3]
pgt2 = pred_mask[-1][..., :3]
pred_mask = pred_mask[-3][..., :mask.shape[-1]]
mask = mask[0]
pred_mask = pred_mask[0]
if gt1 is not None:
if gt1.shape[-1] == 2:
gt1 = tf.stack([
gt1[..., 0],
tf.ones_like(gt1[..., 0]), gt1[..., 1]
],
axis=-1)
pgt1 = tf.stack([
pgt1[..., 0],
tf.ones_like(pgt1[..., 0]), pgt1[..., 1]
],
axis=-1)
gt2 = tf.stack([
gt2[..., 0],
tf.ones_like(gt2[..., 0]), gt2[..., 1]
],
axis=-1)
pgt2 = tf.stack([
pgt2[..., 0],
tf.ones_like(pgt2[..., 0]), pgt2[..., 1]
],
axis=-1)
gt1 = visualizer.create_mask(gt1[0], (10, 10))
gt2 = visualizer.create_mask(gt2[0], (10, 10))
pgt1 = visualizer.create_mask(pgt1[0], (10, 10))
pgt2 = visualizer.create_mask(pgt2[0], (10, 10))
if gt1 is not None:
visualizer.visualize([
img[0, :, :, :3], img[0, :, :, 3:6],
tf.squeeze(mask[..., :3]),
tf.squeeze(pred_mask[..., :3]), gt1, gt2, pgt1, pgt2
])
else:
visualizer.visualize([
img[0, :, :, :3], img[0, :, :, 3:6],
tf.squeeze(mask[..., :3]),
tf.squeeze(pred_mask[..., :3])
])
print('/nSample Prediction after epoch {}\n'.format(epoch + 1))
def multi_segmentation_mapping(*x, rb=False, clustering=False, regression=False):
mask = x[1]
img = x[0]
gt = x[-1]
gt = tf.reshape(gt, (-1,3))
gt = tf.nn.l2_normalize(gt, axis=-1)
gt = tf.reshape(gt, (-1,))
h = tf.image.rgb_to_hsv(img)[..., 0]
if rb:
_, Iy = histogram.to_rb(img)
# img = tf.stack([img[..., 0], img[..., 1], Iy],axis=-1)
gt, _ = histogram.to_rb(gt)
else:
_, Iy = histogram.to_uv(img)
Iy = tf.where(Iy < 0.01, 0., Iy)
img_hy = tf.stack([Iy, h, Iy], axis=-1)
if clustering:
features = tf.concat((img, img_hy), axis=-1)
gt = visualizer.create_mask(gt, sz =img.shape[-3:-1])
mask = tf.concat([tf.cast(mask, float), gt], axis=-1)
elif regression:
features = tf.concat((img, img_hy), axis=-1)
mask = mask / (tf.linalg.norm(mask, axis=-1, ord=2, keepdims=True) + 1e-7)
else:
img_cor = img / gt[0]
features = tf.concat((img_cor, img_hy), axis=-1)
return features, \
mask, \
tf.where(Iy == 0., 0., 1.)
def multi_representation_histogram(*x, depth, shrink, known_illuminant=False):
images = __process_images__(x[0])
hists = __hr__(images, depth, shrink)
if known_illuminant:
ct = histogram.color_table(depth)
gt1 = visualizer.create_mask(x[-1][:3], (depth, depth))
ct = tf.sqrt(cosine_similarity(ct, gt1, keepdims=True))
ct = tf.tile(ct[tf.newaxis, :, :], (hists.shape[0], 1, 1, 1))
print(ct.shape, hists.shape)
hists = hists * ct
return (hists, *x[1:], images)
def transformation_histogram(*x, depth, shrink, known_illuminant=False):
images = __process_images__(x[0], indecies=[0,3])
hists = __hr__(images, depth, shrink)
images = None
images_corrected = __process_images__(x[-1])
hists2 = __hr__(images_corrected, depth, shrink)
if known_illuminant:
ct = histogram.color_table(depth)
gt1 = visualizer.create_mask(x[-1][:3], (depth, depth))
ct = tf.sqrt(cosine_similarity(ct, gt1, keepdims=True))
ct = tf.tile(ct[tf.newaxis, :, :], (hists.shape[0], 1, 1, 1))
print(ct.shape, hists.shape)
hists = hists * ct
return (hists, hists2, *x[1:], x[0])
plt.bar(x, yb[0, 0], width=1 / d)
#HISTOGRAM PEAKS
peaks, props = s.find_peaks(yb[0, 0], distance=18)
peak_heights = np.array(list(map(lambda x: yb[0, 0, x], peaks)))
pph = sorted(zip(peaks, peak_heights),
key=lambda x: x[1],
reverse=True)
peaks = np.array(list(map(lambda x: x[0], pph)))
colors = ["red", "green", "yellow", "cyan", "magenta"]
ills = []
for p, c in zip(peaks, colors):
p1 = yb[0, 0, p]
ys = np.arange(0., p1, p1 / 100)[:100]
plt.plot(np.ones(100) * p / d, ys, color=c)
ill = v.create_mask(tf.convert_to_tensor([p / d, 1, 1]), [10, 10])
ills.append(tf.image.hsv_to_rgb(ill))
# inp = inpt_rg[Iy > 0]
# kde = st.gaussian_kde(dataset=tf.reshape(inp, (-1,)))
# plt.plot(x, kde.evaluate(x))
plt.show()
v.visualize(ills)
v.visualize([Cube2Dataset.get_image(img_path, 'gt.png', 256, 512)])
#2D HISTOGRAM
# plt.bar(x, y[0,1], width=1/64)
# plt.show()
# v.visualize(inpt)
#
# hist2d = DiffHist2D(64, (-2, 2))
def plot_prediction(img, mask, pred_mask):
gt1, gt2 = None, None
if type(mask) == tuple:
gt1 = tf.cast(mask[-2], tf.float32)
gt2 = tf.cast(mask[-1], tf.float32)
pgt1 = pred_mask[-2]
pgt2 = pred_mask[-1]
m = mask[-3]
y = pred_mask[-3]
if len(mask) == 6:
m += 1 - mask[-4]
m /= 2
y += 1 - pred_mask[-4]
y /= 2
pred_mask = y
mask = m
mask = mask[0]
pred_mask = pred_mask[0]
if gt1 is not None:
corrected, gt_mask = correct(img[0, :, :, :3], pred_mask[..., :3],
pgt1[0], pgt2[0])
corrected1, gt_mask1 = correct(img[0, :, :, :3], mask[..., :3],
gt1[0], gt2[0])
corrected11, _ = correct(img[0, :, :, :3], mask[..., :3], pgt1[0],
pgt1[0])
corrected12, _ = correct(img[0, :, :, :3], mask[..., :3], pgt2[0],
pgt2[0])
# visualizer.visualize(img[:, :, :, :3])
# visualizer.visualize([corrected1, corrected])
# visualizer.visualize([corrected11, corrected12])
# visualizer.visualize([gt_mask1, gt_mask])
# visualizer.visualize([tf.squeeze(mask[..., :3]), tf.squeeze(pred_mask[..., :3])], cb=False)
if gt1.shape[-1] == 2:
gt1 = tf.stack(
[gt1[..., 0],
tf.ones_like(gt1[..., 0]), gt1[..., 1]],
axis=-1)
pgt1 = tf.stack(
[pgt1[..., 0],
tf.ones_like(pgt1[..., 0]), pgt1[..., 1]],
axis=-1)
gt2 = tf.stack(
[gt2[..., 0],
tf.ones_like(gt2[..., 0]), gt2[..., 1]],
axis=-1)
pgt2 = tf.stack(
[pgt2[..., 0],
tf.ones_like(pgt2[..., 0]), pgt2[..., 1]],
axis=-1)
gt1 = visualizer.create_mask(gt1[0], (10, 10))
gt2 = visualizer.create_mask(gt2[0], (10, 10))
pgt1 = visualizer.create_mask(pgt1[0], (10, 10))
pgt2 = visualizer.create_mask(pgt2[0], (10, 10))
if gt1 is not None:
visualizer.visualize([
img[0, :, :, :3], img[0, :, :, 3:6],
tf.squeeze(mask[..., :3]),
tf.squeeze(pred_mask[..., :3]), gt1, gt2, pgt1, pgt2,
corrected, gt_mask
])
else:
visualizer.visualize([
img[0, :, :, :3], img[0, :, :, 3:6],
tf.squeeze(mask[..., :3]),
tf.squeeze(pred_mask[..., :3])
])
ds_base = dataset(paths,
type=dp.TEST,
bs=1,
cache=False,
regression=False,
round=False,
gt=True,
shuffle_buffer_size=1)
for data in iter(ds_base):
img, mask, gt = data
norm = tf.linalg.norm(img, axis=(1, 2, 3))
img = img * 16
emask = encode_mask(mask, img)
visualizer.visualize([
img[0], mask[0, :, :, 0], emask[0],
visualizer.create_mask(gt[0, :3], (10, 10)),
visualizer.create_mask(gt[0, 3:], (10, 10))
])
# p = 32
# ds2 = dp.patch_dataset(uv_ds, p, img_shape, mask_shape)
# ds3 = dp.reduce_dataset(ds2, None)
# ds4 = dp.reduce_dataset(dp.histogram_dataset(ds, 64), None)
# ds5 = dp.reduce_dataset(dp.histogram_dataset(dp.patch_dataset(ds, p, mask_shape, mask_shape), 64), None)
# ds6 = dp.timestamp_dataset(ds5, IMG_HEIGHT//2 * IMG_WIDTH//2 // p**2, (1024, 4096), (1024, 1))
# # data5 = next(iter(ds5))
# data6 = next(iter(ds6))
# data1 = next(iter(ds))
# data2 = next(iter(ds2))
# data3 = next(iter(ds3))
# data4 = next(iter(ds4))