def image_histogram_mapping_segmentation(*x, uv=False, normalize=False, reg_head=True, weighted=False, multioutput=False, clustering=False, correct=True):
mask = x[1]
img = x[0]
gt = tf.cast(x[-1], dtype=tf.float32)
if gt != None and correct:
gt1 = tf.reshape(gt, (-1, 3))
img = img / gt1[0]
gt = gt1[1] / gt1[0]
gt = tf.reshape(gt, (3,))
h = tf.image.rgb_to_hsv(img)[..., 0]
img_uv, Iy = histogram.to_uv(img)
Iy = tf.where(Iy < 0.01, 0., Iy)
if normalize:
Iy = tf.image.per_image_standardization(Iy[:, :, tf.newaxis])
Iy = Iy[..., 0]
img_hy = tf.stack([Iy, h, Iy], axis=-1)
img_uv = tf.stack([Iy, img_uv[..., 0], img_uv[..., 1]], axis=-1)
img = img_uv if uv else img
if multioutput:
features = tf.concat((img, img_hy), axis=-1)
if weighted:
return features, mask, gt, tf.where(Iy == 0., 0., 1.), tf.ones((1,))
return features, mask, gt
elif clustering:
gt1, gt2 = gt[:3], gt[3:]
gt1 = gt1 / (tf.linalg.norm(gt1, axis=-1, ord=2, keepdims=True) + 1e-7)
gt2 = gt2 / (tf.linalg.norm(gt2, axis=-1, ord=2, keepdims=True) + 1e-7)
gt1 = visualizer.create_mask(gt1, sz =img.shape[-3:-1])
gt2 = visualizer.create_mask(gt2, sz =img.shape[-3:-1])
features = tf.concat((img, img_hy, gt1, gt2), axis=-1)
# gt_mask = mask[..., :3]
# gt_mask = gt_mask / (tf.linalg.norm(gt_mask, axis=-1, ord=2, keepdims=True) + 1e-7)
# mask = mask[..., -1:]
if weighted:
return features, mask, tf.where(Iy == 0., 0., 1.)
return features, mask
elif reg_head:
features = tf.concat((img, img_hy), axis=-1)
gt_uv, gty = histogram.to_uv(tf.reshape(gt, (-1, 3)))
gt_uv = tf.reshape(gt_uv, (2,))
gt = gt_uv if uv else gt
gt = [mask, gt] if multioutput else tf.concat([mask, tf.broadcast_to(gt, (*mask.shape[:-1], *gt.shape))], axis=-1)
else:
features = tf.concat((img, img_hy), axis=-1)
gt = x[1]
if weighted:
return features, gt, tf.where(Iy == 0., 0., 1.)
return features, gt
def __single_loss__(self, e, y_pred, y_true):
if y_true.shape[-1] == 3:
y_true, _ = histogram.to_uv(y_true)
# ytrue: (bs x 2)
y_true = tf.map_fn(self.__bn__, y_true)
# print(y_true.shape)
index_true = tf.cast(y_true, tf.int32)
tf.debugging.assert_all_finite(
y_true,
'Target tensor must not contain nan or inf values',
name=None)
rgb_true = tf.gather(self.color_table_flat, index_true)
if len(rgb_true.shape) == 3 and rgb_true.shape[1] == 1:
rgb_true = rgb_true[:, 0, :]
cos = create_loss_ct(rgb_true, self.os, self.depth,
self.color_table_flat) #/ 3.14 * 180
l = cos * y_pred
if e == None:
e = tf.reduce_sum(l)
else:
e = e + tf.reduce_sum(l)
return e
def multi_ese_mapping(*x, corrected_input=False, repeat_gt2=False, invert_mask=False):
mask = x[1]
img = x[0]
gt = x[-1]
gt = tf.reshape(gt, (-1,3))
gt = tf.nn.l2_normalize(gt, axis=-1)
h = tf.image.rgb_to_hsv(img)[..., 0]
_, Iy = histogram.to_uv(img)
Iy = tf.where(Iy < 0.01, 0., Iy)
img_hy = tf.stack([Iy, h, Iy], axis=-1)
masks = mask
masks_weights = tf.where(Iy == 0., 0., 1.)
gts = tf.reshape(gt, (-1,))
gt_weights = tf.ones((1,))
if corrected_input:
img_cor = img / gt[0]
features = tf.concat((img, img_cor, img_hy), axis=-1)
return features, \
mask, gts, \
tf.where(Iy == 0., 0., 1.), tf.ones((1,))
else:
features = tf.concat((img, img_hy), axis=-1)
return (features, \
gt[0], masks, gts, \
gt_weights, masks_weights, gt_weights)
def multi_segmentation_mapping(*x, rb=False, clustering=False, regression=False):
mask = x[1]
img = x[0]
gt = x[-1]
gt = tf.reshape(gt, (-1,3))
gt = tf.nn.l2_normalize(gt, axis=-1)
gt = tf.reshape(gt, (-1,))
h = tf.image.rgb_to_hsv(img)[..., 0]
if rb:
_, Iy = histogram.to_rb(img)
# img = tf.stack([img[..., 0], img[..., 1], Iy],axis=-1)
gt, _ = histogram.to_rb(gt)
else:
_, Iy = histogram.to_uv(img)
Iy = tf.where(Iy < 0.01, 0., Iy)
img_hy = tf.stack([Iy, h, Iy], axis=-1)
if clustering:
features = tf.concat((img, img_hy), axis=-1)
gt = visualizer.create_mask(gt, sz =img.shape[-3:-1])
mask = tf.concat([tf.cast(mask, float), gt], axis=-1)
elif regression:
features = tf.concat((img, img_hy), axis=-1)
mask = mask / (tf.linalg.norm(mask, axis=-1, ord=2, keepdims=True) + 1e-7)
else:
img_cor = img / gt[0]
features = tf.concat((img_cor, img_hy), axis=-1)
return features, \
mask, \
tf.where(Iy == 0., 0., 1.)
def process_path_regression(file_path, type, uv, map_fn, sz):
mask = get_mask(file_path)
print(mask.shape)
# load the raw data from the file as a string
img = get_image(file_path)
if type == TRAIN:
img = augmentations.augment(img)[0]
img = tf.image.resize(img, sz)
if uv:
img, Iy = histogram.to_uv(img)
img = tf.stack([Iy, img[..., 0], img[..., 1]], axis=-1)
mask = tf.reshape(mask, (-1, 3))
mask, masky = histogram.to_uv(mask)
mask = tf.reshape(mask, (-1, ))
return map_fn(img, mask)
def process_path_regression(file_path, ind, type, uv, gts, map_fn, sz, mi):
gt = get_mask(ind, gts)
# load the raw data from the file as a string
img = get_image(file_path)
if type == TRAIN:
img = augmentations.augment(img, mask_image=True)[0]
img = tf.image.resize(img, sz)
if uv:
img, Iy = histogram.to_uv(img)
img = tf.stack([Iy, img[...,0], img[...,1]], axis=-1)
gt = tf.reshape(gt, (-1, 3))
gt, masky = histogram.to_uv(gt)
gt = tf.reshape(gt, (-1,))
if mi:
return map_fn(img, tf.ones_like(img) * 0.5, gt)
return map_fn(img, gt)
def ese_mapping(*x, corrected_input=False, pivot_gt=None, repeat_gt2=False, invert_mask=False, rb=False):
mask = x[1]
img = x[0]
gt = tf.cast(x[-1], dtype=tf.float32)
gt = tf.reshape(gt, (-1,3))
gt = tf.nn.l2_normalize(gt, axis=-1)
h = tf.image.rgb_to_hsv(img)[..., 0]
if rb:
_, Iy = histogram.to_rb(img)
# img = tf.stack([img[..., 0], img[..., 1], Iy],axis=-1)
gt, _ = histogram.to_rb(gt)
else:
_, Iy = histogram.to_uv(img)
Iy = tf.where(Iy < 0.01, 0., Iy)
img_hy = tf.stack([Iy, h, Iy], axis=-1)
if pivot_gt is not None:
pivot_gt = tf.constant(pivot_gt)
c1 = cosine_similarity(gt[0], pivot_gt)
c2 = cosine_similarity(gt[1], pivot_gt)
if c2 < c1:
gt = tf.stack([gt[1], gt[0]], axis=0)
mask = tf.ones_like(mask) - mask
masks = [mask] if not invert_mask else [mask, tf.ones_like(mask) - mask]
masks_weights = [tf.where(Iy == 0., 0., 1.)] if not invert_mask else [tf.where(Iy == 0., 0., 1.), tf.where(Iy == 0., 0., 1.)]
gts = [gt[0]] if not repeat_gt2 else [gt[0], gt[1]]
gt_weights = [tf.ones((1,))] if not repeat_gt2 else [tf.ones((1,)), tf.ones((1,))]
if corrected_input:
img_cor = img / gt[0]
features = tf.concat((img, img_cor, img_hy), axis=-1)
return features, \
mask, gt[0], gt[1], \
tf.where(Iy == 0., 0., 1.), tf.ones((1,)), tf.ones((1,))
else:
features = tf.concat((img, img_hy), axis=-1)
return (features, \
*gts, *masks, gt[0], gt[1], \
*gt_weights, *masks_weights, tf.ones((1,)), tf.ones((1,)))
def __multi_loss__(self, e, y_pred, y_true):
if y_true.shape[-1] == 6:
y_true = tf.reshape(y_true, (-1, 2, 3))
y_true, _ = histogram.to_uv(y_true)
elif y_true.shape[-1] == 4:
y_true = tf.reshape(y_true, (-1, 2, 2))
# ytrue: (bs x 2 x 2)
y_true = tf.map_fn(
lambda x: tf.stack((self.__bn__(x[0]), self.__bn__(x[1]))), y_true)
# print(y_true.shape)
index_true = tf.cast(y_true, tf.int32)
tf.debugging.assert_all_finite(
y_true,
'Target tensor must not contain nan or inf values',
name=None)
rgb_true = tf.gather(self.color_table_flat, index_true)
cos1 = create_loss_ct(rgb_true[:, 0, :], self.os, self.depth,
self.color_table_flat)
cos2 = create_loss_ct(rgb_true[:, 1, :], self.os, self.depth,
self.color_table_flat)
cos = 2 / (1 / cos1 + 1 / cos2)
# l1 = tf.sqrt(cos1) * y_pred
# l2 = tf.sqrt(cos2) * y_pred
l = cos * y_pred
tf.debugging.assert_all_finite(l,
'Loss must not have nan values',
name=None)
# l = l1 + l2
if e == None:
e = tf.reduce_sum(l)
else:
e = e + tf.reduce_sum(l)
return e # + self.cc(y_true, y_pred)
return np.concatenate(([0 for k in range(n - 1)], mv))
with tf.device('/device:cpu:0'):
img_path = 'D:/fax/Cube2/outdoor/canon_550d/outdoor1/4'
inpt = tf.ones((1, 100, 100, 5))
inpt2 = tf.random.uniform((1, 100, 100, 5))
inpt = tf.concat((inpt, inpt2), axis=0)
d = 256
hist = DiffHist(d, (0, 1), w_init=1 / d)
inpt = Cube2Dataset.get_image(img_path, 'img.png', 256, 512)
inpt = tf.expand_dims(inpt, axis=0)
# inpt = dp.__process_images__(inpt, [1, 4])
inpt_rg = tf.math.divide_no_nan(inpt[..., 0], inpt[..., 1])
inpt_bg = tf.math.divide_no_nan(inpt[..., 2], inpt[..., 1])
inpt_rb = tf.stack((inpt_rg, inpt_bg), axis=-1)
inpt_uv, Iy = histogram.to_uv(inpt)
inpt_hsv = tf.image.rgb_to_hsv(inpt)
inpt_h = inpt_hsv[..., 0:1]
# inpt_rb = tf.expand_dims(inpt_rb, axis=0)
# inpt_rb = tf.concat((inpt_rb[:,0,:,:,:], inpt_rb[:,1,:,:,:]), axis=-1)
# Iy = tf.transpose(Iy, (1, 2, 0))
Iy = tf.where(Iy < 0.05, 0., Iy)
w = tf.where(Iy == 0., Iy, tf.ones_like(Iy))
y = hist(inpt_h)
y = y * tf.expand_dims(tf.expand_dims(w, axis=-1), axis=-1)
y = tf.reduce_sum(y, axis=[1, 2])
y = tf.sqrt(y / tf.reduce_sum(y, axis=-1, keepdims=True))
rb = tf.argmax(y, axis=-1)
v.visualize(y)
def f(img):
return histogram.to_uv(img)[0]