def add_diff_rot90(g_outmap):
g_out = g_outmap["output"]
grid_idx = g_outmap['grid_idx']
z_rot90 = g_outmap['z_rot90']
alphas = binary_mask(grid_idx, black=0.5, ignore=1.0, white=0.5)
bw_mask = binary_mask(grid_idx, black=0., ignore=0, white=0.5)
combined = (alphas * g_out) + bw_mask
return rotate_by_multiple_of_90(combined, z_rot90)
def add_diff_rot90(g_outmap):
g_out = g_outmap["output"]
grid_idx = g_outmap['grid_idx']
z_rot90 = g_outmap['z_rot90']
alphas = binary_mask(grid_idx, black=0.5, ignore=1.0, white=0.5)
bw_mask = binary_mask(grid_idx, black=0., ignore=0, white=0.5)
combined = (alphas * g_out) + bw_mask
return rotate_by_multiple_of_90(combined, z_rot90)
def grid_loss(g_outmap):
g_out = g_outmap['output']
grid_idx = g_outmap["grid_idx"]
m = g_out[:, :1]
b = binary_mask(grid_idx, ignore=0, black=0,
white=1 - variation_weight)
return grid_loss_weight*mse(b, m)
def grid_loss(g_outmap):
g_out = g_outmap['output']
grid_idx = g_outmap["grid_idx"]
m = g_out[:, :1]
b = binary_mask(grid_idx,
ignore=0,
black=0,
white=1 - variation_weight)
return grid_loss_weight * mse(b, m)
def reconstruct(g_outmap):
g_out = g_outmap["output"]
grid_idx = g_outmap["grid_idx"]
z_rot90 = g_outmap['z_rot90']
alphas = binary_mask(grid_idx, black=variation_weight,
ignore=1.0, white=variation_weight)
m = theano.gradient.disconnected_grad(g_out[:, :1])
v = g_out[:, 1:]
combined = v # T.clip(m + alphas*v, 0., 1.)
return rotate_by_multiple_of_90(combined, z_rot90)
def reconstruct(g_outmap):
g_out = g_outmap["output"]
grid_idx = g_outmap["grid_idx"]
z_rot90 = g_outmap['z_rot90']
alphas = binary_mask(grid_idx,
black=variation_weight,
ignore=1.0,
white=variation_weight)
m = theano.gradient.disconnected_grad(g_out[:, :1])
v = g_out[:, 1:]
combined = v # T.clip(m + alphas*v, 0., 1.)
return rotate_by_multiple_of_90(combined, z_rot90)
def mask_loss_mse(grid_idx, image):
indicies = T.bitwise_and(T.neq(grid_idx, MASK["IGNORE"]),
T.neq(grid_idx, MASK["BACKGROUND_RING"]))
bw = binary_mask(grid_idx, ignore=0.0)
diff = (bw - image)
loss = (diff[indicies.nonzero()]**2).mean()
visual_diff = T.zeros_like(diff)
visual_diff = T.set_subtensor(visual_diff[indicies.nonzero()],
diff[indicies.nonzero()]**2)
return DotMap({
'loss': loss,
'visual': {
'diff': visual_diff,
'bw_grid': bw
}
})
def mask_loss_mse(grid_idx, image):
indicies = T.bitwise_and(T.neq(grid_idx, MASK["IGNORE"]),
T.neq(grid_idx, MASK["BACKGROUND_RING"]))
bw = binary_mask(grid_idx, ignore=0.0)
diff = (bw - image)
loss = (diff[indicies.nonzero()]**2).mean()
visual_diff = T.zeros_like(diff)
visual_diff = T.set_subtensor(visual_diff[indicies.nonzero()],
diff[indicies.nonzero()]**2)
return DotMap({
'loss': loss,
'visual': {
'diff': visual_diff,
'bw_grid': bw
}
})
def mask_loss_sobel(grid_idx, image, impl='auto', diff_type='mse', scale=10.):
def norm_by_max(x):
maxs = T.max(x, axis=[1, 2, 3])
return x / maxs.dimshuffle(0, 'x', 'x', 'x')
def clip_around_zero(x, threshold=0.2):
indicies = T.bitwise_and(x < threshold, x > -threshold)
return T.set_subtensor(x[indicies.nonzero()], 0)
bw = binary_mask(grid_idx, ignore=0.5)
bw = gaussian_filter_2d(bw, sigma=2.)
sobel_bw = sobel(bw)
sobel_img = [gaussian_filter_2d(s, sigma=1.) for s in sobel(image)]
sobel_bw = [norm_by_max(s) for s in sobel_bw]
sobel_img = [norm_by_max(s) for s in sobel_img]
if diff_type == 'correlation':
loss_x = -sobel_bw[0] * sobel_img[0]
loss_y = -sobel_bw[1] * sobel_img[1]
loss = T.mean(loss_x + loss_y) / 2
elif diff_type == 'mse':
sobel_bw = [clip_around_zero(s) for s in sobel_bw]
sobel_img = [clip_around_zero(s) for s in sobel_img]
loss_x = (sobel_bw[0] - sobel_img[0])**2
loss_y = (sobel_bw[1] - sobel_img[1])**2
loss = T.mean(loss_x + loss_y) / 2
else:
raise ValueError("unknown diff_type: {}".format(diff_type))
return DotMap({
'loss': scale * T.mean(loss),
'visual': {
'loss_x': loss_x,
'loss_y': loss_y,
'sobel_img_x': sobel_img[0],
'sobel_img_y': sobel_img[1],
'sobel_mask_x': sobel_bw[0],
'sobel_mask_y': sobel_bw[1],
},
'loss_x_mean': loss_x.mean(),
'loss_y_mean': loss_x.mean(),
})
def mask_loss_sobel(grid_idx, image, impl='auto', diff_type='mse', scale=10.):
def norm_by_max(x):
maxs = T.max(x, axis=[1, 2, 3])
return x / maxs.dimshuffle(0, 'x', 'x', 'x')
def clip_around_zero(x, threshold=0.2):
indicies = T.bitwise_and(x < threshold, x > -threshold)
return T.set_subtensor(x[indicies.nonzero()], 0)
bw = binary_mask(grid_idx, ignore=0.5)
bw = gaussian_filter_2d(bw, sigma=2.)
sobel_bw = sobel(bw)
sobel_img = [gaussian_filter_2d(s, sigma=1.) for s in sobel(image)]
sobel_bw = [norm_by_max(s) for s in sobel_bw]
sobel_img = [norm_by_max(s) for s in sobel_img]
if diff_type == 'correlation':
loss_x = -sobel_bw[0] * sobel_img[0]
loss_y = -sobel_bw[1] * sobel_img[1]
loss = T.mean(loss_x + loss_y) / 2
elif diff_type == 'mse':
sobel_bw = [clip_around_zero(s) for s in sobel_bw]
sobel_img = [clip_around_zero(s) for s in sobel_img]
loss_x = (sobel_bw[0] - sobel_img[0])**2
loss_y = (sobel_bw[1] - sobel_img[1])**2
loss = T.mean(loss_x + loss_y) / 2
else:
raise ValueError("unknown diff_type: {}".format(diff_type))
return DotMap({
'loss': scale*T.mean(loss),
'visual': {
'loss_x': loss_x,
'loss_y': loss_y,
'sobel_img_x': sobel_img[0],
'sobel_img_y': sobel_img[1],
'sobel_mask_x': sobel_bw[0],
'sobel_mask_y': sobel_bw[1],
},
'loss_x_mean': loss_x.mean(),
'loss_y_mean': loss_x.mean(),
})
def get_output(self, train=False):
tag_mean = get_output(self.tag_mean, train,
self.layer_cache)
black_mean = tag_mean[:, 0]
white_mean = K.abs(tag_mean[:, 1]) + black_mean + \
self.min_black_white_distance
nb_pyramid_layers = 3
input = self.get_input(train)
image = input[:, :1]
grid_idx = input[:, 1:]
selection_mask = binary_mask(grid_idx, ignore=0, black=0.8, white=0.8)
pattern = (0, 'x', 'x', 'x')
tag = adaptive_mask(grid_idx, ignore=0,
black=black_mean.dimshuffle(*pattern),
white=white_mean.dimshuffle(*pattern))
gauss_pyr_tag = list(pyramid_gaussian(tag, nb_pyramid_layers))
gauss_pyr_image = list(pyramid_gaussian(image, nb_pyramid_layers))
gauss_pyr_mask = list(pyramid_gaussian(selection_mask,
nb_pyramid_layers))
pyr_masks = [0]*(len(gauss_pyr_mask) - 1) + gauss_pyr_mask[-1:]
lap_pyr_tag = pyramid_laplace(gauss_pyr_tag) + gauss_pyr_tag[-1:]
lap_pyr_image = pyramid_laplace(gauss_pyr_image) + gauss_pyr_image[-1:]
blend_pyr = []
for mask, lap_tag, lap_image in zip(pyr_masks, lap_pyr_tag,
lap_pyr_image):
blend = lap_tag*mask + lap_image*(1 - mask)
blend_pyr.append(blend)
img = None
for low, high in pairwise(reversed(blend_pyr)):
if img is None:
img = low
img = upsample(img) + high
return img
def pyramid_loss(grid_idx, image):
def mean(mask):
return T.sum(image * mask, axis=(1, 2, 3)) / T.sum(mask,
axis=(1, 2, 3))
min_mean_distance = 0.2
max_grayness = 0.3
# use 16x16 as last resolution
max_layer = 3
black_mask = binary_mask(grid_idx, ignore=0, black=1, white=0)
white_mask = binary_mask(grid_idx, ignore=0, black=0, white=1)
black_mean = mean(black_mask)
white_mean = mean(white_mask)
mean_diff = white_mean - black_mean
gray_mean = (white_mean + black_mean) / 2
black_mean = T.where(mean_diff < min_mean_distance,
gray_mean - min_mean_distance / 2, black_mean)
white_mean = T.where(mean_diff < min_mean_distance,
gray_mean + min_mean_distance / 2, white_mean)
dimshuffle = (0, 'x', 'x', 'x')
white_mean = white_mean.dimshuffle(*dimshuffle)
black_mean = black_mean.dimshuffle(*dimshuffle)
mean_half_dist = (white_mean - black_mean) / 2
gauss_pyr_black = list(pyramid_gaussian(black_mask, max_layer))
gauss_pyr_white = list(pyramid_gaussian(white_mask, max_layer))
gauss_pyr_image = list(pyramid_gaussian(image, max_layer))
white_diff = white_mean*gauss_pyr_white[-1] - \
gauss_pyr_image[-1]*gauss_pyr_white[-1]
black_diff = gauss_pyr_image[-1]*gauss_pyr_black[-1] - \
black_mean*gauss_pyr_black[-1]
white_diff_thres = T.maximum(white_diff - max_grayness * (mean_half_dist),
0)
black_diff_thres = T.maximum(black_diff - max_grayness * (mean_half_dist),
0)
loss_white = white_diff_thres.sum(axis=(1, 2, 3)) / \
gauss_pyr_white[-1].sum(axis=(1, 2, 3))
loss_black = black_diff_thres.sum(axis=(1, 2, 3)) / \
gauss_pyr_black[-1].sum(axis=(1, 2, 3))
loss = (loss_white + loss_black).mean()
return DotMap({
'loss': loss,
'visual': {
'gauss_pyr_image': gauss_pyr_image[-1],
'gauss_pyr_black': gauss_pyr_black[-1],
'black_diff': black_diff,
'black_diff_thres': black_diff_thres,
'gauss_pyr_white': gauss_pyr_white[-1],
'white_diff': white_diff,
'white_diff_thres': white_diff_thres,
},
'print': {
'0 black_mean': black_mean,
'1 mean_half_dist': mean_half_dist,
'2 white_mean': white_mean,
'3 loss_black': loss_black,
'4 loss_white': loss_white,
'5 loss': loss_black + loss_white,
}
})
def grid_loss(grid_idx, g_outmap):
g_out = g_outmap['output']
m = g_out[:, :1]
b = binary_mask(grid_idx, ignore=0.0, white=1.)
return grid_loss_weight*mse(b, m)
def pyramid_loss(grid_idx, image):
def mean(mask):
return T.sum(image*mask, axis=(1, 2, 3)) / T.sum(mask, axis=(1, 2, 3))
min_mean_distance = 0.2
max_grayness = 0.3
# use 16x16 as last resolution
max_layer = 3
black_mask = binary_mask(grid_idx, ignore=0, black=1, white=0)
white_mask = binary_mask(grid_idx, ignore=0, black=0, white=1)
black_mean = mean(black_mask)
white_mean = mean(white_mask)
mean_diff = white_mean - black_mean
gray_mean = (white_mean + black_mean) / 2
black_mean = T.where(mean_diff < min_mean_distance,
gray_mean - min_mean_distance/2,
black_mean)
white_mean = T.where(mean_diff < min_mean_distance,
gray_mean + min_mean_distance/2,
white_mean)
dimshuffle = (0, 'x', 'x', 'x')
white_mean = white_mean.dimshuffle(*dimshuffle)
black_mean = black_mean.dimshuffle(*dimshuffle)
mean_half_dist = (white_mean - black_mean) / 2
gauss_pyr_black = list(pyramid_gaussian(black_mask, max_layer))
gauss_pyr_white = list(pyramid_gaussian(white_mask, max_layer))
gauss_pyr_image = list(pyramid_gaussian(image, max_layer))
white_diff = white_mean*gauss_pyr_white[-1] - \
gauss_pyr_image[-1]*gauss_pyr_white[-1]
black_diff = gauss_pyr_image[-1]*gauss_pyr_black[-1] - \
black_mean*gauss_pyr_black[-1]
white_diff_thres = T.maximum(white_diff - max_grayness*(mean_half_dist), 0)
black_diff_thres = T.maximum(black_diff - max_grayness*(mean_half_dist), 0)
loss_white = white_diff_thres.sum(axis=(1, 2, 3)) / \
gauss_pyr_white[-1].sum(axis=(1, 2, 3))
loss_black = black_diff_thres.sum(axis=(1, 2, 3)) / \
gauss_pyr_black[-1].sum(axis=(1, 2, 3))
loss = (loss_white + loss_black).mean()
return DotMap({
'loss': loss,
'visual': {
'gauss_pyr_image': gauss_pyr_image[-1],
'gauss_pyr_black': gauss_pyr_black[-1],
'black_diff': black_diff,
'black_diff_thres': black_diff_thres,
'gauss_pyr_white': gauss_pyr_white[-1],
'white_diff': white_diff,
'white_diff_thres': white_diff_thres,
},
'print': {
'0 black_mean': black_mean,
'1 mean_half_dist': mean_half_dist,
'2 white_mean': white_mean,
'3 loss_black': loss_black,
'4 loss_white': loss_white,
'5 loss': loss_black + loss_white,
}
})
def grid_loss(grid_idx, g_outmap):
g_out = g_outmap['output']
m = g_out[:, :1]
b = binary_mask(grid_idx, ignore=0.0, white=1.)
return grid_loss_weight * mse(b, m)