def call(self, x, mask=None):
if self.mode == 0 or self.mode == 2:
assert self.built, 'Layer must be built before being called'
input_shape = K.int_shape(x)
reduction_axes = list(range(len(input_shape)))
del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
broadcast_shape[self.axis] = input_shape[self.axis]
# mean_batch, var_batch = K.moments(x, reduction_axes, shift=None, keep_dims=False)
normed, mean_batch, var_batch = K.normalize_batch_in_training(
x, self.gamma, self.beta, reduction_axes, epsilon=self.epsilon)
std_batch = (K.sqrt(var_batch + self.epsilon))
r_max_value = K.get_value(self.r_max)
r = std_batch / (K.sqrt(self.running_std + self.epsilon))
r = K.stop_gradient(K.clip(r, 1 / r_max_value, r_max_value))
d_max_value = K.get_value(self.d_max)
d = (mean_batch - self.running_mean) / K.sqrt(self.running_std +
self.epsilon)
d = K.stop_gradient(K.clip(d, -d_max_value, d_max_value))
if sorted(reduction_axes) == range(K.ndim(x))[:-1]:
x_normed_batch = (x - mean_batch) / std_batch
x_normed = (x_normed_batch * r + d) * self.gamma + self.beta
else:
# need broadcasting
broadcast_mean = K.reshape(mean_batch, broadcast_shape)
broadcast_std = K.reshape(std_batch, broadcast_shape)
broadcast_r = K.reshape(r, broadcast_shape)
broadcast_d = K.reshape(d, broadcast_shape)
broadcast_beta = K.reshape(self.beta, broadcast_shape)
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
x_normed_batch = (x - broadcast_mean) / broadcast_std
x_normed = (x_normed_batch * broadcast_r +
broadcast_d) * broadcast_gamma + broadcast_beta
# explicit update to moving mean and standard deviation
self.add_update([
K.moving_average_update(self.running_mean, mean_batch,
self.momentum),
K.moving_average_update(self.running_std, std_batch**2,
self.momentum)
], x)
# update r_max and d_max
t_val = K.get_value(self.t)
r_val = self.r_max_value / (
1 + (self.r_max_value - 1) * np.exp(-t_val))
d_val = self.d_max_value / (1 + (
(self.d_max_value / 1e-3) - 1) * np.exp(-(2 * t_val)))
t_val += float(self.t_delta)
self.add_update([
K.update(self.r_max, r_val),
K.update(self.d_max, d_val),
K.update(self.t, t_val)
], x)
if self.mode == 0:
if sorted(reduction_axes) == range(K.ndim(x))[:-1]:
x_normed_running = K.batch_normalization(
x,
self.running_mean,
self.running_std,
self.beta,
self.gamma,
epsilon=self.epsilon)
else:
# need broadcasting
broadcast_running_mean = K.reshape(self.running_mean,
broadcast_shape)
broadcast_running_std = K.reshape(self.running_std,
broadcast_shape)
broadcast_beta = K.reshape(self.beta, broadcast_shape)
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
x_normed_running = K.batch_normalization(
x,
broadcast_running_mean,
broadcast_running_std,
broadcast_beta,
broadcast_gamma,
epsilon=self.epsilon)
# pick the normalized form of x corresponding to the training phase
# for batch renormalization, inference time remains same as batchnorm
x_normed = K.in_train_phase(x_normed, x_normed_running)
elif self.mode == 1:
# sample-wise normalization
m = K.mean(x, axis=self.axis, keepdims=True)
std = K.sqrt(
K.var(x, axis=self.axis, keepdims=True) + self.epsilon)
x_normed_batch = (x - m) / (std + self.epsilon)
r_max_value = K.get_value(self.r_max)
r = std / (self.running_std + self.epsilon)
r = K.stop_gradient(K.clip(r, 1 / r_max_value, r_max_value))
d_max_value = K.get_value(self.d_max)
d = (m - self.running_mean) / (self.running_std + self.epsilon)
d = K.stop_gradient(K.clip(d, -d_max_value, d_max_value))
x_normed = ((x_normed_batch * r) + d) * self.gamma + self.beta
# update r_max and d_max
t_val = K.get_value(self.t)
r_val = self.r_max_value / (
1 + (self.r_max_value - 1) * np.exp(-t_val))
d_val = self.d_max_value / (1 + (
(self.d_max_value / 1e-3) - 1) * np.exp(-(2 * t_val)))
t_val += float(self.t_delta)
self.add_update([
K.update(self.r_max, r_val),
K.update(self.d_max, d_val),
K.update(self.t, t_val)
], x)
return x_normed
def __call__(self, p):
return K.clip(p, -self.c, self.c)
def _loss_generator(y_true, y_pred): y_pred = K.clip(y_pred, _EPSILON, 1.0-_EPSILON) out = -(K.log(y_pred)) return K.mean(out, axis=-1)
sigma = model['sigma']
z = model['z']
encoder = model['encoder']
decoder = model['decoder']
plot_model(encoder, to_file='vae_encoder.pdf', show_shapes=True)
plot_model(decoder, to_file='vae_decoder.pdf', show_shapes=True)
vae = model['vae']
# kl_div = -0.5 * K.mean(1 + 2 * sigma - K.square(mu) - K.exp(2 * sigma))
voxel_loss = K.cast(
K.mean(
weighted_binary_crossentropy(
inputs, K.clip(sigmoid(outputs), 1e-7, 1.0 - 1e-7))),
'float32') # + kl_div
vae.add_loss(voxel_loss)
sgd = SGD(lr=learning_rate_1, momentum=momentum, nesterov=True)
vae.compile(optimizer=sgd, metrics=['accuracy'])
plot_model(vae, to_file='vae.pdf', show_shapes=True)
data_train = data_loader('datasets/shapenet10_chairs_nr.tar')
vae.fit(data_train,
epochs=epoch_num,
batch_size=batch_size,
validation_data=(data_train, None),
callbacks=[LearningRateScheduler(learning_rate_scheduler)])
def prec(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
return precision
def mean_squared_error_p(y_true, y_pred):
""" Modified mean square error that clips
"""
return K.clip(K.max(K.square(y_pred - y_true), axis=-1) - 1, 0.,
100.) # = modified mse error L_inf
def loss_diff_s_s_(y_true, y_pred):
return K.square(
1. - K.sqrt(K.clip(K.sum(y_pred, axis=-1, keepdims=True), 0.000001,
1.))) # tend to increase y_pred --> loss -1
def __call__(self, w):
norms = K.sqrt(K.sum(w, axis=self.axis, keepdims=True))
desired = (self.rate * K.clip(norms, self.min_value, self.max_value) +
(1 - self.rate) * norms)
w = w * (desired / (K.epsilon() + norms))
return w
def kullback_leibler_div2(c):
p = K.clip(c[0], K.epsilon(), 1)
q = K.clip(c[1], K.epsilon(), 1)
return K.sum(p * K.log(p / q), axis=-1)
def __call__(self, p):
return K.clip(p, self.min, self.max)
def __call__(self, p):
return K.clip(p, self.lower, self.upper)
def entropy(y):
# Clip to avoid Zeros.
y = K.clip(y, 1e-20, 1.0)
return -K.sum(y * K.log(y))
def _interpolate(self, image, sampled_grids, output_size):
batch_size = K.shape(image)[0]
height = K.shape(image)[1]
width = K.shape(image)[2]
num_channels = K.shape(image)[3]
x = K.cast(K.flatten(sampled_grids[:, 0:1, :]), dtype='float32')
y = K.cast(K.flatten(sampled_grids[:, 1:2, :]), dtype='float32')
x = .5 * (x + 1.0) * K.cast(width, dtype='float32')
y = .5 * (y + 1.0) * K.cast(height, dtype='float32')
x0 = K.cast(x, 'int32')
x1 = x0 + 1
y0 = K.cast(y, 'int32')
y1 = y0 + 1
max_x = int(K.int_shape(image)[2] - 1)
max_y = int(K.int_shape(image)[1] - 1)
x0 = K.clip(x0, 0, max_x)
x1 = K.clip(x1, 0, max_x)
y0 = K.clip(y0, 0, max_y)
y1 = K.clip(y1, 0, max_y)
pixels_batch = K.arange(0, batch_size) * (height * width)
pixels_batch = K.expand_dims(pixels_batch, axis=-1)
flat_output_size = output_size[0] * output_size[1]
base = K.repeat_elements(pixels_batch, flat_output_size, axis=1)
base = K.flatten(base)
# base_y0 = base + (y0 * width)
base_y0 = y0 * width
base_y0 = base + base_y0
# base_y1 = base + (y1 * width)
base_y1 = y1 * width
base_y1 = base_y1 + base
indices_a = base_y0 + x0
indices_b = base_y1 + x0
indices_c = base_y0 + x1
indices_d = base_y1 + x1
flat_image = K.reshape(image, shape=(-1, num_channels))
flat_image = K.cast(flat_image, dtype='float32')
pixel_values_a = K.gather(flat_image, indices_a)
pixel_values_b = K.gather(flat_image, indices_b)
pixel_values_c = K.gather(flat_image, indices_c)
pixel_values_d = K.gather(flat_image, indices_d)
x0 = K.cast(x0, 'float32')
x1 = K.cast(x1, 'float32')
y0 = K.cast(y0, 'float32')
y1 = K.cast(y1, 'float32')
area_a = K.expand_dims(((x1 - x) * (y1 - y)), 1)
area_b = K.expand_dims(((x1 - x) * (y - y0)), 1)
area_c = K.expand_dims(((x - x0) * (y1 - y)), 1)
area_d = K.expand_dims(((x - x0) * (y - y0)), 1)
values_a = area_a * pixel_values_a
values_b = area_b * pixel_values_b
values_c = area_c * pixel_values_c
values_d = area_d * pixel_values_d
return values_a + values_b + values_c + values_d
def __call__(self, weights):
return K.clip(weights, -self.value, self.value)
def recall(y_true, y_pred):
tp = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
rec = tp / (K.sum(K.round(K.clip(y_true, 0, 1))) + K.epsilon())
return rec
def kullback_leibler_div(p, q):
p = K.clip(p, K.epsilon(), 1)
q = K.clip(q, K.epsilon(), 1)
return K.sum(p * K.log(p / q), axis=-1)
def precision(y_true, y_pred):
tp = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
prec = tp / (K.sum(K.round(K.clip(y_pred, 0, 1))) + K.epsilon())
return prec
def custom_loss(y_true, y_pred):
out = K.clip(y_pred, 1e-8, 1 - 1e-8)
log_lik = y_true * K.log(out)
return K.sum(-log_lik * delta)
def exp_dec_error(y_true, y_pred):
return K.exp(-5. * K.sqrt(
K.clip(K.sum(K.square(y_pred), axis=-1, keepdims=True), 0.000001,
10))) # tend to increase y_pred
def TP_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
return true_positives
def create_train_pconv_unet():
input_image = layers.Input((320,240,3))
input_mask = layers.Input((320,240,3))
input_groundtruth = layers.Input((320,240,3))
# マスクの拡大
expanded_mask = layers.Lambda(lambda x: 1.0-K.sign(1.0-x))(input_mask)
## U-Net
# Encoder
conv1, mask1 = conv_bn_relu(input_image, expanded_mask,
filters=32, kernel_size=3, downsampling=1, reps=2) # 320x240
conv2, mask2 = conv_bn_relu(conv1, mask1,
filters=64, kernel_size=7, downsampling=5, reps=1)
conv2, mask2 = conv_bn_relu(conv2, mask2,
filters=64, kernel_size=3, downsampling=1, reps=2) # 64x48
conv3, mask3 = conv_bn_relu(conv2, mask2,
filters=128, kernel_size=3, downsampling=2, reps=3) # 32x24
conv4, mask4 = conv_bn_relu(conv3, mask3,
filters=256, kernel_size=3, downsampling=2, reps=3) # 16x12
conv5, mask5 = conv_bn_relu(conv4, mask4,
filters=512, kernel_size=3, downsampling=2, reps=3) # 8x6
## Decoder
img, mask = conv_bn_relu(conv5, mask5,
filters=256, kernel_size=3, upsampling=2, reps=3,
concat_img=conv4, concat_mask=mask4) # 16x12
img, mask = conv_bn_relu(img, mask,
filters=128, kernel_size=3, upsampling=2, reps=3,
concat_img=conv3, concat_mask=mask3) # 32x24
img, mask = conv_bn_relu(img, mask,
filters=64, kernel_size=3, upsampling=2, reps=3,
concat_img=conv2, concat_mask=mask2) # 64x48
img, mask = conv_bn_relu(img, mask,
filters=32, kernel_size=7, upsampling=5, reps=1,
concat_img=conv1, concat_mask=mask1) # 320x240
img, mask = conv_bn_relu(img, mask,
filters=32, kernel_size=3, upsampling=1, reps=2)
img, mask = conv_bn_relu(img, mask,
filters=3, kernel_size=1, reps=1, act="tanh") # 差分出力
# skip connection(Alpha ぼかしに応じた合成)、収束が早くなるはず
img = layers.Lambda(lambda inputs: inputs[0]*inputs[1] + (1-inputs[0])*inputs[2])(
[input_mask, input_image, img])
img = layers.Lambda(lambda x : K.clip(x, -1.0, 1.0), name="unmasked")(img)
## 損失関数
# マスクしていない部分の真の画像＋マスク部分の予測画像
y_comp = layers.Lambda(lambda inputs: inputs[0]*inputs[1] + (1-inputs[0])*inputs[2])(
[expanded_mask, input_groundtruth, img])
# Caffeカラースケールに変換
vgg_in_pred = layers.Lambda(convert_caffe_color_space)(img)
vgg_in_groundtruth = layers.Lambda(convert_caffe_color_space)(input_groundtruth)
vgg_in_comp = layers.Lambda(convert_caffe_color_space)(y_comp)
# vggの特徴量
vgg_pred_1, vgg_pred_2, vgg_pred_3 = extract_vgg_features(vgg_in_pred, (320,240,3), 0)
vgg_true_1, vgg_true_2, vgg_true_3 = extract_vgg_features(vgg_in_groundtruth, (320,240,3), 1)
vgg_comp_1, vgg_comp_2, vgg_comp_3 = extract_vgg_features(vgg_in_comp, (320,240,3), 2)
# 画像＋損失
join = LossLayer()([expanded_mask,
img, input_groundtruth, y_comp,
vgg_pred_1, vgg_pred_2, vgg_pred_3,
vgg_true_1, vgg_true_2, vgg_true_3,
vgg_comp_1, vgg_comp_2, vgg_comp_3])
# lossやmetricsの表示がうまくいかないので出力は1つにする
model = Model([input_image, input_mask, input_groundtruth], join) # このモデルは>100MBだが、推論用モデルは93MB
return model
def FP_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
FP = predicted_positives - true_positives
return FP
def log(x):
return K.log(K.clip(x, min_value = 1e-7, max_value = 10000))
def FN_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
FN = possible_positives - true_positives
return FN
def rec(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
def recall_m(y_true, y_pred):
true_positives = k.sum(k.round(k.clip(y_true * y_pred, 0, 1)))
possible_positives = k.sum(k.round(k.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + k.epsilon())
return recall
def __call__(self, weights):
return backend.clip(weights, -self.clip_value, self.clip_value)
def precision_m(y_true, y_pred):
true_positives = k.sum(k.round(k.clip(y_true * y_pred, 0, 1)))
predicted_positives = k.sum(k.round(k.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + k.epsilon())
return precision
def specificity(y_true, y_pred):
true_negatives = K.sum(K.round(K.clip((1-y_true) * (1-y_pred), 0, 1)))
possible_negatives = K.sum(K.round(K.clip(1-y_true, 0, 1)))
spec = true_negatives / (possible_negatives + K.epsilon())
return spec
def y2var(y_pred):
var = k.square(gather(y_pred, [1], axis=1))
put_strike = None # .5**2
var = k.clip(var, strike, put_strike)
return var
