def call(self, x, mask=None): if self.mode == 0 or self.mode == 2: assert self.built, 'Layer must be built before being called' input_shape = K.int_shape(x) reduction_axes = list(range(len(input_shape))) del reduction_axes[self.axis] broadcast_shape = [1] * len(input_shape) broadcast_shape[self.axis] = input_shape[self.axis] # mean_batch, var_batch = K.moments(x, reduction_axes, shift=None, keep_dims=False) normed, mean_batch, var_batch = K.normalize_batch_in_training( x, self.gamma, self.beta, reduction_axes, epsilon=self.epsilon) std_batch = (K.sqrt(var_batch + self.epsilon)) r_max_value = K.get_value(self.r_max) r = std_batch / (K.sqrt(self.running_std + self.epsilon)) r = K.stop_gradient(K.clip(r, 1 / r_max_value, r_max_value)) d_max_value = K.get_value(self.d_max) d = (mean_batch - self.running_mean) / K.sqrt(self.running_std + self.epsilon) d = K.stop_gradient(K.clip(d, -d_max_value, d_max_value)) if sorted(reduction_axes) == range(K.ndim(x))[:-1]: x_normed_batch = (x - mean_batch) / std_batch x_normed = (x_normed_batch * r + d) * self.gamma + self.beta else: # need broadcasting broadcast_mean = K.reshape(mean_batch, broadcast_shape) broadcast_std = K.reshape(std_batch, broadcast_shape) broadcast_r = K.reshape(r, broadcast_shape) broadcast_d = K.reshape(d, broadcast_shape) broadcast_beta = K.reshape(self.beta, broadcast_shape) broadcast_gamma = K.reshape(self.gamma, broadcast_shape) x_normed_batch = (x - broadcast_mean) / broadcast_std x_normed = (x_normed_batch * broadcast_r + broadcast_d) * broadcast_gamma + broadcast_beta # explicit update to moving mean and standard deviation self.add_update([ K.moving_average_update(self.running_mean, mean_batch, self.momentum), K.moving_average_update(self.running_std, std_batch**2, self.momentum) ], x) # update r_max and d_max t_val = K.get_value(self.t) r_val = self.r_max_value / ( 1 + (self.r_max_value - 1) * np.exp(-t_val)) d_val = self.d_max_value / (1 + ( (self.d_max_value / 1e-3) - 1) * np.exp(-(2 * t_val))) t_val += float(self.t_delta) self.add_update([ K.update(self.r_max, r_val), K.update(self.d_max, d_val), K.update(self.t, t_val) ], x) if self.mode == 0: if sorted(reduction_axes) == range(K.ndim(x))[:-1]: x_normed_running = K.batch_normalization( x, self.running_mean, self.running_std, self.beta, self.gamma, epsilon=self.epsilon) else: # need broadcasting broadcast_running_mean = K.reshape(self.running_mean, broadcast_shape) broadcast_running_std = K.reshape(self.running_std, broadcast_shape) broadcast_beta = K.reshape(self.beta, broadcast_shape) broadcast_gamma = K.reshape(self.gamma, broadcast_shape) x_normed_running = K.batch_normalization( x, broadcast_running_mean, broadcast_running_std, broadcast_beta, broadcast_gamma, epsilon=self.epsilon) # pick the normalized form of x corresponding to the training phase # for batch renormalization, inference time remains same as batchnorm x_normed = K.in_train_phase(x_normed, x_normed_running) elif self.mode == 1: # sample-wise normalization m = K.mean(x, axis=self.axis, keepdims=True) std = K.sqrt( K.var(x, axis=self.axis, keepdims=True) + self.epsilon) x_normed_batch = (x - m) / (std + self.epsilon) r_max_value = K.get_value(self.r_max) r = std / (self.running_std + self.epsilon) r = K.stop_gradient(K.clip(r, 1 / r_max_value, r_max_value)) d_max_value = K.get_value(self.d_max) d = (m - self.running_mean) / (self.running_std + self.epsilon) d = K.stop_gradient(K.clip(d, -d_max_value, d_max_value)) x_normed = ((x_normed_batch * r) + d) * self.gamma + self.beta # update r_max and d_max t_val = K.get_value(self.t) r_val = self.r_max_value / ( 1 + (self.r_max_value - 1) * np.exp(-t_val)) d_val = self.d_max_value / (1 + ( (self.d_max_value / 1e-3) - 1) * np.exp(-(2 * t_val))) t_val += float(self.t_delta) self.add_update([ K.update(self.r_max, r_val), K.update(self.d_max, d_val), K.update(self.t, t_val) ], x) return x_normed
def __call__(self, p): return K.clip(p, -self.c, self.c)
def _loss_generator(y_true, y_pred): y_pred = K.clip(y_pred, _EPSILON, 1.0-_EPSILON) out = -(K.log(y_pred)) return K.mean(out, axis=-1)
sigma = model['sigma'] z = model['z'] encoder = model['encoder'] decoder = model['decoder'] plot_model(encoder, to_file='vae_encoder.pdf', show_shapes=True) plot_model(decoder, to_file='vae_decoder.pdf', show_shapes=True) vae = model['vae'] # kl_div = -0.5 * K.mean(1 + 2 * sigma - K.square(mu) - K.exp(2 * sigma)) voxel_loss = K.cast( K.mean( weighted_binary_crossentropy( inputs, K.clip(sigmoid(outputs), 1e-7, 1.0 - 1e-7))), 'float32') # + kl_div vae.add_loss(voxel_loss) sgd = SGD(lr=learning_rate_1, momentum=momentum, nesterov=True) vae.compile(optimizer=sgd, metrics=['accuracy']) plot_model(vae, to_file='vae.pdf', show_shapes=True) data_train = data_loader('datasets/shapenet10_chairs_nr.tar') vae.fit(data_train, epochs=epoch_num, batch_size=batch_size, validation_data=(data_train, None), callbacks=[LearningRateScheduler(learning_rate_scheduler)])
def prec(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) precision = true_positives / (predicted_positives + K.epsilon()) return precision
def mean_squared_error_p(y_true, y_pred): """ Modified mean square error that clips """ return K.clip(K.max(K.square(y_pred - y_true), axis=-1) - 1, 0., 100.) # = modified mse error L_inf
def loss_diff_s_s_(y_true, y_pred): return K.square( 1. - K.sqrt(K.clip(K.sum(y_pred, axis=-1, keepdims=True), 0.000001, 1.))) # tend to increase y_pred --> loss -1
def __call__(self, w): norms = K.sqrt(K.sum(w, axis=self.axis, keepdims=True)) desired = (self.rate * K.clip(norms, self.min_value, self.max_value) + (1 - self.rate) * norms) w = w * (desired / (K.epsilon() + norms)) return w
def kullback_leibler_div2(c): p = K.clip(c[0], K.epsilon(), 1) q = K.clip(c[1], K.epsilon(), 1) return K.sum(p * K.log(p / q), axis=-1)
def __call__(self, p): return K.clip(p, self.min, self.max)
def __call__(self, p): return K.clip(p, self.lower, self.upper)
def entropy(y): # Clip to avoid Zeros. y = K.clip(y, 1e-20, 1.0) return -K.sum(y * K.log(y))
def _interpolate(self, image, sampled_grids, output_size): batch_size = K.shape(image)[0] height = K.shape(image)[1] width = K.shape(image)[2] num_channels = K.shape(image)[3] x = K.cast(K.flatten(sampled_grids[:, 0:1, :]), dtype='float32') y = K.cast(K.flatten(sampled_grids[:, 1:2, :]), dtype='float32') x = .5 * (x + 1.0) * K.cast(width, dtype='float32') y = .5 * (y + 1.0) * K.cast(height, dtype='float32') x0 = K.cast(x, 'int32') x1 = x0 + 1 y0 = K.cast(y, 'int32') y1 = y0 + 1 max_x = int(K.int_shape(image)[2] - 1) max_y = int(K.int_shape(image)[1] - 1) x0 = K.clip(x0, 0, max_x) x1 = K.clip(x1, 0, max_x) y0 = K.clip(y0, 0, max_y) y1 = K.clip(y1, 0, max_y) pixels_batch = K.arange(0, batch_size) * (height * width) pixels_batch = K.expand_dims(pixels_batch, axis=-1) flat_output_size = output_size[0] * output_size[1] base = K.repeat_elements(pixels_batch, flat_output_size, axis=1) base = K.flatten(base) # base_y0 = base + (y0 * width) base_y0 = y0 * width base_y0 = base + base_y0 # base_y1 = base + (y1 * width) base_y1 = y1 * width base_y1 = base_y1 + base indices_a = base_y0 + x0 indices_b = base_y1 + x0 indices_c = base_y0 + x1 indices_d = base_y1 + x1 flat_image = K.reshape(image, shape=(-1, num_channels)) flat_image = K.cast(flat_image, dtype='float32') pixel_values_a = K.gather(flat_image, indices_a) pixel_values_b = K.gather(flat_image, indices_b) pixel_values_c = K.gather(flat_image, indices_c) pixel_values_d = K.gather(flat_image, indices_d) x0 = K.cast(x0, 'float32') x1 = K.cast(x1, 'float32') y0 = K.cast(y0, 'float32') y1 = K.cast(y1, 'float32') area_a = K.expand_dims(((x1 - x) * (y1 - y)), 1) area_b = K.expand_dims(((x1 - x) * (y - y0)), 1) area_c = K.expand_dims(((x - x0) * (y1 - y)), 1) area_d = K.expand_dims(((x - x0) * (y - y0)), 1) values_a = area_a * pixel_values_a values_b = area_b * pixel_values_b values_c = area_c * pixel_values_c values_d = area_d * pixel_values_d return values_a + values_b + values_c + values_d
def __call__(self, weights): return K.clip(weights, -self.value, self.value)
def recall(y_true, y_pred): tp = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) rec = tp / (K.sum(K.round(K.clip(y_true, 0, 1))) + K.epsilon()) return rec
def kullback_leibler_div(p, q): p = K.clip(p, K.epsilon(), 1) q = K.clip(q, K.epsilon(), 1) return K.sum(p * K.log(p / q), axis=-1)
def precision(y_true, y_pred): tp = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) prec = tp / (K.sum(K.round(K.clip(y_pred, 0, 1))) + K.epsilon()) return prec
def custom_loss(y_true, y_pred): out = K.clip(y_pred, 1e-8, 1 - 1e-8) log_lik = y_true * K.log(out) return K.sum(-log_lik * delta)
def exp_dec_error(y_true, y_pred): return K.exp(-5. * K.sqrt( K.clip(K.sum(K.square(y_pred), axis=-1, keepdims=True), 0.000001, 10))) # tend to increase y_pred
def TP_m(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) return true_positives
def create_train_pconv_unet(): input_image = layers.Input((320,240,3)) input_mask = layers.Input((320,240,3)) input_groundtruth = layers.Input((320,240,3)) # マスクの拡大 expanded_mask = layers.Lambda(lambda x: 1.0-K.sign(1.0-x))(input_mask) ## U-Net # Encoder conv1, mask1 = conv_bn_relu(input_image, expanded_mask, filters=32, kernel_size=3, downsampling=1, reps=2) # 320x240 conv2, mask2 = conv_bn_relu(conv1, mask1, filters=64, kernel_size=7, downsampling=5, reps=1) conv2, mask2 = conv_bn_relu(conv2, mask2, filters=64, kernel_size=3, downsampling=1, reps=2) # 64x48 conv3, mask3 = conv_bn_relu(conv2, mask2, filters=128, kernel_size=3, downsampling=2, reps=3) # 32x24 conv4, mask4 = conv_bn_relu(conv3, mask3, filters=256, kernel_size=3, downsampling=2, reps=3) # 16x12 conv5, mask5 = conv_bn_relu(conv4, mask4, filters=512, kernel_size=3, downsampling=2, reps=3) # 8x6 ## Decoder img, mask = conv_bn_relu(conv5, mask5, filters=256, kernel_size=3, upsampling=2, reps=3, concat_img=conv4, concat_mask=mask4) # 16x12 img, mask = conv_bn_relu(img, mask, filters=128, kernel_size=3, upsampling=2, reps=3, concat_img=conv3, concat_mask=mask3) # 32x24 img, mask = conv_bn_relu(img, mask, filters=64, kernel_size=3, upsampling=2, reps=3, concat_img=conv2, concat_mask=mask2) # 64x48 img, mask = conv_bn_relu(img, mask, filters=32, kernel_size=7, upsampling=5, reps=1, concat_img=conv1, concat_mask=mask1) # 320x240 img, mask = conv_bn_relu(img, mask, filters=32, kernel_size=3, upsampling=1, reps=2) img, mask = conv_bn_relu(img, mask, filters=3, kernel_size=1, reps=1, act="tanh") # 差分出力 # skip connection(Alpha ぼかしに応じた合成)、収束が早くなるはず img = layers.Lambda(lambda inputs: inputs[0]*inputs[1] + (1-inputs[0])*inputs[2])( [input_mask, input_image, img]) img = layers.Lambda(lambda x : K.clip(x, -1.0, 1.0), name="unmasked")(img) ## 損失関数 # マスクしていない部分の真の画像+マスク部分の予測画像 y_comp = layers.Lambda(lambda inputs: inputs[0]*inputs[1] + (1-inputs[0])*inputs[2])( [expanded_mask, input_groundtruth, img]) # Caffeカラースケールに変換 vgg_in_pred = layers.Lambda(convert_caffe_color_space)(img) vgg_in_groundtruth = layers.Lambda(convert_caffe_color_space)(input_groundtruth) vgg_in_comp = layers.Lambda(convert_caffe_color_space)(y_comp) # vggの特徴量 vgg_pred_1, vgg_pred_2, vgg_pred_3 = extract_vgg_features(vgg_in_pred, (320,240,3), 0) vgg_true_1, vgg_true_2, vgg_true_3 = extract_vgg_features(vgg_in_groundtruth, (320,240,3), 1) vgg_comp_1, vgg_comp_2, vgg_comp_3 = extract_vgg_features(vgg_in_comp, (320,240,3), 2) # 画像+損失 join = LossLayer()([expanded_mask, img, input_groundtruth, y_comp, vgg_pred_1, vgg_pred_2, vgg_pred_3, vgg_true_1, vgg_true_2, vgg_true_3, vgg_comp_1, vgg_comp_2, vgg_comp_3]) # lossやmetricsの表示がうまくいかないので出力は1つにする model = Model([input_image, input_mask, input_groundtruth], join) # このモデルは>100MBだが、推論用モデルは93MB return model
def FP_m(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) FP = predicted_positives - true_positives return FP
def log(x): return K.log(K.clip(x, min_value = 1e-7, max_value = 10000))
def FN_m(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) FN = possible_positives - true_positives return FN
def rec(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) recall = true_positives / (possible_positives + K.epsilon()) return recall
def recall_m(y_true, y_pred): true_positives = k.sum(k.round(k.clip(y_true * y_pred, 0, 1))) possible_positives = k.sum(k.round(k.clip(y_true, 0, 1))) recall = true_positives / (possible_positives + k.epsilon()) return recall
def __call__(self, weights): return backend.clip(weights, -self.clip_value, self.clip_value)
def precision_m(y_true, y_pred): true_positives = k.sum(k.round(k.clip(y_true * y_pred, 0, 1))) predicted_positives = k.sum(k.round(k.clip(y_pred, 0, 1))) precision = true_positives / (predicted_positives + k.epsilon()) return precision
def specificity(y_true, y_pred): true_negatives = K.sum(K.round(K.clip((1-y_true) * (1-y_pred), 0, 1))) possible_negatives = K.sum(K.round(K.clip(1-y_true, 0, 1))) spec = true_negatives / (possible_negatives + K.epsilon()) return spec
def y2var(y_pred): var = k.square(gather(y_pred, [1], axis=1)) put_strike = None # .5**2 var = k.clip(var, strike, put_strike) return var