def vae_loss(x, x_mean):
x = flatten(x)
x_mean = flatten(x_mean)
xent_loss = input_shape[0] * binary_crossentropy(x, x_mean)
kl_loss = -0.5 * mean(
1 + z_log_var - square(z_mean) - exp(z_log_var), axis=-1)
return xent_loss + kl_loss
def bce_dice_loss(y_true, y_pred):
y_true = tf.cast(y_true, tf.float32)
loss = 20 * losses.binary_crossentropy(y_true, y_pred) + dice_loss(
y_true, y_pred)
# loss = dice_loss(y_true, y_pred)
# loss = 100 * binary_focal_loss(y_true, y_pred, 1,0.25) + dice_loss(y_true, y_pred)
return loss
def create_img2attr(self, kernel_initializer = 'he_normal', img_flat_len = 1024):
attr_input = layers.Input(shape = (50,), name = 'attr')
word_emb = layers.Input(shape = (600,), name = 'wv')
imag_classifier = layers.Input(shape = (img_flat_len,), name = 'img')
attr_dense = layers.Dense(600, use_bias = True, kernel_initializer=kernel_initializer,
kernel_regularizer = l2(1e-4), name = 'attr_dense')(attr_input)
attr_word_emb = layers.Concatenate(name = 'attr_word_emb')([word_emb, attr_dense])
out_size = 50
attr_preds = self.full_connect_layer(imag_classifier, hidden_dim = [
int(out_size * 20),
int(out_size * 15),
# int(out_size * 7),
# int(img_flat_len * 1.125),
# int(img_flat_len * 1.0625)
], \
activation = 'relu', resnet = False, drop_out_ratio = 0.2)
attr_preds = self.full_connect_layer(attr_preds, hidden_dim = [out_size], activation = 'sigmoid')
log_loss = K.mean(binary_crossentropy(attr_input, attr_preds))
model = Model([attr_input, word_emb, imag_classifier], outputs = [attr_preds]) #, vgg_output])
model.add_loss(log_loss)
model.compile(optimizer=Adam(lr=1e-5), loss=None)
return model
def get_gradients(self, model):
"""
This function outputs a function to calculate the gradients based on the loss function, current weights/biases
Input:
- model: model that is training.
Returns:
- func: a function that uses input of features and true labels to calculate loss and hence gradients
"""
model_weights = model.trainable_weights
if self.only_weights:
weights = [
weight for weight in model_weights if 'kernel' in weight.name
]
else:
weights = [weight for weight in model_weights]
if self.num_classes > 1:
loss = K.mean(categorical_crossentropy(self.y_true, model.output))
else:
loss = K.mean(binary_crossentropy(self.y_true, model.output))
func = K.function([model.input, self.y_true],
K.gradients(loss, weights))
return func
def train(self, data):
"""Pretrain the latent layers of the model."""
# network parameters
original_dim = data.shape[1]
input_shape = (original_dim,)
# build encoder model
inputs = Input(shape=input_shape, name='encoder_input')
hidden = inputs
for i, hidden_dim in enumerate(self.hidden_dim, 1):
hidden = Dense(hidden_dim, activation='sigmoid', name='hidden_e_{}'.format(i))(hidden)
logger.debug("Hooked up hidden layer with %d neurons" % hidden_dim)
z_mean = Dense(params_training.num_latent, activation=None, name='z_mean')(hidden)
z_log_sigma = Dense(params_training.num_latent, activation=None, name='z_log_sigma')(hidden)
z = Lambda(self.sampling, output_shape=(params_training.num_latent,), name='z')(
[z_mean, z_log_sigma])
encoder = Model(inputs, [z_mean, z_log_sigma, z], name='encoder')
self.encoder_z_mean = encoder.predict(data)[0]
# build decoder model
latent_inputs = Input(shape=(params_training.num_latent,), name='z_sampling')
hidden = latent_inputs
for i, hidden_dim in enumerate(self.hidden_dim[::-1], 1): # Reverse because decoder.
hidden = Dense(hidden_dim, activation='sigmoid', name='hidden_d_{}'.format(i))(hidden)
logger.debug("Hooked up hidden layer with %d neurons" % hidden_dim)
# if hidden == latent_inputs:
# logger.warning("No Hidden layers hooked up.")
outputs = Dense(original_dim, activation='sigmoid')(hidden)
decoder = Model(latent_inputs, outputs, name='decoder')
# Build the CVAE auto-encoder
outputs = decoder(encoder(inputs)[2])
cvae_model = Model(inputs, outputs, name='cvae')
# Load the pre-trained weights.
self.load_pretrain_weights(cvae_model)
reconstruction_loss = binary_crossentropy(inputs, outputs) * original_dim
kl_loss = 1 + z_log_sigma - tf.square(z_mean) - tf.exp(z_log_sigma)
kl_loss = -0.5 * tf.reduce_sum(kl_loss, axis=-1)
cvae_model.add_loss(tf.reduce_mean(reconstruction_loss + kl_loss))
cvae_model.compile(optimizer='adam')
cvae_model.summary()
# First load the weights from the pre-training
if self.pretrain_weights:
cvae_model = self.load_pretrain_weights(cvae_model)
saver = ModelCheckpoint(
check_path(TEMPORARY_CVAE_PATH),
save_weights_only=True,
verbose=1
)
tensorboard_config = TensorBoard(log_dir=check_path(TEMPORARY_CVAE_PATH))
# train the auto-encoder
cvae_model.fit(data, epochs=params_training.num_epochs,
batch_size=params_training.batch_size,
callbacks=[saver, tensorboard_config])
return self.encoder_z_mean
def loss_high_order(self, y_true, y_pred):
# y_true = tf.reshape(y_true, shape=[-1])
# y_pred = tf.reshape(y_pred, shape=[-1])
label_smoothing, eps = 0.01, 0.00001
y_true = y_true * self.beta + label_smoothing
y_pred = y_pred + eps
# loss = -tf.reduce_mean(y_true*tf.log(y_pred))
loss = binary_crossentropy(y_true, y_pred)
return loss
def train(self, data):
"""Pretrain the latent layers of the model."""
# network parameters
original_dim = data.shape[1]
input_shape = (original_dim, )
batch_size = train_params.batch_size
latent_dim = train_params.num_latent
epochs = train_params.num_epochs
# build encoder model
inputs = Input(shape=input_shape, name='encoder_input')
inputs_noisy = inputs
z_mean = Dense(latent_dim, activation=None, name='z_mean')
z_mean = z_mean(inputs_noisy)
z_log_sigma = Dense(latent_dim, activation=None, name='z_log_sigma')
z_log_sigma = z_log_sigma(inputs_noisy)
z = Lambda(self.sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_sigma])
encoder = Model(inputs, [z_mean, z_log_sigma, z], name='encoder')
# build decoder model
latent_inputs = Input(shape=(latent_dim, ), name='z_sampling')
outputs = Dense(original_dim, activation='sigmoid',
name="decoder_l")(latent_inputs)
decoder = Model(latent_inputs, outputs, name='decoder')
# Build the DAE
outputs = decoder(encoder(inputs)[2])
latent_model = Model(inputs, outputs, name='vae_mlp')
reconstruction_loss = binary_crossentropy(inputs,
outputs) * original_dim
kl_loss = 1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
latent_model.add_loss(vae_loss)
latent_model.compile(optimizer='adam')
saver = ModelCheckpoint(check_path(TEMPORARY_LATENT_PATH),
save_weights_only=True,
verbose=1)
tensorboard_config = TensorBoard(
log_dir=check_path(TEMPORARY_LATENT_PATH))
logger.info("Model checkpoints has ben saved.")
# train the autoencoder
latent_model.fit(data,
epochs=epochs,
batch_size=batch_size,
callbacks=[saver, tensorboard_config])
# Collect the weights for z_log_sigma and z_mean, the layers being pretrained.
self.weights.append(
latent_model.get_layer("encoder").get_layer(
"z_mean").get_weights())
self.weights.append(
latent_model.get_layer("encoder").get_layer(
"z_log_sigma").get_weights())
self.de_weights.append(
latent_model.get_layer("decoder").get_layer(
"decoder_l").get_weights())
logger.info("Weights has been updated successfully.")
def yolo_loss(y_true, y_pred):
# 1. transform all pred outputs
# y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))
pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(
y_pred, anchors, classes)
pred_xy = pred_xywh[..., 0:2]
pred_wh = pred_xywh[..., 2:4]
# 2. transform all true outputs
# y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))
true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1),
axis=-1)
true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2
true_wh = true_box[..., 2:4] - true_box[..., 0:2]
# give higher weights to small boxes
box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]
# 3. inverting the pred box equations
grid_size = tf.shape(y_true)[1]
grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))
grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)
true_xy = true_xy * tf.cast(grid_size, tf.float32) - \
tf.cast(grid, tf.float32)
true_wh = tf.math.log(true_wh / anchors)
true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh),
true_wh)
# 4. calculate all masks
obj_mask = tf.squeeze(true_obj, -1)
# ignore false positive when iou is over threshold
true_box_flat = tf.boolean_mask(true_box, tf.cast(obj_mask, tf.bool))
best_iou = tf.reduce_max(broadcast_iou(pred_box, true_box_flat),
axis=-1)
ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)
# 5. calculate all losses
xy_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)
wh_loss = obj_mask * box_loss_scale * \
tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)
obj_loss = binary_crossentropy(true_obj, pred_obj)
obj_loss = obj_mask * obj_loss + \
(1 - obj_mask) * ignore_mask * obj_loss
# TODO: use binary_crossentropy instead
class_loss = obj_mask * sparse_categorical_crossentropy(
true_class_idx, pred_class)
# 6. sum over (batch, gridx, gridy, anchors) => (batch, 1)
xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))
wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))
obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))
class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))
return xy_loss + wh_loss + obj_loss + class_loss
def kl_reconstruction_loss(self, true, pred):
# Reconstruction loss
reconstruction_loss = binary_crossentropy(K.flatten(true),
K.flatten(pred)) * 64**2
# KL divergence loss
kl_loss = 1 + self.log_variance - K.square(self.mu) - K.exp(
self.log_variance)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
# Total loss = 50% rec + 50% KL divergence loss
return K.mean(reconstruction_loss + kl_loss)
def vae_loss(x, x_decoded_mean):
"""
Two losses:
1) Reconstruction loss (as specified by binary_crossentropy)
2) KL divergence of the distributions
"""
xent_loss = ORIGINAL_DIM * losses.binary_crossentropy(x, x_decoded_mean)
kl_loss = -0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var),
axis=-1)
vae_loss = K.mean(xent_loss + kl_loss)
return vae_loss
def test_add_entropy_loss_on_functional_model(self):
inputs = Input(shape=(1, ))
targets = Input(shape=(1, ))
outputs = testing_utils.Bias()(inputs)
model = Model([inputs, targets], outputs)
model.add_loss(losses.binary_crossentropy(targets, outputs))
model.compile('sgd', run_eagerly=testing_utils.should_run_eagerly())
with test.mock.patch.object(logging, 'warning') as mock_log:
model.fit([self.x, self.y], batch_size=3, epochs=5)
self.assertNotIn('Gradients do not exist for variables',
str(mock_log.call_args))
def create_model(self):
"""
"""
# VAE model = encoder + decoder
# build encoder model
input_shape = (self.original_dim, )
inputs = Input(shape=input_shape, name='encoder_input')
x = Dense(self.intermediate_dim, activation='relu')(inputs)
z_mean = Dense(self.latent_dim, name='z_mean')(x)
z_log_var = Dense(self.latent_dim, name='z_log_var')(x)
# use reparameterization trick to push the sampling out as input
# note that "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(self.sampling, name='z')([z_mean, z_log_var])
# instantiate encoder model
# encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
# print(encoder.summary())
# plot_model(encoder, to_file='vae_mlp_encoder.png', show_shapes=True)
# build decoder model
# latent_inputs = Input(shape=(self.latent_dim,), name='z_sampling')
# x = Dense(self.intermediate_dim, activation='relu')(latent_inputs)
x = Dense(self.intermediate_dim, activation='relu')(z)
outputs = Dense(self.original_dim, activation='sigmoid')(x)
# instantiate decoder model
# decoder = Model(latent_inputs, outputs, name='decoder')
# print(decoder.summary())
# plot_model(decoder, to_file='vae_mlp_decoder.png', show_shapes=True)
# instantiate VAE model
# outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae_mlp')
# VAE loss = mse_loss or xent_loss + kl_loss
if self.mse:
reconstruction_loss = mean_squared_error(inputs, outputs)
else:
reconstruction_loss = binary_crossentropy(inputs,
outputs)
reconstruction_loss *= self.original_dim
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam', loss = None)
# print (vae.summary())
return vae
def bce_loss(y_true, y_pred):
"""Binary cross entropy loss function.
Arguments:
y_true: label tensor of dtype float.
y_pred: predicted tensor of dtype float.
Returns:
a scalar tensor of dtype float
"""
loss = losses.binary_crossentropy(y_true, y_pred)
return loss
def heatmap_loss(target_mask, heatmap_3d):
X = 2
Y = 3
Z = 4
dims = heatmap_3d.shape # batches, joints, x, y, z
batch_size = dims[0]
nb_joints = dims[1]
heatmap_ = tf.reshape(heatmap_3d, [batch_size, nb_joints, -1])
heatmap_ = Softmax(axis=2)(heatmap_)
target_mask = tf.reshape(target_mask, [batch_size, nb_joints, -1])
return binary_crossentropy(target_mask, heatmap_)
def bce_dice_loss(y_true, y_pred):
"""loss function consisting of binary cross entropy plus dice loss
Arguments:
y_true: label tensor of dtype float.
y_pred: predicted tensor of dtype float.
Returns:
a scalar tensor of dtype float
"""
loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(
y_true, y_pred)
return loss
def create_dem_bc_aug(self, kernel_initializer = 'he_normal', img_flat_len = 1024, only_emb = False):
attr_input = layers.Input(shape = (self.attr_len,), name = 'attr')
word_emb = layers.Input(shape = (self.wv_len,), name = 'wv')
img_input = layers.Input(shape = (self.pixel, self.pixel, 3))
label = layers.Input(shape = (1,), name = 'label')
# img_flat_model = Model(inputs = self.img_model[0].inputs, outputs = self.img_model[0].get_layer(name = 'avg_pool').output)
imag_classifier = self.img_flat_model(img_input)
if self.attr_emb_transform == 'flat':
attr_emb = layers.Embedding(294, self.attr_emb_len)(attr_input)
attr_dense = layers.Flatten()(attr_emb) #layers.GlobalAveragePooling1D()(attr_emb)
elif self.attr_emb_transform == 'dense':
attr_dense = layers.Dense(self.attr_emb_len, use_bias = True, kernel_initializer=kernel_initializer,
kernel_regularizer = l2(1e-4), name = 'attr_dense')(attr_input)
if only_emb:
attr_word_emb = word_emb
else:
attr_word_emb = layers.Concatenate(name = 'attr_word_emb')([word_emb, attr_dense])
attr_word_emb_dense = self.full_connect_layer(attr_word_emb, hidden_dim = [
# int(img_flat_len * 4),
int(img_flat_len * 2),
int(img_flat_len * 1.5),
int(img_flat_len * 1.25),
# int(img_flat_len * 1.125),
int(img_flat_len)
], \
activation = 'relu', resnet = False, drop_out_ratio = 0.2)
# attr_word_emb_dense = self.full_connect_layer(attr_word_emb_dense, hidden_dim = [img_flat_len],
# activation = 'relu')
attr_x_img = layers.Lambda(lambda x: x[0] * x[1], name = 'attr_x_img')([attr_word_emb_dense, imag_classifier])
# attr_x_img = layers.Concatenate(name = 'attr_x_img')([attr_word_emb_dense, imag_classifier])
attr_img_input = layers.Input(shape = (img_flat_len,), name = 'attr_img_input')
# attr_img_input = layers.Input(shape = (img_flat_len * 2,), name = 'attr_img_input')
proba = self.full_connect_layer(attr_img_input, hidden_dim = [1], activation = 'sigmoid')
attr_img_model = Model(inputs = attr_img_input, outputs = proba, name = 'attr_x_img_model')
out = attr_img_model([attr_x_img])
# dem_bc_model = self.create_dem_bc(kernel_initializer = 'he_normal',
# img_flat_len = img_flat_len,
# only_emb = only_emb)
# attr_word_emb_dense, out = dem_bc_model([imag_classifier, attr_input, word_emb, label])
bc_loss = K.mean(binary_crossentropy(label, out))
model = Model([img_input, attr_input, word_emb, label], outputs = [attr_word_emb_dense, out, imag_classifier])
model.add_loss(bc_loss)
model.compile(optimizer=Adam(lr=1e-4), loss=None)
return model
def create_res_dem_bc(self, kernel_initializer = 'he_normal', img_flat_len = 1024, only_emb = False):
attr_input = layers.Input(shape = (50,), name = 'attr')
word_emb = layers.Input(shape = (self.wv_len,), name = 'wv')
imag_classifier = layers.Input(shape = (img_flat_len,), name = 'img')
label = layers.Input(shape = (1,), name = 'label')
attr_dense = layers.Dense(self.wv_len, use_bias = True, kernel_initializer=kernel_initializer,
kernel_regularizer = l2(1e-4), name = 'attr_dense')(attr_input)
ini_dem_model = self.create_dem_bc(kernel_initializer = 'he_normal',
img_flat_len = img_flat_len,
only_emb = True)
ini_dem_model.load_weights('./only_emb.h5')
ini_dem_model_part = Model(inputs = ini_dem_model.inputs[2],
outputs = ini_dem_model.outputs[0])
ini_dem_model_part.trainable = False
ini_attr_word_emb_dense = ini_dem_model_part([word_emb])
if only_emb:
attr_word_emb = word_emb
else:
attr_word_emb = layers.Concatenate(name = 'attr_word_emb')([word_emb, attr_dense])
attr_word_emb_dense = self.full_connect_layer(attr_word_emb, hidden_dim = [
int(img_flat_len * 2),
int(img_flat_len * 1.5),
int(img_flat_len * 1.25),
int(img_flat_len)
], \
activation = 'relu', resnet = False, drop_out_ratio = 0.2)
attr_word_emb_dense = layers.Lambda(lambda x: x[0] + x[1])([attr_word_emb_dense, ini_attr_word_emb_dense])
attr_x_img = layers.Lambda(lambda x: x[0] * x[1], name = 'attr_x_img')([attr_word_emb_dense, imag_classifier])
# attr_x_img = layers.Concatenate(name = 'attr_x_img')([attr_word_emb_dense, imag_classifier])
attr_img_input = layers.Input(shape = (img_flat_len,), name = 'attr_img_input')
# attr_img_input = layers.Input(shape = (img_flat_len * 2,), name = 'attr_img_input')
proba = self.full_connect_layer(attr_img_input, hidden_dim = [1], activation = 'sigmoid')
attr_img_model = Model(inputs = attr_img_input, outputs = proba, name = 'attr_x_img_model')
out = attr_img_model([attr_x_img])
bc_loss = K.mean(binary_crossentropy(label, out))
model = Model([imag_classifier, attr_input, word_emb, label], outputs = [attr_word_emb_dense, out])
model.add_loss(bc_loss)
model.compile(optimizer=Adam(lr=1e-4), loss=None)
return model
def total_loss(y_true, y_pred, DL=1, BCE=20, FL=0, gamma=1, alpha=0.25):
'''Total loss between two arrays. Can be linear superposition of multiple loss functions,
including dice_loss, binary_corss_entropy, and focal_loss.
Inputs:
y_true (tf.TensorArray): GT array
y_pred (tf.TensorArray): Predicted array by CNN
DL(float, default to 1): Coefficient of dice loss in the total loss
BCE(float, default to 20): Coefficient of binary cross entropy in the total loss
FL(float, default to 0): Coefficient of focal loss in the total loss
gamma (float, default to 1): first parameter of focal loss
alpha (float, default to 0.25): second parameter of focal loss
Outputs:
loss (tf.float32): binary focal loss.
'''
y_true = tf.cast(y_true, tf.float32)
loss = DL * dice_loss(y_true, y_pred)
if BCE:
loss += BCE * losses.binary_crossentropy(y_true, y_pred)
if DL:
loss += FL * binary_focal_loss(y_true, y_pred, gamma, alpha)
return loss
def bce_dice_loss(y_true, y_pred):
from tensorflow.python.keras import losses
loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(
y_true, y_pred)
return loss
def bce_dice_loss(y_true, y_pred):
loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)
return loss
def total_loss(y_true, y_pred):
loss = binary_crossentropy(y_true, y_pred) + (3*dice_loss(y_true, y_pred))
return loss
def bce_dice_loss(y_true, y_pred):
loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(
y_true, y_pred)
#assert np.isnan(loss.eval(session=tf.compat.v1.Session()))
tf.print(loss, output_stream=sys.stderr)
return loss
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:, 0],
(-1, 1)), y_pred) * y_true[:, 1]
def bce_dice_loss(y_true, y_pred):
loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)+losses.mean_squared_error(y_true,y_pred)
return loss
def train(self, data):
"""Pretrain the latent layers of the model."""
# network parameters
original_dim = data.shape[1]
input_shape = (original_dim, )
batch_size = params_training.batch_size
latent_dim = params_training.num_latent
epochs = params_training.num_epochs
layer_num = 0
# build encoder model
inputs = Input(shape=input_shape, name='encoder_input')
inputs_noisy = GaussianNoise(stddev=0.1)(inputs)
hidden = inputs_noisy
for i, hidden_dim in enumerate(self.hidden_dim, 1):
hidden_layer = Dense(hidden_dim,
activation='sigmoid',
name='hidden_e_{}'.format(i),
weights=self.pretrain[layer_num])
hidden = hidden_layer(hidden)
layer_num += 1
logger.debug("Hooked up hidden layer with %d neurons" % hidden_dim)
z_mean = Dense(latent_dim,
activation=None,
name='z_mean',
weights=self.pretrain[layer_num])(hidden)
layer_num += 1
z_log_sigma = Dense(latent_dim,
activation=None,
name='z_log_sigma',
weights=self.pretrain[layer_num])(hidden)
layer_num += 1
z = Lambda(self.sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_sigma])
encoder = Model(inputs, [z_mean, z_log_sigma, z], name='encoder')
# build decoder model
latent_inputs = Input(shape=(latent_dim, ), name='z_sampling')
hidden = latent_inputs
for i, hidden_dim in enumerate(self.hidden_dim[::-1],
1): # Reverse because decoder.
hidden = Dense(hidden_dim,
activation='sigmoid',
name='hidden_d_{}'.format(i),
weights=self.pretrain[layer_num])(hidden)
layer_num += 1
logger.debug("Hooked up hidden layer with %d neurons" % hidden_dim)
outputs = Dense(original_dim, activation='sigmoid')(hidden)
decoder = Model(latent_inputs, outputs, name='decoder')
# Build the DAE
outputs = decoder(encoder(inputs)[2])
sdae = Model(inputs, outputs, name='vae_mlp')
reconstruction_loss = binary_crossentropy(inputs,
outputs) * original_dim
kl_loss = 1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
sdae.add_loss(vae_loss)
sdae.compile(optimizer='adam')
saver = ModelCheckpoint(check_path(TEMPORARY_SDAE_PATH),
save_weights_only=True,
verbose=1)
tensorboard_config = TensorBoard(
log_dir=check_path(TEMPORARY_SDAE_PATH))
logger.info("Checkpoint has been saved for SDAE.")
# train the autoencoder
logger.warning("Pretraining started, Don't interrupt.")
sdae.fit(data,
epochs=epochs,
batch_size=batch_size,
callbacks=[saver, tensorboard_config])
logger.info("Model has been pretrained successfully.")
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)
def bce_dice_loss(y_true, y_pred):
loss = losses.binary_crossentropy(y_true, y_pred) + K.math.log(
dice_loss(y_true, y_pred))
return loss
def binary_boundary_crossentropy(
y_true,
y_pred,
from_logits=False,
label_smoothing=0,
range: int = 1,
max: float = 2.0,
):
"""
[summary]
Parameters
----------
y_true : [type]
[description]
y_pred : [type]
[description]
from_logits : bool, optional
[description], by default False
label_smoothing : int, optional
[description], by default 0
range : int, optional
[description], by default 0
max : float, optional
[description], by default 1.0
Returns
-------
[type]
[description]
Examples
--------
>>> from image_keras.custom.losses_binary_boundary_crossentropy import binary_boundary_crossentropy
>>> import cv2
>>> a = cv2.imread("tests/test_resources/a.png", cv2.IMREAD_GRAYSCALE)
>>> a_modified = (a / 255).reshape(1, a.shape[0], a.shape[1], 1)
>>> binary_boundary_crossentropy(a_modified, a_modified, range=1, max=2)
"""
bce = binary_crossentropy(
y_true=y_true,
y_pred=y_pred,
from_logits=from_logits,
label_smoothing=label_smoothing,
)
def count_around_blocks(arr, range: int = 1):
ones = tf.ones_like(arr)
size = range * 2 + 1
if range < 1:
size = 1
extracted = tf.image.extract_patches(
images=ones,
sizes=[1, size, size, 1],
strides=[1, 1, 1, 1],
rates=[1, 1, 1, 1],
padding="SAME",
)
result = tf.reduce_sum(extracted, axis=-1)
if range > 0:
result -= 1
return result
def count_around_blocks2(arr, range: int = 1):
size = range * 2 + 1
if range < 1:
size = 1
extracted = tf.image.extract_patches(
images=arr,
sizes=[1, size, size, 1],
strides=[1, 1, 1, 1],
rates=[1, 1, 1, 1],
padding="SAME",
)
e_base = extracted[:, :, :, tf.shape(extracted)[-1] // 2]
e_base = tf.reshape(e_base, (-1, tf.shape(arr)[1], tf.shape(arr)[2], 1))
e_base_expanded = tf.reshape(
tf.repeat(e_base, tf.shape(extracted)[-1]),
(-1, tf.shape(arr)[1], tf.shape(arr)[2], tf.shape(extracted)[-1]),
)
same_values = tf.math.equal(extracted, e_base_expanded)
result_1 = tf.shape(extracted)[-1] - tf.cast(
tf.math.count_nonzero(same_values, axis=-1), tf.int32
)
result_1 = tf.reshape(result_1, (-1, tf.shape(arr)[1], tf.shape(arr)[2], 1))
result_1 = tf.cast(result_1, tf.float32)
result_1 += arr
block_counts = tf.reshape(
count_around_blocks(arr, range), (-1, tf.shape(arr)[1], tf.shape(arr)[2], 1)
)
modify_result_1 = -(size ** 2 - block_counts)
modify_result_1 = modify_result_1 * arr
modify_result_1 = tf.cast(modify_result_1, tf.float32)
diff_block_count = result_1 + modify_result_1
return diff_block_count
around_block_count = count_around_blocks(y_true, range=range)
around_block_count = tf.reshape(
around_block_count, (-1, tf.shape(y_true)[1], tf.shape(y_true)[2], 1)
)
around_block_count = tf.cast(around_block_count, tf.float32)
diff_block_count = count_around_blocks2(y_true, range=range)
diff_block_count = tf.reshape(
diff_block_count, (-1, tf.shape(y_true)[1], tf.shape(y_true)[2], 1)
)
diff_block_count = tf.cast(diff_block_count, tf.float32)
diff_ratio = diff_block_count / around_block_count
diff_ratio = 1.0 + tf.cast(tf.math.maximum(max - 1.0, 0), tf.float32) * diff_ratio
diff_ratio = tf.reshape(diff_ratio, (-1, tf.shape(y_true)[1], tf.shape(y_true)[2]))
return bce * diff_ratio
def bce_dice_loss(y_true, y_pred):
loss = losses.binary_crossentropy(
y_true, y_pred) + UnetModelGenerator.dice_loss(y_true, y_pred)
return loss
#from VAE
# parser = argparse.ArgumentParser()
# help_ = "Load h5 model trained weights"
# parser.add_argument("-w", "--weights", help=help_)
# help_ = "Use mse loss instead of binary cross entropy (default)"
# parser.add_argument("-m", "--mse", help=help_, action='store_true')
# args = parser.parse_args()
models = (encoder, decoder)
#data = (x_test, y_test)
# VAE loss = mse_loss or xent_loss + kl_loss
# if args.mse:
# reconstruction_loss = mse(K.flatten(inputs), K.flatten(outputs))
# else:
reconstruction_loss = binary_crossentropy(K.flatten(inputs),
K.flatten(outputs))
reconstruction_loss *= 120 * 160 * 3
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='rmsprop')
vae.summary()
plot_model(vae, to_file='vae_cnn.png', show_shapes=True)
train_gen = generator(opts, gen_records, cfg.BATCH_SIZE, True)
val_gen = generator(opts, gen_records, cfg.BATCH_SIZE, False)
# if args.weights:
# vae.load_weights(args.weights)
