from tslearn.clustering import KernelKMeans
from sklearn.decomposition import NMF, PCA
from sklearn.manifold import TSNE
from sklearn.metrics.cluster import *
from sklearn.preprocessing import normalize
# from keras.layers import GaussianNoise, Dense, Activation
from tensorflow.keras import regularizers
# import keras.backend as K
from tensorflow.keras.layers import Activation, Dense, GaussianNoise
from collections import defaultdict
from scipy.spatial import distance
# from dca.api import dca
MeanAct = lambda x: tf.clip_by_value(K.exp(x), 1e-5, 1e6)
DispAct = lambda x: tf.clip_by_value(tf.nn.softplus(x), 1e-4, 1e4)
imAct = lambda x: tf.nn.softplus(x)
#experiment
unannotated_exp = "pollen"
def Y_to_clusters(Y, cnum):
# cnum=max(Y)+1
clusters = []
for i in range(cnum):
clusters.append([])
for i in range(len(Y)):
clusters[Y[i]].append(i)
return clusters
def exponent_neg_manhattan_distance(left, right):
''' Helper function for the similarity estimate of the LSTMs outputs'''
return K.exp(-K.sum(K.abs(left - right), axis=1, keepdims=True))
def train_step_tf(self, inp, true_lens):
start_trace = time.time()
recon_loss = 0.0
acc = tf.constant([0, 0], dtype=tf.float32)
with tf.GradientTape() as tape:
masked_inp = inp * tf.expand_dims(
tf.sequence_mask(
true_lens, dtype=tf.float32, maxlen=inp.shape[1]), -1)
ctx_mean, ctx_log_var, context = self.encoder(masked_inp)
kl_loss = 1 + ctx_log_var - K.square(ctx_mean) - K.exp(ctx_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
kl_loss = K.mean(kl_loss)
pred_len = self.len_decoder(context)
len_loss = tf.reduce_mean(loss_object(true_lens, pred_len))
dec_input = tf.expand_dims([self.start_word] * self.batch_size, 1)
dec_hidden = self.initialize_hidden_state()
# No Teacher forcing
for t in range(inp.shape[1]):
# passing enc_output to the self.decoder
predictions, dec_hidden, = self.decoder(
(dec_input, context, dec_hidden))
recon_loss += loss_function(inp[:, t], predictions, pred_len)
acc += acc_metric(inp[:, t], predictions)
# not using teacher forcing
dec_input = predictions
recons_loss = tf.reduce_mean(tf.divide(recon_loss,
pred_len)) * self.msl
loss = recons_loss + len_loss + kl_loss
batch_loss = (loss / int(inp.shape[1]))
variables = self.encoder.trainable_variables + self.decoder.trainable_variables + self.len_decoder.trainable_variables
gradients = tape.gradient(loss, variables)
self.optimizer.apply_gradients(zip(gradients, variables))
trace_time = time.time() - start_trace
print('Time taken for trace (train):\t\t{:.2f}\n'.format(trace_time))
####
acc_m = 1 - acc[0] / acc[1]
kl_m = (kl_loss / int(inp.shape[1]))
len_m = (len_loss / int(inp.shape[1]))
recons_m = (recons_loss / int(inp.shape[1]))
metrics = [(name, x)
for x, name in zip([acc_m, kl_m, len_m, recons_m],
["acc_m", "kl_m", "len_m", "recons_m"])]
return batch_loss, metrics
def vae_loss(y_true, y_pred):
xent_loss = losses.MSE(y_true, y_pred)
kl_loss = - 0.5 * K.mean(1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma))
loss = xent_loss + kl_loss
return loss
def target_layer(x):
import keras.backend as K
return K.exp(x)
def task_6_var_autoencoder_alt(data, parameters):
x_train = data['x_train']
x_train = x_train.reshape(60000, 28, 28, 1).astype('float32')
y_train = to_categorical(data['y_train'])
x_test = data['x_test']
x_test = x_test.reshape(10000, 28, 28, 1).astype('float32')
y_test = data['y_test']
# Extract parameters
latent_dim = parameters["latent_dim"]
learning_rate = parameters["learning_rate"]
mini_batch_size = parameters["mini_batch_size"]
epochs = parameters["epochs"]
loss_func = parameters["loss_func"]
# Initialize a tensorflow session for evaluating tensors
sess = tf.Session()
with sess.as_default():
# Build Encoder
inputs = Input(shape=(28, 28, 1), name='encoder_input')
c1 = Conv2D(filters=10,
kernel_size=3,
activation='relu',
strides=2,
padding='same',
input_shape=(28, 28, 1))(inputs)
c2 = Conv2D(filters=20,
kernel_size=3,
activation='relu',
strides=2,
padding='same')(c1)
f1 = Flatten()(c2)
d1 = Dense(16, activation='relu')(f1)
z_mean = Dense(latent_dim, name='z_mean')(d1)
z_log_var = Dense(latent_dim, name='z_log_var')(d1)
z = Lambda(sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_var])
encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder_output')
encoder.summary()
# Build Decoder
latent_inputs = Input(shape=(latent_dim, ), name='z_sampling')
x = Dense(7 * 7 * 20, activation='relu',
name='decoder_first_dense')(latent_inputs)
r = Reshape(target_shape=(7, 7, 20))(x)
dc1 = Conv2DTranspose(filters=20,
kernel_size=3,
activation='relu',
strides=2,
padding='same')(r)
dc2 = Conv2DTranspose(filters=10,
kernel_size=3,
activation='relu',
strides=2,
padding='same')(dc1)
y = Conv2DTranspose(filters=1,
kernel_size=3,
padding='same',
activation='sigmoid')(dc2)
decoder = Model(latent_inputs, y, name='decoder_output')
decoder.summary()
# Instantiate VAE model
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='VAE')
if (loss_func == "mean_squared_error"):
loss = mse(K.flatten(inputs), K.flatten(outputs))
else:
loss = binary_crossentropy(K.flatten(inputs), K.flatten(outputs))
# Add KL Divergence term
loss *= x_train.shape[1] * x_train.shape[1]
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(loss + kl_loss)
vae.add_loss(vae_loss)
# Compile VAE model
sgd = keras.optimizers.SGD(lr=learning_rate)
vae.compile(optimizer=sgd, metrics=['accuracy'])
time_callback = TimeHistory()
train_history = vae.fit(x_train,
epochs=epochs,
batch_size=mini_batch_size,
callbacks=[time_callback])
times = np.cumsum(time_callback.times) # Cumulative sum for plotting
# (each time is roughly the same value, but want to plot over the entire period)
# Generate 10 random latent vectors following a N(0, 1) distribution
random_test_vectors = np.random.normal(0, 1, (10, 20))
test_images = np.zeros(
(28, 28, 10)
) # Initialize variable for decoder outputs based on the random latent vectors
# Generate images based on ten random latent vectors
for lv in range(0, random_test_vectors.shape[0]):
test_images[:, :, lv] = decoder(random_test_vectors[lv].reshape(
1, 20)).eval().reshape(28, 28)
# Plotting
test_plot_idcs = [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (1, 0),
(1, 1), (1, 2), (1, 3), (1, 4)]
plt.figure(0)
for im in range(0, random_test_vectors.shape[0]):
plt.subplot2grid((2, 5), test_plot_idcs[im])
plt.imshow(test_images[:, :, im])
plt.show()
plt.subplot(1, 2, 1)
plt.plot(train_history.history['loss'])
plt.grid()
plt.title('Epoch-Loss Plot - Task 5')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.subplot(1, 2, 2)
plt.plot(times / 60, train_history.history['loss'])
plt.grid()
plt.title('Time-Loss Plot - Task 5')
plt.xlabel('Time (min)')
plt.ylabel('Loss')
plt.show()
SAVE_PATH + "decoder_{}.png".format(INPUT_FORM),
show_shapes=True,
show_dtype=True,
show_layer_names=True,
rankdir='TB',
expand_nested=False,
dpi=48)
# instantiate VAE model
outputs = decoder(encoder([encoder_inputs] + param_inputs)[2])
vae = keras.Model([encoder_inputs] + param_inputs, outputs, name='vae_model')
# customize loss of VAE model
recstr_loss = binary_crossentropy(K.flatten(encoder_inputs),
K.flatten(outputs))
kl_loss = -0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var),
axis=-1)
label_losses = []
all_label_loss = 0
for i_param, param in enumerate(PARAMS_TO_INFER):
l = 0.01 * mape(param_inputs[i_param], y_predictions[i_param])
all_label_loss += l
label_losses.append(l)
# vae_loss = K.mean(reconstruction_loss+kl_loss+label_loss)
vae_loss = K.mean(recstr_loss + kl_loss + all_label_loss)
vae.add_loss(vae_loss)
# add metrics
vae.add_metric(recstr_loss, name='recstr_loss', aggregation='mean')
vae.add_metric(kl_loss, name='kl_loss', aggregation='mean')
def call(self, inputs):
diff = K.expand_dims(inputs) - self.mu
l2 = K.sum(K.pow(diff,2), axis=1)
res = K.exp(-1 * self.gamma * l2)
return res
def on_epoch_end(self, epoch, logs={}):
self.lambdap = 2 / (1 + K.exp(-self.gamma * epoch)) - 1
def variational_autoencoder_cnn_mnist_example():
is_mse_used = True # Use MSE loss or binary cross entropy loss.
is_trained_model_used = False
weight_filepath = 'vae_cnn_mnist.h5' # h5 model trained weights.
#--------------------
# MNIST dataset.
(x_train, y_train), (x_test, y_test) = mnist.load_data()
image_height, image_width = x_train.shape[1:3]
x_train = np.expand_dims(x_train, axis=-1)
x_test = np.expand_dims(x_test, axis=-1)
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
#--------------------
# Network parameters.
input_shape = (image_height, image_width, 1)
batch_size = 128
kernel_size = 3
layer_filters = [32, 64]
latent_dim = 2
epochs = 30
#--------------------
# VAE model = encoder + decoder.
#--------------------
# Build encoder model.
inputs = Input(shape=input_shape, name='encoder_input')
x = inputs
for filters in layer_filters:
x = Conv2D(filters=filters,
kernel_size=kernel_size,
activation='relu',
strides=2,
padding='same')(x)
# Shape info needed to build decoder model.
shape = K.int_shape(x)
# Generate latent vector Q(z|X).
x = Flatten()(x)
x = Dense(16, activation='relu')(x)
z_mean = Dense(latent_dim, name='z_mean')(x)
z_log_var = Dense(latent_dim, name='z_log_var')(x)
# Use reparameterization trick to push the sampling out as input.
# Note that 'output_shape' isn't necessary with the TensorFlow backend.
z = Lambda(sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_var])
# Instantiate encoder model.
encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
encoder.summary()
plot_model(encoder, to_file='vae_cnn_encoder.png', show_shapes=True)
#--------------------
# Build decoder model.
latent_inputs = Input(shape=(latent_dim, ), name='z_sampling')
x = Dense(shape[1] * shape[2] * shape[3], activation='relu')(latent_inputs)
x = Reshape((shape[1], shape[2], shape[3]))(x)
for filters in layer_filters[::-1]:
x = Conv2DTranspose(filters=filters,
kernel_size=kernel_size,
activation='relu',
strides=2,
padding='same')(x)
outputs = Conv2DTranspose(filters=1,
kernel_size=kernel_size,
activation='sigmoid',
padding='same',
name='decoder_output')(x)
# Instantiate decoder model.
decoder = Model(latent_inputs, outputs, name='decoder')
decoder.summary()
plot_model(decoder, to_file='vae_cnn_decoder.png', show_shapes=True)
#--------------------
# Instantiate VAE model.
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae')
#--------------------
# VAE loss = mse_loss or xent_loss + kl_loss
if is_mse_used:
reconstruction_loss = mse(K.flatten(inputs), K.flatten(outputs))
else:
reconstruction_loss = binary_crossentropy(K.flatten(inputs),
K.flatten(outputs))
reconstruction_loss *= image_height * image_width
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='rmsprop')
vae.summary()
plot_model(vae, to_file='vae_cnn.png', show_shapes=True)
#--------------------
if is_trained_model_used:
vae.load_weights(weight_filepath)
else:
# Train the autoencoder.
vae.fit(x_train,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_test, None))
vae.save_weights(weight_filepath)
#--------------------
models = (encoder, decoder)
data = (x_test, y_test)
plot_results(models, data, batch_size=batch_size, model_name='vae_cnn')
def gauss_lr(min_lr, max_lr, center, lrsigma, i):
return (min_lr + max_lr * K.exp(-(i - center)**2 /
(center * lrsigma)**2))
def variational_autoencoder_mlp_mnist_example():
is_mse_used = True # Use MSE loss or binary cross entropy loss.
is_trained_model_used = False
weight_filepath = 'vae_mlp_mnist.h5' # h5 model trained weights.
#--------------------
# MNIST dataset.
(x_train, y_train), (x_test, y_test) = mnist.load_data()
image_height, image_width = x_train.shape[1:3]
image_size = image_height * image_width
x_train = np.reshape(x_train, [-1, image_size])
x_test = np.reshape(x_test, [-1, image_size])
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
#--------------------
# Network parameters.
input_shape = (image_size, )
intermediate_dim = 512
batch_size = 128
latent_dim = 2
epochs = 50
#--------------------
# VAE model = encoder + decoder.
#--------------------
# Build encoder model.
inputs = Input(shape=input_shape, name='encoder_input')
x = Dense(intermediate_dim, activation='relu')(inputs)
z_mean = Dense(latent_dim, name='z_mean')(x)
z_log_var = Dense(latent_dim, name='z_log_var')(x)
# Use reparameterization trick to push the sampling out as input.
# Note that 'output_shape' isn't necessary with the TensorFlow backend.
z = Lambda(sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_var])
# Instantiate encoder model.
encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
encoder.summary()
plot_model(encoder, to_file='vae_mlp_encoder.png', show_shapes=True)
#--------------------
# Build decoder model.
latent_inputs = Input(shape=(latent_dim, ), name='z_sampling')
x = Dense(intermediate_dim, activation='relu')(latent_inputs)
outputs = Dense(image_size, activation='sigmoid')(x)
# Instantiate decoder model.
decoder = Model(latent_inputs, outputs, name='decoder')
decoder.summary()
plot_model(decoder, to_file='vae_mlp_decoder.png', show_shapes=True)
#--------------------
# Instantiate VAE model.
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae_mlp')
#--------------------
# VAE loss = mse_loss or xent_loss + kl_loss.
if is_mse_used:
reconstruction_loss = mse(inputs, outputs)
else:
reconstruction_loss = binary_crossentropy(inputs, outputs)
reconstruction_loss *= image_size
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
vae.summary()
plot_model(vae, to_file='vae_mlp.png', show_shapes=True)
#--------------------
if is_trained_model_used:
vae.load_weights(weight_filepath)
else:
# Train the autoencoder.
vae.fit(x_train,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_test, None))
vae.save_weights(weight_filepath)
#--------------------
models = (encoder, decoder)
data = (x_test, y_test)
plot_results(models, data, batch_size=batch_size, model_name='vae_mlp')
def keras_perplexity(y_true, y_pred):
cross_entropy = K.mean(tf.keras.losses.categorical_crossentropy(y_true, y_pred))
perplexity = K.exp(cross_entropy)
return perplexity
def custom_loss(label_pair, e_w):
loss_p = (1 - label_pair) * 2 * e_w**2 / Q
loss_n = label_pair * 2 * Q * K.exp(-2.77 * e_w / Q)
loss = K.mean(loss_p + loss_n)
return loss
def _calculate_kl_loss(self, y_target, y_predicted):
kl_loss = -0.5 * K.sum(1 + self.log_variance - K.square(self.mu) -
K.exp(self.log_variance),
axis=1)
return kl_loss
def make_k_functions(vol_shape, mapping, max_feats=None, norm_post=True):
"""
Utility to build keras (gpu-runnable) functions that will warp the atlas and compute
posteriors given the original full atlas and unnormalized log likelihood.
norm_post (True): normalize posterior? Thi sis faster on GPU, so if possible should
be set to True
max_feats (None): since atlas van be very large, warping full atlas can run out of memory on
current GPUs.
Providing a number here will avoid OOM error, and will return several keras functions that
each provide the posterior computation for at most max_feats nb_full_labels. Stacking the
result of calling these functions will provide an *unnormalized* posterior (since it can't
properly normalize)
"""
nb_full_labels = len(mapping)
nb_labels = np.max(mapping) + 1
# compute maximum features and whether to return single function
return_single_fn = max_feats is None
if max_feats is None:
max_feats = nb_full_labels
else:
assert not norm_post, 'cannot do normalized posterior if providing max_feats'
# prepare ull and
ull = tf.placeholder(tf.float32, vol_shape + (nb_labels, ))
input_flow = tf.placeholder(tf.float32, vol_shape + (len(vol_shape), ))
ul_pred = K.exp(ull)
funcs = []
for i in range(0, nb_full_labels, max_feats):
end = min(i + max_feats, nb_full_labels)
this_atlas_full_shape = vol_shape + (end - i, )
this_mapping = mapping[i:end]
# prepare atlas input
input_atlas = tf.placeholder(tf.float32, this_atlas_full_shape)
# warp atlas
warped = vxm.tf.ne.transform(input_atlas,
input_flow,
interp_method='linear',
indexing='ij')
# normalized posterior
post_lst = [
ul_pred[..., this_mapping[j]] * warped[..., j]
for j in range(len(this_mapping))
]
posterior = K.stack(post_lst, -1)
funcs.append(
K.function([input_atlas, ull, input_flow], [posterior, warped]))
if return_single_fn:
if norm_post:
posterior = posterior / (1e-12 +
K.sum(posterior, -1, keepdims=True))
funcs = funcs.append(
K.function([input_atlas, ull, input_flow], [posterior, warped]))
return funcs
# instantiate VAE model
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae_mlp')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
help_ = "Load h5 model trained weights"
parser.add_argument("-w", "--weights", help=help_)
help_ = "Use mse loss instead of binary cross entropy (default)"
parser.add_argument("-r", "--ran", help=help_, action='store_true')
args = parser.parse_args()
models = (encoder, decoder)
reconstruction_loss = mse(inputs, outputs)
reconstruction_loss *= original_dim
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
vae.summary()
plot_model(vae, to_file='vae_mlp.png', show_shapes=True)
vae.load_weights('./me')
decoded = []
file_path = ""
if args.ran:
d_input = []
for i in range(0, 386):
mu = (random.random() * 5.0)
def _likelihood(self, y_true, y_pred):
data = K.sum((-y_pred) * (1. - y_true)) / K.sum(1. - y_true)
gen = K.log(K.sum(K.exp(-y_pred) * y_true)) - K.log(K.sum(y_true))
return gen - data
def call(self, x):
"""Postprocess part for YOLOv3 model except NMS."""
assert isinstance(x, list)
#num_layers = len(anchors)//3 # default setting
yolo_outputs, image_shape = x
#anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [0,1,2]] # default setting
#input_shape = K.shape(yolo_outputs[0])[1:3] * 32
batch_size = K.shape(image_shape)[0] # batch size, tensor
boxes = []
box_scores = []
for l in range(self.num_layers):
# get anchor set for each feature layer
if self.num_layers == 3: #YOLOv3 arch
if l == 0:
anchorset = self.anchors[6:]
grid_shape = [
self.input_dim[0] // 32, self.input_dim[1] // 32
]
elif l == 1:
anchorset = self.anchors[3:6]
grid_shape = [
self.input_dim[0] // 16, self.input_dim[1] // 16
]
elif l == 2:
anchorset = self.anchors[:3]
grid_shape = [
self.input_dim[0] // 8, self.input_dim[1] // 8
]
elif self.num_layers == 2: # Tiny YOLOv3 arch
if l == 0:
anchorset = self.anchors[3:]
grid_shape = [
self.input_dim[0] // 32, self.input_dim[1] // 32
]
elif l == 1:
anchorset = self.anchors[:3]
grid_shape = [
self.input_dim[0] // 16, self.input_dim[1] // 16
]
else:
raise ValueError('Invalid layer number')
feats = yolo_outputs[l]
# Convert final layer features to bounding box parameters
num_anchors = len(anchorset)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchorset),
[1, 1, 1, num_anchors, 2])
#grid_shape = K.shape(feats)[1:3] # height, width
# get total anchor number for each feature layer
total_anchor_num = grid_shape[0] * grid_shape[1] * num_anchors
grid_y = K.tile(
K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(
K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
reshape_feats = K.reshape(feats, [
-1, grid_shape[0], grid_shape[1], num_anchors,
self.num_classes + 5
])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (K.sigmoid(reshape_feats[..., :2]) + grid) / K.cast(
grid_shape[::-1], K.dtype(reshape_feats))
box_wh = K.exp(reshape_feats[..., 2:4]) * anchors_tensor / K.cast(
self.input_dim[::-1], K.dtype(reshape_feats))
box_confidence = K.sigmoid(reshape_feats[..., 4:5])
box_class_probs = K.sigmoid(reshape_feats[..., 5:])
# correct boxes to the original image shape
input_shape = K.cast(self.input_dim, K.dtype(box_xy))
image_shape = K.cast(image_shape, K.dtype(box_xy))
#new_shape = K.round(image_shape * K.min(input_shape/image_shape))
new_shape = K.cast(image_shape * K.min(input_shape / image_shape),
dtype='int32')
new_shape = K.cast(new_shape, dtype='float32')
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_xy = (box_xy - offset) * scale
box_wh *= scale
box_mins = box_xy - (box_wh / 2.)
box_maxes = box_xy + (box_wh / 2.)
_boxes = K.concatenate([
box_mins[..., 0:1], # x_min
box_mins[..., 1:2], # y_min
box_maxes[..., 0:1], # x_max
box_maxes[..., 1:2] # y_max
])
# Scale boxes back to original image shape.
_boxes *= K.concatenate([image_shape, image_shape])
# Reshape boxes to flatten the boxes
_boxes = K.reshape(_boxes, [-1, total_anchor_num, 4])
_box_scores = box_confidence * box_class_probs
_box_scores = K.reshape(_box_scores,
[-1, total_anchor_num, self.num_classes])
boxes.append(_boxes)
box_scores.append(_box_scores)
# Merge boxes for all feature layers, for further NMS option
boxes = K.concatenate(boxes, axis=1)
box_scores = K.concatenate(box_scores, axis=1)
return boxes, box_scores
def tf_lognormal(self, z_true, mu, log_sigma):
logSqrtTwoPI = np.log(np.sqrt(2.0 * np.pi))
return -0.5 * (
(z_true - mu) / K.exp(log_sigma))**2 - log_sigma - logSqrtTwoPI
def shifted_softplus(x):
return K.log(0.5 * K.exp(x) + 0.5)
def exponent_neg_manhattan_distance(self, left, right):
return K.exp(-K.sum(K.abs(left - right), axis=1, keepdims=True))
def correntropy(y_true, y_pred, gamma=1):
a = y_pred - y_true
a = K.abs(a)
return (1 - K.exp(-(K.pow(a, 2))))
def z_sampling(args):
z_mu, z_log_var = args
eps = K.random_normal(shape=(K.shape(z_mu)[0], latent_dim), mean=0., stddev=1.)
return z_mu + K.exp(z_log_var) * eps
def batch_hard_loss(outputs,
loss_batch,
loss_margin,
soft=False,
metric='euclidian'):
"""
An implementation of the batch hard loss (eq. 5 from arXiv:1703.07737 "In Defense of the Triplet Loss for Person
Re-Identification" by Hermans, Beyer & Leibe)
Args:
outputs: the encoded features. We expecte that the batch size of the outputs tensor is a multiple of loss_batch
loss_batch: the size of the minibatch for the loss function
loss_margin: the margin for the triple loss formula (m in the paper)
soft: use soft margin, i.e. log(1+exp(distance))
metric: 'euclidian' or 'binaryce' (binary cross entropy)
Returns:
the batch hard loss
"""
import tensorflow.keras.backend as K
# group images by examples (each example contains loss_batch images)
examples = K.reshape(outputs, (-1, loss_batch, K.shape(outputs)[1]))
# get the anchor image and expand the second dimension
anchors = K.expand_dims(examples[:, 0, :], 1)
positives = examples[:, 1:loss_batch //
4, :] # the next loss_batch-1 images are positives
negatives = examples[:, loss_batch //
4:, :] # the last loss_batch images are nagatives
if metric == 'euclidian':
# compute the maximum positive distance
pos_dist = \
K.max(
K.mean(
K.square(
K.repeat_elements(anchors, loss_batch // 4 - 1, axis=1) - positives
),
axis=2
)
)
# compute the minimum negative distance
neg_dist = \
K.min(
K.mean(
K.square(
K.repeat_elements(anchors, loss_batch - loss_batch // 4, axis=1) - negatives
),
axis=2
)
)
else:
# compute the maximum positive distance
pos_dist = \
K.max(
K.mean(
K.binary_crossentropy(
K.repeat_elements(anchors, loss_batch // 4 - 1, axis=1), positives
),
axis=2
)
)
# compute the minimum negative distance
neg_dist = \
K.min(
K.mean(
K.binary_crossentropy(
K.repeat_elements(anchors, loss_batch - loss_batch // 4, axis=1), negatives
), axis=2
)
)
# compute the average true batch loss
if not soft:
loss = K.mean(K.maximum(loss_margin + pos_dist - neg_dist, 0))
else:
import tensorflow as tf
loss = K.mean(tf.math.log1p(K.exp(pos_dist - neg_dist)))
return loss
def samplingfun(z_mean, z_log_var):
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim), mean=0.,
stddev=1.0)
return z_mean + K.exp(z_log_var / 2) * epsilon
def KL_loss(y_true, y_pred):
mean = y_pred[:, :128]
logsigma = y_pred[:, :128]
loss = -logsigma + .5 * (-1 + K.exp(2. * logsigma) + K.square(mean))
loss = K.mean(loss)
return loss
def call(self, inputs):
mu, log_var = inputs
epsilon = K.random_normal(shape=K.shape(mu), mean=0., stddev=1.)
return mu + K.exp(log_var / 2) * epsilon
def minb_disc(x):
diffs = K.expand_dims(x, 3) - K.expand_dims(K.permute_dimensions(x, [1, 2, 0]), 0)
abs_diffs = K.sum(K.abs(diffs), 2)
x = K.sum(K.exp(-abs_diffs), 2)
return x
def call(self, inputs):
"""
Creates the layer as a Keras graph.
Note that the inputs are tensors with a batch dimension of 1:
Keras requires this batch dimension, and for full-batch methods
we only have a single "batch".
There are two inputs required, the node features,
and the graph adjacency matrix
Notes:
This does not add self loops to the adjacency matrix.
Args:
inputs (list): list of inputs with 3 items:
node features (size 1 x N x F),
graph adjacency matrix (size N x N),
where N is the number of nodes in the graph,
F is the dimensionality of node features
M is the number of output nodes
"""
X = inputs[0] # Node features (1 x N x F)
A = inputs[1] # Adjacency matrix (N x N)
N = K.int_shape(A)[-1]
batch_dim, n_nodes, _ = K.int_shape(X)
if batch_dim != 1:
raise ValueError(
"Currently full-batch methods only support a batch dimension of one"
)
else:
# Remove singleton batch dimension
X = K.squeeze(X, 0)
outputs = []
for head in range(self.attn_heads):
kernel = self.kernels[head] # W in the paper (F x F')
attention_kernel = self.attn_kernels[
head] # Attention kernel a in the paper (2F' x 1)
# Compute inputs to attention network
features = K.dot(X, kernel) # (N x F')
# Compute feature combinations
# Note: [[a_1], [a_2]]^T [[Wh_i], [Wh_2]] = [a_1]^T [Wh_i] + [a_2]^T [Wh_j]
attn_for_self = K.dot(
features, attention_kernel[0]) # (N x 1), [a_1]^T [Wh_i]
attn_for_neighs = K.dot(
features, attention_kernel[1]) # (N x 1), [a_2]^T [Wh_j]
# Attention head a(Wh_i, Wh_j) = a^T [[Wh_i], [Wh_j]]
dense = attn_for_self + K.transpose(
attn_for_neighs) # (N x N) via broadcasting
# Add nonlinearity
dense = LeakyReLU(alpha=0.2)(dense)
# Mask values before activation (Vaswani et al., 2017)
# YT: this only works for 'binary' A, not for 'weighted' A!
# YT: if A does not have self-loops, the node itself will be masked, so A should have self-loops
# YT: this is ensured by setting the diagonal elements of A tensor to 1 above
if not self.saliency_map_support:
mask = -10e9 * (1.0 - A)
dense += mask
dense = K.softmax(dense) # (N x N), Eq. 3 of the paper
else:
# dense = dense - tf.reduce_max(dense)
# GAT with support for saliency calculations
W = (self.delta * A
) * K.exp(dense - K.max(dense, axis=1, keepdims=True)) * (
1 - self.non_exist_edge) + self.non_exist_edge * (
A + self.delta * (tf.ones((N, N)) - A) + tf.eye(N)
) * K.exp(dense - K.max(dense, axis=1, keepdims=True))
dense = W / K.sum(W, axis=1, keepdims=True)
# Apply dropout to features and attention coefficients
dropout_feat = Dropout(self.in_dropout_rate)(features) # (N x F')
dropout_attn = Dropout(self.attn_dropout_rate)(dense) # (N x N)
# Linear combination with neighbors' features [YT: see Eq. 4]
node_features = K.dot(dropout_attn, dropout_feat) # (N x F')
if self.use_bias:
node_features = K.bias_add(node_features, self.biases[head])
# Add output of attention head to final output
outputs.append(node_features)
# Aggregate the heads' output according to the reduction method
if self.attn_heads_reduction == "concat":
output = K.concatenate(outputs) # (N x KF')
else:
output = K.mean(K.stack(outputs), axis=0) # N x F')
# Nonlinear activation function
output = self.activation(output)
# Add batch dimension back if we removed it
if batch_dim == 1:
output = K.expand_dims(output, 0)
return output