def call(self, X):
if type(X) is not list or len(X) != 2:
raise Exception("GaussianAttention must be called on a list of two tensors. Got: " + str(X))
frame, position = X[0], X[1]
# Reshaping the input to exclude the time dimension
frameShape = K.shape(frame)
positionShape = K.shape(position)
(chans, height, width) = frameShape[-3:]
targetDim = positionShape[-1]
frame = K.reshape(frame, (-1, chans, height, width))
position = K.reshape(position, (-1, ) + (targetDim, ))
cx = (position[:, 0] + position[:, 2]) / 2.0
cy = (position[:, 1] + position[:, 3]) / 2.0
sx = (position[:, 2] - cx) * 0.60
sy = (position[:, 3] - cy) * 0.60
rX = Data.linspace(-1.0, 1.0, width)
rY = Data.linspace(-1.0, 1.0, height)
FX = K.exp(-(rX - cx.dimshuffle(0, 'x')) ** 2 / (2.0 * (sx.dimshuffle(0, 'x') ** 2 + self.epsilon)))
FY = K.exp(-(rY - cy.dimshuffle(0, 'x')) ** 2 / (2.0 * (sy.dimshuffle(0, 'x') ** 2 + self.epsilon)))
m = (FY.dimshuffle(0, 1, 'x') * FX.dimshuffle(0, 'x', 1))
m = m + self.alpha
m = m - K.greater(m, 1.0) * (m - 1.0)
frame = frame * m.dimshuffle(0, 'x', 1, 2)
# Reshaping the frame to include time dimension
output = K.reshape(frame, frameShape)
return output
def __call__(self, loss):
# See Variational Auto-Encoding Bayes by Kingma and Welling.
mean, logsigma = self.mean, self.logsigma
kl = (self.prior_logsigma - logsigma
+ 0.5 * (K.exp(2 * logsigma) + (mean - self.prior_mean) ** 2)
/ K.exp(2 * self.prior_logsigma))
loss += kl.mean() * self.regularizer_scale
return loss
def kl_normal(q, p=0):
dim = q.shape[1]//2
qmu, qlv = q[:, :dim], q[:, dim:]
if p is 0:
pmu, plv = 0, 0
else:
pmu, plv = p[:, :dim], p[:, dim:]
return 0.5 * K.sum(plv - qlv + K.exp(qlv - plv) + K.square(qmu - pmu) * K.exp(-plv) - 1, axis=-1)
def get_similarity(self):
''' Specify similarity in configuration under 'similarity_params' -> 'mode'
If a parameter is needed for the model, specify it in 'similarity_params'
Example configuration:
config = {
... other parameters ...
'similarity_params': {
'mode': 'gesd',
'gamma': 1,
'c': 1,
}
}
cosine: dot(a, b) / sqrt(dot(a, a) * dot(b, b))
polynomial: (gamma * dot(a, b) + c) ^ d
sigmoid: tanh(gamma * dot(a, b) + c)
rbf: exp(-gamma * l2_norm(a-b) ^ 2)
euclidean: 1 / (1 + l2_norm(a - b))
exponential: exp(-gamma * l2_norm(a - b))
gesd: euclidean * sigmoid
aesd: (euclidean + sigmoid) / 2
'''
params = self.similarity_params
similarity = params['mode']
axis = lambda a: len(a._keras_shape) - 1
dot = lambda a, b: K.batch_dot(a, b, axes=axis(a))
l2_norm = lambda a, b: K.sqrt(K.sum((a - b) ** 2, axis=axis(a), keepdims=True))
l1_norm = lambda a, b: K.sum(K.abs(a - b), axis=axis(a), keepdims=True)
if similarity == 'cosine':
return lambda x: dot(x[0], x[1]) / K.sqrt(dot(x[0], x[0]) * dot(x[1], x[1]))
elif similarity == 'polynomial':
return lambda x: (params['gamma'] * dot(x[0], x[1]) + params['c']) ** params['d']
elif similarity == 'sigmoid':
return lambda x: K.tanh(params['gamma'] * dot(x[0], x[1]) + params['c'])
elif similarity == 'rbf':
return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1]) ** 2)
elif similarity == 'euclidean':
return lambda x: 1 / (1 + l2_norm(x[0], x[1]))
elif similarity == 'l1':
return lambda x: -l1_norm(x[0], x[1])
elif similarity == 'exponential':
return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1]))
elif similarity == 'gesd':
euclidean = lambda x: 1 / (1 + l2_norm(x[0], x[1]))
sigmoid = lambda x: 1 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c'])))
return lambda x: euclidean(x) * sigmoid(x)
elif similarity == 'aesd':
euclidean = lambda x: 0.5 / (1 + l2_norm(x[0], x[1]))
sigmoid = lambda x: 0.5 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c'])))
return lambda x: euclidean(x) + sigmoid(x)
else:
raise Exception('Invalid similarity: {}'.format(similarity))
def __call__(self, loss):
# See Variational Auto-Encoding Bayes by Kingma and Welling.
mean, logsigma = self.mean, self.logsigma
kl = (self.prior_logsigma - logsigma +
0.5 * (-1 + K.exp(2 * logsigma) + (mean - self.prior_mean) ** 2) /
K.exp(2 * self.prior_logsigma))
## TEMPORARY CHANGE: scale regularizer more slowly than sampling
loss += K.mean(kl) * (self.regularizer_scale) ** 2
## TEMPORARY CHANGE: print kl raw and scaled contribution to loss
return loss
def get_output(self, train=False):
print "LogAnyBoundOcc", self.output_shape
X = self.get_input(train)
log_none_bnd = K.sum(
K.log(1-K.clip(K.exp(X), 1e-6, 1-1e-6)), axis=3, keepdims=True)
at_least_1_bnd = 1-K.exp(log_none_bnd)
max_occ = K.max(K.exp(X), axis=3, keepdims=True)
# we take the weighted sum because the max is easier to fit, and
# thus this helps to regularize the optimization procedure
rv = K.log(0.05*max_occ + 0.95*at_least_1_bnd)
return rv
def __call__(self, x):
xshape = K.int_shape(x)
if self.axis is 'last':
x = K.reshape(x, (-1, xshape[-1]))
x /= K.sqrt(K.sum(K.square(x), axis=0, keepdims=True))
xx = K.dot(K.transpose(x), x)
return self.gamma * K.sum(K.log(1.0 + K.exp(self.lam * (xx - 1.0))) * (1.0 - K.eye(xshape[-1])))
elif self.axis is 'first':
x = K.reshape(x, (xshape[0], -1))
x /= K.sqrt(K.sum(K.square(x), axis=1, keepdims=True))
xx = K.dot(x, K.transpose(x))
return self.gamma * K.sum(K.log(1.0 + K.exp(self.lam * (xx - 1.0))) * (1.0 - K.eye(xshape[0])))
def normals_metric(y_true, y_pred):
y_true = K.variable(y_true)
y_pred = K.variable(y_pred)
y_true = K.expand_dims(y_true,0)
filter_y = K.variable(np.array([[ 0., -0.5 , 0.],
[0., 0., 0.],
[0., 0.5, 0.]]).reshape(3, 3, 1, 1))
filter_x = K.variable(np.array([ [0, 0., 0.],
[0.5, 0., -0.5],
[0., 0., 0.]]).reshape(3, 3, 1, 1))
dzdx = K.conv2d(K.exp(y_true), filter_x, padding='same')
dzdy = K.conv2d(K.exp(y_true), filter_y, padding='same')
dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx))
dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy))
mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx))
mag_norm = K.sqrt(mag_norm)
N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm
N1 = dzdx_ / mag_norm
N2 = dzdy_ / mag_norm
normals = K.concatenate(tensors=[N1,N2,N3],axis=-1)
dzdx_pred = K.conv2d(K.exp(y_pred), filter_x, padding='same')
dzdy_pred = K.conv2d(K.exp(y_pred), filter_y, padding='same')
mag_norm_pred = K.pow(dzdx_pred,2) + K.pow(dzdy_pred,2) + 1.0
mag_norm_pred = K.sqrt(mag_norm_pred)
grad_x = K.concatenate(tensors=[1.0/ mag_norm_pred,
0.0/ mag_norm_pred, dzdx_pred/ mag_norm_pred],axis=-1)
grad_y = K.concatenate(tensors=[0.0/ mag_norm_pred,
1.0/ mag_norm_pred, dzdy_pred/ mag_norm_pred],axis=-1)
dot_term_x = K.mean(K.sum(normals[0,:,:,:] * grad_x[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_y = K.mean(K.sum(normals[0,:,:,:] * grad_y[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_x = K.abs(dot_term_x)
dot_term_y = K.abs(dot_term_y)
return K.eval(K.mean(dot_term_x)),K.eval(K.mean(dot_term_y))
def dssm_loss(d, y):
q = y[:, :y.shape[1]/3]
d_positive = y[:, y.shape[1]/3:2*y.shape[1]/3]
d_negative = y[:, 2*y.shape[1]/3:]
eps = 0.00001
inf = 100000
q_len = K.sqrt(K.sum(K.square(q), axis=1))
d_positive_len = K.sqrt(K.sum(K.square(d_positive), axis=1))
d_negative_len = K.sqrt(K.sum(K.square(d_negative), axis=1))
positive_cosine_distance = K.sum(q * d_positive, axis=1) / (q_len * d_positive_len)
negative_cosine_distance = K.sum(q * d_negative, axis=1) / (q_len * d_negative_len)
all_cosine_distance = K.exp(positive_cosine_distance) + K.exp(negative_cosine_distance)
positive_cosine_distance = K.exp(positive_cosine_distance) / (all_cosine_distance)
loss = -K.mean(K.log(positive_cosine_distance))
return ifelse(T.isnan(loss), 0., loss)
def A_network_output(x):
# The input of this layer is [L, mu, a] in concatenated form. We first split
# those up.
idx = 0
L_flat = x[:, idx:idx + (self.nb_actions * self.nb_actions + self.nb_actions) / 2]
idx += (self.nb_actions * self.nb_actions + self.nb_actions) / 2
mu = x[:, idx:idx + self.nb_actions]
idx += self.nb_actions
a = x[:, idx:idx + self.nb_actions]
idx += self.nb_actions
# Create L and L^T matrix, which we use to construct the positive-definite matrix P.
Ls = []
LTs = []
for idx in xrange(self.batch_size):
L = K.zeros((self.nb_actions, self.nb_actions))
L = T.set_subtensor(L[np.tril_indices(self.nb_actions)], L_flat[idx, :])
diag = K.exp(T.diag(L))
L = T.set_subtensor(L[np.diag_indices(self.nb_actions)], diag)
Ls.append(L)
LTs.append(K.transpose(L))
# TODO: diagonal elements exp
L = K.pack(Ls)
LT = K.pack(LTs)
P = K.batch_dot(L, LT, axes=(1, 2))
assert K.ndim(P) == 3
# Combine a, mu and P into a scalar (over the batches).
A = -.5 * K.batch_dot(K.batch_dot(a - mu, P, axes=(1, 2)), a - mu, axes=1)
assert K.ndim(A) == 2
return A
def softmax(x):
ndim = K.ndim(x)
if ndim == 2:
return K.softmax(x)
elif ndim == 3:
e = K.exp(x - K.max(x, axis=-1, keepdims=True))
s = K.sum(e, axis=-1, keepdims=True)
return e / s
elif ndim == 4:
e = K.exp(x - K.max(x, axis=-1, keepdims=True))
s = K.sum(e, axis=-1, keepdims=True)
return e / s
else:
raise ValueError('Cannot apply softmax to a tensor '
'that is not 2D or 3D. '
'Here, ndim=' + str(ndim))
def loglik_discrete(y, u, a, b, epsilon=1e-35):
hazard0 = K.pow((y + epsilon) / a, b)
hazard1 = K.pow((y + 1.0) / a, b)
loglikelihoods = u * \
K.log(K.exp(hazard1 - hazard0) - 1.0) - hazard1
return loglikelihoods
def call(self, x):
assert(K.backend() == 'tensorflow')
temp = K.permute_dimensions(x, (0, 2, 1))
for i in range(0, self.attention_depth):
temp = K.sigmoid(K.dot(temp, self.Ws[i]) + self.bs[i])
temp = K.permute_dimensions(temp, (0, 2, 1))
estimated_weight = K.squeeze(K.dot(temp, K.expand_dims(self.Wf, -1)), -1)
biased_weight = estimated_weight + self.bias
non_linear_weight = K.tanh(biased_weight)
# For each hidded state calculate how much should it contribute
# to the context vector. This is the main part of attention.
# In order to convert weights to "probabilities" use a sigmoid
# based function: exp(x) / sum(exp(xi)).
prob = K.exp(non_linear_weight)
# Compute the total sum for each batch.
total_sum = K.sum(prob, axis=1, keepdims=True)
prob /= K.cast(total_sum, K.floatx())
# Enable this if you want access to internal probabilities.
# Should only be used for testing that Attention works as expected.
# return prob
# Multiply each hidden value by the corresponding probability.
prob = K.expand_dims(prob, -1)
new_hidden_values = x * prob
return K.sum(new_hidden_values, axis=1)
def sampling(args):
z_mean, z_log_var = args
batch = K.shape(z_mean)[0]
dim = K.int_shape(z_mean)[1]
# by default, random_normal has mean=0 and std=1.0
epsilon = K.random_normal(shape=(batch, dim))
return z_mean + K.exp(0.5 * z_log_var) * epsilon
def vae_loss(x, x_decoded_mean):
# NOTE: binary_crossentropy expects a batch_size by dim for x and x_decoded_mean, so we MUST flatten these!
x = K.flatten(x)
x_decoded_mean = K.flatten(x_decoded_mean)
xent_loss = np.dot(original_dim, original_dim) * objectives.binary_crossentropy(x, x_decoded_mean)
kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
def kl_loss(truth_dummy, x_mean_log_var_output):
x_mean, x_log_var = tf.split(x_mean_log_var_output, 2, axis=1)
print('x_mean shape in kl_loss: ', x_mean.get_shape())
kl_loss = - 0.5 * \
K.mean(1 + x_log_var - K.square(x_mean) -
K.exp(x_log_var), axis=-1)
return kl_loss
def _get_output(self, X, train=False):
mean, logsigma = self.get_mean_logsigma(X)
if train:
eps = K.random_normal((self.batch_size, self.output_dim))
return mean + K.exp(logsigma) * eps
else:
return mean
def convolution_softmax(volume):
"""Like K.softmax, but we handle arbitrary volumes by only computing down
axis 1."""
# The subtraction of K.max is for numeric stability. See T.nnet docs for
# more.
exps = K.exp(volume - K.max(volume, axis=1, keepdims=True))
return exps / K.sum(exps, axis=1, keepdims=True)
def sigmoid_cross_entropy(y_true, y_pred):
z = K.flatten(y_true)
x = K.flatten(y_pred)
q = 10
l = (1 + (q - 1) * z)
loss = (K.sum((1 - z) * x) + K.sum(l * (K.log(1 + K.exp(- K.abs(x))) + K.max(-x, 0)))) / 500
return loss
def vae_loss(x, x_decoded_mean):
x = K.flatten(x)
x_decoded_mean = K.flatten(x_decoded_mean)
xent_loss = max_length * objectives.binary_crossentropy(x, x_decoded_mean)
kl_loss = -0.5 * K.mean(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
def get_output(self, train=False):
print "LogNormalizedOccupancy", self.output_shape
X = self.get_input(train)
# calculate the log occupancies
log_occs = theano_calc_log_occs(-X, self.chem_affinity)
# reshape the output so that the forward and reverse complement
# occupancies are viewed as different tracks
log_occs = K.reshape(log_occs, (X.shape[0], 1, 2*X.shape[1], X.shape[3]))
if self.steric_hindrance_win_len == 0:
log_norm_factor = 0
else:
# correct occupancies for overlapping binding sites
occs = K.exp(log_occs)
kernel = K.ones((1, 1, 1, 2*self.steric_hindrance_win_len-1), dtype='float32')
win_occ_sum = K.conv2d(occs, kernel, border_mode='same').sum(axis=2, keepdims=True)
win_prb_all_unbnd = TT.exp(
K.conv2d(K.log(1-occs), kernel, border_mode='same')).sum(axis=2, keepdims=True)
log_norm_factor = TT.log(win_occ_sum + win_prb_all_unbnd)
#start = max(0, self.steric_hindrance_win_len-1)
#stop = min(self.output_shape[3],
# self.output_shape[3]-(self.steric_hindrance_win_len-1))
#rv = log_occs[:,:,:,start:stop] - log_norm_factor
rv = (log_occs - log_norm_factor)
return K.reshape(
rv,
(X.shape[0], 2*X.shape[1], 1, X.shape[3])
)
def log_normal(x, p=0):
if p is 0:
mu, lv = 0, 0
else:
dim = p.shape[1]//2
mu, lv = p[:, :dim], p[:, dim:]
return - 0.5 * K.sum(K.log(2*np.pi) + lv + K.square(x - mu) * K.exp(-lv), axis=-1)
def call(self, x, mask=None):
# eij = K.dot(x, self.W) TF backend doesn't support it
# features_dim = self.W.shape[0]
# step_dim = x._keras_shape[1]
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
a = K.exp(eij)
# apply mask after the exp. will be re-normalized next
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
a *= K.cast(mask, K.floatx())
# in some cases especially in the early stages of training the sum may be almost zero
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
a = K.expand_dims(a)
weighted_input = x * a
# print weigthted_input.shape
return K.sum(weighted_input, axis=1)
def yolo_head(feats, anchors, num_classes, input_shape):
"""Convert final layer features to bounding box parameters."""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
feats = K.reshape(
feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (box_xy + grid) / K.cast(grid_shape[::-1], K.dtype(feats))
box_wh = box_wh * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))
return box_xy, box_wh, box_confidence, box_class_probs
def sampling(args):
"""Reparameterization trick by sampling fr an isotropic unit Gaussian.
# Arguments:
args (tensor): mean and log of variance of Q(z|X)
# Returns:
z (tensor): sampled latent vector
"""
print("args")
print(args)
# print("z_mean")
# print(z_mean)
# print("z_log_var")
# print(z_log_var)
z_mean, z_log_var = args
batch = K.shape(z_mean)[0]
print("batch")
print(z_mean.shape[0])
print(batch)
dim = K.int_shape(z_mean)[1]
print("dim")
print(z_mean.shape[1])
print(dim)
# by default, random_normal has mean=0 and std=1.0
epsilon = K.random_normal(shape=(batch, dim))
return z_mean + K.exp(0.5 * z_log_var) * epsilon
def vae_loss(y_true, y_pred):
recon = binary_crossentropy(y_true, y_pred)
recon *= original_dim
kl = 0.5 * K.sum(-1. - log_sigma + K.exp(log_sigma) + K.square(mu), axis=-1)
loss = K.mean(kl + recon)
return loss
def call(self, x, mask=None):
eij = dot_product(x, self.W)
if self.bias:
eij += self.b
eij = K.tanh(eij)
a = K.exp(eij)
# apply mask after the exp. will be re-normalized next
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
a *= K.cast(mask, K.floatx())
# in some cases especially in the early stages of training the sum may be almost zero
# and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
# a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
weighted_input = x * K.expand_dims(a)
result = K.sum(weighted_input, axis=1)
if self.return_attention:
return [result, a]
return result
def call(self, x, training = None):
eps = 0.01
ax = K.abs(x)
M = K.mean((ax+eps) ** 4, axis = self.channel_axis) ** (1./4) # Minkowsky's average to focus more on the (few) large values than on (many) smaller ones
noise = K.random_uniform(shape = K.shape(x), minval = -1.0, maxval = 1.0, seed = self.seed)
# xr = ax * K.exp(self.reduction)
# red = xr / (1 + xr**2)
red = 1 / (1 + ax) # individual noise reduction for each element of input
mag = K.exp(-M / K.exp(self.sensitivity)) * self.scale # global magnitude: if M = 0.0 -> large magnitude (1.0) ... if M >> 0.0 -> low magnitude (~0.0)
noisy = x + noise * red * mag[...,None]
return noisy
def create_vae(batch_size, base_filters=64, latent=8,
image_size=64, learning_rate=0.001,
reconstruction_weight=1000, layers=4):
'''
Constructs VAE model with given parameters.
:param batch_size: size of a batch (used for placeholder)
:param base_filters: number of filters after first layer. Other layers will double this number
:param latent: latent space dimension
:param image_size: size of input image
Returns compiled Keras model along with encoder and decoder
'''
if isinstance(image_size, int):
image_size = (image_size, image_size)
x = Input(batch_shape=(batch_size, image_size[0], image_size[1], 3))
encoder = create_encoder([image_size[0], image_size[1], 3], base_filters=base_filters, latent=latent, layers=layers)
decoder = create_decoder([image_size[0], image_size[1], 3], base_filters=base_filters, latent=latent, layers=layers)
mean_log_var = encoder(x)
mean_size = mean_log_var.shape[1]//2
mean = Lambda(lambda h: h[:, :mean_size])(mean_log_var)
log_var = Lambda(lambda h: h[:, mean_size:])(mean_log_var)
z = Lambda(sample)([mean, log_var])
reconstruction = decoder(z)
loss_reconstruction = K.mean(metrics.mean_squared_error(x, reconstruction))
loss_KL = - K.mean(0.5 * K.sum(1 + log_var - K.square(mean) - K.exp(log_var), axis=1))
loss = reconstruction_weight*loss_reconstruction + loss_KL
vae = Model(x, reconstruction)
vae.compile(optimizer=keras.optimizers.Adam(lr=learning_rate), loss=lambda x, y: loss)
return vae, encoder, decoder
def vae_loss(input_phono,phono_decoded):
mse_loss_phono = objectives.mse(input_phono, phono_decoded)
kl_loss = - 0.5 * K.mean(1 + z_log_std - K.square(z_mean) - K.exp(z_log_std), axis=-1)
return (
mse_loss_phono
+ kl_loss
)
# 重参数层，相当于给输入加入噪声
z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
# 解码层，也就是生成器部分
decoder_h = Dense(intermediate_dim, activation='relu')
decoder_mean = Dense(original_dim, activation='sigmoid')
h_decoded = decoder_h(z)
x_decoded_mean = decoder_mean(h_decoded)
# 建立模型
vae = Model(x, x_decoded_mean)
# xent_loss是重构loss，kl_loss是KL loss
xent_loss = K.sum(K.binary_crossentropy(x, x_decoded_mean), axis=-1)
kl_loss = -0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var),
axis=-1)
vae_loss = K.mean(xent_loss + kl_loss)
# add_loss是新增的方法，用于更灵活地添加各种loss
vae.add_loss(vae_loss)
vae.compile(optimizer='rmsprop')
vae.summary()
vae.fit(x_train,
shuffle=True,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_test, None))
# 构建encoder，然后观察各个数字在隐空间的分布
def softmaxNd(x, axis=-1):
m = K.max(x, axis=axis, keepdims=True)
exp_x = K.exp(x - m)
prob_x = exp_x / K.sum(exp_x, axis=axis, keepdims=True)
return prob_x
def yolo_head(feats, anchors, num_classes):
"""Convert final layer features to bounding box parameters.
Parameters
----------
feats : tensor
Final convolutional layer features.
anchors : array-like
Anchor box widths and heights.
num_classes : int
Number of target classes.
Returns
-------
box_xy : tensor
x, y box predictions adjusted by spatial location in conv layer.
box_wh : tensor
w, h box predictions adjusted by anchors and conv spatial resolution.
box_conf : tensor
Probability estimate for whether each box contains any object.
box_class_pred : tensor
Probability distribution estimate for each box over class labels.
"""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.variable(anchors), [1, 1, 1, num_anchors, 2])
# Static implementation for fixed models.
# TODO: Remove or add option for static implementation.
# _, conv_height, conv_width, _ = K.int_shape(feats)
# conv_dims = K.variable([conv_width, conv_height])
# Dynamic implementation of conv dims for fully convolutional model.
conv_dims = K.shape(feats)[1:3] # assuming channels last
# In YOLO the height index is the inner most iteration.
conv_height_index = K.arange(0, stop=conv_dims[0])
conv_width_index = K.arange(0, stop=conv_dims[1])
conv_height_index = K.tile(conv_height_index, [conv_dims[1]])
# TODO: Repeat_elements and tf.split doesn't support dynamic splits.
# conv_width_index = K.repeat_elements(conv_width_index, conv_dims[1], axis=0)
conv_width_index = K.tile(K.expand_dims(conv_width_index, 0),
[conv_dims[0], 1])
conv_width_index = K.flatten(K.transpose(conv_width_index))
conv_index = K.transpose(K.stack([conv_height_index, conv_width_index]))
conv_index = K.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2])
conv_index = K.cast(conv_index, K.dtype(feats))
feats = K.reshape(
feats, [-1, conv_dims[0], conv_dims[1], num_anchors, num_classes + 5])
conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]), K.dtype(feats))
# Static generation of conv_index:
# conv_index = np.array([_ for _ in np.ndindex(conv_width, conv_height)])
# conv_index = conv_index[:, [1, 0]] # swap columns for YOLO ordering.
# conv_index = K.variable(
# conv_index.reshape(1, conv_height, conv_width, 1, 2))
# feats = Reshape(
# (conv_dims[0], conv_dims[1], num_anchors, num_classes + 5))(feats)
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.softmax(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
# Note: YOLO iterates over height index before width index.
box_xy = (box_xy + conv_index) / conv_dims
box_wh = box_wh * anchors_tensor / conv_dims
return box_xy, box_wh, box_confidence, box_class_probs
def kl_loss(truth_dummy, z_mean_log_var_output):
z_mean, z_log_var = tf.split(z_mean_log_var_output, 2, axis=1)
#print('x_mean shape in kl_loss: ', x_mean.get_shape())
kl = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl *= -0.5
return (kl)
def sampling(inputs):
z_mean, z_log_var = inputs
epsilon = K.random_normal(shape=output_shape, mean=0.)
return z_mean + K.exp(z_log_var / 2) * epsilon
def vae_loss(self, x, x_decoded_mean):
xent_loss = input_shape * metrics.binary_crossentropy(x, x_decoded_mean)
kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return K.mean(xent_loss + kl_loss)
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], encoding_layer_dim), mean=0.,
stddev=1.)
return z_mean + K.exp(z_log_var / 2) * epsilon
def lossfun(self, z_avg, z_log_var):
kl_loss = -0.5 * K.mean(1.0 + z_log_var - K.square(z_avg) -
K.exp(z_log_var))
return kl_loss
#def vae_loss(y_true, y_pred):
# """loss = reconstruction loss + KL loss for each data batch"""
#
# # E[log P(X|z)]
# recon = K.sum(binary_crossentropy(y_pred, y_true))
#
# # D_KL(Q(z|x) || P(z|X))
# kl = -0.5 * K.sum( 1. + log_sigma - K.exp(log_sigma) - K.square(mu), axis = -1)
#
# return K.mean(recon + kl)
#define loss
xent_loss = img_rows * img_cols * channels * metrics.binary_crossentropy(
K.flatten(img), K.flatten(h_decoded))
kl_loss = -0.5 * K.sum(1 + log_sigma - K.square(mu) - K.exp(log_sigma),
axis=-1)
cvae_loss = K.mean(xent_loss + kl_loss)
cvae.add_loss(cvae_loss)
cvae.compile(optimizer='adam')
cvae.summary()
cvae.fit([x_train, y_train], batch_size=m, epochs=n_epoch)
#%%
encoder = Model([img, label], [mu, log_sigma, z])
decoder_input = Input(shape=(latent_dim, ))
d = Concatenate(axis=-1)([decoder_input, label])
d = Dense(4 * 4 * 128, activation='relu')(decoder_input)
d = Reshape((4, 4, 128))(d)
d = BatchNormalization()(d)
def call(self, x, mask=None):
e = K.exp(x - K.max(x, axis=self.axis, keepdims=True))
s = K.sum(e, axis=self.axis, keepdims=True)
return e / s
def call(self, inputs):
mean, log_var = inputs
return K.random_normal(tf.shape(log_var)) * K.exp(log_var / 2) + mean
activation=decoder_layers_activations[i])(previous_layer)
previous_layer = x
outputs = Dense(input_shape[0], activation='sigmoid')(previous_layer)
# instantiate decoder model
decoder = Model(latent_inputs, outputs, name='decoder')
decoder.summary()
plot_model(decoder, to_file='vae_mlp_decoder.png', show_shapes=True)
# instantiate VAE model
outputs_vae = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs_vae, name='vae_mlp')
beta = 1
reconstruction_loss = binary_crossentropy(inputs, outputs_vae)
reconstruction_loss *= input_shape[0] #OBS WHY??
kl_loss = 1 + z_log_var - z_mean**2 - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5 * beta
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
x_train_sets = []
for i in range(7):
x_train_sets.append(np.load('data/data_matrix' + str(i) + ".npy"))
x_train0 = x_train_sets[0]
x_train1 = x_train_sets[1]
x_train2 = x_train_sets[2]
x_train3 = x_train_sets[3]
x_train4 = x_train_sets[4]
def tanh(x):
return (K.exp(x) - K.exp(-x)) / (K.exp(x) + K.exp(-x))
def vae_loss(x, x_decoded_mean):
xent_loss = original_dim * metrics.binary_crossentropy(
x, x_decoded_mean)
kl_loss = -0.5 * K.mean(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(batch_size, latent_dim),
mean=0.,
stddev=epsilon_std)
return z_mean + K.exp(z_log_var / 2) * epsilon
def vae_loss(x, x_decoded_mean):
xent_loss = objectives.mse(x, x_decoded_mean)
kl_loss = -0.5 * K.mean(1 + z_log_sigma - K.square(z_mean) -
K.exp(z_log_sigma))
loss = xent_loss + kl_loss
return loss
def create_model(self):
codings_size = self.codings_size
input_features = self.input_features
patience = self.patience
dropout_p = self.dropout_p
optimizer = self.optimizer
input_features_float = np.float32(input_features)
d1_size = self.d1_size
d2_size = self.d2_size
d3_size = self.d3_size
d4_size = self.d4_size
inputs = keras.layers.Input(shape=(input_features, ),
name="encoder_input")
dropout = keras.layers.Dropout(dropout_p)
d1 = keras.layers.Dense(d1_size, activation="selu")
d2 = keras.layers.Dense(d2_size, activation="selu")
d3 = keras.layers.Dense(d3_size, activation="selu")
d4 = keras.layers.Dense(d4_size, activation="selu")
z = dropout(inputs)
z = d1(z)
z = d2(z)
z = d3(z)
z = d4(z)
codings_mean = keras.layers.Dense(codings_size, name="codings_mean")(z)
codings_log_var = keras.layers.Dense(codings_size,
name="codings_log_var")(z)
sampling_layer = Sampling(name="sampling_layer")
codings = sampling_layer([codings_mean, codings_log_var])
variational_encoder = keras.models.Model(
inputs=[inputs],
outputs=[codings_mean, codings_log_var, codings],
name="encoder")
#variational_encoder._name = "encoder"
decoder_inputs = keras.layers.Input(shape=[codings_size],
name="decoder_input")
dc = keras.layers.Dense(d4_size, activation="selu")
d4_t = DenseTranspose(d4, activation="selu")
d3_t = DenseTranspose(d3, activation="selu")
d2_t = DenseTranspose(d2, activation="selu")
d1_t = DenseTranspose(d1, activation="linear", name="decoder_outputs")
x = dc(decoder_inputs)
x = d4_t(x)
x = d3_t(x)
x = d2_t(x)
outputs = d1_t(x)
#outputs = keras.layers.Dense(input_features, activation="linear", name="decoder_outputs")(x)
variational_decoder = keras.models.Model(inputs=[decoder_inputs],
outputs=[outputs],
name="decoder")
#variational_decoder._name = "decoder"
_, _, codings = variational_encoder(inputs)
reconstructions = variational_decoder(codings)
#LAST COLUMN IS THE TARGET
input_features_m_1 = self.input_features - 1
def apply_slice_numeric(x):
return slice(x, (0, 0), (-1, input_features_m_1))
def apply_slice_target(x):
return slice(x, (0, input_features_m_1), (-1, 1))
l1 = Lambda(apply_slice_numeric, name='numeric')
l2 = Lambda(apply_slice_target, name='lambda_categorical')
d2a_layer = keras.layers.Activation(keras.activations.sigmoid,
name='categorical')
self.l1 = l1
self.l2 = l2
self.d2a_layer = d2a_layer
d1 = l1(reconstructions)
d2 = l2(reconstructions)
d2a = d2a_layer(d2)
variational_ae = keras.models.Model(inputs=[inputs], outputs=[d1, d2a])
latent_loss = -0.5 * K.sum(1 + codings_log_var - K.exp(codings_log_var)
- K.square(codings_mean),
axis=-1)
variational_ae.add_loss(
K.mean(latent_loss) / np.float32(input_features))
if self.use_mse_loss:
variational_ae.compile(
loss=[keras.losses.mse, keras.losses.binary_crossentropy],
optimizer=optimizer,
loss_weights=[
(input_features_float - 1.0) / input_features_float,
10 / input_features_float
])
else:
variational_ae.compile(loss=[
masked_loss_function_weighted, keras.losses.binary_crossentropy
],
optimizer=optimizer,
loss_weights=[(input_features_float - 1.0) /
input_features_float,
10 / input_features_float])
#variational_ae.compile(loss=[masked_loss_function,keras.losses.binary_crossentropy], optimizer=optimizer, loss_weights=[(input_features_float-1.0)/input_features_float, 10/input_features_float])
#variational_ae.compile(loss=[keras.losses.mse,keras.losses.binary_crossentropy], optimizer=optimizer, loss_weights=[(input_features_float-1.0)/input_features_float, 10/input_features_float])
self.variational_ae = variational_ae
self.variational_encoder = variational_encoder
self.variational_decoder = variational_decoder
self.latent_loss = latent_loss
self.codings_mean = codings_mean
self.codings_log_var = codings_log_var
input_dim = 784
latent_dim = 2
learning_rate = 0.001
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape([x_train.shape[0], -1]).astype('float32') / 255
x_test = x_test.reshape([x_test.shape[0], -1]).astype('float32') / 255
# encoder
input = Input(shape=(input_dim, ))
layer = Dense(512, activation='relu', kernel_initializer='he_normal')(input)
z_mean = Dense(latent_dim, name='z_mean')(layer)
z_log_var = Dense(latent_dim, name='z_variance')(layer)
z = Lambda(
lambda x: x[0] + K.exp(0.5 * x[1]) * K.random_normal(shape=K.shape(x[0])),
name='z')([z_mean, z_log_var])
encoder = Model(input, [z_mean, z_log_var, z], name='encoder')
encoder.summary()
# decoder
input = Input(shape=(latent_dim, ))
layer = Dense(512, activation='relu', kernel_initializer='he_normal')(input)
layer = Dense(input_dim, activation='sigmoid')(layer)
decoder = Model(input, layer, name='decoder')
decoder.summary()
# variational autoencoder
VAE = Model(encoder.input, decoder(encoder(encoder.input)[2]))
def kl_loss(x, x_decoded_mean):
kl_loss = -0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var),
axis=-1)
kl_loss = kl_weight * kl_loss
return kl_loss
def my_vae_loss(y_true, y_pred):
xent_loss = keras.metrics.mse(K.flatten(y_true), K.flatten(y_pred))
kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
vae_loss = K.mean(xent_loss + kl_loss)
return vae_loss
def vae_loss(x, x_decoded_mean):
xent_loss = K.mean(K.square((x- x_decoded_mean)))
kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
def vae_loss(x, x_decoded_mean):
xent_loss = metrics.binary_crossentropy(x, x_decoded_mean)
kl_loss = -0.5 * K.mean(
1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma), axis=-1)
return xent_loss + kl_loss
def KL_loss(y_true, y_pred):
mean = y_pred[:, :128]
logsigma = y_pred[:, :128]
loss = -logsigma + .5 * (-1 + K.exp(2. * logsigma) + K.square(mean))
loss = K.mean(loss)
return loss
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=K.shape(z_mean))
return z_mean + K.exp(z_log_var / 2) * epsilon
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(n_components,))
return z_mean + K.exp(z_log_var) * epsilon
def reg_loss(mean, log_var):
return K.mean(
0.5 * K.sum(-1 - log_var + K.square(mean) + K.exp(log_var), axis=-1))
def augmented_variance_loss(mean, log_var):
variance = K.exp(z_log_var)
mean_variance = K.var(mean, axis=0, keepdims=True)
total_variance = variance + mean_variance
loss = 0.5 * K.sum(-1 - K.log(total_variance) + total_variance, axis=-1)
return K.mean(loss)
x = layers.Dense(width, activation="linear")(decoder_input) #was elu
for i in range(nlayers - 1):
x = layers.Dense(width, activation="elu")(x)
output = layers.Dense(traingen.shape[1], activation="sigmoid")(
x
) #hard sigmoid seems natural here but appears to lead to more left-skewed decoder outputs.
decoder = Model(decoder_input, output, name='decoder')
#end-to-end vae
output_seq = decoder(encoder(input_seq)[2])
vae = Model(input_seq, output_seq, name='vae')
#add loss function
reconstruction_loss = keras.losses.binary_crossentropy(input_seq, output_seq)
reconstruction_loss *= traingen.shape[1]
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
checkpointer = keras.callbacks.ModelCheckpoint(filepath=out + "_weights.hdf5",
verbose=1,
save_best_only=True,
monitor="val_loss",
period=1)
earlystop = keras.callbacks.EarlyStopping(monitor="val_loss",
min_delta=0,
patience=patience)
def vae_loss(x, x_decoded_mean):
# xent_loss = original_dim * objectives.binary_crossentropy(x, x_decoded_mean)
xent_loss = objectives.binary_crossentropy(x, x_decoded_mean)
kl_loss = -0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var),
axis=-1) / original_dim
return xent_loss + kl_loss
def exponent_neg_manhattan_distance(left, right):
''' Helper function for the similarity estimate of the LSTMs outputs
FROM https://medium.com/mlreview/implementing-malstm-on-kaggles-quora-question-pairs-competition-8b31b0b16a07
'''
return K.exp(-K.sum(K.abs(left - right), axis=1, keepdims=True))