def eigen_loss(y_true, y_pred):
y_true = tf.Print(y_true, [y_true], message='y_true', summarize=30)
y_pred = tf.Print(y_pred, [y_pred], message='y_pred', summarize=30)
y_true_clipped = K.clip(y_true, K.epsilon(), None)
y_pred_clipped = K.clip(y_pred, K.epsilon(), None)
first_log = K.log(y_pred_clipped + 1.)
second_log = K.log(y_true_clipped + 1.)
w_x = K.variable(np.array([[-1., 0., 1.],
[-1., 0., 1.],
[-1., 0., 1.]]).reshape(3, 3, 1, 1))
grad_x_pred = K.conv2d(first_log, w_x, padding='same')
grad_x_true = K.conv2d(second_log, w_x, padding='same')
w_y = K.variable(np.array([[-1., -1., -1.],
[0., 0., 0.],
[1., 1., 1.]]).reshape(3, 3, 1, 1))
grad_y_pred = K.conv2d(first_log, w_y, padding='same')
grad_y_true = K.conv2d(second_log, w_y, padding='same')
diff_x = grad_x_pred - grad_x_true
diff_y = grad_y_pred - grad_y_true
log_term = K.mean(K.square((first_log - second_log)), axis=-1)
sc_inv_term = K.square(K.mean((first_log - second_log),axis=-1))
grad_loss = K.mean(K.square(diff_x) + K.square(diff_y), axis=-1)
return log_term - (0.5 * sc_inv_term) + grad_loss
def create_vae(batch_size, base_filters=64, latent=8,
image_size=64, learning_rate=0.001,
reconstruction_weight=1000, layers=4):
'''
Constructs VAE model with given parameters.
:param batch_size: size of a batch (used for placeholder)
:param base_filters: number of filters after first layer. Other layers will double this number
:param latent: latent space dimension
:param image_size: size of input image
Returns compiled Keras model along with encoder and decoder
'''
if isinstance(image_size, int):
image_size = (image_size, image_size)
x = Input(batch_shape=(batch_size, image_size[0], image_size[1], 3))
encoder = create_encoder([image_size[0], image_size[1], 3], base_filters=base_filters, latent=latent, layers=layers)
decoder = create_decoder([image_size[0], image_size[1], 3], base_filters=base_filters, latent=latent, layers=layers)
mean_log_var = encoder(x)
mean_size = mean_log_var.shape[1]//2
mean = Lambda(lambda h: h[:, :mean_size])(mean_log_var)
log_var = Lambda(lambda h: h[:, mean_size:])(mean_log_var)
z = Lambda(sample)([mean, log_var])
reconstruction = decoder(z)
loss_reconstruction = K.mean(metrics.mean_squared_error(x, reconstruction))
loss_KL = - K.mean(0.5 * K.sum(1 + log_var - K.square(mean) - K.exp(log_var), axis=1))
loss = reconstruction_weight*loss_reconstruction + loss_KL
vae = Model(x, reconstruction)
vae.compile(optimizer=keras.optimizers.Adam(lr=learning_rate), loss=lambda x, y: loss)
return vae, encoder, decoder
def call(self, x, mask=None):
input_shape = self.input_spec[0].shape
reduction_axes = list(range(len(input_shape)))
del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
broadcast_shape[self.axis] = input_shape[self.axis]
# case: train mode (uses stats of the current batch)
mean = K.mean(x, axis=reduction_axes)
brodcast_mean = K.reshape(mean, broadcast_shape)
std = K.mean(K.square(x - brodcast_mean) + self.epsilon, axis=reduction_axes)
std = K.sqrt(std)
brodcast_std = K.reshape(std, broadcast_shape)
mean_update = self.momentum * self.running_mean + (1 - self.momentum) * mean
std_update = self.momentum * self.running_std + (1 - self.momentum) * std
self.updates = [(self.running_mean, mean_update),
(self.running_std, std_update)]
x_normed = (x - brodcast_mean) / (brodcast_std + self.epsilon)
# case: test mode (uses running averages)
brodcast_running_mean = K.reshape(self.running_mean, broadcast_shape)
brodcast_running_std = K.reshape(self.running_std, broadcast_shape)
x_normed_running = ((x - brodcast_running_mean) / (brodcast_running_std + self.epsilon))
# pick the normalized form of x corresponding to the training phase
x_normed = K.in_train_phase(x_normed, x_normed_running)
out = K.reshape(self.gamma, broadcast_shape) * x_normed + K.reshape(self.beta, broadcast_shape)
return out
def criterion_GAN(output, target, use_lsgan=True):
if use_lsgan:
diff = output - target
dims = list(range(1, K.ndim(diff)))
return K.expand_dims((K.mean(diff ** 2, dims)), 0)
else:
return K.mean(K.log(output + 1e-12) * target + K.log(1 - output + 1e-12) * (1 - target))
def mutual_info_loss(self, c, c_given_x):
"""The mutual information metric we aim to minimize"""
eps = 1e-8
conditional_entropy = K.mean(- K.sum(K.log(c_given_x + eps) * c, axis=1))
entropy = K.mean(- K.sum(K.log(c + eps) * c, axis=1))
return conditional_entropy + entropy
def func(y_true, y_pred):
Gxx = _get_kernel(y_true, y_true)
Gzz = _get_kernel(y_pred, y_pred)
Gxz = _get_kernel(y_true, y_pred)
cost = K.log(K.sqrt(K.mean(Gxx)*K.mean(Gzz)) /
K.mean(Gxz))
return cost
def get_loss(self):
loss = 0.0
if self.l1:
loss += K.mean(K.abs(self.p)) * self.l1
if self.l2:
loss += K.mean(K.square(self.p)) * self.l2
return loss
def visualize_mser_classifier():
model = load_model('mser_classifier')
# Get the symbolic outputs of each "key" layer
layer_dict = dict([(layer.name, layer) for layer in model.layers])
layer_name = 'block5_conv3'
# Can be any integer from 0 to 511, as there are 512 filters in that layer
kept_filters = []
for filter_index in range(200):
print(filter_index)
# Loss function that maximizes the activation
# of the nth filter of the layer considered
layer_output = layer_dict[layer_name].output
loss = K.mean(layer_output[:, :, :, filter_index])
# Placeholder for the input images
input_img = model.input
# Dimensions of the generated pictures for each filter.
img_width = 128
img_height = 128
# Compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, input_img)[0]
# Normalize the gradient
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
# This function returns the loss and grads given the input picture
iterate = K.function([input_img], [loss, grads])
# Step size for gradient ascent
step = 1
# Start with a random gray image
input_img_data = np.random.random((1, img_width, img_height, 3)) * 20 + 128
# run gradient ascent for 20 steps
for i in range(20):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
print('Current loss value:', loss_value)
if loss_value <= 0.:
# some filters get stuck to 0, we can skip them
break
# Append generated image
if loss_value > 0:
img = deprocess_image(input_img_data[0])
kept_filters.append((img, loss_value))
# Stich the best 16 filters on a 4 x 4 grid
n = 4
# The filters that have the highest loss are assumed to be better-looking.
# Keep the best 64 filters.
kept_filters.sort(key=lambda x: x[1], reverse=True)
kept_filters = kept_filters[:n * n]
# Build a black picture with enough space for
# all 4 x 4 filters of size 128 x 128, with a 5px margin in between
margin = 5
width = n * img_width + (n - 1) * margin
height = n * img_height + (n - 1) * margin
stitched_filters = np.zeros((width, height, 3))
# Fill the picture with the saved filters
for i in range(n):
for j in range(n):
img, loss = kept_filters[i * n + j]
stitched_filters[(img_width + margin) * i: (img_width + margin) * i + img_width,
(img_height + margin) * j: (img_height + margin) * j + img_height, :] = img
# Save the result to disk
imsave('mser_classifier_stitched_filters_%dx%d.png' % (n, n), stitched_filters)
def visualize_layer(self, layer, model, keep_filters, out_path, vis_size):
layer_output = layer.output
num_filters = layer.nb_filter
filters = []
img_width = vis_size[0]
img_height = vis_size[1]
for filter_index in xrange(num_filters):
loss = K.mean(K.mean(layer_output[:, filter_index, :, :]))
# compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, model.layers[0].input)[0]
# normalization trick: we normalize the gradient
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
# this function returns the loss and grads given the input picture
iterate = K.function([model.layers[0].input], [loss, grads])
# step size for gradient ascent
step = 1.
input_img_data = np.random.random((1, 1, img_width, img_height)) * 20 + 128.
for i in xrange(50):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
if loss_value <= 0:
break
img = self.deprocess_image(input_img_data[0])
filters.append((img, loss_value))
filters.sort(key=lambda x: x[1], reverse=True)
filters = filters[:keep_filters]
# get number of grid rows and columns
grid_r, grid_c = helpers.get_grid_dim(keep_filters)
# create figure and axes
fig, axes = plt.subplots(min([grid_r, grid_c]),
max([grid_r, grid_c]))
# iterate filters inside every channel
for l, ax in enumerate(axes.flat):
# get a single filter
img = filters[l][0][:, :, 0]
# put it on the grid
ax.imshow(img, interpolation='bicubic', cmap='Greys')
# remove any labels from the axes
ax.set_xticks([])
ax.set_yticks([])
# save figure
out_path = os.path.join(out_path, layer.name) + '.png'
plt.savefig(out_path, bbox_inches='tight')
"""
def label_reg_loss(y_true, y_pred): # KL-div y_true = K.clip(y_true, K.epsilon(), 1) y_pred = K.clip(y_pred, K.epsilon(), 1) y_true_mean = K.mean(y_true, axis=0) y_pred_mean = K.mean(y_pred, axis=0) return K.sum(y_true_mean * K.log(y_true_mean / y_pred_mean), axis=-1)
def nme(self, y_true, y_pred):
y_pred_reshape = K.reshape(y_pred, (-1, self.config.num_output//2, 2))
y_true_reshape = K.reshape(y_true, (-1, self.config.num_output//2, 2))
interocular_distance = K.sqrt(K.sum(
(K.mean(y_true_reshape[:, 36:42, :], axis=1) - K.mean(y_true_reshape[:, 42:48, :], axis=1)) ** 2, axis=-1))
return K.mean(K.sum(
K.sqrt(K.sum((y_pred_reshape - y_true_reshape) ** 2, axis=-1))
, axis=-1)) / K.mean((interocular_distance * self.config.num_output / 2))
def pearson_loss(y_true_rank, y_pred_rank, eps=1e-10):
y_true_mean = K.mean(y_true_rank)
y_pred_mean = K.mean(y_pred_rank)
u1 = (y_true_rank - y_true_mean)
u2 = (y_pred_rank - y_pred_mean)
u=K.sum(tf.multiply(u1,u2))
d=K.sqrt(K.sum(K.square(u1))*K.sum(K.square(u2)))
rou=tf.div(u,d+eps)
return 1.-rou
def focal_loss_fixed(y_true, y_pred): if(K.backend()=="tensorflow"): import tensorflow as tf pt = tf.where(tf.equal(y_true, 1), y_pred, 1 - y_pred) return -K.mean(alpha * K.pow(1. - pt, gamma) * K.log(pt)) if(K.backend()=="theano"): import theano.tensor as T pt = T.where(T.eq(y_true, 1), y_pred, 1 - y_pred) return -K.mean(alpha * K.pow(1. - pt, gamma) * K.log(pt))
def setup(self):
distorted_A, fake_A, fake_sz64_A, mask_A, self.path_A, self.path_mask_A, self.path_abgr_A, self.path_bgr_A = self.cycle_variables(self.model.netGA)
distorted_B, fake_B, fake_sz64_B, mask_B, self.path_B, self.path_mask_B, self.path_abgr_B, self.path_bgr_B = self.cycle_variables(self.model.netGB)
real_A = Input(shape=self.model.img_shape)
real_B = Input(shape=self.model.img_shape)
if self.use_lsgan:
self.loss_fn = lambda output, target : K.mean(K.abs(K.square(output-target)))
else:
self.loss_fn = lambda output, target : -K.mean(K.log(output+1e-12)*target+K.log(1-output+1e-12)*(1-target))
# ========== Define Perceptual Loss Model==========
if self.use_perceptual_loss:
from keras.models import Model
from keras_vggface.vggface import VGGFace
vggface = VGGFace(include_top=False, model='resnet50', input_shape=(224, 224, 3))
vggface.trainable = False
out_size55 = vggface.layers[36].output
out_size28 = vggface.layers[78].output
out_size7 = vggface.layers[-2].output
vggface_feat = Model(vggface.input, [out_size55, out_size28, out_size7])
vggface_feat.trainable = False
else:
vggface_feat = None
loss_DA, loss_GA = self.define_loss(self.model.netDA, real_A, fake_A, fake_sz64_A, distorted_A, vggface_feat)
loss_DB, loss_GB = self.define_loss(self.model.netDB, real_B, fake_B, fake_sz64_B, distorted_B, vggface_feat)
if self.use_mask_refinement:
loss_GA += 1e-3 * K.mean(K.square(mask_A))
loss_GB += 1e-3 * K.mean(K.square(mask_B))
else:
loss_GA += 3e-3 * K.mean(K.abs(mask_A))
loss_GB += 3e-3 * K.mean(K.abs(mask_B))
w_fo = 0.01
loss_GA += w_fo * K.mean(self.first_order(mask_A, axis=1))
loss_GA += w_fo * K.mean(self.first_order(mask_A, axis=2))
loss_GB += w_fo * K.mean(self.first_order(mask_B, axis=1))
loss_GB += w_fo * K.mean(self.first_order(mask_B, axis=2))
weightsDA = self.model.netDA.trainable_weights
weightsGA = self.model.netGA.trainable_weights
weightsDB = self.model.netDB.trainable_weights
weightsGB = self.model.netGB.trainable_weights
# Adam(..).get_updates(...)
training_updates = Adam(lr=self.lrD, beta_1=0.5).get_updates(weightsDA,[],loss_DA)
self.netDA_train = K.function([distorted_A, real_A],[loss_DA], training_updates)
training_updates = Adam(lr=self.lrG, beta_1=0.5).get_updates(weightsGA,[], loss_GA)
self.netGA_train = K.function([distorted_A, real_A], [loss_GA], training_updates)
training_updates = Adam(lr=self.lrD, beta_1=0.5).get_updates(weightsDB,[],loss_DB)
self.netDB_train = K.function([distorted_B, real_B],[loss_DB], training_updates)
training_updates = Adam(lr=self.lrG, beta_1=0.5).get_updates(weightsGB,[], loss_GB)
self.netGB_train = K.function([distorted_B, real_B], [loss_GB], training_updates)
def calc_loss(pred, target, loss='l2'):
""" Calculate Loss from Shoanlu GAN """
if loss.lower() == "l2":
return K.mean(K.square(pred - target))
if loss.lower() == "l1":
return K.mean(K.abs(pred - target))
if loss.lower() == "cross_entropy":
return -K.mean(K.log(pred + K.epsilon()) * target +
K.log(1 - pred + K.epsilon()) * (1 - target))
raise ValueError('Recieve an unknown loss type: {}.'.format(loss))
def pearsonobj(y_true, y_pred):
""" Pearson's r objective for STS grade correlation """
ny_true = _y2num(y_true)
ny_pred = _y2num(y_pred)
my_true = K.mean(ny_true)
my_pred = K.mean(ny_pred)
var_true = (ny_true - my_true)**2
var_pred = (ny_pred - my_pred)**2
return - K.sum((ny_true - my_true) * (ny_pred - my_pred), axis=-1) / \
(K.sqrt(K.sum(var_true, axis=-1) * K.sum(var_pred, axis=-1)))
def linear_correlation_loss(y_true, y_pred):
mean_y_true = K.mean(y_true)
mean_y_pred = K.mean(y_pred)
std_y_true = K.std(y_true)+1e-6
std_y_pred = K.std(y_pred)+1e-6
nSamples = K.shape(y_true)[0]
firstTerm = (y_true - mean_y_true)/std_y_true
secondTerm = (y_pred - mean_y_pred)/std_y_pred
pearsonCorr = K.sum(firstTerm*secondTerm)/(nSamples-1)
maeLoss = K.abs(y_true-y_pred)
return maeLoss*(1-K.maximum(0.,pearsonCorr))
def vae_loss(self, x, x_decoded_mean_squash, z_mean, z_log_var):
x = K.flatten(x)
x_decoded_mean_squash = K.flatten(x_decoded_mean_squash)
img_rows, img_cols = self._img_rows, self._img_cols
# generative or reconstruction loss
xent_loss = img_rows * img_cols * \
metrics.binary_crossentropy(x, x_decoded_mean_squash)
# Kullback-Leibler divergence loss
kl_loss = - 0.5 * K.mean(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return K.mean(xent_loss + kl_loss)
def call(self, x):
# statistics computed along features dimension, on every spatial position of the input tensor
A = K.mean(K.abs(x), axis = self.channel_axis) # mean absolute value
M1 = K.mean(x, axis = self.channel_axis) # mean value
M2 = K.mean(x**2, axis = self.channel_axis) # squared quadratic average
V = M2 - M1**2 # variance: V[X] = E[X^2] - E[X]^2
eps = 0.001 #K.epsilon()
norm = K.pow(V + eps, self.norm_dev/2) * K.pow(M2 + eps, self.norm_mag/2) * K.pow(A + eps, self.norm_abs)
return x / norm[...,None]
def call(self, x, mask=None):
if mask is not None:
mask = K.cast(mask, K.floatx())
mask = K.expand_dims(mask, axis=-1)
s = K.sum(mask, axis=1)
if K.equal(s, K.zeros_like(s)) is None:
return K.mean(x, axis=1)
else:
return K.cast(K.sum(x * mask, axis=1) / (K.sqrt(s) + K.constant(1e-10, dtype=K.floatx())), K.floatx())
else:
print (x)
return K.mean(x, axis=1)
def normals_metric(y_true, y_pred):
y_true = K.variable(y_true)
y_pred = K.variable(y_pred)
y_true = K.expand_dims(y_true,0)
filter_y = K.variable(np.array([[ 0., -0.5 , 0.],
[0., 0., 0.],
[0., 0.5, 0.]]).reshape(3, 3, 1, 1))
filter_x = K.variable(np.array([ [0, 0., 0.],
[0.5, 0., -0.5],
[0., 0., 0.]]).reshape(3, 3, 1, 1))
dzdx = K.conv2d(K.exp(y_true), filter_x, padding='same')
dzdy = K.conv2d(K.exp(y_true), filter_y, padding='same')
dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx))
dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy))
mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx))
mag_norm = K.sqrt(mag_norm)
N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm
N1 = dzdx_ / mag_norm
N2 = dzdy_ / mag_norm
normals = K.concatenate(tensors=[N1,N2,N3],axis=-1)
dzdx_pred = K.conv2d(K.exp(y_pred), filter_x, padding='same')
dzdy_pred = K.conv2d(K.exp(y_pred), filter_y, padding='same')
mag_norm_pred = K.pow(dzdx_pred,2) + K.pow(dzdy_pred,2) + 1.0
mag_norm_pred = K.sqrt(mag_norm_pred)
grad_x = K.concatenate(tensors=[1.0/ mag_norm_pred,
0.0/ mag_norm_pred, dzdx_pred/ mag_norm_pred],axis=-1)
grad_y = K.concatenate(tensors=[0.0/ mag_norm_pred,
1.0/ mag_norm_pred, dzdy_pred/ mag_norm_pred],axis=-1)
dot_term_x = K.mean(K.sum(normals[0,:,:,:] * grad_x[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_y = K.mean(K.sum(normals[0,:,:,:] * grad_y[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_x = K.abs(dot_term_x)
dot_term_y = K.abs(dot_term_y)
return K.eval(K.mean(dot_term_x)),K.eval(K.mean(dot_term_y))
def weighted(y_true, y_pred, weights, mask=None):
assert mask is None
assert weights is not None
score_array = fn(y_true, y_pred)
# reduce score_array to same ndim as weight array
ndim = K.ndim(score_array)
weight_ndim = K.ndim(weights)
score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim)))
# apply sample weighting
score_array *= weights
word_scores = K.sum(score_array, axis=-1)
return K.mean(word_scores)
def linear_correlation_loss(y_true, y_pred):
mean_y_true = K.mean(y_true)
mean_y_pred = K.mean(y_pred)
std_y_true = K.std(y_true)+1e-6
std_y_pred = K.std(y_pred)+1e-6
nSamples = K.shape(y_true)[0]
firstTerm = (y_true - mean_y_true)/std_y_true
secondTerm = (y_pred - mean_y_pred)/std_y_pred
pearsonCorr = K.sum(firstTerm*secondTerm)/(nSamples-1)
pearsonCorr = K.clip(pearsonCorr,-1.,1.)
maeLoss = K.mean(K.abs(y_true-y_pred))
# loss = 1./(0.1+K.exp(-0.5*K.log(maeLoss+(1-pearsonCorr))))
loss = (1./(0.1+K.exp(-0.5*K.log(maeLoss))))*(2-pearsonCorr)
return loss
def generate_pattern(modle, layer_name, filter_index, size=150): '''builds a loss func that maximizes the activation of the nth filter of the layer under consideration.''' layer_output = model.get_layer(layer_name).output loss = K.mean(layer_output[:, :, :, filter_index]) grads = K.gradients(loss, model.input)[0] grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) iterate = K.function([model.input], [loss, grads]) input_img_data = np.random.random((1, size, size, 3)) * 20 + 128. step = 1 for i in range(40): loss_value, grads_value = iterate([input_img_data]) input_img_data += grads_value * step img = input_img_data[0] return deprocess_image(img)
def obj_mix(Y_true,Y_pred):
"""
Calculates the binary cross entropy on the aggregated outputs of each
batch. Use the max when y_true == 1 but the mean when y_true== 0
"""
y_true = K.mean(Y_true,axis=0)
if y_true == 1:
y_pred = K.max(Y_pred,axis=0)
return(K.mean(K.binary_crossentropy(y_pred, y_true)))
elif y_true == 0:
return(K.mean(K.binary_crossentropy(Y_pred,Y_true)))
else:
print('unexpected value of y_true',y_true)
return(K.mean(K.binary_crossentropy(Y_pred,Y_true)))
def _build(self):
fake, _, _, g_additional_losses = self.g.run_internal_graph(self.g.inputs)
real = self.d.inputs[0]
data = concat([fake, real], axis=0)
realness, _, _, d_additional_losses = self.d.run_internal_graph(
[data] + self.d.inputs[1:])
nb_fakes = fake.shape[0]
fake_realness = realness[:nb_fakes]
real_realness = realness[nb_fakes:]
split = 2*nb_fakes // 3
g_fake_realness = fake_realness[:split]
d_fake_realness = fake_realness[split:]
outputs = OrderedDict()
g_loss = K.mean(K.binary_crossentropy(g_fake_realness, K.ones_like(real_realness)))
outputs['g_loss'] = g_loss
g_reg_loss = sum([v for v in g_additional_losses.values()])
if g_reg_loss != 0:
outputs['g_reg_loss'] = g_reg_loss
g_total_loss = g_loss + g_reg_loss
d_loss = K.mean(K.binary_crossentropy(real_realness, K.ones_like(real_realness)))
d_loss += K.mean(K.binary_crossentropy(d_fake_realness, K.zeros_like(real_realness)))
outputs['d_loss'] = d_loss
d_reg_loss = sum([v for v in d_additional_losses.values()])
if d_reg_loss != 0:
outputs['d_reg_loss'] = d_reg_loss
d_total_loss = d_loss + d_reg_loss
inputs = {i.name: i for i in self.g.inputs + self.d.inputs}
inputs_list = []
for name in self.input_names:
inputs_list.append(inputs[name])
g_updates = self.g_optimizer.get_updates(
collect_trainable_weights(self.g), self.g.constraints, g_total_loss)
d_updates = self.d_optimizer.get_updates(
collect_trainable_weights(self.d), self.d.constraints, d_total_loss)
if self.uses_learning_phase:
lr_phase = [K.learning_phase()]
else:
lr_phase = []
self.metrics_names = list(outputs.keys())
self._train_function = K.function(inputs_list + lr_phase, list(outputs.values()),
updates=g_updates + d_updates)
def total_loss(y_true, y_pred): output_list = [] temp = [] num_post = [1]*self.dim_feature for i in range(self.dim_feature): temp.append(y_pred[:,i*self.dim_feature+i]) p0 = tf.transpose(tf.stack(temp)) for j in range(len(yy)): temp[yy[j]] += y_pred[:,xx[j]*self.dim_feature+yy[j]] num_post[yy[j]] += 1 for k in range(len(num_post)): temp[k] /= num_post[k] output_list = temp p = tf.transpose(tf.stack(output_list)) return 100*(K.mean(K.square(K.mean(p, axis = 0)-self.center_prob),axis = -1)-self.alpha*K.mean(K.log(K.abs(p-self.center_prob)+self.relax), axis = -1)+self.beta*K.mean(K.square(p0-p), axis = -1))
def binary_crossentropy_with_ranking(y_true, y_pred):
""" Trying to combine ranking loss with numeric precision"""
# first get the log loss like normal
logloss = K.mean(K.binary_crossentropy(y_pred, y_true), axis=-1)
# next, build a rank loss
# clip the probabilities to keep stability
y_pred_clipped = K.clip(y_pred, K.epsilon(), 1-K.epsilon())
# translate into the raw scores before the logit
y_pred_score = K.log(y_pred_clipped / (1 - y_pred_clipped))
# determine what the maximum score for a zero outcome is
y_pred_score_zerooutcome_max = K.max(y_pred_score * (y_true <1))
# determine how much each score is above or below it
rankloss = y_pred_score - y_pred_score_zerooutcome_max
# only keep losses for positive outcomes
rankloss = rankloss * y_true
# only keep losses where the score is below the max
rankloss = K.square(K.clip(rankloss, -100, 0))
# average the loss for just the positive outcomes
rankloss = K.sum(rankloss, axis=-1) / (K.sum(y_true > 0) + 1)
# return (rankloss + 1) * logloss - an alternative to try
return rankloss + logloss
def recall_loss(y_true, y_pred):
'''
input: y_true (theano Tensor), y_pred (theano Tensor)
output: recall_loss (float)
'''
# print(K.ndim(y_true), K.ndim(y_pred))
return -np.log(K.mean(K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1))))
def kl_loss(truth_dummy, x_mean_log_var_output):
x_mean, x_log_var = tf.split(x_mean_log_var_output, 2, axis=1)
print('x_mean shape in kl_loss: ', x_mean.get_shape())
kl_loss = - 0.5 * \
K.mean(1 + x_log_var - K.square(x_mean) -
K.exp(x_log_var), axis=-1)
return kl_loss
def accuracy(y_true, y_pred):
return K.mean(K.equal(y_true, K.cast(y_pred < 0.5, y_true.dtype)))
def style_transfer(content_image_path, style_image_path):
mydir = './Outputs/style_transfer_details'
# 1: load the content and style images, then rescale the style image to the scale of content image
content_img = Tools.load_img_and_preprocess_resize(content_image_path, resize=512)
h, w = content_img.shape[1:3]
# test_content_reconstruction(content_img[0], mydir, [16, 17, 18, 19], 0)
style_img = Tools.load_img_and_preprocess_shape(style_image_path, (h, w))
# show all blocks output
# test_style_reconstruction(style_img[0], mydir, 1)
batch_shape = content_img.shape
shape = content_img.shape[1:]
vgg = Tools.VGG19_AvgPool(shape)
print(vgg.summary())
# 2: get content and style features + features extractor model
content_features, content_features_extractor_model = ContentReconstruction.get_content_image_features(content_img,
14, vgg)
style_layers_features_outputs, symbolic_conv_outputs, style_features_extractor_model = StyleReconstruction.get_style_image_features(
style_img, 5, vgg)
# we will assume the weight of the content loss is 1
# and only weight the style losses
style_weights = [0.2, 0.4, 0.3, 0.5, 0.2]
# style_weights = [0.4, 0.6, 0.6, 0.7, 0.4]
# create the total loss which is the sum of content + style loss
loss = 1 * K.mean(K.square(content_features_extractor_model.output - content_features))
for w, symbolic, actual in zip(style_weights, symbolic_conv_outputs, style_layers_features_outputs):
# gram_matrix() expects a (H, W, C) as input
loss += w * Tools.style_loss(symbolic[0], actual[0])
# loss += 0.0001 * tf.image.total_variation(vgg.input)
# once again, create the gradients and loss + grads function
# note: it doesn't matter which model's input you use
# they are both pointing to the same keras Input layer in memory
grads = K.gradients(loss, vgg.input)
# just like theano.function
get_loss_and_grads = K.function(
inputs=[vgg.input],
outputs=[loss] + grads
)
def get_loss_and_grads_wrapper(x_vec):
l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)])
return l.astype(np.float64), g.flatten().astype(np.float64)
final_image, losses = Tools.LBFGS_Optimizer(get_loss_and_grads_wrapper, 10, batch_shape)
# plot loss
plt.plot(losses)
# plt.savefig(plot_name)
plt.show()
# save image
final_image = Tools.scale_img(final_image)
plt.imshow(final_image)
# plt.imsave(file_name, final_image)
plt.show()
def loss(chain_start, x):
x_rec, _, _ = self.mcmc_chain(chain_start, self.nb_gibbs_steps)
cd = K.mean(self.free_energy(x)) - K.mean(self.free_energy(x_rec))
return cd, x_rec
def build_decoder(self, latent, name='conv', n_layers=1):
decoder = latent
if name == 'conv':
if n_layers == 3:
decoder = Dense(self.window_size // 64 * 256,
activation='relu',
name='decoder_dense1')(decoder)
decoder = Reshape((self.window_size // 64, 256),
name='decoder_reshape1')(decoder)
decoder = UpSampling1D(4, name='decoder_upsample1')(decoder)
decoder = Conv1D(128,
3,
padding='same',
activation='relu',
name='decoder_conv1')(decoder)
decoder = UpSampling1D(4, name='decocer_upsample2')(decoder)
decoder = Conv1D(64,
3,
padding='same',
activation='relu',
name='decoder_conv2')(decoder)
if n_layers == 2:
decoder = Dense(self.window_size // 16 * 128,
activation='relu',
name='decoder_dense1')(decoder)
decoder = Reshape((self.window_size // 16, 128),
name='decoder_reshape1')(decoder)
decoder = UpSampling1D(4, name='decocer_upsample2')(decoder)
decoder = Conv1D(64,
3,
padding='same',
activation='relu',
name='decoder_conv2')(decoder)
if n_layers == 1:
decoder = Dense(self.window_size // 4 * 64,
activation='relu',
name='decoder_dense1')(decoder)
decoder = Reshape((self.window_size // 4, 64),
name='decoder_reshape1')(decoder)
decoder = UpSampling1D(4, name='decoder_upsample3')(decoder)
decoder = Conv1D(4, 1, padding='same',
name='decoder_conv3')(decoder)
decoder = Lambda(lambda x: K.softmax(x, axis=-1),
name='output_softmax')(decoder)
decoder = Lambda(lambda x: K.mean(K.reshape(
x, (-1, self.n_sampler, self.window_size, self.n_channels)),
axis=1),
name='output_mean')(decoder)
elif name == 'mlp':
if n_layers >= 2:
decoder = Dense(128, activation='relu',
name='decoder_dense2')(decoder)
decoder = Dense(self.window_size * self.n_channels,
name='decoder_dense3')(decoder)
decoder = Lambda(lambda x: K.softmax(x, axis=-1),
name='output_softmax')(decoder)
decoder = Lambda(lambda x: K.mean(K.reshape(
x, (-1, self.n_sampler, self.window_size, self.n_channels)),
axis=1),
name='output_mean')(decoder)
elif name == 'lstm':
decoder = LSTM(64, name='encoder_lstm1',
return_sequences=True)(decoder)
return decoder
def mae_tol(y_true,y_pred):
return K.mean( K.maximum(K.abs(y_true - y_pred) - tol, 0) )
def earth_mover_loss(y_true, y_pred):
cdf_ytrue = K.cumsum(y_true, axis=-1)
cdf_ypred = K.cumsum(y_pred, axis=-1)
samplewise_emd = K.sqrt(
K.mean(K.square(K.abs(cdf_ytrue - cdf_ypred)), axis=-1))
return K.mean(samplewise_emd)
x = Dense(intermediate_dim, activation='relu')(latent_inputs)
outputs = Dense(original_dim, activation='sigmoid')(x)
# instantiate decoder model
decoder = Model(latent_inputs, outputs, name='decoder')
# instantiate VAE model
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae_mlp')
models = (encoder, decoder)
data = (x_test, y_test)
reconstruction_loss = mse(inputs, outputs)
reconstruction_loss *= original_dim
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
vae.load_weights("vae_mlp_mnist_D_" + str(latent_dim) + ".h5")
### find the latent encoding (i.e., the mean vector) of the target image
z_mean, _, _ = encoder.predict(x_test[target_img_ind, :, :, 0].reshape(1, -1),
batch_size=batch_size)
print("z_mean: ", z_mean.shape)
dim_high = 28
# eps defines the search space for the perturbations with bounded inf norm
eps = 0.2
target_class_ind = 0 # The target class if targeted-attack is used; it is not used in the current versionn, but can be easily adapted
def cosine_distance(left, right):
left = K.l2_normalize(left, axis=-1)
right = K.l2_normalize(right, axis=-1)
return -K.mean(left * right, axis=-1, keepdims=True)
def weighted_loss(y_true, y_pred):
weights = y_true * weight_of_ones
clipped_y_pred = K.clip(y_pred, K.epsilon(), None)
weighted_cross_entropy = -(y_true * K.log(clipped_y_pred) * weights)
result = K.mean(weighted_cross_entropy)
return result
def weighted_accuracy(y_true, y_pred):
weights = y_true * weight_of_ones
weighted_equal = K.cast(K.equal(y_true, K.round(y_pred)),
K.floatx()) * weights
return K.mean(weighted_equal)
def _huber_loss(self, target, prediction):
# sqrt(1+error^2)-1
error = prediction - target
return K.mean(K.sqrt(1 + K.square(error)) - 1, axis=-1)
def loss(y_true, y_pred):
return 1. / (image_sigma**2) * K.mean(K.square(y_true - y_pred))
def vae_r_loss(y_true, y_pred):
y_true_flat = K.flatten(y_true)
y_pred_flat = K.flatten(y_pred)
return 10 * K.mean(K.square(y_true_flat - y_pred_flat), axis = -1)
def negGrowthRateLoss(b, q):
""" Customized loss function. """
return (K.mean(-K.log(b + pow(-1, b) + pow(-1, b + 1) * q) / K.log(2.0)))
def normalization(x):
mean = KR.mean(x**2)
return x / KR.sqrt(2 * mean) # 2 = number of NN into the channel
def normalize(x):
# utility function to normalized a tensor by its L2 norm
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
print("\n\tDECODER STRUCTURE")
decoder.summary()
#**********AUTOENCODER STRUCTURE***************
out = decoder(encoder(inputs)[2])
vae = Model(inputs, out, name='vae')
vae.summary()
# Loss definition
rec_loss = mse(K.flatten(inputs), K.flatten(out))
rec_loss *= original_dim
KL_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
KL_loss = K.sum(KL_loss, axis=-1)
KL_loss *= -0.5
vae_loss = K.mean(rec_loss + KL_loss) / original_dim
vae.add_loss(vae_loss)
my_opt = Adam(lr=1e-4)
vae.compile(optimizer=my_opt)
def my_generator(input_data, target, batch_size):
# ----The generator for the train set----#
while True:
# Create empty arrays to contain batch of features and labels #
batch_data = np.zeros(
(batch_size, dataset.shape[1], dataset.shape[2], 1))
batch_target = np.zeros(
(batch_size, dataset.shape[1], dataset.shape[2], 1))
def filter_visualisation(iterations=20,
weights_path='imagenet',
layer='block5_conv1',
max_filters=200,
filters=8,
output_shape=224):
""" Visualise convolutional filters of a CNN model by maximising their activations with gradient ascent.
Args:
iterations -- number of iterations of the gradient ascent procedure (Default: 20)
weights_path -- path to the Keras CNN model (Default: imagenet model in Keras models directory)
layer -- the network layer name whose activations are being shown. Use model.summary() to get names
max_filters -- number of filters to be checked in layer
filters -- determine the number of filters to be displayed (total: filters x filters)
output_shape -- height and width of each output image showing activations (arranged in a grid)
"""
img_width, img_height = output_shape, output_shape
filter_indexes = range(0, max_filters)
# By default, load the VGG16 network with ImageNet weights
if weights_path == 'imagenet':
model = vgg16.VGG16(weights=weights_path, include_top=False)
else:
model = load_model(weights_path)
print('Model loaded.')
# Display a summary of all the blocks
model.summary()
# This is the placeholder for the input images
input_img = model.input
# Get the symbolic outputs of each "key" layer (we gave them unique names)
layer_dict = dict([(layer.name, layer) for layer in model.layers[0:]])
kept_filters = []
for filter_index in filter_indexes:
# To speed up, scan only the first max_filters
print('Processing filter %d' % filter_index)
start_time = time.time()
# Build a loss function that maximizes the activation of the nth filter of the layer considered
layer_output = layer_dict[layer].output
if K.image_data_format() == 'channels_first':
loss = K.mean(layer_output[:, filter_index, :, :])
else:
loss = K.mean(layer_output[:, :, :, filter_index])
# Compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, input_img)[0]
def normalize(x):
# Utility function to normalize a tensor by its L2 norm
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
# Normalization trick: we normalize the gradient
grads = normalize(grads)
# This function returns the loss and grads given the input picture
iterate = K.function([input_img], [loss, grads])
# Step size for gradient ascent
step = 1.
# Start from a gray image with some random noise
if K.image_data_format() == 'channels_first':
input_img_data = np.random.random((1, 3, img_width, img_height))
else:
input_img_data = np.random.random(
(1, img_width, img_height, model.input.shape[-1]))
input_img_data = (input_img_data - 0.5) * 20 + 128
# Run gradient ascent for 20 steps
for i in range(iterations):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
print('Current loss value:', loss_value)
if loss_value <= 0.:
# some filters get stuck to 0, we can skip them
break
# Decode the resulting input image
if loss_value > 0:
img = deprocess_image(input_img_data[0])
kept_filters.append((img, loss_value))
end_time = time.time()
print('Filter %d processed in %ds' %
(filter_index, end_time - start_time))
# Stich the best filters**2 filters on a filters x filters grid
n = filters
if filters**2 > len(kept_filters):
n = int(np.floor(np.sqrt(len(kept_filters))))
else:
n = int(filters)
# Filters that have the highest loss are assumed to be better-looking.
kept_filters.sort(key=lambda x: x[1], reverse=True)
kept_filters = kept_filters[:n * n]
# Black picture with space for filters**2 filters with size output_shape x output_shape, 5px margin in between
margin = 5
width = n * img_width + (n - 1) * margin
height = n * img_height + (n - 1) * margin
stitched_filters = np.zeros((width, height, 3))
# Fill the picture with our saved filters
for i in range(n):
for j in range(n):
img, loss = kept_filters[i * n + j]
stitched_filters[(img_width + margin) *
i:(img_width + margin) * i + img_width,
(img_height + margin) *
j:(img_height + margin) * j + img_height, :] = img
# Save the result to disk
imsave('stitched_filters_%dx%d.png' % (n, n), stitched_filters)
return stitched_filters
def myloss(y_true, y_pred):
return K.mean(K.square((y_pred - y_true) / y_true))
def wasserstein_loss(self, y_true, y_pred):
return K.mean(y_true * y_pred)
def free_energy_gap(self, x_train, x_test):
return K.mean(self.free_energy(x_train)) - K.mean(
self.free_energy(x_test))
def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true)))
def mean_error(y_true, y_pred):
return K.mean(y_true - y_pred)
def mse(y_true, y_pred):
return K.mean(K.square((y_pred - avg_of_play_no_noise) - y_true))
def mseT(y_true, y_pred):
return K.mean(K.square(y_pred - y_true), axis=(-4, -1))
def wasserstein(self, y_true, y_pred):
return -K.mean(y_true * y_pred)
def call(self, x):
mean = K.mean(x, axis=self.axis, keepdims=True)
std = K.std(x, axis=self.axis, keepdims=True) + self.epsilon
return (x - mean) / std
# 解码层，也就是生成器部分
decoder_h = Dense(intermediate_dim, activation='relu')
decoder_mean = Dense(original_dim, activation='sigmoid')
h_decoded = decoder_h(z)
x_decoded_mean = decoder_mean(h_decoded) # 估计的是每一个变量维度输出的均值
# 建立模型
vae = Model([x, y], [x_decoded_mean, yh])
# xent_loss是重构loss，kl_loss是KL loss
xent_loss = original_dim * metrics.binary_crossentropy(x, x_decoded_mean)
# 只需要修改K.square(z_mean)为K.square(z_mean - yh)，也就是让隐变量向类内均值看齐
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean - yh) - K.exp(z_log_var), axis=-1)
vae_loss = K.mean(xent_loss + kl_loss)
# add_loss是新增的方法，用于更灵活地添加各种loss
vae.add_loss(vae_loss)
vae.compile(optimizer='rmsprop')
vae.summary()
vae.fit([x_train, y_train],
shuffle=True,
epochs=epochs,
batch_size=batch_size,
validation_data=([x_test, y_test], None))
# 构建encoder，然后观察各个数字在隐空间的分布
encoder = Model(x, z_mean)
def mase_loss(y_true, y_pred):
return K.mean(
K.abs(y_true - y_pred) /
K.mean(K.abs(y_true - tf.gather(batches, global_step))))
def vae_kl_loss(y_true, y_pred):
return - 0.5 * K.mean(1 + vae_z_log_var - K.square(vae_z_mean) - K.exp(vae_z_log_var), axis = -1)
