def __init__(self, model, layer_name):
self.model = model
self.layer_name = layer_name
dream = model.input
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layers_all = [layer.name for layer in model.layers]
if layer_name not in layers_all:
raise ValueError('Layer ' + layer_name + ' not found in model.')
# Define the loss.
loss = K.variable(0.)
for layer_local in model.layers:
if layer_local.name == layer_name:
x = layer_local.output
# We avoid border artifacts by only involving non-border pixels in the loss.
if K.image_data_format() == 'channels_first':
scaling = K.prod(K.cast(K.shape(x), 'float32'))
loss = loss + K.sum(K.square(x[:, :, 2:-2,
2:-2])) / scaling
else:
scaling = K.prod(K.cast(K.shape(x), 'float32'))
loss = loss + K.sum(K.square(x[:, 2:-2,
2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
self.fetch_loss_and_grads = K.function([dream], outputs)
def compute_output_shape(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape).as_list()
if None not in input_shape[1:]:
if self.data_format == "channels_first":
total = K.prod(input_shape[2:4]) * self.num_anchors
else:
total = K.prod(input_shape[1:3]) * self.num_anchors
total = K.get_value(total)
return tensor_shape.TensorShape((input_shape[0], total, 4))
return tensor_shape.TensorShape((input_shape[0], None, 4))
def compute_output_shape(self, input_shape):
boxes_shape, other_shape = input_shape
if len(boxes_shape) == 3:
output_shape = tuple(
list(boxes_shape[:2]) +
[K.prod([s for s in other_shape[2:]]) + 4])
return tensor_shape.TensorShape(output_shape)
elif len(boxes_shape) == 4:
output_shape = tuple(
list(boxes_shape[:3]) +
[K.prod([s for s in other_shape[3:]]) + 4])
return tensor_shape.TensorShape(output_shape)
def compute_output_shape(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape).as_list()
if None not in input_shape[1:]:
if self.data_format == "channels_first":
total = K.prod(input_shape[2:4]) * self.num_anchors
else:
total = K.prod(input_shape[1:3]) * self.num_anchors
# TODO: fails time_distributed tests for RetinaMask
# AttributeError: 'Tensor' object has no attribute 'numpy'
total = K.get_value(total)
return tensor_shape.TensorShape((input_shape[0], total, 4))
return tensor_shape.TensorShape((input_shape[0], None, 4))
def define_deepDream_model_layerBased(model):
dream = model.input
print('Model loaded.')
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layer_dict = dict([(layer.name, layer) for layer in model.layers])
# Define the loss.
loss = K.variable(0.)
for layer_name in settings['features']:
# Add the L2 norm of the features of a layer to the loss.
if layer_name not in layer_dict:
raise ValueError('Layer ' + layer_name + ' not found in model.')
coeff = settings['features'][layer_name]
x = layer_dict[layer_name].output
# We avoid border artifacts by only involving non-border pixels in the loss.
scaling = K.prod(K.cast(K.shape(x), 'float32'))
if K.image_data_format() == 'channels_first':
loss = loss + coeff * K.sum(K.square(x[:, :, 2:-2,
2:-2])) / scaling
else:
loss = loss + coeff * K.sum(K.square(x[:, 2:-2,
2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
fetch_loss_and_grads = K.function([dream], outputs)
def training_phase():
mean_batch = K.mean(mean_instance, axis=0, keepdims=True)
variance_batch = K.mean(temp, axis=0,
keepdims=True) - K.square(mean_batch)
mean_batch_reshaped = K.flatten(mean_batch)
variance_batch_reshaped = K.flatten(variance_batch)
if K.backend() != 'cntk':
sample_size = K.prod(
[K.shape(inputs)[axis] for axis in reduction_axes])
sample_size = K.cast(sample_size, dtype=K.dtype(inputs))
# sample variance - unbiased estimator of population variance
variance_batch_reshaped *= sample_size / (sample_size -
(1.0 + self.epsilon))
self.add_update([
K.moving_average_update(self.moving_mean, mean_batch_reshaped,
self.momentum),
K.moving_average_update(self.moving_variance,
variance_batch_reshaped, self.momentum)
], inputs)
return normalize_func(mean_batch, variance_batch)
def gram_matrix(self, X):
"""グラム行列の算出"""
X_sw = K.permute_dimensions(
X, (0, 3, 2, 1)
) # 軸の入れ替え
s = K.shape(X_sw)
new_shape = (s[0], s[1], s[2]*s[3])
X_rs = K.reshape(X_sw, new_shape)
X_rs_t = K.permute_dimensions(
X_rs, (0, 2, 1)
) # 行列の転置
dot = K.batch_dot(X_rs, X_rs_t) # 内積の計算
norm = K.prod(K.cast(s[1:], 'float32'))
return dot/norm
def gram_matrix(X):
# 軸の入れ替え => batch, channel, height, width
axis_replaced_X = K.permute_dimensions(X, (0, 3, 2, 1))
replaced_shape = K.shape(axis_replaced_X)
# 特徴マップ（高さと幅を1つの軸に展開）の内積をとるためのshape
dot_shape = (replaced_shape[0], replaced_shape[1],
replaced_shape[2] * replaced_shape[3])
# 実際に内積を計算する行列
dot_X = K.reshape(axis_replaced_X, dot_shape)
# 転置行列
dot_X_t = K.permute_dimensions(dot_X, (0, 2, 1))
# 行列の内積
dot = K.batch_dot(dot_X, dot_X_t)
norm = K.prod(K.cast(replaced_shape[1:], 'float32'))
return dot / norm
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if len(input_shape) != 5:
raise ValueError('Inputs should have rank 5, '
'received input shape: %s' % input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape.dims[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined, found None: %s' % input_shape)
input_dim = int(input_shape[channel_axis])
self.input_spec = InputSpec(ndim=5, axes={channel_axis: input_dim})
if self.data_format == 'channels_first':
depth = int(input_shape[2])
else:
depth = int(input_shape[1])
kernel_shape = (depth, self.filter_size, self.filter_size, input_dim,
1)
# self.kernel = self.add_weight(
# 'kernel',
# shape=kernel_shape,
# initializer=self.kernel_initializer,
# regularizer=self.kernel_regularizer,
# constraint=self.kernel_constraint,
# trainable=False,
# dtype=self.dtype)
W = K.ones(kernel_shape, dtype=K.floatx())
W = W / K.cast(K.prod(K.int_shape(W)), dtype=K.floatx())
self.kernel = W
# self.set_weights([W])
if self.use_bias:
self.bias = self.add_weight('bias',
shape=(depth, self.filter_size,
self.filter_size),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=False,
dtype=self.dtype)
else:
self.bias = None
self.built = True
def compute_output_shape(self, input_shape):
boxes_shape, other_shape = input_shape
n = len(boxes_shape) - 1
output_shape = tuple(
list(boxes_shape[:n]) + [K.prod([s for s in other_shape[n:]]) + 4])
return tensor_shape.TensorShape(output_shape)
def contents_feature_loss(y_contents, contents_pred):
norm = K.prod(K.cast(K.shape(y_contents)[1:], 'float32'))
# 二乗誤差
return K.sum(K.square(contents_pred - y_contents), axis=(1, 2, 3)) / norm
def __init__(self,
model,
layer_name,
index_feature,
kind_of_optim='squared'):
"""
Initialisation function of the Optimization of the feature map
Parameters
----------
model : keras model
layer_name : string : the layer you want to use
index_feature : string : the index of the feature in the given layer
kind_of_optim : string
The kind of optimisation maded on the given feature. The default is 'squared'.
Use 'pos' for the positive feature maximisation and 'neg' for the negative one
Returns
-------
None
"""
self.model = model
self.layer_name = layer_name
self.index_feature = index_feature
self.kind_of_optim = kind_of_optim
dream = model.input
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layers_all = [layer.name for layer in model.layers]
if layer_name not in layers_all:
raise ValueError('Layer ' + layer_name + ' not found in model.')
# Define the loss.
loss = K.variable(0.)
for layer_local in model.layers:
if layer_local.name == layer_name:
x = layer_local.output
# We avoid border artifacts by only involving non-border pixels in the loss.
if K.image_data_format() == 'channels_first':
raise (NotImplementedError)
x_index_feature = x[:, index_feature, :, :]
x_index_feature = K.expand_dims(x_index_feature, axis=1)
scaling = K.prod(
K.cast(K.shape(x_index_feature), 'float32'))
loss = loss + K.sum(
K.square(x_index_feature[:, :, 2:-2, 2:-2])) / scaling
else:
x_index_feature = x[:, :, :, index_feature]
x_index_feature = K.expand_dims(x_index_feature, axis=-1)
scaling = K.prod(
K.cast(K.shape(x_index_feature), 'float32'))
if self.kind_of_optim == 'squared':
loss = loss + K.sum(
K.square(x_index_feature[:, 2:-2,
2:-2, :])) / scaling
elif self.kind_of_optim == 'pos':
loss = loss + K.sum(x_index_feature[:, 2:-2,
2:-2, :]) / scaling
elif self.kind_of_optim == 'neg':
loss = loss - K.sum(x_index_feature[:, 2:-2,
2:-2, :]) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
self.fetch_loss_and_grads = K.function([dream], outputs)
def feature_loss(self, y_true, y_pred):
"""コンテンツ特徴量の損失関数"""
norm = K.prod(K.cast(K.shape(y_true)[1:], 'float32'))
return K.sum(
K.square(y_pred - y_true), axis=(1, 2, 3)
)/norm
def msssim(self, y_true, y_pred):
'''
Compute multiscale ssim according to Zhao 2016.
Has only been tested with tensorflow backend (channels last) so far!
Uses convolutions to do the calculations in one go.
This function takes proper 2D Keras Tensors (NWHC or NCWH)
# Arguments
y_true: Keras Tensor with Rank 4: Image to compare to
y_pred: Keras Tensor with Rank 4: Image to compare
'''
# some useful inits
channels = self.__int_shape(y_pred)[self.channel_dim]
# repeat kernel for each channel
kernel = K.tile(self.kernels, [1, 1, channels, 1])
# compute means
mu_true = K.depthwise_conv2d(y_true, kernel, padding='same')
mu_pred = K.depthwise_conv2d(y_pred, kernel, padding='same')
# compute mean squares
mu_true_sq = K.square(mu_true)
mu_pred_sq = K.square(mu_pred)
mu_true_pred = mu_true * mu_pred
# compute input square
y_true_sq = K.square(y_true)
y_pred_sq = K.square(y_pred)
y_true_pred = y_true * y_pred
# compute variances/covariance
sigma_true_sq = K.depthwise_conv2d(y_true_sq, kernel, padding='same')
sigma_pred_sq = K.depthwise_conv2d(y_pred_sq, kernel, padding='same')
sigma_true_pred = K.depthwise_conv2d(y_true_pred, kernel, padding='same')
# centered squares of variances
sigma_true_sq -= mu_true_sq
sigma_pred_sq -= mu_pred_sq
sigma_true_pred -= mu_true_pred
# compute luminance term (l), select only maximum kernel for each channel
l = (2 * mu_true_pred + self.c1) / (mu_true_sq + mu_pred_sq + self.c1)
if self.dim_ordering == 'channels_last':
l_max = l[:,:,:,(self.num - 1)::self.num]
else:
l_max = l[:,(self.num - 1)::self.num,:,:]
# compute contrast-structure term (cs)
cs = (2 * sigma_true_pred + self.c2) / (sigma_true_sq + sigma_pred_sq +
self.c2)
# compute product of different scale cs
if self.dim_ordering == 'channels_last':
pcs = [K.prod(cs[:,:,:,i*self.num:(i+1)*self.num], axis=-1,
keepdims=True) for i in range(channels)]
else:
pcs = [K.prod(cs[:,i*self.num:(i+1)*self.num,:,:], axis=1,
keepdims=True) for i in range(channels)]
pcs = K.concatenate(pcs, axis=self.channel_dim)
# compute msssim map
msssim = l_max * pcs # do normalization?
return msssim
