def call(self, inputs):
input_shape = self.in_shape
if self.data_format == 'channels_first':
x = K.arange(0, input_shape[1], dtype=K.floatx())
y = K.arange(0, input_shape[2], dtype=K.floatx())
else:
x = K.arange(0, input_shape[0], dtype=K.floatx())
y = K.arange(0, input_shape[1], dtype=K.floatx())
x = x / K.max(x)
y = y / K.max(y)
loc_x, loc_y = tf.meshgrid(x, y, indexing='ij')
if self.data_format == 'channels_first':
loc = K.stack([loc_x, loc_y], axis=0)
else:
loc = K.stack([loc_x, loc_y], axis=-1)
location = K.expand_dims(loc, axis=0)
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 2, 3, 1])
location = tf.tile(location, [K.shape(inputs)[0], 1, 1, 1])
if self.data_format == 'channels_first':
location = K.permute_dimensions(location, pattern=[0, 3, 1, 2])
return location
def call(self, x, mask=None):
'''
shape=(batch_size,new_time_step,filters)
x_cont=Tensor("layer_dropout_5/cond/Identity:0", shape=(None, None, 128), dtype=float32)
x_ques=Tensor("layer_dropout_11/cond/Identity:0", shape=(None, None, 128), dtype=float32)
c_mask=Tensor("batch_slice_4/Slice:0", shape=(None, None), dtype=bool)#
q_mask=Tensor("batch_slice_5/Slice:0", shape=(None, None), dtype=bool)
'''
x_cont, x_ques, c_mask, q_mask = x
# get similarity matrix S
##K.dot(x_cont, self.W0)维度变化： [batch_size,time_step,dim] *[dim,1] =[batch_size,time_step,1]
subres0 = K.tile(K.dot(x_cont, self.W0), [1, 1, self.q_maxlen])
subres1 = K.tile(
K.permute_dimensions(K.dot(x_ques, self.W1), pattern=(0, 2, 1)),
[1, self.c_maxlen, 1])
subres2 = K.batch_dot(x_cont * self.W2,
K.permute_dimensions(x_ques, pattern=(0, 2, 1)))
S = subres0 + subres1 + subres2
S += self.bias
q_mask = tf.expand_dims(q_mask, 1)
#默认是对最后一维度，即axis=-1
S_ = tf.nn.softmax(self.mask_logits(S, q_mask))
c_mask = tf.expand_dims(c_mask, 2)
S_T = K.permute_dimensions(
tf.nn.softmax(self.mask_logits(S, c_mask), axis=1), (0, 2, 1))
c2q = tf.matmul(S_, x_ques)
q2c = tf.matmul(tf.matmul(S_, S_T), x_cont)
result = K.concatenate([x_cont, c2q, x_cont * c2q, x_cont * q2c],
axis=-1)
return result
def gram_matrix(x, norm_by_channels=False):
'''
Returns the Gram matrix of the tensor x.
'''
if K.ndim(x) == 3:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
shape = K.shape(x)
C, H, W = shape[0], shape[1], shape[2]
gram = K.dot(features, K.transpose(features))
elif K.ndim(x) == 4:
# Swap from (H, W, C) to (B, C, H, W)
x = K.permute_dimensions(x, (0, 3, 1, 2))
shape = K.shape(x)
B, C, H, W = shape[0], shape[1], shape[2], shape[3]
# Reshape as a batch of 2D matrices with vectorized channels
features = K.reshape(x, K.stack([B, C, H * W]))
# This is a batch of Gram matrices (B, C, C).
gram = K.batch_dot(features, features, axes=2)
else:
raise ValueError(
'The input tensor should be either a 3d (H, W, C) or 4d (B, H, W, C) tensor.'
)
# Normalize the Gram matrix
if norm_by_channels:
denominator = C * H * W # Normalization from Johnson
else:
denominator = H * W # Normalization from Google
gram = gram / K.cast(denominator, x.dtype)
return gram
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
#xw = K.reshape(K.dot(x[0], K.reshape(self.W, (features_dim, features_dim))), (-1, features_dim))
#yavg=K.reshape(K.mean(K.mean(x[1], axis=1, keepdims=True),axis=0, keepdims=True), (features_dim,-1))
xw1 = K.dot(x[0], K.reshape(self.W1, (features_dim, features_dim)))
xw2 = K.dot(x[1], K.reshape(self.W2, (features_dim, features_dim)))
xw1t = K.permute_dimensions(xw1, [0, 2, 1])
xw2t = K.permute_dimensions(xw2, [0, 2, 1])
xw11 = K.batch_dot(xw1, xw1t) / (step_dim**0.5)
xw12 = K.batch_dot(xw1, xw2t) / (step_dim**0.5)
s11 = self.ll * K.softmax(xw11)
s12 = (1 - self.ll) * K.softmax(xw12)
eij = s11 + s12
print(eij.get_shape())
V = x[0] * K.mean(eij, axis=2, keepdims=True)
if self.get_alpha:
return eij
else:
if self.get_sequence:
return V
else:
return K.sum(V, axis=1)
def call(self, inputs, mask=None):
# output = softmax(score)
k, q = inputs
if len(q.shape) == 2:
q = K.expand_dims(q, axis=1)
# k: (?, K_LEN, EMBED_DIM,)
# q: (?, Q_LEN, EMBED_DIM,)
# score: (?, Q_LEN, K_LEN,)
if self.score_function == 'scaled_dot_product':
kt = K.permute_dimensions(k, (0, 2, 1))
qkt = K.batch_dot(q, kt)
score = qkt / self.EMBED_DIM
elif self.score_function == 'mlp':
kq = K.concatenate([k, q], axis=1)
kqw2 = K.tanh(K.dot(kq, self.W2))
score = K.permute_dimensions(K.dot(self.W1, kqw2), (1, 0, 2))
elif self.score_function == 'bi_linear':
qw = K.dot(q, self.W)
kt = K.permute_dimensions(k, (0, 2, 1))
score = K.batch_dot(qw, kt)
else:
raise RuntimeError('invalid score_function')
score = K.softmax(score)
# if mask is not None:
# score *= K.cast(mask[0], K.floatx())
# output: (?, Q_LEN, EMBED_DIM,)
output = K.batch_dot(score, k)
return output
def call(self, inputs, **kwargs):
if not inputs.shape[0]:
return inputs
recurrent_input = ops.convert_to_tensor(inputs)
if not self._mixed_precision_policy.should_cast_variables:
recurrent_input = math_ops.cast(recurrent_input, self.dtype)
batch_size = recurrent_input.shape[0]
# Flatten last two dimensions, but along dimension [2]
flat_recurrent = K.reshape(
K.permute_dimensions(recurrent_input, (0, 2, 1)), (batch_size, -1))
outputs = gen_math_ops.mat_mul(
flat_recurrent,
tf.math.multiply(self.recurrent_kernel, self.recurrent_mask))
if self.use_bias:
outputs = nn.bias_add(outputs, self.bias)
if self.activation is not None:
outputs = self.activation(outputs)
# Transform back outputs to original shape
outputs = K.reshape(
K.transpose(outputs),
(self.target_shape[0], self.target_shape[1], batch_size))
outputs = K.reshape(
outputs, (self.target_shape[1], self.target_shape[0], batch_size))
outputs = K.permute_dimensions(outputs, (2, 1, 0))
return outputs
def channel_shuffle(x, groups):
"""
Channel shuffle operation from 'ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices,'
https://arxiv.org/abs/1707.01083.
Parameters:
----------
x : keras.backend tensor/variable/symbol
Input tensor/variable/symbol.
groups : int
Number of groups.
Returns
-------
keras.backend tensor/variable/symbol
Resulted tensor/variable/symbol.
"""
if is_channels_first():
batch, channels, height, width = x.shape
else:
batch, height, width, channels = x.shape
# assert (channels % groups == 0)
channels_per_group = channels // groups
if is_channels_first():
x = K.reshape(x, shape=(-1, groups, channels_per_group, height, width))
x = K.permute_dimensions(x, pattern=(0, 2, 1, 3, 4))
x = K.reshape(x, shape=(-1, channels, height, width))
else:
x = K.reshape(x, shape=(-1, height, width, groups, channels_per_group))
x = K.permute_dimensions(x, pattern=(0, 1, 2, 4, 3))
x = K.reshape(x, shape=(-1, height, width, channels))
updateshape(x)
return x
def call(self, inputs, mask=None):
x = K.permute_dimensions(inputs, (0, 2, 1))
a = K.softmax(K.tanh(K.dot(x, self.W)))
a = K.permute_dimensions(a, (0, 2, 1))
outputs = a * inputs
outputs = K.sum(outputs, axis=1)
return outputs
def call(self, inputs):
if self.data_format == 'channels_first':
inputs = K.permute_dimensions(inputs, pattern=[0, 2, 3, 1])
dilation_rate = conv_utils.normalize_tuple(
self.dilation_rate, 2, 'dilation_rate')
if self.padding == 'valid':
outputs = tf.nn.pool(inputs,
window_shape=self.pool_size,
pooling_type='MAX',
padding=self.padding.upper(),
dilation_rate=dilation_rate,
strides=self.strides,
data_format='NHWC')
elif self.padding == 'same':
input_shape = K.int_shape(inputs)
rows = input_shape[1]
cols = input_shape[2]
rows_unpadded = conv_utils.conv_output_length(
rows, self.pool_size[0],
padding='valid',
stride=self.strides[0],
dilation=self.dilation_rate)
cols_unpadded = conv_utils.conv_output_length(
cols, self.pool_size[1],
padding='valid',
stride=self.strides[1],
dilation=self.dilation_rate)
w_pad = (rows - rows_unpadded) // 2
h_pad = (cols - cols_unpadded) // 2
w_pad = (w_pad, w_pad)
h_pad = (h_pad, h_pad)
pattern = [[0, 0], list(w_pad), list(h_pad), [0, 0]]
# Pad the image
outputs = tf.pad(inputs, pattern, mode='REFLECT')
# Perform pooling
outputs = tf.nn.pool(inputs,
window_shape=self.pool_size,
pooling_type='MAX',
padding='VALID',
dilation_rate=dilation_rate,
strides=self.strides,
data_format='NHWC')
if self.data_format == 'channels_first':
outputs = K.permute_dimensions(outputs, pattern=[0, 3, 1, 2])
return outputs
def call(self, inputs, mask=None):
# inputs.shape = (batch_size, time_steps, seq_len)
x = K.permute_dimensions(inputs, (0, 2, 1))
# x.shape = (batch_size, seq_len, time_steps)
# general
a = K.softmax(K.tanh(K.dot(x, self.W)))
a = K.permute_dimensions(a, (0, 2, 1))
outputs = a * inputs
outputs = K.sum(outputs, axis=1)
return outputs
def _average_filter(self, inputs):
if self.data_format == 'channels_first':
inputs = K.permute_dimensions(inputs, pattern=[0, 2, 3, 1])
outputs = tf.nn.depthwise_conv2d(inputs,
self.kernel, [1, 1, 1, 1],
padding='SAME',
data_format='NHWC')
if self.data_format == 'channels_first':
outputs = K.permute_dimensions(outputs, pattern=[0, 3, 1, 2])
return outputs
def call(self, inputs, **kwargs):
backend = retina_net_tensorflow_backend()
source, target = inputs
target_shape = K.shape(target)
# hack to deal with channels being first
permuted = K.permute_dimensions(source, [0, 2, 3, 1])
resized = backend.resize_images(permuted,
(target_shape[2], target_shape[3]))
output = K.permute_dimensions(resized, [0, 3, 1, 2])
return output
def gram_matrix(self, X):
"""グラム行列の算出"""
X_sw = K.permute_dimensions(
X, (0, 3, 2, 1)
) # 軸の入れ替え
s = K.shape(X_sw)
new_shape = (s[0], s[1], s[2]*s[3])
X_rs = K.reshape(X_sw, new_shape)
X_rs_t = K.permute_dimensions(
X_rs, (0, 2, 1)
) # 行列の転置
dot = K.batch_dot(X_rs, X_rs_t) # 内積の計算
norm = K.prod(K.cast(s[1:], 'float32'))
return dot/norm
def call(self, u_vecs, scores=None):
# if self.share_weights:
# u_hat_vecs = K.conv1d(u_vecs, self.W)
# else:
# u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
u_hat_vecs = u_vecs
batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
if scores is not None:
scores = K.permute_dimensions(scores, (0, 2, 1))
u_hat_vecs = u_hat_vecs * scores
u_hat_vecs = K.reshape(u_hat_vecs,
(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))
b = K.zeros_like(
u_hat_vecs[:, :, :,
0]) # shape = [None, num_capsule, input_num_capsule]
# biases = self.add_weight(name='capsule_kernel',
# shape=(batch_size1, self.num_capsule, self.dim_capsule),
# # shape=self.kernel_size,
# dtype=tf.float32,
# initializer='glorot_uniform',
# trainable=True)
# biases = tf.get_variable(name='bias',
# shape=(self.num_capsule, self.dim_capsule), initializer='glorot_uniform',)
for i in range(self.routings):
# b = K.permute_dimensions(b, (0, 2, 1)) # shape = [None, input_num_capsule, num_capsule]
# c = K.softmax(b)
leak = tf.zeros_like(b, optimize=True)
leak = tf.reduce_sum(leak, axis=1, keep_dims=True)
leaky_logits = tf.concat([leak, b], axis=1)
leaky_routing = tf.nn.softmax(leaky_logits, dim=1)
c = tf.split(leaky_routing, [1, self.num_capsule], axis=1)[1]
# c = K.permute_dimensions(c, (0, 2, 1))
# b = K.permute_dimensions(b, (0, 2, 1))
o = K.batch_dot(c, u_hat_vecs, [2, 2]) # + self.biases
outputs = self.activation(o)
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
# self.c = scores
return outputs
def gram_matrix(X):
# 軸の入れ替え => batch, channel, height, width
axis_replaced_X = K.permute_dimensions(X, (0, 3, 2, 1))
replaced_shape = K.shape(axis_replaced_X)
# 特徴マップ（高さと幅を1つの軸に展開）の内積をとるためのshape
dot_shape = (replaced_shape[0], replaced_shape[1],
replaced_shape[2] * replaced_shape[3])
# 実際に内積を計算する行列
dot_X = K.reshape(axis_replaced_X, dot_shape)
# 転置行列
dot_X_t = K.permute_dimensions(dot_X, (0, 2, 1))
# 行列の内積
dot = K.batch_dot(dot_X, dot_X_t)
norm = K.prod(K.cast(replaced_shape[1:], 'float32'))
return dot / norm
def call(self, u_vecs, **kwargs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,
(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))
# final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]
b = K.zeros_like(
u_hat_vecs[:, :, :,
0]) # shape = [None, num_capsule, input_num_capsule]
for i in range(self.routings):
c = softmax(b, 1)
o = K.batch_dot(c, u_hat_vecs, [2, 2])
if K.backend() == 'theano':
o = K.sum(o, axis=1)
if i < self.routings - 1:
o = K.l2_normalize(o, -1)
b = K.batch_dot(o, u_hat_vecs, [2, 3])
if K.backend() == 'theano':
b = K.sum(b, axis=1)
return self.activation(o)
def make_patches_grid(x, patch_size, patch_stride):
'''Break image `x` up into a grid of patches.
input shape: (channels, rows, cols)
output shape: (rows, cols, channels, patch_rows, patch_cols)
'''
from theano.tensor.nnet.neighbours import images2neibs # TODO: all K, no T
x = K.expand_dims(x, 0)
xs = K.shape(x)
num_rows = 1 + (xs[-2] - patch_size) // patch_stride
num_cols = 1 + (xs[-1] - patch_size) // patch_stride
num_channels = xs[-3]
patches = images2neibs(x, (patch_size, patch_size),
(patch_stride, patch_stride),
mode='valid')
# neibs are sorted per-channel
patches = K.reshape(patches,
(num_channels, K.shape(patches)[0] // num_channels,
patch_size, patch_size))
patches = K.permute_dimensions(patches, (1, 0, 2, 3))
# arrange in a 2d-grid (rows, cols, channels, px, py)
patches = K.reshape(
patches, (num_rows, num_cols, num_channels, patch_size, patch_size))
patches_norm = K.sqrt(
K.sum(K.square(patches), axis=(2, 3, 4), keepdims=True))
return patches, patches_norm
def call(self, inputs):
data_format = conv_utils.convert_data_format(self.data_format,
self.rank + 2)
inputs, tf_data_format = K._preprocess_conv2d_input(
inputs, self.data_format)
inputs = tf.extract_image_patches(
inputs,
ksizes=(1, ) + K.int_shape(self.kernel)[:2] + (1, ),
strides=(1, ) + self.strides + (1, ),
rates=(1, ) + self.dilation_rate + (1, ),
padding=self.padding.upper(),
)
kernel = K.reshape(self.kernel, (-1, self.filters))
outputs = self.kernel_function([inputs, kernel])
if self.data_format == 'channels_first':
outputs = K.permute_dimensions(outputs, (0, 3, 1, 2))
if self.use_bias:
outputs = nn.bias_add(outputs, self.bias, data_format=data_format)
if self.activation is not None:
outputs = self.activation(outputs)
return outputs
def energy_step(inputs, states):
""" Step function for computing energy for a single decoder state """
# input: (batch_size, latent_dim)
assert_msg = "States must be a list. However states {} is of type {}".format(
states, type(states))
assert isinstance(states, list) or isinstance(states,
tuple), assert_msg
""" Computing sj.Ua """
# (batch_size, 1, d3)
U_a_dot_s = K.expand_dims(K.dot(inputs, self.U_a), 1)
if verbose:
print('Ua.h>', K.int_shape(U_a_dot_s))
""" tanh(h.Wa + s.Ua) """
# (batch_size, h1*h2*...*hn, d3) = (batch_size, h1*h2*...*hn, d3) + (batch_size, 1, d3)
Wh_plus_Us = K.tanh(W_hi + U_a_dot_s)
# (batch_size, d3, h1*h2*...*hn)
Wh_plus_Us = K.permute_dimensions(Wh_plus_Us, (0, 2, 1))
if verbose:
print('Wh+Us>', K.int_shape(Wh_plus_Us))
""" softmax(va.tanh(S.Wa + hj.Ua)) """
# (1, batch_size, h1*h2*...*hn) = (1, d3) . (batch_size, d3, h1*h2*...*hn)
Wh_plus_Us_dot_Va = K.dot(self.V_a, Wh_plus_Us)
# (batch_size, h1*h2*...*hn)
e_i = K.squeeze(Wh_plus_Us_dot_Va, 0)
e_i = K.softmax(e_i)
if verbose:
print('ei>', K.int_shape(e_i))
# (batch_size, h1*h2*...*hn)
return e_i, states
def resnet(num_blocks, img_input=None, classes=10, training=None):
"""Instantiates the ResNet architecture.
Arguments:
num_blocks: integer, the number of conv/identity blocks in each block.
The ResNet contains 3 blocks with each block containing one conv block
followed by (layers_per_block - 1) number of idenity blocks. Each
conv/idenity block has 2 convolutional layers. With the input
convolutional layer and the pooling layer towards the end, this brings
the total size of the network to (6*num_blocks + 2)
classes: optional number of classes to classify images into
training: Only used if training keras model with Estimator. In other
scenarios it is handled automatically.
Returns:
A Keras model instance.
"""
if backend.image_data_format() == 'channels_first':
x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)),
name='transpose')(img_input)
bn_axis = 1
else: # channel_last
x = img_input
bn_axis = 3
x = layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(x)
x = layers.Conv2D(16, (3, 3),
strides=(1, 1),
padding='valid', use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name='bn_conv1',)(x, training=training)
x = layers.Activation('relu')(x)
x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[16, 16],
stage=2, conv_strides=(1, 1), training=training)
x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[32, 32],
stage=3, conv_strides=(2, 2), training=training)
x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[64, 64],
stage=4, conv_strides=(2, 2), training=training)
rm_axes = [1, 2] if backend.image_data_format() == 'channels_last' else [2, 3]
x = layers.Lambda(lambda x: backend.mean(x, rm_axes), name='reduce_mean')(x)
x = layers.Dense(classes,
activation='softmax',
kernel_initializer=initializers.RandomNormal(stddev=0.01),
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='fc10')(x)
inputs = img_input
# Create model.
model = tf.keras.models.Model(inputs, x, name='resnet56')
return model
def gram_matrix(x):
assert K.ndim(x) == 3
if K.image_data_format() == "channels_first":
features = K.batch_flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def gram_matrix(x):
assert K.ndim(x) == 3
if K.image_dim_ordering() == "th":
features = K.batch_flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def region_style_loss(style_image, target_image, style_mask, target_mask):
'''Calculate style loss between style_image and target_image,
for one common region specified by their (boolean) masks
'''
assert 3 == K.ndim(style_image) == K.ndim(target_image)
assert 2 == K.ndim(style_mask) == K.ndim(target_mask)
if K.image_dim_ordering() == 'th':
masked_style = style_image * style_mask
masked_target = target_image * target_mask
nb_channels = K.shape(style_image)[0]
else:
masked_style = K.permute_dimensions(style_image,
(2, 0, 1)) * style_mask
masked_target = K.permute_dimensions(target_image,
(2, 0, 1)) * target_mask
nb_channels = K.shape(style_image)[-1]
s = gram_matrix(masked_style) / K.mean(style_mask) / nb_channels
c = gram_matrix(masked_target) / K.mean(target_mask) / nb_channels
return K.mean(K.square(s - c))
def call(self, inputs, **kwargs):
x = inputs
_, height, width, n_ch = K.int_shape(x)
factor = self.factor
x = K.reshape(
x, [-1, height // factor, factor, width // factor, factor, n_ch])
x = K.permute_dimensions(x, [0, 1, 3, 5, 2, 4])
x = K.reshape(
x, [-1, height // factor, width // factor, n_ch * factor * factor])
return x
def call(self, inputs):
if self.data_format == 'channels_first':
channel_last = K.permute_dimensions(inputs, (0, 2, 3, 1))
else:
channel_last = inputs
input_shape = tf.shape(channel_last)
rows = self.scale * input_shape[1]
cols = self.scale * input_shape[2]
resized = tf.image.resize_images(channel_last, (rows, cols))
if self.data_format == 'channels_first':
output = K.permute_dimensions(resized, (0, 3, 1, 2))
else:
output = resized
return output
def region_style_loss(style_image, target_image, style_mask, target_mask):
'''Calculate style loss between style_image and target_image,
for one common region specified by their (boolean) masks
'''
assert 3 == K.ndim(style_image) == K.ndim(target_image)
assert 2 == K.ndim(style_mask) == K.ndim(target_mask)
if K.image_data_format() == 'channels_first':
masked_style = style_image * style_mask
masked_target = target_image * target_mask
num_channels = K.shape(style_image)[0]
else:
masked_style = K.permute_dimensions(
style_image, (2, 0, 1)) * style_mask
masked_target = K.permute_dimensions(
target_image, (2, 0, 1)) * target_mask
num_channels = K.shape(style_image)[-1]
num_channels = K.cast(num_channels, dtype='float32')
s = gram_matrix(masked_style) / K.mean(style_mask) / num_channels
c = gram_matrix(masked_target) / K.mean(target_mask) / num_channels
return K.mean(K.square(s - c))
def call(self, inputs):
if self.data_format == 'channels_first':
inputs = K.permute_dimensions(inputs, pattern=[0, 2, 3, 1])
padding_input = self.padding.upper()
dilation_rate = conv_utils.normalize_tuple(self.dilation_rate, 2,
'dilation_rate')
outputs = tf.nn.pool(inputs,
window_shape=self.pool_size,
pooling_type='MAX',
padding=padding_input,
dilation_rate=dilation_rate,
strides=self.strides,
data_format='NHWC')
if self.data_format == 'channels_first':
outputs = K.permute_dimensions(outputs, pattern=[0, 3, 1, 2])
return outputs
def call(self, inputs, **kwargs):
x = inputs
factor = self.factor
bs, height, width, n_ch = K.int_shape(inputs)
assert n_ch >= factor**2 and n_ch % factor**2 == 0, f'n_ch={n_ch}, input_shape={K.int_shape(inputs)}'
x = K.reshape(x,
[-1, height, width, n_ch // (factor**2), factor, factor])
x = K.permute_dimensions(x, [0, 1, 4, 2, 5, 3])
x = K.reshape(
x, [-1, height * factor, width * factor, n_ch // (factor**2)])
return x
def call(self, inputs, **kwargs):
# Get the Dynamic Shape
shape = tf.shape(inputs)
batch_size = shape[0]
f_width = shape[2]
# Get the Static Shape
_, f_height, _, f_num = inputs.shape.as_list()
inputs = K.permute_dimensions(inputs, (0, 2, 1, 3))
return tf.reshape(inputs,
shape=[batch_size, f_width, f_height * f_num])
def gram_matrix(x, y):
"""
shift = -1
features = tf.reshape(x, tf.shape(x)[ 1 : ])
features = backend.batch_flatten(backend.permute_dimensions(features, (2, 0, 1)))
return backend.dot(features + shift, backend.transpose(features + shift))
"""
y_up = tf.image.resize_images(y, tf.shape(x)[1:3])
shift = -1
features_A = tf.reshape(x, tf.shape(x)[1:]) + shift
features_A = backend.batch_flatten(
backend.permute_dimensions(features_A, (2, 0, 1)))
features_B = tf.reshape(y_up, tf.shape(y_up)[1:]) + shift
features_B = backend.batch_flatten(
backend.permute_dimensions(features_B, (2, 0, 1)))
gram = backend.dot(features_A, backend.transpose(features_B))
return gram
def resnet56(classes=100, training=None):
"""Instantiates the ResNet56 architecture.
Arguments:
classes: optional number of classes to classify images into
training: Only used if training keras model with Estimator. In other
scenarios it is handled automatically.
Returns:
A Keras model instance.
"""
input_shape = (32, 32, 3)
img_input = layers.Input(shape=input_shape)
if backend.image_data_format() == 'channels_first':
x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)),
name='transpose')(img_input)
bn_axis = 1
else: # channel_last
x = img_input
bn_axis = 3
x = tf.keras.layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(x)
x = tf.keras.layers.Conv2D(16, (3, 3),
strides=(1, 1),
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
name='conv1')(x)
x = tf.keras.layers.BatchNormalization(axis=bn_axis, name='bn_conv1',
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON)(
x, training=training)
x = tf.keras.layers.Activation('relu')(x)
x = conv_building_block(x, 3, [16, 16], stage=2, block='a', strides=(1, 1),
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='b',
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='c',
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='d',
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='e',
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='f',
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='g',
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='h',
training=training)
x = identity_building_block(x, 3, [16, 16], stage=2, block='i',
training=training)
x = conv_building_block(x, 3, [32, 32], stage=3, block='a',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='b',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='c',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='d',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='e',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='f',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='g',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='h',
training=training)
x = identity_building_block(x, 3, [32, 32], stage=3, block='i',
training=training)
x = conv_building_block(x, 3, [64, 64], stage=4, block='a',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='b',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='c',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='d',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='e',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='f',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='g',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='h',
training=training)
x = identity_building_block(x, 3, [64, 64], stage=4, block='i',
training=training)
x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = tf.keras.layers.Dense(classes, activation='softmax',
kernel_initializer='he_normal',
kernel_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
name='fc10')(x)
inputs = img_input
# Create model.
model = tf.keras.models.Model(inputs, x, name='resnet56')
return model
def resnet50(num_classes, dtype='float32'):
# TODO(tfboyd): add training argument, just lik resnet56.
"""Instantiates the ResNet50 architecture.
Args:
num_classes: `int` number of classes for image classification.
Returns:
A Keras model instance.
"""
input_shape = (224, 224, 3)
img_input = layers.Input(shape=input_shape, dtype=dtype)
if backend.image_data_format() == 'channels_first':
x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)),
name='transpose')(img_input)
bn_axis = 1
else: # channels_last
x = img_input
bn_axis = 3
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(x)
x = layers.Conv2D(64, (7, 7),
strides=(2, 2),
padding='valid', use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name='bn_conv1')(x)
x = layers.Activation('relu')(x)
x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(
num_classes,
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='fc1000')(x)
# TODO(reedwm): Remove manual casts once mixed precision can be enabled with a
# single line of code.
x = backend.cast(x, 'float32')
x = layers.Activation('softmax')(x)
# Create model.
return models.Model(img_input, x, name='resnet50')
def resnet(num_blocks, classes=10, training=None):
"""Instantiates the ResNet architecture.
Arguments:
num_blocks: integer, the number of conv/identity blocks in each block.
The ResNet contains 3 blocks with each block containing one conv block
followed by (layers_per_block - 1) number of idenity blocks. Each
conv/idenity block has 2 convolutional layers. With the input
convolutional layer and the pooling layer towards the end, this brings
the total size of the network to (6*num_blocks + 2)
classes: optional number of classes to classify images into
training: Only used if training keras model with Estimator. In other
scenarios it is handled automatically.
Returns:
A Keras model instance.
"""
input_shape = (32, 32, 3)
img_input = layers.Input(shape=input_shape)
if backend.image_data_format() == 'channels_first':
x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)),
name='transpose')(img_input)
bn_axis = 1
else: # channel_last
x = img_input
bn_axis = 3
x = tf.keras.layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(x)
x = tf.keras.layers.Conv2D(16, (3, 3),
strides=(1, 1),
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
name='conv1')(x)
x = tf.keras.layers.BatchNormalization(axis=bn_axis, name='bn_conv1',
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON)(
x, training=training)
x = tf.keras.layers.Activation('relu')(x)
x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[16, 16],
stage=2, conv_strides=(1, 1), training=training)
x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[32, 32],
stage=3, conv_strides=(2, 2), training=training)
x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[64, 64],
stage=4, conv_strides=(2, 2), training=training)
x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = tf.keras.layers.Dense(classes, activation='softmax',
kernel_initializer='he_normal',
kernel_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=
tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
name='fc10')(x)
inputs = img_input
# Create model.
model = tf.keras.models.Model(inputs, x, name='resnet56')
return model
