def call(self, x):
features = K.depthwise_conv2d(x,
self.kernel,
strides=self.strides,
padding=self.padding)
# normalize the features
mask = tf.ones_like(x)
norm = K.depthwise_conv2d(mask,
self.kernel,
strides=self.strides,
padding=self.padding)
features = tf.multiply(features, 1. / norm)
return features
def call(self, inputs):
if self.padding == 'causal':
inputs = array_ops.pad(inputs, self._compute_causal_padding())
if self.data_format == 'channels_last':
spatial_start_dim = 1
else:
spatial_start_dim = 2
# Explicitly broadcast inputs and kernels to 4D.
strides = self.strides * 2
inputs = array_ops.expand_dims(inputs, spatial_start_dim)
depthwise_kernel = array_ops.expand_dims(self.depthwise_kernel, 0)
dilation_rate = (1, ) + self.dilation_rate
outputs = backend.depthwise_conv2d(inputs,
depthwise_kernel,
strides=strides,
padding=self.padding,
dilation_rate=dilation_rate,
data_format=self.data_format)
if self.use_bias:
outputs = backend.bias_add(outputs,
self.bias,
data_format=self.data_format)
outputs = array_ops.squeeze(outputs, [spatial_start_dim])
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, inputs, params=None):
if params[self.name + '/depthwise_kernel:0'] is None:
return super(layers.DepthwiseConv2D, self).call(inputs)
else:
depthwise_kernel = params.get(self.name + '/depthwise_kernel:0')
bias = params.get(self.name + '/bias:0')
outputs = backend.depthwise_conv2d(
inputs,
depthwise_kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
if self.use_bias:
outputs = backend.bias_add(
outputs,
bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, inputs, training=None):
if training is None:
training = K.learning_phase()
conv_out = super(_DepthwiseConvBatchNorm2D, self).call(inputs)
self.batchnorm.call(conv_out)
folded_conv_kernel_multiplier = self.batchnorm.gamma * math_ops.rsqrt(
self.batchnorm.moving_variance + self.batchnorm.epsilon)
folded_conv_bias = math_ops.subtract(
self.batchnorm.beta,
self.batchnorm.moving_mean * folded_conv_kernel_multiplier,
name='folded_conv_bias')
depthwise_weights_shape = [
self.depthwise_kernel.get_shape().as_list()[2],
self.depthwise_kernel.get_shape().as_list()[3]
]
folded_conv_kernel_multiplier = array_ops.reshape(
folded_conv_kernel_multiplier, depthwise_weights_shape)
folded_conv_kernel = math_ops.mul(
folded_conv_kernel_multiplier,
self.depthwise_kernel,
name='folded_conv_kernel')
if self.is_quantized:
folded_conv_kernel = self._apply_weight_quantizer(training,
folded_conv_kernel)
# TODO(alanchiao): this is an internal API.
# See if Keras would make this public, like
# backend.conv2d is.
#
# From DepthwiseConv2D layer call() function.
folded_conv_out = K.depthwise_conv2d(
inputs,
folded_conv_kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format,
)
outputs = K.bias_add(
folded_conv_out, folded_conv_bias, data_format=self.data_format)
if self.post_activation is not None:
outputs = self.post_activation(outputs)
if self.is_quantized:
outputs = self._apply_activation_quantizer(training, outputs)
return outputs
def call(self, inputs, total_runtime, training=None):
outputs = K.depthwise_conv2d(inputs,
self.depthwise_kernel_masked,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
if self.use_bias:
outputs = K.bias_add(outputs,
self.bias,
data_format=self.data_format)
total_runtime = total_runtime + self.runtime_reg
if self.activation is not None:
return self.activation(outputs), total_runtime
return outputs, total_runtime
def call(self, inputs, training=None):
outputs = depthwise_conv2d(
inputs,
self.depthwise_kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
if self.bias:
outputs = K.bias_add(
outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, inputs, **kwargs):
if type(inputs) is list:
features = inputs[0]
else:
features = inputs
features = K.depthwise_conv2d(features,
self.kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate)
if self.use_bias:
features = tf.add(features, self.bias)
if self.activation is not None:
features = self.activation(features)
return features
def call(self, inputs, training=None):
inputs = _fake_cast_float16(inputs)
quant_kernel = _fake_cast_float16(self.depthwise_kernel)
outputs = backend.depthwise_conv2d(_float16_cast(inputs),
_float16_cast(quant_kernel),
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
outputs = tf.cast(outputs, tf.float32)
if self.use_bias:
bias = _fake_cast_float16(self.bias)
outputs = backend.bias_add(outputs,
bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, inputs, states, training=None):
h_tm1 = states[0]
c_tm1 = states[1]
# dropout matrices for input units
dp_mask = self.get_dropout_mask_for_cell(inputs, training, count=4)
# dropout matrices for recurrent units
rec_dp_mask = self.get_recurrent_dropout_mask_for_cell(h_tm1,
training,
count=4)
if 0 < self.dropout < 1.:
inputs_i = inputs * dp_mask[0]
inputs_f = inputs * dp_mask[1]
inputs_c = inputs * dp_mask[2]
inputs_o = inputs * dp_mask[3]
else:
inputs_i = inputs
inputs_f = inputs
inputs_c = inputs
inputs_o = inputs
if 0 < self.recurrent_dropout < 1.:
h_tm1_i = h_tm1 * rec_dp_mask[0]
h_tm1_f = h_tm1 * rec_dp_mask[1]
h_tm1_c = h_tm1 * rec_dp_mask[2]
h_tm1_o = h_tm1 * rec_dp_mask[3]
else:
h_tm1_i = h_tm1
h_tm1_f = h_tm1
h_tm1_c = h_tm1
h_tm1_o = h_tm1
(kernel_i, kernel_f, kernel_c,
kernel_o) = array_ops.split(self.kernel, 4,
axis=3) # (3, 3, input_dim, filters)
(recurrent_kernel_i, recurrent_kernel_f, recurrent_kernel_c,
recurrent_kernel_o) = array_ops.split(self.recurrent_kernel,
4,
axis=3)
if self.use_bias:
bias_i, bias_f, bias_c, bias_o = array_ops.split(self.bias, 4)
else:
bias_i, bias_f, bias_c, bias_o = None, None, None, None
# input_i: batch
x_i = self.input_conv(inputs_i, kernel_i, bias_i, padding=self.padding)
x_f = self.input_conv(inputs_f, kernel_f, bias_f, padding=self.padding)
x_c = self.input_conv(inputs_c, kernel_c, bias_c, padding=self.padding)
x_o = self.input_conv(inputs_o, kernel_o, bias_o, padding=self.padding)
h_i = self.recurrent_conv(h_tm1_i, recurrent_kernel_i)
h_f = self.recurrent_conv(h_tm1_f, recurrent_kernel_f)
h_c = self.recurrent_conv(h_tm1_c, recurrent_kernel_c)
h_o = self.recurrent_conv(h_tm1_o, recurrent_kernel_o)
i = self.recurrent_activation(x_i + h_i)
f = self.recurrent_activation(x_f + h_f)
c = f * c_tm1 + i * self.activation(x_c + h_c)
o = self.recurrent_activation(x_o + h_o)
h = o * self.activation(c)
# sa computation
m_t_minus_one = states[2] # h, w, filters
h_t, c_t = h, c
(kernel_hv, kernel_hk, kernel_hq, kernel_mk,
kernel_mv) = array_ops.split(
self.sa_kernel, 5,
axis=3) # kernel_size, filters, 1, turn to one layer
if self.use_bias:
bias_i, bias_g, bias_o = array_ops.split(self.sa_bias, 3)
else:
bias_i, bias_g, bias_o = None, None, None
v_h = self.sa_conv(h_t, kernel_hv)
k_h = self.sa_conv(h_t, kernel_hk)
q_h = self.sa_conv(h_t, kernel_hq)
k_m = self.sa_conv(m_t_minus_one, kernel_mk)
v_m = self.sa_conv(m_t_minus_one, kernel_mv) # h, w, 1
q_h = K.squeeze(q_h, 3)
k_m = K.squeeze(k_m, 3)
k_h = K.squeeze(k_h, 3)
e_m = tf.matmul(q_h, k_m)
alpha_m = K.softmax(e_m)
e_h = tf.matmul(q_h, k_h)
alpha_h = K.softmax(e_h)
v_m = K.squeeze(v_m, 3)
v_h = K.squeeze(v_h, 3)
z_m = tf.matmul(alpha_m, v_m)
z_h = tf.matmul(alpha_h, v_h)
z_m = K.expand_dims(z_m, 3)
z_h = K.expand_dims(z_h, 3)
zi = self.sa_conv(K.concatenate((z_h, z_m), 3), self.kernel_z)
(kernel_m_zi, kernel_m_hi, kernel_m_zg, kernel_m_hg, kernel_m_zo,
kernel_m_ho) = array_ops.split(self.depth_wise_kernel, 6, axis=3) #
i = K.sigmoid(
K.depthwise_conv2d(zi, kernel_m_zi, padding='same') +
K.depthwise_conv2d(h_t, kernel_m_hi, padding='same') + bias_i)
g = K.tanh(
K.depthwise_conv2d(zi, kernel_m_zg, padding='same') +
K.depthwise_conv2d(h_t, kernel_m_hg, padding='same') + bias_g)
o = K.sigmoid(
K.depthwise_conv2d(zi, kernel_m_zo, padding='same') +
K.depthwise_conv2d(h_t, kernel_m_ho, padding='same') + bias_o)
m_t = (1 - i) * m_t_minus_one + i * g
h_hat_t = m_t * o
# sa computation end
return h_hat_t, [c_t, h_hat_t, m_t]
def msssim(self, y_true, y_pred):
'''
Compute multiscale ssim according to Zhao 2016.
Has only been tested with tensorflow backend (channels last) so far!
Uses convolutions to do the calculations in one go.
This function takes proper 2D Keras Tensors (NWHC or NCWH)
# Arguments
y_true: Keras Tensor with Rank 4: Image to compare to
y_pred: Keras Tensor with Rank 4: Image to compare
'''
# some useful inits
channels = self.__int_shape(y_pred)[self.channel_dim]
# repeat kernel for each channel
kernel = K.tile(self.kernels, [1, 1, channels, 1])
# compute means
mu_true = K.depthwise_conv2d(y_true, kernel, padding='same')
mu_pred = K.depthwise_conv2d(y_pred, kernel, padding='same')
# compute mean squares
mu_true_sq = K.square(mu_true)
mu_pred_sq = K.square(mu_pred)
mu_true_pred = mu_true * mu_pred
# compute input square
y_true_sq = K.square(y_true)
y_pred_sq = K.square(y_pred)
y_true_pred = y_true * y_pred
# compute variances/covariance
sigma_true_sq = K.depthwise_conv2d(y_true_sq, kernel, padding='same')
sigma_pred_sq = K.depthwise_conv2d(y_pred_sq, kernel, padding='same')
sigma_true_pred = K.depthwise_conv2d(y_true_pred, kernel, padding='same')
# centered squares of variances
sigma_true_sq -= mu_true_sq
sigma_pred_sq -= mu_pred_sq
sigma_true_pred -= mu_true_pred
# compute luminance term (l), select only maximum kernel for each channel
l = (2 * mu_true_pred + self.c1) / (mu_true_sq + mu_pred_sq + self.c1)
if self.dim_ordering == 'channels_last':
l_max = l[:,:,:,(self.num - 1)::self.num]
else:
l_max = l[:,(self.num - 1)::self.num,:,:]
# compute contrast-structure term (cs)
cs = (2 * sigma_true_pred + self.c2) / (sigma_true_sq + sigma_pred_sq +
self.c2)
# compute product of different scale cs
if self.dim_ordering == 'channels_last':
pcs = [K.prod(cs[:,:,:,i*self.num:(i+1)*self.num], axis=-1,
keepdims=True) for i in range(channels)]
else:
pcs = [K.prod(cs[:,i*self.num:(i+1)*self.num,:,:], axis=1,
keepdims=True) for i in range(channels)]
pcs = K.concatenate(pcs, axis=self.channel_dim)
# compute msssim map
msssim = l_max * pcs # do normalization?
return msssim