def call(self, x): features = K.depthwise_conv2d(x, self.kernel, strides=self.strides, padding=self.padding) # normalize the features mask = tf.ones_like(x) norm = K.depthwise_conv2d(mask, self.kernel, strides=self.strides, padding=self.padding) features = tf.multiply(features, 1. / norm) return features
def call(self, inputs): if self.padding == 'causal': inputs = array_ops.pad(inputs, self._compute_causal_padding()) if self.data_format == 'channels_last': spatial_start_dim = 1 else: spatial_start_dim = 2 # Explicitly broadcast inputs and kernels to 4D. strides = self.strides * 2 inputs = array_ops.expand_dims(inputs, spatial_start_dim) depthwise_kernel = array_ops.expand_dims(self.depthwise_kernel, 0) dilation_rate = (1, ) + self.dilation_rate outputs = backend.depthwise_conv2d(inputs, depthwise_kernel, strides=strides, padding=self.padding, dilation_rate=dilation_rate, data_format=self.data_format) if self.use_bias: outputs = backend.bias_add(outputs, self.bias, data_format=self.data_format) outputs = array_ops.squeeze(outputs, [spatial_start_dim]) if self.activation is not None: return self.activation(outputs) return outputs
def call(self, inputs, params=None): if params[self.name + '/depthwise_kernel:0'] is None: return super(layers.DepthwiseConv2D, self).call(inputs) else: depthwise_kernel = params.get(self.name + '/depthwise_kernel:0') bias = params.get(self.name + '/bias:0') outputs = backend.depthwise_conv2d( inputs, depthwise_kernel, strides=self.strides, padding=self.padding, dilation_rate=self.dilation_rate, data_format=self.data_format) if self.use_bias: outputs = backend.bias_add( outputs, bias, data_format=self.data_format) if self.activation is not None: return self.activation(outputs) return outputs
def call(self, inputs, training=None): if training is None: training = K.learning_phase() conv_out = super(_DepthwiseConvBatchNorm2D, self).call(inputs) self.batchnorm.call(conv_out) folded_conv_kernel_multiplier = self.batchnorm.gamma * math_ops.rsqrt( self.batchnorm.moving_variance + self.batchnorm.epsilon) folded_conv_bias = math_ops.subtract( self.batchnorm.beta, self.batchnorm.moving_mean * folded_conv_kernel_multiplier, name='folded_conv_bias') depthwise_weights_shape = [ self.depthwise_kernel.get_shape().as_list()[2], self.depthwise_kernel.get_shape().as_list()[3] ] folded_conv_kernel_multiplier = array_ops.reshape( folded_conv_kernel_multiplier, depthwise_weights_shape) folded_conv_kernel = math_ops.mul( folded_conv_kernel_multiplier, self.depthwise_kernel, name='folded_conv_kernel') if self.is_quantized: folded_conv_kernel = self._apply_weight_quantizer(training, folded_conv_kernel) # TODO(alanchiao): this is an internal API. # See if Keras would make this public, like # backend.conv2d is. # # From DepthwiseConv2D layer call() function. folded_conv_out = K.depthwise_conv2d( inputs, folded_conv_kernel, strides=self.strides, padding=self.padding, dilation_rate=self.dilation_rate, data_format=self.data_format, ) outputs = K.bias_add( folded_conv_out, folded_conv_bias, data_format=self.data_format) if self.post_activation is not None: outputs = self.post_activation(outputs) if self.is_quantized: outputs = self._apply_activation_quantizer(training, outputs) return outputs
def call(self, inputs, total_runtime, training=None): outputs = K.depthwise_conv2d(inputs, self.depthwise_kernel_masked, strides=self.strides, padding=self.padding, dilation_rate=self.dilation_rate, data_format=self.data_format) if self.use_bias: outputs = K.bias_add(outputs, self.bias, data_format=self.data_format) total_runtime = total_runtime + self.runtime_reg if self.activation is not None: return self.activation(outputs), total_runtime return outputs, total_runtime
def call(self, inputs, training=None): outputs = depthwise_conv2d( inputs, self.depthwise_kernel, strides=self.strides, padding=self.padding, dilation_rate=self.dilation_rate, data_format=self.data_format) if self.bias: outputs = K.bias_add( outputs, self.bias, data_format=self.data_format) if self.activation is not None: return self.activation(outputs) return outputs
def call(self, inputs, **kwargs): if type(inputs) is list: features = inputs[0] else: features = inputs features = K.depthwise_conv2d(features, self.kernel, strides=self.strides, padding=self.padding, dilation_rate=self.dilation_rate) if self.use_bias: features = tf.add(features, self.bias) if self.activation is not None: features = self.activation(features) return features
def call(self, inputs, training=None): inputs = _fake_cast_float16(inputs) quant_kernel = _fake_cast_float16(self.depthwise_kernel) outputs = backend.depthwise_conv2d(_float16_cast(inputs), _float16_cast(quant_kernel), strides=self.strides, padding=self.padding, dilation_rate=self.dilation_rate, data_format=self.data_format) outputs = tf.cast(outputs, tf.float32) if self.use_bias: bias = _fake_cast_float16(self.bias) outputs = backend.bias_add(outputs, bias, data_format=self.data_format) if self.activation is not None: return self.activation(outputs) return outputs
def call(self, inputs, states, training=None): h_tm1 = states[0] c_tm1 = states[1] # dropout matrices for input units dp_mask = self.get_dropout_mask_for_cell(inputs, training, count=4) # dropout matrices for recurrent units rec_dp_mask = self.get_recurrent_dropout_mask_for_cell(h_tm1, training, count=4) if 0 < self.dropout < 1.: inputs_i = inputs * dp_mask[0] inputs_f = inputs * dp_mask[1] inputs_c = inputs * dp_mask[2] inputs_o = inputs * dp_mask[3] else: inputs_i = inputs inputs_f = inputs inputs_c = inputs inputs_o = inputs if 0 < self.recurrent_dropout < 1.: h_tm1_i = h_tm1 * rec_dp_mask[0] h_tm1_f = h_tm1 * rec_dp_mask[1] h_tm1_c = h_tm1 * rec_dp_mask[2] h_tm1_o = h_tm1 * rec_dp_mask[3] else: h_tm1_i = h_tm1 h_tm1_f = h_tm1 h_tm1_c = h_tm1 h_tm1_o = h_tm1 (kernel_i, kernel_f, kernel_c, kernel_o) = array_ops.split(self.kernel, 4, axis=3) # (3, 3, input_dim, filters) (recurrent_kernel_i, recurrent_kernel_f, recurrent_kernel_c, recurrent_kernel_o) = array_ops.split(self.recurrent_kernel, 4, axis=3) if self.use_bias: bias_i, bias_f, bias_c, bias_o = array_ops.split(self.bias, 4) else: bias_i, bias_f, bias_c, bias_o = None, None, None, None # input_i: batch x_i = self.input_conv(inputs_i, kernel_i, bias_i, padding=self.padding) x_f = self.input_conv(inputs_f, kernel_f, bias_f, padding=self.padding) x_c = self.input_conv(inputs_c, kernel_c, bias_c, padding=self.padding) x_o = self.input_conv(inputs_o, kernel_o, bias_o, padding=self.padding) h_i = self.recurrent_conv(h_tm1_i, recurrent_kernel_i) h_f = self.recurrent_conv(h_tm1_f, recurrent_kernel_f) h_c = self.recurrent_conv(h_tm1_c, recurrent_kernel_c) h_o = self.recurrent_conv(h_tm1_o, recurrent_kernel_o) i = self.recurrent_activation(x_i + h_i) f = self.recurrent_activation(x_f + h_f) c = f * c_tm1 + i * self.activation(x_c + h_c) o = self.recurrent_activation(x_o + h_o) h = o * self.activation(c) # sa computation m_t_minus_one = states[2] # h, w, filters h_t, c_t = h, c (kernel_hv, kernel_hk, kernel_hq, kernel_mk, kernel_mv) = array_ops.split( self.sa_kernel, 5, axis=3) # kernel_size, filters, 1, turn to one layer if self.use_bias: bias_i, bias_g, bias_o = array_ops.split(self.sa_bias, 3) else: bias_i, bias_g, bias_o = None, None, None v_h = self.sa_conv(h_t, kernel_hv) k_h = self.sa_conv(h_t, kernel_hk) q_h = self.sa_conv(h_t, kernel_hq) k_m = self.sa_conv(m_t_minus_one, kernel_mk) v_m = self.sa_conv(m_t_minus_one, kernel_mv) # h, w, 1 q_h = K.squeeze(q_h, 3) k_m = K.squeeze(k_m, 3) k_h = K.squeeze(k_h, 3) e_m = tf.matmul(q_h, k_m) alpha_m = K.softmax(e_m) e_h = tf.matmul(q_h, k_h) alpha_h = K.softmax(e_h) v_m = K.squeeze(v_m, 3) v_h = K.squeeze(v_h, 3) z_m = tf.matmul(alpha_m, v_m) z_h = tf.matmul(alpha_h, v_h) z_m = K.expand_dims(z_m, 3) z_h = K.expand_dims(z_h, 3) zi = self.sa_conv(K.concatenate((z_h, z_m), 3), self.kernel_z) (kernel_m_zi, kernel_m_hi, kernel_m_zg, kernel_m_hg, kernel_m_zo, kernel_m_ho) = array_ops.split(self.depth_wise_kernel, 6, axis=3) # i = K.sigmoid( K.depthwise_conv2d(zi, kernel_m_zi, padding='same') + K.depthwise_conv2d(h_t, kernel_m_hi, padding='same') + bias_i) g = K.tanh( K.depthwise_conv2d(zi, kernel_m_zg, padding='same') + K.depthwise_conv2d(h_t, kernel_m_hg, padding='same') + bias_g) o = K.sigmoid( K.depthwise_conv2d(zi, kernel_m_zo, padding='same') + K.depthwise_conv2d(h_t, kernel_m_ho, padding='same') + bias_o) m_t = (1 - i) * m_t_minus_one + i * g h_hat_t = m_t * o # sa computation end return h_hat_t, [c_t, h_hat_t, m_t]
def msssim(self, y_true, y_pred): ''' Compute multiscale ssim according to Zhao 2016. Has only been tested with tensorflow backend (channels last) so far! Uses convolutions to do the calculations in one go. This function takes proper 2D Keras Tensors (NWHC or NCWH) # Arguments y_true: Keras Tensor with Rank 4: Image to compare to y_pred: Keras Tensor with Rank 4: Image to compare ''' # some useful inits channels = self.__int_shape(y_pred)[self.channel_dim] # repeat kernel for each channel kernel = K.tile(self.kernels, [1, 1, channels, 1]) # compute means mu_true = K.depthwise_conv2d(y_true, kernel, padding='same') mu_pred = K.depthwise_conv2d(y_pred, kernel, padding='same') # compute mean squares mu_true_sq = K.square(mu_true) mu_pred_sq = K.square(mu_pred) mu_true_pred = mu_true * mu_pred # compute input square y_true_sq = K.square(y_true) y_pred_sq = K.square(y_pred) y_true_pred = y_true * y_pred # compute variances/covariance sigma_true_sq = K.depthwise_conv2d(y_true_sq, kernel, padding='same') sigma_pred_sq = K.depthwise_conv2d(y_pred_sq, kernel, padding='same') sigma_true_pred = K.depthwise_conv2d(y_true_pred, kernel, padding='same') # centered squares of variances sigma_true_sq -= mu_true_sq sigma_pred_sq -= mu_pred_sq sigma_true_pred -= mu_true_pred # compute luminance term (l), select only maximum kernel for each channel l = (2 * mu_true_pred + self.c1) / (mu_true_sq + mu_pred_sq + self.c1) if self.dim_ordering == 'channels_last': l_max = l[:,:,:,(self.num - 1)::self.num] else: l_max = l[:,(self.num - 1)::self.num,:,:] # compute contrast-structure term (cs) cs = (2 * sigma_true_pred + self.c2) / (sigma_true_sq + sigma_pred_sq + self.c2) # compute product of different scale cs if self.dim_ordering == 'channels_last': pcs = [K.prod(cs[:,:,:,i*self.num:(i+1)*self.num], axis=-1, keepdims=True) for i in range(channels)] else: pcs = [K.prod(cs[:,i*self.num:(i+1)*self.num,:,:], axis=1, keepdims=True) for i in range(channels)] pcs = K.concatenate(pcs, axis=self.channel_dim) # compute msssim map msssim = l_max * pcs # do normalization? return msssim