def cross_input_both(X): tensor_left = X[0] tensor_right = X[1] x_length = K.int_shape(tensor_left)[1] y_length = K.int_shape(tensor_left)[2] cross_y_left = [] cross_x_left = [] cross_y_right = [] cross_x_right = [] scalar = K.ones([5,5]) tensor_left_padding = K.spatial_2d_padding(tensor_left,padding=(2,2)) tensor_right_padding = K.spatial_2d_padding(tensor_right,padding=(2,2)) for i_x in range(2, x_length + 2): for i_y in range(2, y_length + 2): cross_y_left.append(tensor_left_padding[:,i_x-2:i_x+3,i_y-2:i_y+3,:] - concat_iterat(tensor_right_padding[:,i_x,i_y,:])) cross_y_right.append(tensor_right_padding[:,i_x-2:i_x+3,i_y-2:i_y+3,:] - concat_iterat(tensor_left_padding[:,i_x,i_y,:])) cross_x_left.append(K.concatenate(cross_y_left, axis=2)) cross_x_right.append(K.concatenate(cross_y_right, axis=2)) cross_y_left = [] cross_y_right = [] cross_out_left = K.concatenate(cross_x_left,axis=1) cross_out_right = K.concatenate(cross_x_right,axis=1) cross_out = K.concatenate([cross_out_left, cross_out_right], axis=3) return K.abs(cross_out)
def call(self, inputs, mask=None): """Define PConv2D layer's logic. Args: inputs: list=[K.tensor, K.tensor] Returns: list=[K.tensor, K.tensor] """ # Both sound field and mask must be supplied if type(inputs) is not list or len(inputs) != 2: raise Exception('PartialConvolution2D must be called on a list of two tensors [sf, mask]. Instead got: ' + str(inputs)) # Padding done explicitly so that padding becomes part of the masked partial convolution sfs = K.spatial_2d_padding(inputs[0], self.pconv_padding, self.data_format) masks = K.spatial_2d_padding(inputs[1], self.pconv_padding, self.data_format) # Apply convolutions to mask mask_output = K.conv2d( masks, self.kernel_mask, strides=self.strides, padding='valid', data_format=self.data_format, dilation_rate=self.dilation_rate ) # Apply convolutions to sound field sf_output = K.conv2d( (sfs*masks), self.kernel, strides=self.strides, padding='valid', data_format=self.data_format, dilation_rate=self.dilation_rate ) # Calculate the mask ratio on each psoition in the output mask mask_ratio = self.window_size / (mask_output + 1e-8) # Clip output to be between 0 and 1 mask_output = K.clip(mask_output, 0, 1) # Remove ratio values where there are holes mask_ratio = mask_ratio * mask_output # Normalize sound field output sf_output = sf_output * mask_ratio # Apply bias only to the sound field (if chosen to do so) if self.use_bias: sf_output = K.bias_add( sf_output, self.bias, data_format=self.data_format) # Apply activations on the sound field if self.activation is not None: sf_output = self.activation(sf_output) return [sf_output, mask_output]
def f(y_true, y_pred): padding_pattern = ((kernal[0]//2, kernal[0]//2), (kernal[1]//2, kernal[1]//2)) y_true = K.spatial_2d_padding(y_true, padding_pattern) y_pred = K.spatial_2d_padding(y_pred, padding_pattern) kernal_x = kernal[0] kernal_y = kernal[1] stride_x, stride_y = strides x = 0 for idx_x in range(0, height, stride_y): print(idx_x) y = 0 for idx_y in range(0, width, stride_x): if K.image_data_format() == 'channels_first': tmp=1-SSMI(y_true[:, :, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y], y_pred[:, :, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y]) else: tmp=1-SSMI(y_true[:, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y, :], y_pred[:, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y, :]) try: loss_ssmi=K.concatenate([loss_ssmi, K.expand_dims(tmp,0)],0) except Exception as e: loss_ssmi= K.expand_dims(tmp,0) y += 1 x += 1 loss_ssmi=K.transpose(loss_ssmi) true_loss_ssmi = loss_ssmi[:K.shape(y_true)[0]] return true_loss_ssmi
def call(self, X, mask=None): if self.data_format == 'channels_last': b, r, c, ch = K.int_shape(X) else: b, ch, r, c = K.int_shape(X) half = self.n // 2 square = K.square(X) if self.data_format != 'channels_last': extra_channels = K.spatial_2d_padding(square, padding=((half, half), (0, 0)), data_format='channels_last') else: extra_channels = K.spatial_2d_padding( K.permute_dimensions(square, (0, 3, 1, 2)), padding=((half, half), (0, 0)), data_format='channels_last') scale = self.k for i in range(self.n): scale += (self.alpha / self.n) * extra_channels[:, i:(i + ch), :, :] scale = scale**self.beta if self.data_format == 'channels_last': scale = K.permute_dimensions(scale, (0, 2, 3, 1)) return X / scale
def call(self, inputs, mask=None): ''' We will be using the Keras conv2d method, and essentially we have to do here is multiply the mask with the input X, before we apply the convolutions. For the mask itself, we apply convolutions with all weights set to 1. Subsequently, we clip mask values to between 0 and 1 ''' # Both image and mask must be supplied if type(inputs) is not list or len(inputs) != 2: raise Exception( 'PartialConvolution2D must be called on a list of two tensors [img, mask]. Instead got: ' + str(inputs)) # Padding done explicitly so that padding becomes part of the masked partial convolution images = K.spatial_2d_padding(inputs[0], self.pconv_padding, self.data_format) masks = K.spatial_2d_padding(inputs[1], self.pconv_padding, self.data_format) # Apply convolutions to mask mask_output = K.conv2d(masks, self.kernel_mask, strides=self.strides, padding='valid', data_format=self.data_format, dilation_rate=self.dilation_rate) # Apply convolutions to image img_output = K.conv2d((images * masks), self.kernel, strides=self.strides, padding='valid', data_format=self.data_format, dilation_rate=self.dilation_rate) # Calculate the mask ratio on each pixel in the output mask mask_ratio = self.window_size / (mask_output + 1e-8) # Clip output to be between 0 and 1 mask_output = K.clip(mask_output, 0, 1) # Remove ratio values where there are holes mask_ratio = mask_ratio * mask_output # Normalize iamge output img_output = img_output * mask_ratio # Apply bias only to the image (if chosen to do so) if self.use_bias: img_output = K.bias_add(img_output, self.bias, data_format=self.data_format) # Apply activations on the image if self.activation is not None: img_output = self.activation(img_output) return [img_output, mask_output]
def call(self, x, mask=None): input_1, input_2 = x input_shape = input_1._keras_shape assert input_shape == input_2._keras_shape self.H = input_shape[1] self.W = input_shape[2] self.C = input_shape[3] padding1 = K.spatial_2d_padding(input_1, padding=((self.k, self.k + 1), (self.k, self.k + 1)), data_format='channels_last') padding2 = K.spatial_2d_padding( input_2, padding=(((self.D_stride - 1) / 2, (self.D_stride - 1) / 2 + 1), ((self.D_stride - 1) / 2, (self.D_stride - 1) / 2 + 1)), data_format='channels_last') # padding1&2: [nS, w, h, c] out = tf.scan(self.single_sample_corr, elems=[padding2, padding1], initializer=(K.zeros( (self.H / self.s1, self.W / self.s1, self.D**2)))) return out
def call(self, inputs): ''' This portion includes the actual implementation : the image is elementally multiplied to the masks which is fed to the network and later the masks are updated on the basis of the output generated''' assert isinstance(inputs, list) images = inputs[0] masks = inputs[1] #padding layer images = K.spatial_2d_padding(inputs[0], self.pconv_padding, self.data_format) masks = K.spatial_2d_padding(inputs[1], self.pconv_padding, self.data_format) X = images * masks out = K.conv2d(X, self.kernel, strides=self.strides, padding='valid', data_format=self.data_format) ################################################# def mask_update(masks): ''' mask update function that updates the mask after a succesful Partial conv layer ''' out = K.conv2d(masks, self.kernel_mask, strides=self.strides, padding='valid', data_format=self.data_format) out = K.clip(out, 0, 1) eps = 1e-6 mask_ratio = self.window_size / (out + eps) mask_ratio *= out return out, mask_ratio ################################################# updated_mask, mask_ratio = mask_update(masks) out = out * mask_ratio if self.use_bias: out = K.bias_add(out, self.bias, data_format=self.data_format) if self.activation: out = self.activation(out) return [out, updated_mask]
def call(self, inputs, **kwargs): """ :param inputs: :param kwargs: :return: """ _, ch, r, c = K.int_shape(inputs) # print("Call Fcn: Channel First Input shape ", K.int_shape(inputs)) # 1. Inputs Formatting # -------------------- # Pad the rows and columns to allow full matrix multiplication # Note that this function is aware of which dimension the columns and rows are padded_inputs = K.spatial_2d_padding(inputs, ((self.n / 2, self.n / 2), (self.n / 2, self.n / 2))) # print("Call Fcn: padded_inputs shape ", K.int_shape(padded_inputs)) # Channel first, batch second. This is done to take the unknown batch size into the matrix # multiply where it can be handled more easily inputs_chan_first = K.permute_dimensions(padded_inputs, [1, 0, 2, 3]) # print("Call Fcn: inputs_chan_first shape: ", inputs_chan_first.shape) # 2. Kernel Formatting # -------------------- # Flatten rows and columns into a single dimension k_ch, k_r, k_c = K.int_shape(self.kernel) apply_kernel = K.reshape(self.kernel, (k_ch, k_r * k_c, 1)) # print("Call Fcn: kernel for matrix multiply: ", apply_kernel.shape) # 3. Get outputs at each spatial location # --------------------------------------- xs = [] for i in range(r): for j in range(c): input_slice = inputs_chan_first[:, :, i:i + self.n, j:j + self.n] input_slice_apply = K.reshape(input_slice, (ch, -1, self.n**2)) output_slice = K.batch_dot(input_slice_apply, apply_kernel) # Reshape the output slice to put batch first output_slice = K.permute_dimensions(output_slice, [1, 0, 2]) xs.append(output_slice) # print("Call Fcn: len of xs", len(xs)) # print("Call Fcn: shape of each element of xs", xs[0].shape) # Reshape the output to correct format outputs = K.concatenate(xs, axis=2) outputs = K.reshape(outputs, (-1, ch, r, c)) # Break into row and column # 4. Add the lateral and the feed-forward activations # ------------------------------------------------------ outputs = outputs * inputs + self.bias outputs = self.activation(outputs) return outputs + inputs
def call(self, inputs): """Performs the actual padding. Parameters ---------- inputs : Tensor, rank 4 4D tensor with shape: - If `data_format` is `"channels_last"`: `(batch, rows, cols, channels)` - If `data_format` is `"channels_first"`: `(batch, channels, rows, cols)` Returns ------- outputs : Tensor, rank 4 4D tensor with shape: - If `data_format` is `"channels_last"`: `(batch, padded_rows, padded_cols, channels)` - If `data_format` is `"channels_first"`: `(batch, channels, padded_rows, padded_cols)` """ outputs = K.spatial_2d_padding(inputs, padding=self.padding, data_format=self.data_format) p00, p01 = self.padding[0][0], self.padding[0][1] p10, p11 = self.padding[1][0], self.padding[1][1] if self.data_format == "channels_last": row0 = K.concatenate([inputs[:, p00:0:-1, p10:0:-1, :], inputs[:, p00:0:-1, :, :], inputs[:, p00:0:-1, -2:-2-p11:-1, :]], axis=2) row1 = K.concatenate([inputs[:, :, p10:0:-1, :], inputs, inputs[:, :, -2:-2-p11:-1, :]], axis=2) row2 = K.concatenate([inputs[:, -2:-2-p01:-1, p10:0:-1, :], inputs[:, -2:-2-p01:-1, :, :], inputs[:, -2:-2-p01:-1, -2:-2-p11:-1, :]], axis=2) outputs = K.concatenate([row0, row1, row2], axis=1) else: # self.data_format == "channels_first" row0 = K.concatenate([inputs[:, :, p00:0:-1, p10:0:-1], inputs[:, :, p00:0:-1, :], inputs[:, :, p00:0:-1, -2:-2-p11:-1]], axis=3) row1 = K.concatenate([inputs[:, :, :, p10:0:-1], inputs, inputs[:, :, :, -2:-2-p11:-1]], axis=3) row2 = K.concatenate([inputs[:, :, -2:-2-p01:-1, p10:0:-1], inputs[:, :, -2:-2-p01:-1, :], inputs[:, :, -2:-2-p01:-1, -2:-2-p11:-1]], axis=3) outputs = K.concatenate([row0, row1, row2], axis=2) return outputs
def __call__(self, inputs): if self.mask: masked = layers.Lambda(capsule.mask)(inputs) masked = layers.Flatten()(masked) elif K.ndim(inputs) > 2: masked = layers.Flatten()(inputs) else: masked = inputs if self.conv: fc = layers.Dense(256, activation='relu')(masked) reshape = layers.Reshape((8, 8, 4))(fc) up1 = layers.Conv2DTranspose(256, 9, strides=2, activation='relu', padding='valid')(reshape) pad = layers.Lambda( lambda x: K.spatial_2d_padding(x, ((0, 1), (0, 1))))(up1) up2 = layers.Conv2DTranspose(256, 9, activation='relu', padding='valid')(pad) outputs = layers.Conv2D(self.image_shape[-1], 1, activation='sigmoid', name='reconstruction')(up2) else: up1 = layers.Dense(512, activation='relu')(masked) up2 = layers.Dense(1024, activation='relu')(up1) outputs = layers.Dense(np.prod(self.image_shape), activation='sigmoid')(up2) outputs = layers.Reshape(self.image_shape, name='reconstruction')(outputs) return outputs
def blur_pool_func(x, filt_size=3, stride=2): stride = (stride, stride) padding = ((int(1. * (filt_size - 1) / 2), int(np.ceil(1. * (filt_size - 1) / 2))), (int(1. * (filt_size - 1) / 2), int(np.ceil(1. * (filt_size - 1) / 2)))) if (filt_size == 1): k = np.array([ 1., ]) elif (filt_size == 2): k = np.array([1., 1.]) elif (filt_size == 3): k = np.array([1., 2., 1.]) elif (filt_size == 4): k = np.array([1., 3., 3., 1.]) elif (filt_size == 5): k = np.array([1., 4., 6., 4., 1.]) elif (filt_size == 6): k = np.array([1., 5., 10., 10., 5., 1.]) elif (filt_size == 7): k = np.array([1., 6., 15., 20., 15., 6., 1.]) #k = a k = k[:, None] * k[None, :] k = k / np.sum(k) k = np.tile(k[:, :, None, None], (1, 1, K.int_shape(x)[-1], 1)) k = K.constant(k, dtype=K.floatx()) x = K.spatial_2d_padding(x, padding=padding) x = K.depthwise_conv2d(x, k, strides=stride, padding='valid') return x
def get_keras_pad(cls, x, pads, dim, data_format=None): """ implement pad in conv or pool operator :param x: input tensor :param pads: pads attribute in conv or pool operator :param dim: the pad dim :param data_format: data format of x :return: the result tensor of input tensor implementing padding operation """ if sum(pads) == 0: return x if len(pads) == dim * 2: pads = list( np.transpose( np.array(pads).reshape([2, dim]).astype(np.int32))) pads = tuple(tuple(i) for i in pads) elif len(pads) == dim: pads = tuple((i, i) for i in pads) if dim == 1: return Lambda(lambda _x: K.temporal_padding(_x, pads))(x) elif dim == 2: return Lambda( lambda _x: K.spatial_2d_padding(_x, pads, data_format))(x) elif dim == 3: return Lambda( lambda _x: K.spatial_3d_padding(_x, pads, data_format))(x) else: raise NotImplementedError( "padding with dim {} is not implemented.".format(dim))
def _deconv(self, X, lname, d_switch, feat_map=None): o_width, o_height = self[lname].output_shape[-2:] # Get filter size f_width = self[lname].W_shape[2] f_height = self[lname].W_shape[3] # Keras 2.0. # f_width = self[lname].kernel_size[0] # f_height = self[lname].kernel_size[1] # Compute padding needed i_width, i_height = X.shape[-2:] pad_width = (o_width - i_width + f_width - 1) / 2 pad_height = (o_height - i_height + f_height - 1) / 2 pad_width = int(pad_width) pad_height = int(pad_height) assert isinstance(pad_width, int), "Pad width size issue at layer %s" % lname assert isinstance(pad_height, int), "Pad height size issue at layer %s" % lname # Set to zero based on switch values X[d_switch[lname]] = 0 # Get activation function activation = self[lname].activation X = activation(X) if feat_map is not None: logger.debug("Setting other feat map to zero") for i in range(X.shape[1]): if i != feat_map: X[:, i, :, :] = 0 logger.debug("Setting non max activations to zero") for i in range(X.shape[0]): iw, ih = np.unravel_index(X[i, feat_map, :, :].argmax(), X[i, feat_map, :, :].shape) m = np.max(X[i, feat_map, :, :]) X[i, feat_map, :, :] = 0 X[i, feat_map, iw, ih] = m # Get filters. No bias for now W = self[lname].W # W = self[lname].kernel # keras 2.0 # Transpose filter W = W.transpose([1, 0, 2, 3]) W = W[:, :, ::-1, ::-1] # CUDNN for conv2d ? conv_out = K.T.nnet.conv2d(input=self.x, filters=W, border_mode='valid') # Add padding to get correct size pad = K.function([self.x], K.spatial_2d_padding(self.x, padding=(pad_width, pad_height), dim_ordering="th")) # Keras 2.0 but not sure # pad = K.function([self.x], K.spatial_2d_padding( # self.x, padding=((pad_width, 0), (0, pad_height)), data_format="channels_first")) X_pad = pad([X]) # Get Deconv output deconv_func = K.function([self.x], conv_out) X_deconv = deconv_func([X_pad]) assert X_deconv.shape[-2:] == (o_width, o_height), "Deconv output at %s has wrong size" % lname return X_deconv
def _initialize_ii_buffer(self, x): x_pad = K.spatial_2d_padding( x, ((self.max_wh // 2 + 1, self.max_wh // 2 + 1), (self.max_ww // 2 + 1, self.max_ww // 2 + 1))) ii_x = K.cumsum(x_pad, axis=1) ii_x2 = K.cumsum(ii_x, axis=2) return ii_x2
def local_response_normalization(x, alpha=1e-4, k=2, beta=0.75, n=5): """ :param x: :param alpha: :param k: :param beta: :param n: :return: """ if K.image_data_format() == 'channel_first': b, ch, r, c = K.int_shape(x) else: b, r, c, ch = K.int_shape(x) x = K.permute_dimensions(x, (0, 3, 1, 2)) square = K.square(x) half = n // 2 extra_channels = K.spatial_2d_padding(square, ((half, half), (0, 0))) scale = k for i in range(n): scale += alpha * extra_channels[:, i:i + ch, :, :] scale**beta output = x / scale if K.image_data_format() == 'channels_last': output = K.permute_dimensions(output, (0, 2, 3, 1)) return output
def inst_weight(output_y, output_x, output_dr, output_dl, config=None): dy = output_y[:,2:,2:]-output_y[:, :-2,2:] + \ 2*(output_y[:,2:,1:-1]- output_y[:,:-2,1:-1]) + \ output_y[:,2:,:-2]-output_y[:,:-2,:-2] dx = output_x[:,2:,2:]- output_x[:,2:,:-2] + \ 2*( output_x[:,1:-1,2:]- output_x[:,1:-1,:-2]) +\ output_x[:,:-2,2:]- output_x[:,:-2,:-2] ddr= (output_dr[:,2:,2:]-output_dr[:,:-2,:-2] +\ output_dr[:,1:-1,2:]-output_dr[:,:-2,1:-1]+\ output_dr[:,2:,1:-1]-output_dr[:,1:-1,:-2])*K.constant(2) ddl= (output_dl[:,2:,:-2]-output_dl[:,:-2,2:] +\ output_dl[:,2:,1:-1]-output_dl[:,1:-1,2:]+\ output_dl[:,1:-1,:-2]-output_dl[:,:-2,1:-1])*K.constant(2) dpred = K.concatenate([dy,dx,ddr,ddl],axis=-1) dpred = K.spatial_2d_padding(dpred) weight_fg = K.cast(K.all(dpred>K.constant(config.GRADIENT_THRES), axis=3, keepdims=True), K.floatx()) weight = K.clip(K.sqrt(weight_fg*K.prod(dpred, axis=3, keepdims=True)), config.WEIGHT_AREA/config.CLIP_AREA_HIGH, config.WEIGHT_AREA/config.CLIP_AREA_LOW) weight +=(1-weight_fg)*config.WEIGHT_AREA/config.BG_AREA weight = K.conv2d(weight, K.constant(config.GAUSSIAN_KERNEL), padding='same') return K.stop_gradient(weight)
def f(X): b, ch, r, c = X.shape half = n // 2 square = K.square(X) # extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0,2,3,1)) # , (0,half)) #extra_channels = K.permute_dimensions(extra_channels, (0,3,1,2)) extra_channels = K.spatial_2d_padding( K.permute_dimensions(square, (0, 2, 3, 1)), ((0, 0), (half, half))) extra_channels = K.spatial_2d_padding( K.permute_dimensions(square, (0, 2, 3, 1)), ((0, 0), (0, n))) scale = k for i in range(n): scale += alpha * extra_channels[:, i:i + ch, :, :] scale = scale**beta return X / scale
def cross_input_asym(X): tensor_left = X[0] tensor_right = X[1] x_length = K.int_shape(tensor_left)[1] y_length = K.int_shape(tensor_left)[2] cross_y = [] cross_x = [] tensor_left_padding = K.spatial_2d_padding(tensor_left,padding=(2,2)) tensor_right_padding = K.spatial_2d_padding(tensor_right,padding=(2,2)) for i_x in range(2, x_length + 2): for i_y in range(2, y_length + 2): cross_y.append(tensor_left_padding[:,i_x-2:i_x+3,i_y-2:i_y+3,:] - concat_iterat(tensor_right_padding[:,i_x,i_y,:])) cross_x.append(K.concatenate(cross_y,axis=2)) cross_y = [] cross_out = K.concatenate(cross_x,axis=1) return K.abs(cross_out)
def call(self, x): k = self.a k = k[:, None] * k[None, :] k = k / K.sum(k) k = K.tile(k[:, :, None, None], (1, 1, K.int_shape(x)[-1], 1)) k = K.constant(k, dtype=K.floatx()) x = K.spatial_2d_padding(x, padding=self.padding) x = K.depthwise_conv2d(x, k, strides=self.strides, padding='valid') return x
def _interpolate(im, x, y): _edge_size = 0 if _wrap_mode == 'border': _edge_size = 1 im = K.spatial_2d_padding(im, padding=((1, 1), (1, 1))) x = x + _edge_size y = y + _edge_size elif _wrap_mode == 'edge': _edge_size = 0 else: return None x = K.clip(x, 0.0, K.eval(_width_f) - 1 + 2 * _edge_size) x0_f = K.round(x) y0_f = K.round(y) x1_f = x0_f + 1 x0 = K.cast(x0_f, 'int32') y0 = K.cast(y0_f, 'int32') x1 = K.cast(K.minimum(x1_f, K.eval(_width_f) - 1 + 2 * _edge_size), 'int32') dim2 = (_width + 2 * _edge_size) dim1 = (_width + 2 * _edge_size) * (_height + 2 * _edge_size) base = _repeat(K.arange(_num_batch) * dim1, _height * _width) base_y0 = base + y0 * dim2 idx_l = base_y0 + x0 idx_r = base_y0 + x1 im_flat = K.reshape(im, K.stack([-1, _num_channels])) pix_l = K.gather(im_flat, idx_l) pix_r = K.gather(im_flat, idx_r) weight_l = K.expand_dims(x1_f - x, 1) weight_r = K.expand_dims(x - x0_f, 1) return weight_l * pix_l + weight_r * pix_r
def f(X): b, ch, r, c = X.shape.as_list() half = n // 2 # Add some zero channels extra_channels = K.spatial_2d_padding(K.square(X), ((half, half), (0, 0)), data_format='channels_last') scale = k for i in range(n): scale += alpha * extra_channels[:, i:i + ch, :, :] scale = scale ** beta return X / scale
def f(X): b, r, c, ch = X.shape half = n // 2 square = K.square(X) extra_channels = K.spatial_2d_padding(square, ((0, 0), (half, half)), data_format='channels_first') scale = k for i in range(n): scale += alpha * extra_channels[:, :, :, i:i + int(ch)] scale = scale ** beta return X / scale
def _concat_features(lf, states): b, f, h, w = lf.get_shape().as_list() rf = states[0] rfs = rf[:, :, :, :-1] disp_rfs = K.spatial_2d_padding(rfs, padding=((0, 0), (1, 0)), data_format='channels_first') concat = K.concatenate([lf, rf], axis=2) output = K.reshape(concat, (-1, 2 * f, h, w)) return output, [disp_rfs]
def call(self, inputs): """ :param inputs: :return: """ _, r, c, ch = K.int_shape(inputs) # print("Call Fcn: Input shape ", r, c, ch) # 1. Inputs Formatting # Channel First, batch second. This is done to take the unknown batch size into the matrix multiply # where it can be handled more easily padded_inputs = K.spatial_2d_padding(inputs, ((self.n / 2, self.n / 2), (self.n / 2, self.n / 2))) inputs_chan_first = K.permute_dimensions(padded_inputs, [3, 0, 1, 2]) # print("Call Fcn: inputs_chan_first shape: ", inputs_chan_first.shape) # 2. Kernel kernel_chan_first = K.permute_dimensions(self.kernel, (2, 0, 1)) # print("Call Fcn: kernel_chan_first shape", kernel_chan_first.shape) k_ch, k_r, k_c = K.int_shape(kernel_chan_first) apply_kernel = K.reshape(kernel_chan_first, (k_ch, k_r * k_c, 1)) # print("Call Fcn: kernel for matrix multiply: ", apply_kernel.shape) # 3. Get outputs at each spatial location xs = [] for i in range(r): for j in range(c): input_slice = inputs_chan_first[:, :, i:i + self.n, j:j + self.n] input_slice_apply = K.reshape(input_slice, (ch, -1, self.n**2)) output_slice = K.batch_dot(input_slice_apply, apply_kernel) # Reshape the output slice to put batch first output_slice = K.permute_dimensions(output_slice, [1, 0, 2]) xs.append(output_slice) # print("Call Fcn: len of xs", len(xs)) # print("Call Fcn: shape of each element of xs", xs[0].shape) # 4. Reshape the output to correct format outputs = K.concatenate(xs, axis=2) outputs = K.reshape(outputs, (-1, ch, r, c)) # Break into row and column outputs = K.permute_dimensions(outputs, [0, 2, 3, 1]) # Back to batch first # print("Call Fcn: shape of output", outputs.shape) # 5. Add the lateral and the feed-forward activations outputs += inputs return outputs
def f(X): b, r, c, ch = X.shape half = n // 2 square = keras.backend.square(X) extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 2, 3, 1)), ((0, 0), (half, half))) extra_channels = K.permute_dimensions(extra_channels, (0, 3, 1, 2)) scale = k for i in range(n): scale += alpha * extra_channels[:, :, :, i:i + int(ch)] scale = scale ** beta return X / scale
def cut_and_paste(x): background = x[0] crop = x[1] mask = x[2] mask_to_paste = mask * crop mask_to_paste = K.spatial_2d_padding(mask_to_paste, padding=((PADDING*2, 0), (PADDING, PADDING)), data_format='channels_last') prep_mask = K.spatial_2d_padding(mask, padding=((PADDING*2, 0), (PADDING, PADDING)), data_format='channels_last') inverted_mask = 1 - prep_mask cp_img = (background * inverted_mask) + mask_to_paste return cp_img
def f(X): b, ch, r, c = X.shape # batch, channel, row, column half = n // 2 square = K.square(X) extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 2, 3, 1)), (0, half)) extra_channels = K.permute_dimensions(extra_channels, (0, 3, 1, 2)) scale = k for i in range(n): scale += alpha * extra_channels[:, i:i + ch, :, :] scale = scale ** beta return X / scale
def CVLayer(inputs, D): lfeatures = inputs[0] rfeatures = inputs[1] output = [] for disparity in np.arange(D): rfs = rfeatures[:, :, :, disparity:] disp_rfs = K.spatial_2d_padding(rfs, padding=((0, 0), (0, disparity)), data_format='channels_first') output.append(K.concatenate([lfeatures, disp_rfs], axis=1)) return K.stack(output, axis=2)
def call(self, x): if self.binarize_input_ == True: x = Binarization(x) self.binary_kernel = Binarization(K.clip(self.float_kernel, -1.0, 1.0)) x = K.spatial_2d_padding(x, padding=((self.pad_, self.pad_), (self.pad_, self.pad_))) return K.conv2d(x, self.binary_kernel, strides=(self.stride_, self.stride_), padding='valid', data_format='channels_last')
def _getCostVolume_(inputs, max_d): left_tensor, right_tensor = inputs shape = K.shape(right_tensor) #(batch, height, width, channel) right_tensor = K.spatial_2d_padding(right_tensor, padding=((0, 0), (max_d, 0))) disparity_costs = [] for d in reversed(range(max_d)): left_tensor_slice = left_tensor right_tensor_slice = tf.slice(right_tensor, begin=[0, 0, d, 0], size=[-1, -1, shape[2], -1]) cost = K.concatenate([left_tensor_slice, right_tensor_slice], axis=3) disparity_costs.append(cost) cost_volume = K.stack(disparity_costs, axis=1) #(batch, D, height, width, 2*channel) return cost_volume
def f(X): if K.image_dim_ordering()=='tf': b, r, c, ch = X.get_shape() else: b, ch, r, c = X.shape half = n // 2 square = K.square(X) scale = k if K.image_dim_ordering() == 'th': extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 2, 3, 1)), (0, half)) extra_channels = K.permute_dimensions(extra_channels, (0, 3, 1, 2)) for i in range(n): scale += alpha * extra_channels[:, i:i+ch, :, :] if K.image_dim_ordering() == 'tf': extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 3, 1, 2)), (half, 0)) extra_channels = K.permute_dimensions(extra_channels, (0, 2, 3, 1)) for i in range(n): scale += alpha * extra_channels[:, :, :, i:i+int(ch)] scale = scale ** beta return X / scale
def _deconv(self, X, lname, d_switch, feat_map=None): o_width, o_height = self[lname].output_shape[-2:] # Get filter size f_width = self[lname].W_shape[2] f_height = self[lname].W_shape[3] # Compute padding needed i_width, i_height = X.shape[-2:] pad_width = (o_width - i_width + f_width - 1) / 2 pad_height = (o_height - i_height + f_height - 1) / 2 assert isinstance( pad_width, int), "Pad width size issue at layer %s" % lname assert isinstance( pad_height, int), "Pad height size issue at layer %s" % lname # Set to zero based on switch values X[d_switch[lname]] = 0 # Get activation function activation = self[lname].activation X = activation(X) if feat_map is not None: print "Setting other feat map to zero" for i in range(X.shape[1]): if i != feat_map: X[:, i, :, :] = 0 print "Setting non max activations to zero" for i in range(X.shape[0]): iw, ih = np.unravel_index( X[i, feat_map, :, :].argmax(), X[i, feat_map, :, :].shape) m = np.max(X[i, feat_map, :, :]) X[i, feat_map, :, :] = 0 X[i, feat_map, iw, ih] = m # Get filters. No bias for now W = self[lname].W # Transpose filter W = W.transpose([1, 0, 2, 3]) W = W[:, :, ::-1, ::-1] # CUDNN for conv2d ? conv_out = K.T.nnet.conv2d( input=self.x, filters=W, border_mode='valid') # Add padding to get correct size pad = K.function([self.x], K.spatial_2d_padding( self.x, padding=(pad_width, pad_height), dim_ordering="th")) X_pad = pad([X]) # Get Deconv output deconv_func = K.function([self.x], conv_out) X_deconv = deconv_func([X_pad]) assert X_deconv.shape[-2:] == (o_width, o_height),\ "Deconv output at %s has wrong size" % lname return X_deconv
def _deconv(self, X, lname, conv_input_shape = None, W = None): if conv_input_shape is None: o_width, o_height = self[lname].output_shape[-2:] else: o_width, o_height = self.input_shape_withoutpadding[lname][-2:] if W is None: # Get filters. No bias for now W = self[lname].W # Get filter size if conv_input_shape is None: f_width = self[lname].W_shape[2] f_height = self[lname].W_shape[3] else: f_width = f_height = conv_input_shape[2] # Compute padding needed i_width, i_height = X.shape[-2:] pad_width = (o_width - i_width + f_width - 1) / 2 pad_height = (o_height - i_height + f_height - 1) / 2 assert isinstance(pad_width, int), "Pad width size issue at layer %s" % lname assert isinstance(pad_height, int), "Pad height size issue at layer %s" % lname if self.lnames.index(lname) != len(self.lnames)-1: # Get activation function, do not apply to last layer (softmax) activation = self[lname].activation X = activation(X) # Transpose filter W = W.transpose([1, 0, 2, 3]) W = W[:,:, ::-1, ::-1] # CUDNN for conv2d ? conv_out = K.T.nnet.conv2d(input=self.x, filters=W, border_mode='valid') # Add padding to get correct size pad = K.function([self.x,K.learning_phase()], K.spatial_2d_padding( self.x, padding=(pad_width, pad_height), dim_ordering="th")) X_pad = pad([X,1]) # Get Deconv output deconv_func = K.function([self.x,K.learning_phase()], conv_out) X_deconv = deconv_func([X_pad,1]) assert X_deconv.shape[-2:] == (o_width, o_height),\ "Deconv output at %s has wrong size" % lname return X_deconv