Beispiel #1
0
 def _pooling_function(self, inputs, pool_size, strides, border_mode, dim_ordering):
     '''
     This pooling operation is basically the sum of both MaxPooling and AveragePooling outputs.
     '''
     output1 = K.pool2d(inputs, pool_size, strides, border_mode, dim_ordering, pool_mode='max')
     output2 = K.pool2d(inputs, pool_size, strides, border_mode, dim_ordering, pool_mode='avg')
     return output1 + output2
Beispiel #2
0
    def _pooling_function(self, back_end, inputs, pool_size, strides,
                          border_mode, dim_ordering):

        output1 = K.pool2d(inputs, pool_size, strides,
                  border_mode, dim_ordering, pool_mode='max')
        output2 = K.pool2d(inputs, pool_size, strides,
                           border_mode, dim_ordering, pool_mode='ave')
        return output1 + output2
Beispiel #3
0
def weighted_bce_dice_loss(y_true, y_pred):
    y_true = K.cast(y_true, 'float32')
    y_pred = K.cast(y_pred, 'float32')
    # if we want to get same size of output, kernel size must be odd number
    averaged_mask = K.pool2d(
            y_true, pool_size=(11, 11), strides=(1, 1), padding='same', pool_mode='avg')
    border = K.cast(K.greater(averaged_mask, 0.005), 'float32') * K.cast(K.less(averaged_mask, 0.995), 'float32')
    weight = K.ones_like(averaged_mask)
    w0 = K.sum(weight)
    weight += border * 2
    w1 = K.sum(weight)
    weight *= (w0 / w1)
    loss = weighted_bce_loss(y_true, y_pred, weight) + \
    weighted_dice_loss(y_true, y_pred, weight)
    return loss
Beispiel #4
0
 def call(self, x, mask=None):
     if K.image_dim_ordering == "th":
         _, f, r, c = self.shape
     else:
         _, r, c, f = self.shape
     half_n = self.n // 2
     squared = K.square(x)
     pooled = K.pool2d(squared, (half_n, half_n), strides=(1, 1),
                      padding="same", pool_mode="avg")
     if K.image_dim_ordering == "th":
         summed = K.sum(pooled, axis=1, keepdims=True)
         averaged = (self.alpha / self.n) * K.repeat_elements(summed, f, axis=1)
     else:
         summed = K.sum(pooled, axis=3, keepdims=True)
         averaged = (self.alpha / self.n) * K.repeat_elements(summed, f, axis=3)
     denom = K.pow(self.k + averaged, self.beta)
     return x / denom
Beispiel #5
0
 def get_output(self, train=False):
     print "OccMaxPool", self.output_shape, self.input_shape
     num_tracks = (
         self.input_shape[1] if self.num_tracks == 'full' 
         else self.num_tracks
     )
     num_bases = (
         self.input_shape[3] if self.num_bases == 'full' 
         else self.num_bases
     )
     X = self.get_input(train)
     X = K.permute_dimensions(X, (0,2,1,3))
     rv = K.pool2d(
         X, 
         pool_size=(num_tracks, num_bases), 
         strides=(num_tracks, num_bases),
         pool_mode='max'
     )
     rv = K.permute_dimensions(rv, (0,2,1,3))
     return rv
Beispiel #6
0
 def call(self, x, mask=None):
     if K.image_dim_ordering == "th":
         _, f, r, c = self.shape
     else:
         _, r, c, f = self.shape
     half_n = self.n // 2
     squared = K.square(x)
     pooled = K.pool2d(squared, (half_n, half_n),
                       strides=(1, 1),
                       padding="same",
                       pool_mode="avg")
     if K.image_dim_ordering == "th":
         summed = K.sum(pooled, axis=1, keepdims=True)
         averaged = (self.alpha / self.n) * K.repeat_elements(
             summed, f, axis=1)
     else:
         summed = K.sum(pooled, axis=3, keepdims=True)
         averaged = (self.alpha / self.n) * K.repeat_elements(
             summed, f, axis=3)
     denom = K.pow(self.k + averaged, self.beta)
     return x / denom
Beispiel #7
0
    def routing_step(feature_map, pool_size, strides, score_map):

        # Compute average vector
        if score_map is None:
            # in first iteration we cannot compute weights average
            # just take input feature map
            weighted_fm = feature_map
        else:
            # normalize scores
            normalized_scores = CapsulePooling2D.normalize_pool_map(
                score_map, pool_size=pool_size, strides=strides, temperature=2)
            weighted_fm = normalized_scores * feature_map

        avg_feature_map = K.pool2d(weighted_fm,
                                   pool_size=pool_size,
                                   strides=strides,
                                   pool_mode='avg',
                                   padding='same')

        avg_feature_map = CapsulePooling2D.squash_fm(avg_feature_map)
        avg_feature_columns = K.resize_images(avg_feature_map,
                                              pool_size[0],
                                              pool_size[1],
                                              data_format='channels_last')
        # fit caps columns to feature map
        input_shape = K.int_shape(feature_map)
        avg_feature_columns = avg_feature_columns[:, :input_shape[
            1], :input_shape[2], :input_shape[3]]

        # compute V * Vs
        new_score_map = K.sum(feature_map * avg_feature_columns,
                              -1,
                              keepdims=True)

        if score_map is None:
            score_map = new_score_map
        else:
            score_map = score_map + new_score_map

        return score_map, weighted_fm
def _do_max_pooling(feature_matrix):
    """Performs max-pooling on 2-D feature map.

    M = number of rows before pooling
    N = number of columns before pooling
    m = number of rows after pooling
    n = number of columns after pooling

    :param feature_matrix: Feature map as M-by-N numpy array.
    :return: feature_matrix: New feature map (m-by-n).
    """

    this_feature_matrix = numpy.expand_dims(feature_matrix, axis=0)
    this_feature_matrix = numpy.expand_dims(this_feature_matrix, axis=-1)

    feature_tensor = K.pool2d(x=K.variable(this_feature_matrix),
                              pool_mode='max',
                              pool_size=(2, 2),
                              strides=(2, 2),
                              padding='valid',
                              data_format='channels_last')
    return feature_tensor.eval(session=K.get_session())[0, ..., 0]
    def _batch_gen(self, traces, spikes, shape, batch_size, nb_steps, margin):

        # Apply the error margin by max pooling the spikes once up-front.
        lens = [len(x) for x in spikes]
        for l in np.unique(lens):
            idxs, = np.where(lens == l)
            x = np.vstack([spikes[i] for i in idxs])
            x = K.variable(x.astype(np.float32))
            x = K.expand_dims(K.expand_dims(x, axis=0), axis=-1)
            x = K.pool2d(x, (1, margin + 1), padding='same')
            x = K.get_value(x[0, :, :, 0])
            for i in range(x.shape[0]):
                spikes[idxs[i]] = x[i]

        while True:

            idxs = np.arange(len(traces))
            cidxs = cycle(rng.choice(idxs, len(idxs), replace=False))

            for _ in range(nb_steps):

                # Empty batches (traces and spikes).
                tb = np.zeros((batch_size, ) + shape, dtype=np.float64)
                sb = np.zeros((batch_size, ) + shape, dtype=np.uint8)

                for bidx in range(batch_size):

                    # Dataset and sample indices.
                    idx = next(cidxs)

                    # Pick start and end point around positive spike index.
                    x0 = rng.randint(0, len(spikes[idx]) - shape[0])
                    x1 = x0 + shape[0]

                    # Populate batch.
                    tb[bidx] = traces[idx][x0:x1]
                    sb[bidx] = spikes[idx][x0:x1]

                yield tb, sb
Beispiel #10
0
    def _get_laplacian_pyramid(self, inputs: plaidml.tile.Value) -> List[plaidml.tile.Value]:
        """ Obtain the Laplacian Pyramid.

        Parameters
        ----------
        inputs: :class:`plaidml.tile.Value`
            The input batch of images to run through the Laplacian Pyramid

        Returns
        -------
        list
            The tensors produced from the Laplacian Pyramid
        """
        pyramid = []
        current = inputs
        for _ in range(self._max_levels):
            gauss = self._conv_gaussian(current)
            diff = current - gauss
            pyramid.append(diff)
            current = K.pool2d(gauss, (2, 2), strides=(2, 2), padding="valid", pool_mode="avg")
        pyramid.append(current)
        return pyramid
Beispiel #11
0
 def call(self, inputs):
     # Input = [X^H, X^L]
     assert len(inputs) == 2
     high_input, low_input = inputs
     # High -> High conv
     high_to_high = K.conv2d(high_input,
                             self.high_to_high_kernel,
                             strides=self.strides,
                             padding=self.padding,
                             data_format="channels_last")
     # High -> Low conv
     high_to_low = K.pool2d(high_input, (2, 2),
                            strides=(2, 2),
                            pool_mode="avg")
     high_to_low = K.conv2d(high_to_low,
                            self.high_to_low_kernel,
                            strides=self.strides,
                            padding=self.padding,
                            data_format="channels_last")
     # Low -> High conv
     low_to_high = K.conv2d(low_input,
                            self.low_to_high_kernel,
                            strides=self.strides,
                            padding=self.padding,
                            data_format="channels_last")
     low_to_high = K.repeat_elements(low_to_high, 2,
                                     axis=1)  # Nearest Neighbor Upsampling
     low_to_high = K.repeat_elements(low_to_high, 2, axis=2)
     # Low -> Low conv
     low_to_low = K.conv2d(low_input,
                           self.low_to_low_kernel,
                           strides=self.strides,
                           padding=self.padding,
                           data_format="channels_last")
     # Cross Add
     high_add = high_to_high + low_to_high
     low_add = high_to_low + low_to_low
     return [high_add, low_add]
Beispiel #12
0
def weighted_dice_loss(y_true, y_pred):
    y_true = K.cast(y_true, 'float32')
    y_pred = K.cast(y_pred, 'float32')
    if K.int_shape(y_pred)[1] == 128:
        kernel_size = 11
    elif K.int_shape(y_pred)[1] == 256:
        kernel_size = 21
    elif K.int_shape(y_pred)[1] == 512:
        kernel_size = 21
    elif K.int_shape(y_pred)[1] == 1024:
        kernel_size = 41
    else:
        raise ValueError('Unexpected image size')
    averaged_mask = K.pool2d(
        y_true, pool_size=(kernel_size, kernel_size), strides=(1, 1), padding='same', pool_mode='avg')
    border = K.cast(K.greater(averaged_mask, 0.005), 'float32') * K.cast(K.less(averaged_mask, 0.995), 'float32')
    weight = K.ones_like(averaged_mask)
    w0 = K.sum(weight)
    weight += border * 2
    w1 = K.sum(weight)
    weight *= (w0 / w1)
    loss = 1 - weighted_dice_coeff(y_true, y_pred, weight)
    return loss
Beispiel #13
0
	def my_loss(x_true, merged_decode):
		#batch, n = merged_decode.get_shape()
		x_decode = merged_decode[:, :n]
		z = merged_decode[:, n:]

		#h_xi_zj = gaussian_cond_ent(x_true, z, invert_sigmoid = True, subtract_log_det =True) # NOW this is a conditional entropy not MI
		mi_ji = gaussian_mi(x_true, z) # prev

		#OLD calculation of h(Xi - g(Z)) term within screening loss
		#xi_entropy_est = error_entropy(x_true, 0, skew = skew, kurt = kurt)
		#mi_xi_z = error_entropy(x_true, x_decode, invert_sigmoid = True, subtract_log_det=True, skew = skew, kurt = kurt)
		
		max_mi = K.pool2d(x=K.reshape(mi_ji, (1, 1, K.cast(z.shape[1], 'int32'), n)),
						  pool_size=(z.shape[1], 1), strides=(1, 1),
						  padding='valid',
						  data_format='channels_first',
						  pool_mode='max')
		max_mi = K.reshape(max_mi, (n,))
		
		# TAKING MIN using smooth min (-alpha)
		#max_mi = tf.divide(K.sum(tf.multiply(mi_ji, K.exp(alpha*mi_ji)), axis = 0), K.sum(K.exp(alpha*mi_ji), axis = 0))

		return K.sum(mi_ji) - K.sum(max_mi)
Beispiel #14
0
    def _pooling_function(self, inputs, pool_size, strides,
                          padding, data_format):

        channels = inputs.get_shape()[-1]
        holder = []
        for c in range(channels):
            init = tf.expand_dims(inputs[:,:,:,c], axis=-1)
            if self.rank == 2:
                outputs = K.conv2d(
                    init,
                    tf.expand_dims(self.kernel[:,:,c,:], 2),
                    strides=self.strides,
                    padding=self.padding,
                    data_format=self.data_format)
		if self.with_softmax:
		    exp = tf.exp(outputs)	
		    prob = exp / tf.reduce_sum(exp, -1, keep_dims=True)
                    outputs = outputs * prob
                holder.append(tf.expand_dims(K.mean(outputs, axis=-1), axis=-1))
        output = K.pool2d(inputs, self.kernel_size, strides,
            padding, data_format, pool_mode='max')
        concat = tf.concat(holder, axis=-1)
        out = concat + output
        return out
Beispiel #15
0
    def call(self, x, **kwargs):
        #make sure the inputs is a list?
        out_shape = K.shape(x)
        input_shape = K.int_shape(x)
        #note the input_shape = (samples,cols,rows,channels)
        num_rows = input_shape[1]
        num_cols = input_shape[2]
        #divided pooling_list
        row_length = [num_rows / i for i in self.pooling_list]
        col_length = [num_cols / i for i in self.pooling_list]

        outputs = []
        for i, pooling_size in enumerate(self.pooling_list):
            #这里需要注意了,看看原文
            h_wid = math.ceil(row_length[i])
            w_wid = math.ceil(col_length[i])
            h_str = math.floor(row_length[i])
            w_str = math.floor(col_length[i])
            out = K.pool2d(x=x,
                           pool_size=(h_wid, w_wid),
                           strides=(h_str, w_str))
            out = K.reshape(x=out, shape=(out_shape[0], -1))
            outputs.append(out)
        return K.concatenate(outputs, 1)
Beispiel #16
0
    def call(self, inputs, **kwargs):

        context = inputs[0]

        x, y = (inputs[1][:, 0], inputs[1][:, 1])
        # preprocessing states to get memory

        # Need this for later computations
        batch_size = tf.shape(context)[0]

        # global read operation
        # r_t = read(M_t) ; output dimension of r_t = self.units
        first_conv = K.conv2d(self.neural_map,
                              self.conv_kernel1,
                              strides=(1, 1),
                              padding='valid')
        #second_conv = K.conv2d(first_conv, self.conv_kernel2, strides=(1, 1), padding='valid')
        pool_conv = K.pool2d(first_conv,
                             pool_size=(2, 2),
                             strides=(1, 1),
                             padding='valid',
                             pool_mode='avg')
        flatten_conv = K.reshape(pool_conv, (1, -1))
        dense1_conv = K.dot(flatten_conv, self.conv_dense1)
        #dense2_conv = K.dot(dense1_conv, self.conv_dense2)
        r_t = dense1_conv
        r_t = K.tile(r_t, K.stack([batch_size, 1]))

        #r_t = K.print_tensor(r_t, "r_t: ")

        # context read operation
        # c_t = context(M_t, s_t, r_t)
        q_t = K.concatenate([context, r_t])  # [1x(s+c)]
        q_t = K.dot(q_t, self.context_kernel)  # [1xc]

        #q_t = K.print_tensor(q_t, "q_t: ")

        # at = K.exp(K.dot(q_t, memory)) # [1xhxw]
        # at_sum = K.sum(at)
        # at_sum_repeated = K.repeat (at_sum, (self.memory_size[0], self.memory_size[1]))
        # at /= at_sum # [1xhxw]

        # Compute attention with softmax, use batch_dot function
        # Keras is garbage, and so am I
        at = tf.tensordot(q_t, self.neural_map, axes=[[1], [3]])
        at /= tf.norm(at, keepdims=True)
        at = K.exp(at)
        #at = tf.Print(at, [at], "at1: ", summarize=1000)

        # Annoying conversion
        neural_map_tensor = tf.convert_to_tensor(self.neural_map)
        #neural_map_tensor = tf.Print(neural_map_tensor, [neural_map_tensor], "neural_map_tensor: ", summarize=1000)
        #at = K.sum(q_t_repeated * neural_map_tensor, axis=3)
        at_sum = K.sum(at, axis=2, keepdims=True)
        at_sum = K.sum(at_sum, axis=3, keepdims=True)
        at_sum = K.repeat_elements(at_sum, self.memory_size[0], 2)
        at_sum = K.repeat_elements(at_sum, self.memory_size[1], 3)
        #at_sum = tf.Print(at_sum, [at_sum], "at_sum: ", summarize=1000)

        at = at / at_sum
        at = K.repeat_elements(at, self.units, 1)
        at = K.permute_dimensions(at, (0, 2, 3, 1))

        #at = K.print_tensor(at, "at: ")
        #at = tf.Print(at, [at], "at2: ", summarize=1000)

        c_t = at * neural_map_tensor

        #c_t = tf.Print(c_t, [c_t], "c_t1: ", summarize=1000)

        c_t = K.sum(c_t, axis=1)
        #c_t = tf.Print(c_t, [c_t], "c_t2: ", summarize=1000)

        c_t = K.sum(c_t, axis=1)

        #c_t = tf.Print(c_t, [c_t], "c_t3: ", summarize=1000)
        # Computing write value
        # m_t+1_x,y = write(s_t, r_t, c_t, m_t_x,y)
        s_t = K.dot(context, self.write_kernel)  # [1xc]
        #s_t = tf.Print(s_t, [s_t], "s_t: ", summarize=1000)
        global_imp = K.batch_dot(s_t, r_t, axes=1)  # [1x1]
        #global_imp = tf.Print(global_imp, [global_imp], "global_imp: ", summarize=1000)

        local_imp = K.batch_dot(s_t, c_t, axes=1)  # [1x1]
        #local_imp = tf.Print(local_imp, [local_imp], "local_imp: ", summarize=1000)

        # Convert batched x,y coordinates into indices for memory
        # We want to index into memory by doing something like
        # memory[sample_number, y, x] so that we get a {units} dimensional
        # vector out. When we do this for every sample in the batch, we'll
        # end up with a {number_batches, units} dimensional matrix.

        # First we copy and reshape the memory into {batch_size x H x W x units}
        zeros = K.zeros_like(y)
        idx = K.stack([zeros, y, x], axis=1)  # {batch_size x 3}

        # And index into the memory using tf.gather_nd
        mem_t = tf.gather_nd(neural_map_tensor, idx)  # {batch_size x units}
        #mem_t = tf.Print(mem_t, [mem_t], "mem_t before update: ", summarize=1000)

        # This line has a divide-by-zero case, in (local_imp + global_imp).
        #mem_t += (local_imp / (local_imp + global_imp)) * K.dot((mem_t - s_t), self.write_update)

        #Check where local_imp + global_imp is zero, then just increase it by 1.
        #local_imp = tf.Print(local_imp, [local_imp], "local_imp: ", summarize=1000)
        #global_imp = tf.Print(global_imp, [global_imp], "global_imp: ", summarize=1000)

        combined_imp = local_imp + global_imp
        more_zeros = K.zeros_like(combined_imp)
        eq = tf.equal(combined_imp, more_zeros)
        combined_imp = tf.where(eq, combined_imp + 1., combined_imp)

        #combined_imp = tf.Print(combined_imp, [combined_imp], "combined_imp: ", summarize=1000)

        mem_t += (local_imp / combined_imp) * K.dot(
            (mem_t - s_t), self.write_update)
        mem_t /= tf.norm(mem_t, keepdims=True)
        #mem_t = tf.Print(mem_t, [mem_t], "mem_t after update: ", summarize=1000)

        # Update memory.
        updates = []
        neural_map_updated = tf.scatter_nd_update(self.neural_map, idx, mem_t)
        updates.append((self.neural_map, neural_map_updated))
        self.add_update(updates, inputs)
        #neural_map_updated = K.print_tensor(neural_map_updated, "neural_map_updated: ")
        # memory[:, :] = mem_t

        return c_t + mem_t
Beispiel #17
0
def conv2d_offset_average_pool_eval(_, padding, x):
    """Perform an average pooling operation on x"""
    return K.pool2d(x, (1, 1),
                    strides=(3, 3),
                    padding=padding,
                    pool_mode='avg')
Beispiel #18
0
    def call(self, input_):
        """
        Forward pass through the layer.

        Args:
            input_: the input tensor to pass through the pyramid pooling module

        Returns:
            the output tensor from the pyramid pooling module

        """
        # the shape to up-sample pooled tensors to
        if self.data_format == 'channels_last':
            channel_axis = -1
            output_shape = K.int_shape(input_)[1:-1]
        if self.data_format == 'channels_first':
            channel_axis = 1
            output_shape = K.int_shape(input_)[2:]
        # create a list of output tensors to concatenate together
        output_tensors = [input_]
        # iterate over the bin sizes in the pooling module
        for (level, bin_size) in enumerate(self.bin_sizes):
            # pass the inputs through the pooling layer with the given bin
            # size, i.e., a square kernel with side matching the bin size and
            # a matching stride
            x = K.pool2d(
                input_,
                tuple(dim // bin_size for dim in output_shape),
                padding=self.padding,
                pool_mode=self.pool_mode,
                data_format=self.data_format,
            )
            # pass the pooled valued through a 1 x 1 convolution
            x = K.conv2d(
                x,
                self.kernels[level],
                strides=(1, 1),
                padding=self.padding,
                data_format=self.data_format,
            )
            # if use bias, apply the bias to the convolved outputs
            if self.use_bias:
                x = K.bias_add(
                    x,
                    self.biases[level],
                    data_format=self.data_format,
                )
            # apply the activation function if there is one
            if self.activation is not None:
                x = self.activation(x)
            # if data format is channels first, have to permute before resizing
            # because resize expects channels last
            if self.data_format == 'channels_first':
                x = K.permute_dimensions(x, [0, 2, 3, 1])
            # up-sample the outputs back to the input shape
            x = tf.compat.v1.image.resize_bilinear(x, output_shape)
            # if data format is channels first, have to permute after resizing
            # because resize output channels last
            if self.data_format == 'channels_first':
                x = K.permute_dimensions(x, [0, 3, 1, 2])
            # concatenate the output tensor with the stack of output tensors
            output_tensors += [x]

        # concatenate the output tensors before returning
        return K.concatenate(output_tensors, axis=channel_axis)
 def pooling_function(inputs, pool_size, strides, padding, data_format):
     return -K.pool2d(
         -inputs, pool_size, strides, padding, data_format, pool_mode='max')
def l2_norm(x):
    x = K.square(x)
    x = K.pool2d(x, pool_size=(3, 3), strides=(1, 1), padding='same', data_format='channels_last', pool_mode='avg')
    x = x * 9
    x = K.sqrt(x)
    return x
 def call(self, x):
     output = K.pool2d(x, self.pool_size, self.strides,
                       self.padding, self.data_format,
                       pool_mode='max')
     return output
 def _pooling_function(self, inputs, pool_size, strides,
                     border_mode, dim_ordering):
     output = K.pool2d(inputs, pool_size, strides,
                     border_mode, dim_ordering, pool_mode='max')
     return output
Beispiel #23
0
def conv2d_offset_average_pool_eval(_, padding, x):
    """Perform an average pooling operation on x"""
    return K.pool2d(x, (1, 1), strides=(3, 3), padding=padding, pool_mode='avg')
Beispiel #24
0
 def min_pool2d(x):
     min_x = -K.pool2d(-x, pool_size=(2, 2), strides=(1, 1), padding='same')
     return min_x
Beispiel #25
0
 def mean2d(y):
     
     y = K.pool2d(y, (self.kernel_size[0], 1), pool_mode = 'avg', padding = 'same')
     y = K.pool2d(y, (1, self.kernel_size[1]), pool_mode = 'avg', padding = 'same')
     return y
Beispiel #26
0
    def call(self, inputs, states, training=None):

        # inputs are going to come in as a tuple of size 2
        # where inputs[0] is the context prior to the neural
        # map, and inputs[1] will be another tuple - (x, y)
        context = inputs
        x, y = (self.pos_input[:, 0], self.pos_input[:, 1])
        # x, y = (K.expand_dims(x, -1), K.expand_dims(y, -1))
        # preprocessing states to get memory
        memory = states[1:((self.memory_size[0] * self.memory_size[1]) + 1)]
        memory = K.transpose(memory)
        memory = K.reshape(
            memory, (-1, self.units, self.memory_size[0], self.memory_size[1]))

        # Need this for later computations
        batch_size = K.shape(memory)[0]

        # global read operation
        # r_t = read(M_t) ; output dimension of r_t = self.units
        first_conv = K.conv2d(memory,
                              self.conv_kernel1,
                              strides=(1, 1),
                              padding='same',
                              data_format='channels_first')
        second_conv = K.conv2d(first_conv,
                               self.conv_kernel2,
                               strides=(1, 1),
                               padding='valid',
                               data_format='channels_first')
        pool_conv = K.pool2d(second_conv,
                             pool_size=(2, 2),
                             strides=(1, 1),
                             padding='valid',
                             data_format='channels_first',
                             pool_mode='avg')
        flatten_conv = K.batch_flatten(pool_conv)
        dense1_conv = K.dot(flatten_conv, self.conv_dense1)
        dense2_conv = K.dot(dense1_conv, self.conv_dense2)
        r_t = dense2_conv

        # context read operation
        # c_t = context(M_t, s_t, r_t)
        q_t = K.concatenate([context, r_t])  # [1x(s+c)]
        q_t = K.dot(q_t, self.context_kernel)  # [1xc]
        # at = K.exp(K.dot(q_t, memory)) # [1xhxw]
        # at_sum = K.sum(at)
        # at_sum_repeated = K.repeat (at_sum, (self.memory_size[0], self.memory_size[1]))
        # at /= at_sum # [1xhxw]

        # Compute attention with softmax, use batch_dot function
        # Keras is garbage, and so am I
        q_t_repeated = K.expand_dims(q_t, axis=2)
        q_t_repeated = K.expand_dims(q_t_repeated, axis=3)
        q_t_repeated = K.repeat_elements(q_t_repeated, self.memory_size[0], 2)
        q_t_repeated = K.repeat_elements(q_t_repeated, self.memory_size[1], 3)

        at = K.sum(q_t_repeated * memory, axis=1)
        at_sum = K.sum(at, axis=1, keepdims=True)
        at_sum = K.sum(at_sum, axis=2, keepdims=True)
        at_sum = K.repeat_elements(at_sum, self.memory_size[0], 1)
        at_sum = K.repeat_elements(at_sum, self.memory_size[1], 2)

        at = at / at_sum
        at = K.expand_dims(at, axis=1)
        at = K.repeat_elements(at, self.units, 1)

        c_t = K.sum(at * memory, axis=2)
        c_t = K.sum(c_t, axis=2)

        # Computing write value
        # m_t+1_x,y = write(s_t, r_t, c_t, m_t_x,y)
        s_t = K.dot(context, self.write_kernel)  # [1xc]
        global_imp = K.batch_dot(s_t, r_t, axes=1)  # [1x1]
        local_imp = K.batch_dot(s_t, c_t, axes=1)  # [1x1]

        # Convert batched x,y coordinates into indices for memory
        # We want to index into memory by doing something like
        # memory[sample_number, y, x] so that we get a {units} dimensional
        # vector out. When we do this for every sample in the batch, we'll
        # end up with a {number_batches, units} dimensional matrix.

        # First we copy and reshape the memory into {batch_size x H x W x units}
        mem_t = K.reshape(
            memory,
            (batch_size, self.memory_size[0], self.memory_size[1], self.units))

        # Now we generate the indices
        B = tf.range(0, batch_size)  # {batch_size}
        idx = tf.stack([B, y, x], axis=1)  # {batch_size x 3}

        # And index into the memory using tf.gather_nd
        mem_t = tf.gather_nd(mem_t, idx)  # {batch_size x units}
        mem_t += (local_imp / (local_imp + global_imp)) * K.dot(
            (mem_t - s_t), self.write_update)

        # We have to reshape the memory b/c of how we calculated indices:
        memory = K.reshape(
            memory,
            (batch_size, self.memory_size[0], self.memory_size[1], self.units))

        # While loop to update memory.
        i = tf.constant(0)
        c = lambda i: tf.less(i, batch_size)

        def body(i):
            memory[i, y[i], x[i]] = mem_t[i]
            return tf.add(i, 1)

        tf.while_loop(c, body, [i])
        # Would love to just use this, but tf has a stick in its butt
        # memory = tf.scatter_nd_update(memory, idx, mem_t)

        # memory[:, :] = mem_t

        # update states
        mem_t = K.reshape(mem_t, (1, self.units))
        new_states = states[:((self.memory_size[0] * self.memory_size[1]) + 1)]
        new_states[0] = r_t
        new_states[((x - 1) * self.units) + y] = mem_t

        return c_t, new_states
Beispiel #27
0
def conv2d_offset_max_pool_eval(_, padding, x):
    """Perform a max pooling operation on x"""
    return K.pool2d(x, (1, 1), strides=(3, 3), padding=padding, pool_mode='max')
    def call(self, PMandHazeIMGandA):

        patchMap = PMandHazeIMGandA[0]
        hazeImg = PMandHazeIMGandA[1]
        A = PMandHazeIMGandA[2]
        x = tf.cast(patchMap, tf.int32)
        s = K.int_shape(patchMap)
        mapList = []
        for i in range(1, 121):
            if i == 1:
                mapList.append(K.greater(1.3, patchMap))
            else:
                mapList.append(K.equal(x, i))
        pmbox = K.concatenate(mapList, axis=3)
        pmbox = tf.cast(pmbox, tf.float32)
        patchConvList = []
        patchMinList = []
        """
        hazeImg=-hazeImg
        
        for i in range(1,121):
            recoveri=K.pool2d(hazeImg, pool_size=(i,i), strides=(1,1),
                          padding='same', data_format=None,
                          pool_mode='max')
            recoveri=-recoveri
            recoveri=K.min(recoveri,axis=3,keepdims=True)
            patchMinList.append(recoveri)
        """
        hazeImgM = K.min(hazeImg, axis=3, keepdims=True)
        hazeImgM = -hazeImgM
        for i in range(1, 121):
            if i <= 5:
                recoveri = K.pool2d(hazeImgM,
                                    pool_size=(i, i),
                                    strides=(1, 1),
                                    padding='same',
                                    data_format=None,
                                    pool_mode='max')
                #recoveri=-recoveri
                #recoveri=K.min(recoveri,axis=3,keepdims=True)
                patchMinList.append(recoveri)
            else:
                #print(len(patchMinList),'len patchMinList')
                #recoveri=-patchMinList[i-2]
                recoveri = K.pool2d(patchMinList[i - 3],
                                    pool_size=(3, 3),
                                    strides=(1, 1),
                                    padding='same',
                                    data_format=None,
                                    pool_mode='max')
                #recoveri=-recoveri
                #recoveri=K.min(recoveri,axis=3,keepdims=True)
                patchMinList.append(recoveri)

        #for i in range(120):
        #    patchMinList[i]=K.min(-patchMinList[i],axis=3,keepdims=True)

        #hazeImg=-hazeImg
        rcbox = K.concatenate(patchMinList, axis=3)
        #print(K.int_shape(rcbox),'int shape')
        rcbox = -rcbox

        A2 = K.expand_dims(A, 2)
        A2 = K.expand_dims(A2, 3)
        A2 = K.tile(A2, [1, 480, 640, 120])
        t = rcbox / A2
        w = 31 / 32
        t = w * t
        t = 1 - t
        pm_rc0 = pmbox * t
        pm_rc = K.sum(pm_rc0, axis=3, keepdims=True)
        pm_rc = K.clip(pm_rc, 0.0, 1.0)
        return [pm_rc]
Beispiel #29
0
    def call(self, ins):
        u   = ins[0]
        img = ins[1]

        ### g learned by net
        g1 = K.conv2d(self.img, self.kernel_1, padding='same')
        g1 = K.bias_add(g1, self.bias_1)
        g1 = K.relu(g1)
        g1 = K.conv2d(g1, self.kernel_2, padding='same')
        g1 = K.bias_add(g1, self.bias_2)
        g1 = K.relu(g1)
        g2 = K.pool2d(g1, (2,2), padding='same')
        g2 = K.conv2d(g2, self.kernel_3, padding='same')
        g2 = K.bias_add(g2, self.bias_3)
        g2 = K.relu(g2)
        g2 = K.conv2d(g2, self.kernel_4, padding='same')
        g2 = K.bias_add(g2, self.bias_4)
        g2 = K.relu(g2)
        g3 = K.pool2d(g2, (2,2), padding='same')
        g3 = K.conv2d(g3, self.kernel_5, padding='same')
        g3 = K.bias_add(g3, self.bias_5)
        g3 = K.relu(g3)
        g3 = K.conv2d(g3, self.kernel_6, padding='same')
        g3 = K.bias_add(g3, self.bias_6)
        g3 = K.relu(g3)
        g3 = K.conv2d(g3, self.kernel_7, padding='same')
        g3 = K.bias_add(g3, self.bias_7)
        g3 = K.relu(g3)
        g3 = K.conv2d(g3, self.kernel_8, padding='same')
        g3 = K.bias_add(g3, self.bias_8)
        g3 = K.relu(g3)
        g4 = K.resize_images(g3, 2, 2, data_format='channels_last', interpolation='bilinear')
        g4 = K.concatenate([g2, g4])
        g4 = K.conv2d(g4, self.kernel_9, padding='same')
        g4 = K.bias_add(g4, self.bias_9)
        g4 = K.relu(g4)
        g4 = K.conv2d(g4, self.kernel_10, padding='same')
        g4 = K.bias_add(g4, self.bias_10)
        g4 = K.relu(g4)
        g5 = K.resize_images(g4, 2, 2, data_format='channels_last', interpolation='bilinear')
        g5 = K.concatenate([g1, g5])
        g5 = K.conv2d(g5, self.kernel_11, padding='same')
        g5 = K.bias_add(g5, self.bias_11)
        g5 = K.relu(g5)
        g5 = K.conv2d(g5, self.kernel_12, padding='same')
        g5 = K.bias_add(g5, self.bias_12)
        g  = K.sigmoid(g5)

        ### grad(g)
        g_x = K.conv2d(g, kXC, padding='same')
        g_x = scale(g_x, self.shp, self.rhp, self.dx)
        g_y = K.conv2d(g, kYC, padding='same')
        g_y = scale(g_y, self.shp, self.rhp, self.dy)

        ### transport - upwind
        xp = K.conv2d(u, xKP, padding='same')
        xn = K.conv2d(u, xKN, padding='same')
        yp = K.conv2d(u, xYP, padding='same')
        yn = K.conv2d(u, xYN, padding='same')
        fxp =        K.relu(        g_x)
        fxn = -1.0 * K.relu( -1.0 * g_x)
        fyp =        K.relu(        g_y)
        fyn = -1.0 * K.relu( -1.0 * g_y)
        xpp = fxp*xp
        xnn = fxn*xn
        ypp = fyp*yp
        ynn = fyn*yn
        xterms = xpp + xnn
        xterms = scale(xterms, self.shp, self.rhp, self.dx)
        yterms = ypp + ynn
        yterms = scale(yterms, self.shp, self.rhp, self.dy)
        transport = xterms + yterms

        ### curvature - learned
        grad_u = K.conv2d(u, self.kernel_k1, padding='same')
        norm_grad_u = K.sqrt(   K.epsilon() + K.sum( K.square(grad_u), axis=-1, keepdims=True) )
        grad_u = grad_u / (norm_grad_u + K.epsilon())
        kappa = K.conv2d(grad_u, self.kernel_k2, padding='same')
        curvature = g*kappa*norm_grad_u

        ### balloon
        balloon = g*norm_grad_u

        return u + K.constant(self.dt)*(   curvature * self.alpha \
                                         + transport * self.beta  \
                                         + balloon   * self.gamma )
Beispiel #30
0
    def _pooling_function(self, inputs, pool_size, strides, padding,
                          data_format, dilation_rate):
        """We illustrate how this method works with an example. Let's have a 
        dilated kernel with pool_size=(3,2), dilation_rate=(2,4). The * 
        represent the non-zero elements
        
        * 0 0 0 *
        0 0 0 0 0
        * 0 0 0 *
        0 0 0 0 0
        * 0 0 0 *
        
        Let's have the following image (each dot represents a pixel)
        
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        · · · · · · · · ·
        
        The pooling kernel will slide over the image, similarly to convolution,
        computing a pooling result at each step. For example, for the vertical 
        slide
        
            dr = 0          dr = 1          dr = 2 
        
        X · · · X · · · ·   · · · · · · · · ·   · · · · · · · · ·
        · · · · · · · · ·   X · · · X · · · ·   · · · · · · · · ·
        X · · · X · · · ·   · · · · · · · · ·   X · · · X · · · ·
        · · · · · · · · ·   X · · · X · · · ·   · · · · · · · · ·
        X · · · X · · · ·   · · · · · · · · ·   X · · · X · · · ·
        · · · · · · · · · , X · · · X · · · · , · · · · · · · · · , etc
        · · · · · · · · ·   · · · · · · · · ·   X · · · X · · · ·
        · · · · · · · · ·   · · · · · · · · ·   · · · · · · · · ·
        · · · · · · · · ·   · · · · · · · · ·   · · · · · · · · ·
        · · · · · · · · ·   · · · · · · · · ·   · · · · · · · · ·
        · · · · · · · · ·   · · · · · · · · ·   · · · · · · · · ·
        · · · · · · · · ·   · · · · · · · · ·   · · · · · · · · ·
        
        Ideally, this process should be implemented at low level in Theano and
        TensorFlow. This here is a high-level implementation, but in order to
        preserve some performance, we reuse K.pool2d() as much as possible.
        
        For that, we are going to subsample each axis of the input D times if
        the dilation factor is D. For the example above (subsampling marked 
        with X):
            
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · · ,  · · · · · · · · · ,  · · · · · · · · · ,  · · · · · · · · · ,
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · · ,  · · · · · · · · · ,  · · · · · · · · · ,  · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·    · · · · · · · · ·
        X · · · X · · · X    · X · · · X · · ·    · · X · · · X · ·    · · · X · · · X ·
        
        We then apply K.pool2d() to each of the subsamplings above (with 
        padding='same' so that each block produces a block of the same size).
        The results are restacked to form an array of the same size as inputs.
        
        Finally, if the user wanted only padding='valid', we crop out the 
        borders of the array.
        """

        if (strides != 1) and (strides != (1, 1)):
            raise Exception('Only implemented for strides=1 or strides=(1,1)')

        sz = K.shape(inputs)
        if data_format == 'channels_first':  # (batch,chan,row,col)
            nbatch = sz[0]
            nchan = sz[1]
            nrows = sz[2]
            ncols = sz[3]
        elif data_format == 'channels_last':  # (batch,row,col,chan)
            nbatch = sz[0]
            nrows = sz[1]
            ncols = sz[2]
            nchan = sz[3]
        else:
            raise ValueError(
                'Expected data_format to be channels_first or channels_last')

        # allocate space for output
        outputs = K.zeros(shape=sz.eval())

        # compute slice objects. Each slice object will be used to split the
        # input into a block. Each block will be pooled with dilation=1 (no
        # dilation). The overall effect is like pooling the whole image with
        # dilation>1

        # first, we compute the slices to split the input in horizontal and
        # vertical blocks, separately
        block_slices_row = [
            slice(i, nrows, dilation_rate[0]) for i in range(dilation_rate[0])
        ]
        block_slices_col = [
            slice(j, ncols, dilation_rate[1]) for j in range(dilation_rate[1])
        ]

        # then, we make all combinations of row-blocks and col-blocks, to get
        # all the blocks that split the inputs. Now we can iterate all the
        # blocks and process them separately
        block_slices_combinatorial = itertools.product(
            *[block_slices_row, block_slices_col])
        total_row_padding_before = 0
        total_row_padding_after = 0
        total_col_padding_before = 0
        total_col_padding_after = 0
        for sl in block_slices_combinatorial:
            # DEBUG: sl = list(itertools.islice(block_slices_combinatorial, 1))[0]

            # extend the slice with the batch and channel
            if data_format == 'channels_first':  # (batch,chan,row,col)
                block_slice = list((slice(0, nbatch, 1), slice(0, nchan, 1)) +
                                   sl)
            elif data_format == 'channels_last':  # (batch,row,col,chan)
                block_slice = list((slice(0, nbatch, 1), ) + sl +
                                   (slice(0, nchan, 1), ))

            # extract block
            block = inputs[block_slice]
            #print(block.eval()) # DEBUG

            # pool each block without dilation
            block_pooled = K.pool2d(block,
                                    pool_size,
                                    strides=(1, 1),
                                    padding='same',
                                    data_format=data_format,
                                    pool_mode='max')
            #print(block_pooled.eval()) # DEBUG

            # compute amount of padding that this blocks contributes
            ((row_padding_before, row_padding_after),
             (col_padding_before, col_padding_after)) = self._block_padding(
                 block=block_pooled,
                 pool_size=pool_size,
                 data_format=data_format)
            total_row_padding_before += row_padding_before
            total_row_padding_after += row_padding_after
            total_col_padding_before += col_padding_before
            total_col_padding_after += col_padding_after

            # put block into output
            if (K.backend() == 'theano'):
                outputs = T.set_subtensor(outputs[block_slice], block_pooled)
            elif (K.backend() == 'tensorflow'):
                outputs[block_slice].assign(block_pooled)
            else:
                raise Exception('not implemented')

        # row blocks contribute redundantly to the amount of padding columns,
        # and column blocks to padding rows. We divide by the number of blocks
        # alon each axis to obtain the actual amount of padding in the input
        total_row_padding_before /= dilation_rate[1]
        total_row_padding_after /= dilation_rate[1]
        total_col_padding_before /= dilation_rate[0]
        total_col_padding_after /= dilation_rate[0]

        total_row_padding_before = int(total_row_padding_before)
        total_row_padding_after = int(total_row_padding_after.eval())
        total_col_padding_before = int(total_col_padding_before)
        total_col_padding_after = int(total_col_padding_after.eval())

        # remove the border where the kernel didn't fully overlap the input
        if padding == 'valid':

            if data_format == 'channels_first':  # (batch,chan,row,col)
                outputs = outputs[:, :, total_row_padding_before:(
                    nrows - total_row_padding_after),
                                  total_col_padding_before:(
                                      ncols - total_col_padding_after)]
            elif data_format == 'channels_last':  # (batch,row,col,chan)
                outputs = outputs[:, total_row_padding_before:(
                    nrows - total_row_padding_after),
                                  total_col_padding_before:(
                                      ncols - total_col_padding_after), :]

        # return tensor
        return outputs
Beispiel #31
0
 def _pooling_function(self, inputs, pool_size, strides,
                       padding, data_format):
     output = K.pool2d(K.square(inputs), pool_size, strides,
                       padding, data_format, pool_mode='avg')
     return K.sqrt(output + K.epsilon())
Beispiel #32
0
def _nms(heat, kernel=3):
    hmax = K.pool2d(heat, (kernel, kernel), padding='same', pool_mode='max')
    keep = K.cast(K.equal(hmax, heat), K.floatx())
    return heat * keep
Beispiel #33
0
    def _pooling_function(self, x, name=None):
        #b = K.shape(x)[0]
        input_shape = K.int_shape(x)
        b, r, c, channel = input_shape[0], input_shape[1], input_shape[
            2], input_shape[3]

        pr, pc = self.pool_size
        sr, sc = self.strides

        # compute number of windows
        num_r = math.ceil(r / sr) if self.padding == 'SAME' else r // sr
        num_c = math.ceil(c / sc) if self.padding == 'SAME' else c // sc

        def _mid_pool(inputs, is_train):
            input_shape = inputs.shape
            batch = input_shape[0]

            # reshape
            w = np.transpose(inputs, (0, 3, 1, 2))
            w = np.reshape(w, (batch * channel, r, c))

            def pool(x):
                # sort
                size = pr * pc
                x_flat = np.reshape(x, [size])
                x_sorted = np.argsort(-x_flat)
                p = x_sorted[math.ceil(size / 2)]

                mid_matrix = np.zeros((pr, pc))
                mid_matrix[p // pr, p % pc] = 1.0

                return mid_matrix

            re = np.zeros((batch * channel, num_r, num_c), dtype=np.float32)
            # extract with pool_size
            for i in range(num_r):
                for j in range(num_c):
                    crop = np.zeros((batch * channel, pr, pc))
                    crop[:, :, :] = w[:, i * sr:i * sr + pr,
                                      j * sc:j * sc + pc]

                    # pool
                    outs = np.array(list(map(pool, crop)))

                    if is_train:
                        re[:, i, j] = (crop * outs).max(axis=(1, 2))
                    else:
                        re[:, i, j] = (crop * outs).sum(axis=(1, 2))

            # reshape
            re = np.reshape(re, (batch, channel, num_r, num_c))
            re = np.transpose(re, (0, 2, 3, 1))

            return re

        def custom_grad(op, grad):
            if self.data_format == 'NHWC':
                ksizes = [1, self.pool_size[0], self.pool_size[1], 1]
                strides = [1, self.strides[0], self.strides[1], 1]
            else:
                ksizes = [1, 1, self.pool_size[0], self.pool_size[1]]
                strides = [1, 1, self.strides[0], self.strides[1]]

            #return gen_nn_ops.max_pool_grad(
            return gen_nn_ops.avg_pool_grad(
                array_ops.shape(op.inputs[0]),
                grad,
                ksizes,
                strides,
                self.padding,
                data_format=self.data_format), K.tf.constant(0.0)

        def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
            # Need to generate a unique name to avoid duplicates:
            rnd_name = 'MidPooling2DGrad' + str(np.random.randint(0, 1E+8))

            K.tf.RegisterGradient(rnd_name)(grad)
            g = K.tf.get_default_graph()
            with g.gradient_override_map({"PyFunc": rnd_name}):
                return K.tf.py_func(func,
                                    inp,
                                    Tout,
                                    stateful=stateful,
                                    name=name)

        def _mid_range_pool(x, name=None):
            with ops.name_scope(name, "mod", [x]) as name:
                z = py_func(_mid_pool, [x, K.learning_phase()], [K.tf.float32],
                            name=name,
                            grad=custom_grad)[0]
                z.set_shape((b, num_r, num_c, channel))

                return z

        """
        Mixed Pooling
        """
        max_pool = K.pool2d(x,
                            self.pool_size,
                            strides=self.strides,
                            padding=self.padding.lower(),
                            pool_mode="max")
        mid_pool = _mid_range_pool(x, name)

        def _train_pool(max_pool, mid_pool):
            self.alpha = random.uniform(0, 1)
            self.alpha_frequencies[0] += self.alpha
            self.alpha_frequencies[1] += 1 - self.alpha

            return self.alpha * max_pool + (1 - self.alpha) * mid_pool

        def _test_pool(max_pool, mid_pool):
            return K.switch(
                K.less(self.alpha_frequencies[0], self.alpha_frequencies[1]),
                mid_pool, max_pool)

        outs = K.in_train_phase(_train_pool(max_pool, mid_pool),
                                _test_pool(max_pool, mid_pool))

        return outs
Beispiel #34
0
    def call(self, inputs):
        boxes = inputs[0]

        # Feature Maps. List of feature maps from different level of the
        # feature pyramid. Each is [batch, height, width, channels]
        score_maps = inputs[1:]

        # Assign each ROI to a level in the pyramid based on the ROI area.
        y1, x1, y2, x2 = tf.split(boxes, 4, axis=2)
        h = y2 - y1
        w = x2 - x1
        # Equation 1 in the Feature Pyramid Networks paper. Account for
        # the fact that our coordinates are normalized here.
        # e.g. a 224x224 ROI (in pixels) maps to P4
        image_area = tf.cast(self.image_shape[0] * self.image_shape[1],
                             tf.float32)
        roi_level = log2_graph(tf.sqrt(h * w) / (224.0 / tf.sqrt(image_area)))
        roi_level = tf.minimum(
            5, tf.maximum(2, 4 + tf.cast(tf.round(roi_level), tf.int32)))
        roi_level = tf.squeeze(roi_level, 2)

        # Loop through levels and apply ROI pooling to each. P2 to P5.
        pooled = []
        box_to_level = []
        for i, level in enumerate(range(2, 6)):
            ix = tf.where(tf.equal(roi_level, level))
            level_boxes = tf.gather_nd(boxes, ix)

            # Box indicies for crop_and_resize.
            box_indices = tf.cast(ix[:, 0], tf.int32)

            # Keep track of which box is mapped to which level
            box_to_level.append(ix)

            # Stop gradient propogation to ROI proposals
            level_boxes = tf.stop_gradient(level_boxes)
            box_indices = tf.stop_gradient(box_indices)

            # Here we use the simplified approach of a single value per bin,
            # which is how it's done in tf.crop_and_resize()
            # Result: [batch * num_boxes, pool_height, pool_width, channels]
            pooled.append(
                tf.image.crop_and_resize(
                    score_maps[i],
                    level_boxes,
                    box_indices,
                    [self.pool_shape * self.k, self.pool_shape * self.k],
                    method="bilinear"))

        # Pack pooled features into one tensor
        pooled = tf.concat(pooled, axis=0)

        # position-sensitive ROI pooling + classify
        score_map_bins = []
        for channel_step in range(self.k * self.k):
            bin_x = K.variable(int(channel_step % self.k) * self.pool_shape,
                               dtype='int32')
            bin_y = K.variable(int(channel_step / self.k) * self.pool_shape,
                               dtype='int32')
            channel_indices = K.variable(list(
                range(channel_step * self.channel_num,
                      (channel_step + 1) * self.channel_num)),
                                         dtype='int32')
            croped = tf.image.crop_to_bounding_box(
                tf.gather(pooled, indices=channel_indices, axis=-1), bin_y,
                bin_x, self.pool_shape, self.pool_shape)
            # [pool_shape, pool_shape, channel_num] ==> [1,1,channel_num] ==> [1, channel_num]
            croped_mean = K.pool2d(croped, (self.pool_shape, self.pool_shape),
                                   strides=(1, 1),
                                   padding='valid',
                                   data_format="channels_last",
                                   pool_mode='avg')
            # [batch * num_rois, 1,1,channel_num] ==> [batch * num_rois, 1, channel_num]
            croped_mean = K.squeeze(croped_mean, axis=1)
            score_map_bins.append(croped_mean)
        # [batch * num_rois, k^2, channel_num]
        score_map_bins = tf.concat(score_map_bins, axis=1)
        # [batch * num_rois, k*k, channel_num] ==> [batch * num_rois,channel_num]
        # because "keepdims=False", the axis 1 will not keep. else will be [batch * num_rois,1,channel_num]
        pooled = K.sum(score_map_bins, axis=1)

        # Pack box_to_level mapping into one array and add another
        # column representing the order of pooled boxes
        box_to_level = tf.concat(box_to_level, axis=0)
        box_range = tf.expand_dims(tf.range(tf.shape(box_to_level)[0]), 1)
        box_to_level = tf.concat([tf.cast(box_to_level, tf.int32), box_range],
                                 axis=1)

        # Rearrange pooled features to match the order of the original boxes
        # Sort box_to_level by batch then box index
        # TF doesn't have a way to sort by two columns, so merge them and sort.
        sorting_tensor = box_to_level[:, 0] * 100000 + box_to_level[:, 1]
        ix = tf.nn.top_k(sorting_tensor,
                         k=tf.shape(box_to_level)[0]).indices[::-1]
        ix = tf.gather(box_to_level[:, 2], ix)
        pooled = tf.gather(pooled, ix)

        # Re-add the batch dimension
        pooled = tf.expand_dims(pooled, 0)

        return pooled
def min_max_pool2d(x):
    max_x = K.pool2d(x, pool_size=(2, 2), strides=(2, 2))
    min_x = min_pool2d(x)
    return K.concatenate([max_x, min_x], axis=3)  # concatenate on channel
Beispiel #36
0
def conv2d_offset_max_pool_eval(_, padding, x):
    """Perform a max pooling operation on x"""
    return K.pool2d(x, (1, 1),
                    strides=(3, 3),
                    padding=padding,
                    pool_mode='max')
def Kpool(x, pool_size):
    return K.pool2d(x, pool_size, pool_mode='avg')
Beispiel #38
0
def points_nms(heat, kernel=2):
    # kernel must be 2
    hmax = K.pool2d(heat, (kernel, kernel), padding='same', pool_mode='max')
    keep = K.cast(K.equal(hmax, heat), K.floatx())
    return heat * keep
def min_max_pool2d(x):
    max_x = K.pool2d(x, pool_size=(7, 7), strides=(2, 2))
    min_x = -K.pool2d(-x, pool_size=(7, 7), strides=(2, 2))
    return K.concatenate([max_x, min_x], axis=1)  # concatenate on channel