Esempio n. 1
0
def accumulate(attend_function, inputs, input_length,
                                mask=None, return_probabilities=False):
    '''get the running attention over a sequence. 

    given a 3dim tensor where the 1st dim is time (or not. whatever.),  calculating the running attended sum.
    in other words, at the first time step, you only have that item.
                    at the second time step, attend over the first two items.
                    at the third..  the third. so on. 

    this basically a mod on keras' rnn implementation
    author: bcm
    '''

    ndim = inputs.ndim
    assert ndim >= 3, 'inputs should be at least 3d'

    axes = [1,0] + list(range(2, ndim))
    inputs = inputs.dimshuffle(axes)

    indices = list(range(input_length))

    successive_outputs = []
    if mask is not None:
        if mask.ndim == ndim-1:
            mask = K.expand_dims(mask)
        assert mask.ndim == ndim
        mask = mask.dimshuffle(axes)
        prev_output = None

    successive_outputs = []
    successive_pvecs = []
    uncover_mask = K.zeros_like(inputs)
    uncover_indices = K.arange(input_length)
    for _ in range(ndim-1):
        uncover_indices = K.expand_dims(uncover_indices)
    make_subset = lambda i,X: K.switch(uncover_indices <= i, X, uncover_mask)
    for i in indices:
        inputs_i = make_subset(i,inputs)
        mask_i = make_subset(i,mask)
        if mask is not None:
            output = attend_function(inputs_i, mask_i) # this should not output the time dimension; it should be marginalized over. 
        else:
            output = attend_function(inputs_i) # this should not output the time dimension; it should be marginalized over. 
        if return_probabilities:
            output, p_vectors = output
            successive_pvecs.append(p_vectors)
        assert output.ndim == 2, "Your attention function is malfunctioning; the attention accumulator should return 2 dimensional tensors"
        successive_outputs.append(output)
    outputs = K.pack(successive_outputs)
    K.squeeze(outputs, -1)
    axes = [1, 0] + list(range(2, outputs.ndim))
    outputs = outputs.dimshuffle(axes)

    if return_probabilities:
        out_pvecs = K.pack(successive_pvecs)
        K.squeeze(out_pvecs, -1)
        out_pvecs = out_pvecs.dimshuffle(axes)
        outputs = [outputs, out_pvecs]

    return outputs
Esempio n. 2
0
	def step(self, x, states):
		M = states[0]  # (nb_samples, nb_slots, memory_size)
		h = states[1]  # (nb_samples, memory_size)
		w = states[2]  # (nb_samples, nb_slots)
		#------Memory read--------#
		k = self.W_k(h)  # (nb_samples, memory_size)
		w_hat = T.batched_tensordot(M, k, axes=[(2), (1)])  # (nb_samples, nb_slots)
		beta = K.sigmoid(self.W_b(h))  # (nb_samples, 1)
		beta = K.repeat(beta, self.nb_slots)  # (nb_samples, nb_slots, 1)
		beta = K.squeeze(beta, 2)  # (nb_samples, nb_slots)
		w_hat = softmax(w_hat * beta)  # (nb_samples, nb_slots)
		g = sigmoid(self.W_hg(h))  # (nb_samples, 1)
		g = K.repeat(g, self.nb_slots)  # (nb_samples, nb_slots, 1)
		g = K.squeeze(g, 2)  # (nb_samples, nb_slots)
		w = (1 - g) * w + g * w_hat  # (nb_samples, nb_slots)
		c = T.batched_tensordot(w, M, axes=[(1), (1)])
		h = tanh(self.W_ih(x) + self.W_c(c))
		y = self.W_ho(h)
		#---------Memory write---------#
		v = self.W_v(h)  # (nb_samples, memory_size)
		v = K.repeat(v, 1)
		e = sigmoid(self.W_he(h))  # (nb_samples, nb_slots)
		f = 1 - w * e  # (nb_samples, nb_slots)
		f = K.repeat(f, self.memory_size)  # (nb_samples, memory_size, nb_slots)
		f = K.permute_dimensions(f, (0, 2, 1))  # (nb_samples, nb_slots, memory_size)
		u = w  # (nb_samples, nb_slots)
		u = K.repeat(u, 1)
		uv = T.batched_tensordot(u, v, axes=[(1), (1)])
		M = M * f + uv
		return y, [M, h, w]
Esempio n. 3
0
def squeezed_accuracy(y_true, y_pred):

        class_id_true = K.argmax(K.squeeze(y_true,axis=0), axis=-1)
        class_id_preds = K.argmax(K.squeeze(y_pred,axis=0), axis=-1)
        # Replace class_id_preds with class_id_true for recall here
#        accuracy_mask = K.cast(K.equal(class_id_preds, interesting_class_id), 'int32')
#        class_acc_tensor = K.cast(K.equal(class_id_true, class_id_preds), 'int32')
        class_acc = K.mean(K.equal(class_id_true, class_id_preds))
        return class_acc
Esempio n. 4
0
    def call(self, x, mask=None):
        x = K.permute_dimensions(x, (0, 2, 1))
        x = K.expand_dims(x, -1)

        output = K.square(K.permute_dimensions(K.squeeze(K.conv2d(x, self.kernel), -1), (0, 2, 1)))

        return output
Esempio n. 5
0
    def conv_step(self, x, W, b=None, border_mode="valid", filter_shape=None, mask_type='b'):

        mask = np.ones(filter_shape, dtype=_FLOATX)

        in_third = self.input_dim//3
        out_third = self.nb_filter//3
        mask[:out_third,in_third:,0,0] = 0
        mask[out_third:2*out_third,2*in_third:,0,0] = 0

        W = W * mask

        input_shape = self.shuffeled_dims

        x = K.expand_dims(x, -1)  # add a dimension of the right

        conv_out = T.nnet.conv2d(x, W, subsample=self.subsample,
                                 border_mode='half',
                                 filter_flip=False,
                                 input_shape=(input_shape[0],
                                              input_shape[2],
                                              input_shape[3],
                                              1),
                                 filter_shape=filter_shape)
        if b:
            conv_out = conv_out + K.reshape(b, (1, filter_shape[0], 1, 1))

        conv_out = K.squeeze(conv_out, 3)  # remove the dummy 3rd dimension

        return conv_out
    def step(self, x, states):  
        h = states[0]
        # states[1] necessary?
        
        # comes from the constants
        X_static = states[-2]
        # equals K.dot(static_x, self._W1) + self._b2 with X.shape=[bs, L, static_input_dim]
        total_x_static_prod = states[-1]

        # expand dims to add the vector which is only valid for this time step
        # to total_x_prod which is valid for all time steps
        hw = K.expand_dims(K.dot(h, self._W2), 1)
        additive_atn = total_x_static_prod + hw
        attention = K.softmax(K.dot(additive_atn, self._V), axis=1)
        static_x_weighted = K.sum(attention * X_static, [1])
        
        x = K.dot(K.concatenate([x, static_x_weighted], 1), self._W3) + self._b3

        h, new_states = self.layer.cell.call(x, states[:-2])
        
        # append attention to the states to "smuggle" it out of the RNN wrapper
        attention = K.squeeze(attention, -1)
        h = K.concatenate([h, attention])

        return h, new_states
    def call(self, X, mask=None):
        # 1D -> 2D
        batch = K.shape(X)[0]
        width = deconv_output_length(K.shape(X)[1],
                                    self.filter_length,
                                    self.padding,
                                    self.strides[2])

        print("Output width: ", width)

        print("Input shape: ", K.shape(X))
        X = K.expand_dims(X,2)
        print("Input shape after expand: ", K.shape(X))
        # X = K.permute_dimensions(X, (0, 2, 3, 1))
        X = K.permute_dimensions(X, (0, 2, 1, 3))
        print("Input shape after permute: ", K.shape(X))
        deconv_shape = tf.pack([batch, 1, width, self.nb_filter])
        print("Deconv shape: ", deconv_shape)
        conv_out = tf.nn.conv2d_transpose(X, self.W, strides=self.strides,
                                          padding=self.padding.upper(),
                                          output_shape=deconv_shape)

        output = conv_out + K.reshape(self.b, (1, 1, 1, self.W_shape[2]))
        print("Output shape: ", K.shape(output))
        # output =  K.permute_dimensions(output, (0, 3, 1, 2))
        output =  K.permute_dimensions(output, (0, 2, 1, 3))
        print("Output shape after permute: ", K.shape(output))
        # 2D -> 1D
        output = K.squeeze(output,2)
        print("Output shape after squeeze: ", K.shape(output))
        return output
    def call(self, x):
        assert(K.backend() == 'tensorflow')
        temp = K.permute_dimensions(x, (0, 2, 1))
        for i in range(0, self.attention_depth):
            temp = K.sigmoid(K.dot(temp, self.Ws[i]) + self.bs[i])
        temp = K.permute_dimensions(temp, (0, 2, 1))
        estimated_weight = K.squeeze(K.dot(temp, K.expand_dims(self.Wf, -1)), -1)
        biased_weight = estimated_weight + self.bias
        non_linear_weight = K.tanh(biased_weight)

        # For each hidded state calculate how much should it contribute
        # to the context vector. This is the main part of attention.
        # In order to convert weights to "probabilities" use a sigmoid
        # based function: exp(x) / sum(exp(xi)).
        prob = K.exp(non_linear_weight)
        # Compute the total sum for each batch.
        total_sum = K.sum(prob, axis=1, keepdims=True)
        prob /= K.cast(total_sum, K.floatx())

        # Enable this if you want access to internal probabilities.
        # Should only be used for testing that Attention works as expected.
        # return prob

        # Multiply each hidden value by the corresponding probability.
        prob = K.expand_dims(prob, -1)
        new_hidden_values = x * prob
        return K.sum(new_hidden_values, axis=1)
Esempio n. 9
0
    def call(self, x, mask=None):
        x = K.permute_dimensions(x, (0, 2, 1))
        x = K.reshape(x, (-1, self.input_length))
        x = K.expand_dims(x, 1)
        x = K.expand_dims(x, -1)
        if self.real_filts is not None:
            conv_out_r = K.conv2d(x, self.W_r, strides=self.subsample,
                                  border_mode=self.border_mode,
                                  dim_ordering='th')
        else:
            conv_out_r = x

        if self.complex_filts is not None:
            conv_out_c1 = K.conv2d(x, self.W_c1, strides=self.subsample,
                                   border_mode=self.border_mode,
                                   dim_ordering='th')
            conv_out_c2 = K.conv2d(x, self.W_c2, strides=self.subsample,
                                   border_mode=self.border_mode,
                                   dim_ordering='th')
            conv_out_c = K.sqrt(K.square(conv_out_c1) + K.square(conv_out_c2) + K.epsilon())
            output = K.concatenate((conv_out_r, conv_out_c), axis=1)
        else:
            output = conv_out_r

        output_shape = self.get_output_shape_for((None, self.input_length, self.input_dim))
        output = K.squeeze(output, 3)  # remove the dummy 3rd dimension
        output = K.permute_dimensions(output, (2, 1, 0))
        output = K.reshape(output, (-1, output_shape[1], output.shape[1]*output.shape[2]))
        return output
Esempio n. 10
0
def Conv1DTranspose(input_tensor, filters, kernel_size, strides=2, padding='same'):
    x = Lambda(lambda x: K.expand_dims(x, axis=2))(input_tensor)
    #x = Reshape( ( -1, 1 ) )( input_tensor )
    #x = Permute( ( 3, 1, 2 ) )( x )
    x = Conv2DTranspose(filters=filters, kernel_size=(kernel_size, 1), strides=(1, 1), padding=padding, kernel_initializer = 'he_normal')(x)
    x = Lambda(lambda x: K.squeeze(x, axis=2))(x)
    x = Permute( ( 2, 1 ) )( x )
    return x
Esempio n. 11
0
 def reverse(x):
     if K.ndim(x) == 2:
         x = K.expand_dims(x, -1)
         rev = K.permute_dimensions(x, (1, 0, 2))[::-1]
         rev = K.squeeze(rev, -1)
     else:
         rev = K.permute_dimensions(x, (1, 0, 2))[::-1]                
     return K.permute_dimensions(rev, (1, 0, 2))
 def _triplet_loss(self, labels: Tensor, pairwise_dist: Tensor) -> Tensor :
     y_true = K.squeeze(labels, axis=1)
     """Triplet loss function"""
     if self.hard_triplets:
         triplet_loss = self._batch_hard_triplet_loss(y_true, pairwise_dist)
     else:
         triplet_loss = self._batch_all_triplet_loss(y_true, pairwise_dist)
     return triplet_loss
Esempio n. 13
0
 def call(self, x, mask=None):
     energy = K.squeeze(self.layer(x), 2)
     p_matrix = softmax(energy)
     if mask is not None:
         mask = self.squash_mask(mask)
         p_matrix = make_safe(p_matrix * mask) # remove unwanted items
         p_matrix = p_matrix / K.sum(p_matrix, axis=-1, keepdims=True) # renormalize
     return make_safe(p_matrix)
    def step(self, x, states):

        ytm, stm = states

        # repeat the hidden state to the length of the sequence
        _stm = K.repeat(stm, self.timesteps)

        # now multiplty the weight matrix with the repeated hidden state
        _Wxstm = K.dot(_stm, self.W_a)

        # calculate the attention probabilities
        # this relates how much other timesteps contributed to this one.
        et = K.dot(activations.tanh(_Wxstm + self._uxpb),
                   K.expand_dims(self.V_a))
        at = K.exp(et)
        at_sum = K.sum(at, axis=1)
        at_sum_repeated = K.repeat(at_sum, self.timesteps)
        at /= at_sum_repeated  # vector of size (batchsize, timesteps, 1)

        # calculate the context vector
        context = K.squeeze(K.batch_dot(at, self.x_seq, axes=1), axis=1)
        # ~~~> calculate new hidden state
        # first calculate the "r" gate:

        rt = activations.sigmoid(
            K.dot(ytm, self.W_r)
            + K.dot(stm, self.U_r)
            + K.dot(context, self.C_r)
            + self.b_r)

        # now calculate the "z" gate
        zt = activations.sigmoid(
            K.dot(ytm, self.W_z)
            + K.dot(stm, self.U_z)
            + K.dot(context, self.C_z)
            + self.b_z)

        # calculate the proposal hidden state:
        s_tp = activations.tanh(
            K.dot(ytm, self.W_p)
            + K.dot((rt * stm), self.U_p)
            + K.dot(context, self.C_p)
            + self.b_p)

        # new hidden state:
        st = (1-zt)*stm + zt * s_tp

        yt = activations.softmax(
            K.dot(ytm, self.W_o)
            + K.dot(stm, self.U_o)
            + K.dot(context, self.C_o)
            + self.b_o)

        if self.return_probabilities:
            return at, [yt, st]
        else:
            return yt, [yt, st]
Esempio n. 15
0
 def call(self, x, mask=None):
     # x[0]: (batch_size, input_length, input_dim)
     # x[1]: (batch_size, 1) indices of prepositions
     # Optional: x[2]: (batch_size, input_length - 2)
     assert isinstance(x, list) or isinstance(x, tuple)
     encoded_sentence = x[0]
     prep_indices = K.squeeze(x[1], axis=-1)  #(batch_size,)
     batch_indices = K.arange(K.shape(encoded_sentence)[0])  # (batch_size,)
     if self.with_attachment_probs:
         # We're essentially doing K.argmax(x[2]) here, but argmax is not differentiable!
         head_probs = x[2]
         head_probs_padding = K.zeros_like(x[2])[:, :2]  # (batch_size, 2)
         # (batch_size, input_length)
         padded_head_probs = K.concatenate([head_probs, head_probs_padding])
         # (batch_size, 1)
         max_head_probs = K.expand_dims(K.max(padded_head_probs, axis=1))
         # (batch_size, input_length, 1)
         max_head_prob_indices = K.expand_dims(K.equal(padded_head_probs, max_head_probs))
         # (batch_size, input_length, input_dim)
         masked_head_encoding = K.switch(max_head_prob_indices, encoded_sentence, K.zeros_like(encoded_sentence))
         # (batch_size, input_dim)
         head_encoding = K.sum(masked_head_encoding, axis=1)
     else:
         head_indices = prep_indices - 1  # (batch_size,)
         head_encoding = encoded_sentence[batch_indices, head_indices, :]  # (batch_size, input_dim)
     prep_encoding = encoded_sentence[batch_indices, prep_indices, :]  # (batch_size, input_dim)
     child_encoding = encoded_sentence[batch_indices, prep_indices+1, :]  # (batch_size, input_dim)
     '''
     prep_indices = x[1]
     sentence_mask = mask[0]
     if sentence_mask is not None:
         if K.ndim(sentence_mask) > 2:
             # This means this layer came after a Bidirectional layer. Keras has this bug which
             # concatenates input masks instead of output masks.
             # TODO: Fix Bidirectional instead.
             sentence_mask = K.any(sentence_mask, axis=(-2, -1))
     head_encoding, prep_encoding, child_encoding = self.get_split_averages(encoded_sentence, sentence_mask,
                                                                            prep_indices)
     '''
     head_projection = K.dot(head_encoding, self.proj_head)  # (batch_size, proj_dim)
     prep_projection = K.dot(prep_encoding, self.proj_prep)  # (batch_size, proj_dim)
     child_projection = K.dot(child_encoding, self.proj_child)  # (batch_size, proj_dim)
     #(batch_size, proj_dim)
     if self.composition_type == 'HPCT':
         composed_projection = K.tanh(head_projection + prep_projection + child_projection)
     elif self.composition_type == 'HPC':
         prep_child_projection = K.tanh(prep_projection + child_projection)  # (batch_size, proj_dim)
         composed_projection = K.tanh(head_projection + prep_child_projection)
     else:
         # Composition type in HC
         composed_projection = K.tanh(head_projection + child_projection)
     for hidden_layer in self.hidden_layers:
         composed_projection = K.tanh(K.dot(composed_projection, hidden_layer))  # (batch_size, proj_dim)
     # (batch_size, num_classes)
     class_scores = K.dot(composed_projection, self.scorer)
     label_probabilities = K.softmax(class_scores)
     return label_probabilities
def create_cnn_network(config):
    #also look at GaussianNoise and GaussianDropout

    # epsilon = 1e-06
    # mode = 0
    # momentum=0.9
    # weights=weights
    # model.add(BatchNormalization(epsilon=epsilon, mode=mode, momentum=momentum, weights=weights))
    #TODO: add dropout and batchnorm to encoders and decoders


    graph = Graph()
    input_shape,depth_axis,steps_axis = calc_input_shape(config)
    graph.add_input(name='input', input_shape=input_shape)
    if config['dim_ordering'] == 'tf':
        steps_axis = 2
    elif config['dim_ordering'] == 'th':
        steps_axis = 3
    else:
        raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
    #make num input steps odd
    graph.add_node(TDSlice1D(input_shape[steps_axis]-1,0), name='sliced_input', input='input')
    #TODO: add input distorter here as data augmentation
    googlenet = TDGoogleNet1D(graph, 'sliced_input', input_shape, config)
    googlenet_output = googlenet.result
    output_shape = googlenet.output_shape
    #make sure we output a flattened layer
    assert output_shape[steps_axis] == 1
    squeezed_shape = (output_shape[0], output_shape[1], output_shape[depth_axis])
    squeezed_output = 'squeezed_output'
    graph.add_node(Lambda(lambda x: K.squeeze(x,steps_axis),output_shape = squeezed_shape),
            name=squeezed_output,input=googlenet_output)
    recurrent_input = squeezed_output
    recurrent_input_shape = squeezed_shape

    #could add a fully connected layer to change dimension instead of using the output dimension
    #graph.add(TimeDistributedDense(input_dim=squeezed_shape[2], output_dim=config['num_hidden_dimensions']),
    #               name="encoder2rnn_fc",input=squeezed_output)
    #recurrent_input = "encoder2rnn_fc"
    #recurrent_input_shape = squeezed_shape
    #recurrent_input_shape[2] = config['num_hidden_dimensions']

    if use_lstm:
        cell = LSTM
    else:
        cell = GRU
    for cur_unit in xrange(num_recurrent_units):
        rnn_cell_name = "rnn_cell_%d"%cur_unit
        graph.add_node(cell(input_dim=recurrent_input_shape[2], output_dim=recurrent_input_shape[2], return_sequences=True),input=recurrent_input,name=name)
        recurrent_input = rnn_cell_name
    graph.add_output(name="output", input=rnn_cell_name)
    graph.compile(optimizer='rmsprop', loss={'output':'mse'})

    #TODO:add classifier here

    return graph
Esempio n. 17
0
 def _step(tensor):
     model_output = model(tensor)
     output = K.squeeze(model_output, axis=1)
     for layer in model.layers:
         if issubclass(type(layer), Recurrent):
             # TODO this is not being entered!
             layer.initial_state = layer.final_states
     feedback_result = feedback_function(output)
     # if type(output) not in (list, tuple):
     #     output = [output]
     return output + [feedback_result]
    def call(self, X,  mask=None):
        # 1D -> 2D
        X = K.expand_dims(X,2)
        X = K.permute_dimensions(X, (0, 2, 3, 1))
        conv_out = tf.nn.conv2d_transpose(X, self.W, strides=self.strides,
                                          padding=self.padding.upper(),
                                          output_shape=self.deconv_shape)

        output = conv_out + K.reshape(self.b, (1, 1, 1, self.W_shape[2]))
        output =  K.permute_dimensions(output, (0, 3, 1, 2))
        # 2D -> 1D
        output = K.squeeze(output,2)
        return output
Esempio n. 19
0
def dot_product(x, kernel):
    """
    Wrapper for dot product operation, in order to be compatible with both
    Theano and Tensorflow
    Args:
        x (): input
        kernel (): weights
    Returns:
    """
    if K.backend() == 'tensorflow':
        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)
    else:
        return K.dot(x, kernel)
Esempio n. 20
0
    def call(self, x, mask=None):
        x = K.permute_dimensions(x, (0, 2, 1))
        x = K.expand_dims(x, -1)

        conv_out = K.permute_dimensions(K.squeeze(K.conv2d(x, self.kernel), -1), (0, 2, 1))

        conv_out_s = conv_out[:,:,:self.nb_simple]

        conv_out_c = K.square(conv_out[:,:,self.nb_simple:])

        output = K.concatenate((conv_out_s, conv_out_c), axis=-1)

        return output
Esempio n. 21
0
 def step(self, x, states):
     h_tm1, c_tm1, y_tm1, B, U, H = states
     s = K.dot(c_tm1, self.W_h) + self.b_h
     s = K.repeat(s, self.input_length)
     energy = time_distributed_dense(s + H, self.W_a, self.b_a)
     energy = K.squeeze(energy, 2)
     alpha = K.softmax(energy)
     alpha = K.repeat(alpha, self.input_dim)
     alpha = K.permute_dimensions(alpha, (0, 2, 1))
     weighted_H = H * alpha
     v = K.sum(weighted_H, axis=1)
     y, new_states = super(AttentionDecoder, self).step(v, states[:-1])
     return y, new_states
 def create_score_model(self) -> Model:
     cr = self.model.inputs
     if self.triplet_mode:
         emb_c = self.model.get_layer("sentence_embedding").get_output_at(0)
         emb_r = self.model.get_layer("sentence_embedding").get_output_at(1)
         dist_score = Lambda(lambda x: self._euclidian_dist(x), name="score_model")
         score = dist_score([emb_c, emb_r])
     else:
         score = self.model.get_layer("score_model").output
         score = Lambda(lambda x: 1. - K.squeeze(x, -1))(score)
     score = Lambda(lambda x: 1. - x)(score)
     model = Model(cr, score)
     return model
Esempio n. 23
0
    def get_output(self, train=False):
        X = train
        X = K.expand_dims(X, -1)  # add a dimension of the right
        X = K.permute_dimensions(X, (0, 2, 1, 3))
        conv_out = K.conv2d(X, self.W, strides=self.subsample,
                            border_mode=self.border_mode,
                            dim_ordering='th')

        output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1))
        output = self.activation(output)
        output = K.squeeze(output, 3)  # remove the dummy 3rd dimension
        output = K.permute_dimensions(output, (0, 2, 1))
        return output
Esempio n. 24
0
def ctc_cost(y_true, y_pred):
    '''
    CTC cost:
    a theano wrapper for warp-ctc
    Arguments:
        y_true : label
        y_pred : acts
    '''
    from theano_ctc import ctc_cost as warp_ctc_cost

    # convert (batch size, timestep, target) to (timestep, batch size, target)
    acts = K.permute_dimensions(y_pred, (1, 0, 2))
    labels = K.cast(K.squeeze(y_true, axis=2), 'int32')
    return warp_ctc_cost(acts, labels)
Esempio n. 25
0
 def call(self, x, mask=None):
     # x: (batch_size, input_length, input_dim) where input_length = head_size + 2
     head_encoding = x[:, :-2, :]  # (batch_size, head_size, input_dim)
     prep_encoding = x[:, -2, :]  # (batch_size, input_dim)
     child_encoding = x[:, -1, :]  # (batch_size, input_dim)
     if self.composition_type == 'HPCD':
         # TODO: The following line may not work with TF.
         # (batch_size, head_size, input_dim, 1) * (1, head_size, input_dim, proj_dim)
         head_proj_prod = K.expand_dims(head_encoding) * K.expand_dims(self.dist_proj_head, dim=0)
         head_projection = K.sum(head_proj_prod, axis=2)  # (batch_size, head_size, proj_dim)
     else:
         head_projection = K.dot(head_encoding, self.proj_head)  # (batch_size, head_size, proj_dim)
     prep_projection = K.expand_dims(K.dot(prep_encoding, self.proj_prep), dim=1)  # (batch_size, 1, proj_dim)
     child_projection = K.expand_dims(K.dot(child_encoding, self.proj_child), dim=1)  # (batch_size, 1, proj_dim)
     #(batch_size, head_size, proj_dim)
     if self.composition_type == 'HPCT':
         composed_projection = K.tanh(head_projection + prep_projection + child_projection)
     elif self.composition_type == 'HPC' or self.composition_type == "HPCD":
         prep_child_projection = K.tanh(prep_projection + child_projection)  # (batch_size, 1, proj_dim)
         composed_projection = K.tanh(head_projection + prep_child_projection)
     else:
         # Composition type in HC
         composed_projection = K.tanh(head_projection + child_projection)
     for hidden_layer in self.hidden_layers:
         composed_projection = K.tanh(K.dot(composed_projection, hidden_layer))  # (batch_size, head_size, proj_dim)
     # (batch_size, head_size)
     head_word_scores = K.squeeze(K.dot(composed_projection, self.scorer), axis=-1)
     if mask is None:
         attachment_probabilities = K.softmax(head_word_scores)  # (batch_size, head_size)
     else:
         if K.ndim(mask) > 2:
             # This means this layer came after a Bidirectional layer. Keras has this bug which
             # concatenates input masks instead of output masks.
             # TODO: Fix Bidirectional instead.
             mask = K.any(mask, axis=(-2, -1))
         # We need to do a masked softmax.
         exp_scores = K.exp(head_word_scores)  # (batch_size, head_size)
         head_mask = mask[:, :-2]  # (batch_size, head_size)
         # (batch_size, head_size)
         masked_exp_scores = switch(head_mask, exp_scores, K.zeros_like(head_encoding[:, :, 0]))
         # (batch_size, 1). Adding epsilon to avoid divison by 0. But epsilon is float64.
         exp_sum = K.cast(K.expand_dims(K.sum(masked_exp_scores, axis=1) + K.epsilon()), 'float32')
         attachment_probabilities = masked_exp_scores / exp_sum  # (batch_size, head_size)
     return attachment_probabilities
Esempio n. 26
0
    def call(self, x, mask=None):
        # size of x :[batch_size, sel_len, attention_dim]
        # size of u :[batch_size, attention_dim]
        # uit = tanh(xW+b)
        uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))
        ait = K.dot(uit, self.u)
        ait = K.squeeze(ait, -1)

        ait = K.exp(ait)

        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            ait *= K.cast(mask, K.floatx())
        ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())
        ait = K.expand_dims(ait)
        weighted_input = x * ait
        output = K.sum(weighted_input, axis=1)

        return output
Esempio n. 27
0
    def conv_step_hidden(self, x, W, border_mode="valid", filters=None, filter_shape=None):

        input_shape = self.shuffeled_dims
        if filters == None:
            filters = self.nb_filter

        x = K.expand_dims(x, -1)  # add a dimension of the right

        conv_out = T.nnet.conv2d(x, W, subsample=(1, 1),
                                 border_mode='valid',
                                 filter_flip=False,
                                 input_shape=(input_shape[0],
                                              filters,
                                              input_shape[3],
                                              1),
                                 filter_shape=filter_shape)
        conv_out = K.squeeze(conv_out, 3)  # remove the dummy 3rd dimension

        return conv_out
Esempio n. 28
0
    def step(self, x_input, states):
        input_shape = self.input_spec[0].shape
        en_seq = states[-1]
        _, [h, c] = super(PointerLSTM, self).step(x_input, states[:-1])

        # vt*tanh(W1*e+W2*d)
        dec_seq = K.repeat(h, input_shape[1])
        #dec_seq = K.repeat(h, 2)
        print ('dec_seq')
        print (dec_seq)
        Eij = time_distributed_dense(en_seq, self.W1, output_dim=1)
        Dij = time_distributed_dense(dec_seq, self.W2, output_dim=1)
        U = self.vt * tanh(Eij + Dij)
        print ('U')
        print (U)
        U = K.squeeze(U, 2)
        print ('U squeezed')
        print (U)
        # make probability tensor
        pointer = softmax(U)
        return pointer, [h, c]
    def special_cube_conv(in_tensor, filter_size):
        """
        Takes in a None (samples) x 54 x ? (filters) tensor.

        It embedds it into 5 x 5 grid, and does a 3D convolution
        using only the nodes in the orginal embedding.

        To speed things up, it actually does the folowing:
        - pads the end with a zero (in the last dimension):
            None (samples) x 55 x ? (filters) (neighbors)
        - align neighbors to get an output of dim:
            None (samples) x 54 x 27 x ? (filters) (neighbors)
        - 2d convolution with filter (1, 27) and no padding to get an output of dim:
            None (samples) x 54 x filter_size
        - reshape to remove last dimension:
            None (samples) x filter_size x 54
        """ 
        print("in    ", in_tensor.shape)
        # pad (output dim: None x 55 x ?)
        padded = Lambda(lambda x: K.temporal_padding(x, (0, 1)))(in_tensor) # just pad end
        print("padded", padded.shape)
        # align neighbors (output dim: None x 54 x 27 x ?)
        #aligned = K.gather(padded, neighbors)
        #aligned = padded[ neighbors[np.newaxis].astype(np.int32), :]
        aligned = Lambda(lambda x: tf.gather(x, neighbors, axis=1))(padded)
        print("align ", aligned.shape)
        # 2D convolution in one axis (output dim: None x 54 x 1 x filter_size)
        conv = Conv2D(filter_size, kernel_size=(1, 27), 
                      strides=(1, 1), 
                      padding='valid', 
                      data_format="channels_last",
                      kernel_regularizer=l2(0.001), 
                      bias_regularizer=l2(0.001))(aligned)

        print("conv  ", conv.shape)
        # reshape (output dim: None x 54 x filter_size)
        out_tensor = Lambda(lambda x: K.squeeze(x, axis=2))(conv)

        return out_tensor
    def call(self, x):
        assert(K.backend() == 'tensorflow')
        # The model is described by the following equations:
        #       estimated_weight_i = dot_product(hidden_state_i, W).
        #       biased_weight = estimated_weight + bias
        #       non_linear_weight = tanh(biased_weight)
        estimated_weight = K.squeeze(K.dot(x, K.expand_dims(self.W, -1)), -1)
        biased_weight = estimated_weight + self.bias
        non_linear_weight = K.tanh(biased_weight)

        # For each hidded state calculate how much should it contribute
        # to the context vector. This is the main part of attention.
        # In order to convert weights to "probabilities" use a sigmoid
        # based function: exp(x) / sum(exp(xi)).
        prob = K.exp(non_linear_weight)
        # Compute the total sum for each batch.
        total_sum = K.sum(prob, axis=1, keepdims=True)
        prob /= K.cast(total_sum, K.floatx())

        # Multiply each hidden value by the corresponding probability.
        prob = K.expand_dims(prob, -1)
        new_hidden_values = x * prob
        return K.sum(new_hidden_values, axis=1)
Esempio n. 31
0
def squeeze(axis):
    return Lambda(lambda x: K.squeeze(x, axis))
Esempio n. 32
0
    def call(self, inputs, mask=None):
        '''
        :param inputs: a list of tensor of length not larger than 2, or a memory tensor of size BxTXD1.
        If a list, the first entry is memory, and the second one is query tensor of size BxD2 if any
        :param mask: the masking entry will be directly discarded
        :return: a tensor of size BxD1, weighted summing along the sequence dimension
        '''
        query = None
        if isinstance(inputs, list):
            memory = inputs[0]
            if len(inputs) > 1:
                query = inputs[1]
            elif len(inputs) > 2:
                raise ValueError('inputs length should not be larger than 2')
            if isinstance(mask, list):
                mask = mask[0]
        else:
            memory = inputs
        
        input_shape = K.int_shape(memory)
        if len(input_shape) >3:
            input_length = input_shape[1]
            memory = K.reshape(memory, (-1,) + input_shape[2:])
            if mask is not None:
                mask = K.reshape(mask, (-1,) + input_shape[2:-1])
            if query is not None:
                raise ValueError('query can be not supported')
        
        last = memory[:,-1,:]
        memory = memory[:,:-1,:]
        if query is None:
            query = last
        else:
            query = K.concatenate([query, last], axis=-1)
        
        if self.method is None:
            if len(input_shape) > 3:
                output_shape = K.int_shape(last)
                return K.reshape(last, (-1, input_shape[1], output_shape[-1]))
            else:
                return last
        elif self.method == 'cba':
            hidden = K.dot(memory, self.Wh) + K.expand_dims(K.dot(query, self.Wq), 1)
            hidden = K.tanh(hidden)
            s = K.squeeze(K.dot(hidden, self.v), -1)
        elif self.method == 'ga':
            s = K.sum(K.expand_dims(K.dot(query, self.Wq), 1) * memory, axis=-1)
        else:
            s = K.squeeze(K.dot(memory, self.v), -1)

        s = K.softmax(s)
        if mask is not None:
            mask = mask[:,:-1]
            s *= K.cast(mask, dtype='float32')
            sum_by_time = K.sum(s, axis=-1, keepdims=True)
            s = s / (sum_by_time + K.epsilon())
        #return [K.concatenate([K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1), s]
        result = K.concatenate([K.sum(memory * K.expand_dims(s), axis=1), last], axis=-1)
        if len(input_shape)>3:
            output_shape = K.int_shape(result)
            return K.reshape(result, (-1, input_shape[1], output_shape[-1]))
        else:
            return result
Esempio n. 33
0
def squeeze(tensor):
    return K.squeeze(tensor, axis=-1)
Esempio n. 34
0
 def compute_similarity(self, repeated_context_vectors, repeated_query_vectors):
     element_wise_multiply = repeated_context_vectors * repeated_query_vectors
     concatenated_tensor = K.concatenate(
         [repeated_context_vectors, repeated_query_vectors, element_wise_multiply], axis=-1)
     dot_product = K.squeeze(K.dot(concatenated_tensor, self.kernel), axis=-1)
     return linear(dot_product + self.bias)
Esempio n. 35
0
def center_caculate(a,b):
    stage1_center = a + K.squeeze(b, axis=1)
    end_points['stage1_center'] = stage1_center
    return stage1_center
Esempio n. 36
0
def get_intersection(y_true, y_pred):
    """ aka TP, assuming binary images that were removed the background """

    return K.sum(K.sum(K.squeeze(y_true * y_pred, axis=3), axis=2), axis=1)
Esempio n. 37
0
 def call(self, q_embed, mask=None):
     q_emb_exp = K.squeeze(q_embed, axis=self.dim)
     # q_emb_exp = K.squeeze(q_embed)
     show_layer_info('squeeze 1', q_emb_exp)
     return q_emb_exp
Esempio n. 38
0
    def __init__(
            self,
            model,
            bounds,
            channel_axis=3,
            preprocessing=(0, 1),
            predicts='probabilities'):

        super(KerasModel, self).__init__(bounds=bounds,
                                         channel_axis=channel_axis,
                                         preprocessing=preprocessing)
        from keras import backend as K
        import keras

        self._task = 'cls'

        if predicts == 'probs':
            predicts = 'probabilities'
        assert predicts in ['probabilities', 'logits']

        images_input = model.input
        label_input = K.placeholder(shape=(1,))

        predictions = model.output

        shape = K.int_shape(predictions)
        _, num_classes = shape
        assert num_classes is not None
        self._num_classes = num_classes

        if predicts == 'probabilities':
            if K.backend() == 'tensorflow':
                predictions, = predictions.op.inputs
                loss = K.sparse_categorical_crossentropy(
                    label_input, predictions, from_logits=True)
        elif predicts == 'logits':
            loss = K.sparse_categorical_crossentropy(
                label_input, predictions, from_logits=True)

        # sparse_categorical_crossentropy returns 1-dim tensor,
        # gradients wants 0-dim tensor

        loss = K.squeeze(loss, axis=0)
        grads = K.gradients(loss, images_input)

        grad_loss_output = K.placeholder(shape=(num_classes, 1))
        external_loss = K.dot(predictions, grad_loss_output)
        # remove batch dimension of predictions
        external_loss = K.squeeze(external_loss, axis=0)
        # remove singleton dimension of grad_loss_output
        external_loss = K.squeeze(external_loss, axis=0)

        grads_loss_input = K.gradients(external_loss, images_input)

        if K.backend() == 'tensorflow':
            # tensorflow backend returns a list with the gradient
            # as the only element, even if loss is a single scalar tensor
            assert isinstance(grads, list)
            assert len(grads) == 1
            grad = grads[0]

            assert isinstance(grads_loss_input, list)
            assert len(grads_loss_input) == 1
            grad_loss_input = grads_loss_input[0]

        self._loss_fn = K.function(
            [images_input, label_input], [loss])
        self._batch_pred_fn = K.function(
            [images_input], [predictions])
        self._pred_grad_fn = K.function(
            [images_input, label_input], [predictions, grad])
        self._bw_grad_fn = K.function(
            [grad_loss_output, images_input], [grad_loss_input])
Esempio n. 39
0
    def call(self, x):
        first_token = K.squeeze(x[:, 0:1, :], axis=1)

        return K.tanh(K.dot(first_token, self.pool))
Esempio n. 40
0
 def toGrey(self, img):
     # returns a 2D grey tensor of the given 4D tensor
     img = K.squeeze(img, 0)
     img = img[:, :, 0] * 0.2126 + img[:, :, 1] * 0.7152 + img[:, :,
                                                               2] * 0.0722
     return img
Esempio n. 41
0
def get_model():
    input_shape = (None, None, 2)
    y0 = Input(shape=input_shape)

    y1_pitch = Conv2D(32, (5, 5),
                      padding='same',
                      activation='relu',
                      name='pitch_layer1')(y0)
    y1a_pitch = BatchNormalization()(y1_pitch)
    y2_pitch = Conv2D(32, (5, 5),
                      padding='same',
                      activation='relu',
                      name='pitch_layer2')(y1a_pitch)
    y2a_pitch = BatchNormalization()(y2_pitch)
    y3_pitch = Conv2D(32, (3, 3),
                      padding='same',
                      activation='relu',
                      name='smoothy2')(y2a_pitch)
    y3a_pitch = BatchNormalization()(y3_pitch)
    y4_pitch = Conv2D(8, (70, 3),
                      padding='same',
                      activation='relu',
                      name='distribute')(y3a_pitch)
    y4a_pitch = BatchNormalization()(y4_pitch)

    y_multif0 = Conv2D(1, (1, 1),
                       padding='same',
                       activation='sigmoid',
                       name='multif0_presqueeze')(y4a_pitch)
    multif0 = Lambda(lambda x: K.squeeze(x, axis=3), name='multif0')(y_multif0)

    y_mask = Multiply(name='mask')([y_multif0, y0])
    y1_timbre = Conv2D(512, (2, 3),
                       padding='same',
                       activation='relu',
                       name='timbre_layer1')(y_mask)
    y1a_timbre = BatchNormalization()(y1_timbre)

    y_concat = Concatenate(name='timbre_and_pitch')([y_multif0, y1a_timbre])
    ya_concat = BatchNormalization()(y_concat)

    y_mel_feat = Conv2D(32, (3, 3),
                        padding='same',
                        activation='relu',
                        name='melody_filters')(ya_concat)  #32
    ya_mel_feat = BatchNormalization()(y_mel_feat)
    y_mel_feat2 = Conv2D(32, (3, 3),
                         padding='same',
                         activation='relu',
                         name='melody_filters2')(ya_mel_feat)  #32
    ya_mel_feat2 = BatchNormalization()(y_mel_feat2)
    y_mel_feat3 = Conv2D(8, (240, 1),
                         padding='same',
                         activation='relu',
                         name='melody_filters3')(ya_mel_feat2)  # 8
    ya_mel_feat3 = BatchNormalization()(y_mel_feat3)
    y_mel_feat4 = Conv2D(16, (7, 7),
                         padding='same',
                         activation='relu',
                         name='melody_filters4')(ya_mel_feat3)  # 16
    ya_mel_feat4 = BatchNormalization()(y_mel_feat4)
    y_mel_feat5 = Conv2D(16, (7, 7),
                         padding='same',
                         activation='relu',
                         name='melody_filters5')(ya_mel_feat4)  #16
    ya_mel_feat5 = BatchNormalization()(y_mel_feat5)

    y_bass_feat = Conv2D(32, (3, 3),
                         padding='same',
                         activation='relu',
                         name='bass_filters')(ya_concat)  #32
    ya_bass_feat = BatchNormalization()(y_bass_feat)
    y_bass_feat2 = Conv2D(32, (3, 3),
                          padding='same',
                          activation='relu',
                          name='bass_filters2')(ya_bass_feat)  #32
    ya_bass_feat2 = BatchNormalization()(y_bass_feat2)
    y_bass_feat3 = Conv2D(8, (240, 1),
                          padding='same',
                          activation='relu',
                          name='bass_filters3')(ya_bass_feat2)  #8
    ya_bass_feat3 = BatchNormalization()(y_bass_feat3)
    y_bass_feat4 = Conv2D(16, (7, 7),
                          padding='same',
                          activation='relu',
                          name='bass_filters4')(ya_bass_feat3)  #16
    ya_bass_feat4 = BatchNormalization()(y_bass_feat4)
    y_bass_feat5 = Conv2D(16, (7, 7),
                          padding='same',
                          activation='relu',
                          name='bass_filters5')(ya_bass_feat4)  #16
    ya_bass_feat5 = BatchNormalization()(y_bass_feat5)

    y_vocal_feat = Conv2D(32, (3, 3),
                          padding='same',
                          activation='relu',
                          name='vocal_filters')(ya_concat)  #32
    ya_vocal_feat = BatchNormalization()(y_vocal_feat)
    y_vocal_feat2 = Conv2D(32, (3, 3),
                           padding='same',
                           activation='relu',
                           name='vocal_filters2')(ya_vocal_feat)  #32
    ya_vocal_feat2 = BatchNormalization()(y_vocal_feat2)
    y_vocal_feat3 = Conv2D(8, (240, 1),
                           padding='same',
                           activation='relu',
                           name='vocal_filters3')(ya_vocal_feat2)  #8
    ya_vocal_feat3 = BatchNormalization()(y_vocal_feat3)
    y_vocal_feat4 = Conv2D(16, (7, 7),
                           padding='same',
                           activation='relu',
                           name='vocal_filters4')(ya_vocal_feat3)  # 16
    ya_vocal_feat4 = BatchNormalization()(y_vocal_feat4)
    y_vocal_feat5 = Conv2D(16, (7, 7),
                           padding='same',
                           activation='relu',
                           name='vocal_filters5')(ya_vocal_feat4)  #16
    ya_vocal_feat5 = BatchNormalization()(y_vocal_feat5)

    y_melody = Conv2D(1, (1, 1),
                      padding='same',
                      activation='sigmoid',
                      name='melody_presqueeze')(ya_mel_feat5)
    melody = Lambda(lambda x: K.squeeze(x, axis=3), name='melody')(y_melody)

    y_bass = Conv2D(1, (1, 1),
                    padding='same',
                    activation='sigmoid',
                    name='bass_presqueeze')(ya_bass_feat5)
    bass = Lambda(lambda x: K.squeeze(x, axis=3), name='bass')(y_bass)

    y_vocal = Conv2D(1, (1, 1),
                     padding='same',
                     activation='sigmoid',
                     name='vocal_presqueeze')(ya_vocal_feat5)
    vocal = Lambda(lambda x: K.squeeze(x, axis=3), name='vocal')(y_vocal)

    model = Model(inputs=y0, outputs=[multif0, melody, bass, vocal])

    model.summary(line_length=120)

    return model
Esempio n. 42
0
    def _sample(self, inputs, mask = None):
        
        substrings_by_size = []
        binaries_by_size = []
        masks_by_size = []
    
        inputs_padded = self._pad(inputs)
        if not mask is None:
            mask_padded = self._pad(mask)
        
        batchsize = K.shape(inputs_padded)[0]
        timesteps = K.shape(inputs_padded)[1]
        int_shape = K.int_shape(inputs)
        
        def sample_length(_):
            return K.random_uniform(shape = (1,), minval=self.low, maxval=self.high, dtype = "int32")
        
        all_lengths = K.arange(self.low, self.high)
        samples_left = self.samples - (self.high - self.low) 
        
        all_lengths = K.concatenate([all_lengths, 
            K.squeeze(K.map_fn(sample_length, K.arange(0, samples_left), dtype = "int32"), axis = -1)], axis = 0)   
        
        for length in range(self.low, self.high):
            times_sampled = K.sum(K.cast(K.equal(all_lengths, length), "int32"))
            
            def sample_start(_):
                return K.random_uniform(shape=(1,), minval=0, maxval=timesteps - length, dtype = "int32")
            starts = K.squeeze(K.map_fn(sample_start, K.arange(times_sampled), dtype = "int32"), axis = -1)
            
            def sample_substring(start):
                return inputs_padded[:,start:start+length]
            
            substrings = K.map_fn(sample_substring, starts, dtype = K.dtype(inputs))
            # (samples, batchsize, length, ...)
            substrings = K.permute_dimensions(substrings, [1,0] + list(range(2, K.ndim(substrings))))
            substrings._keras_shape = (None, None, length) + int_shape[2:]
            substrings._uses_learning_phase = inputs._uses_learning_phase
            substrings_by_size.append(substrings)

            def sample_binary(start):
                before = K.zeros((batchsize, start))
                ones = K.ones((batchsize, length))
                after = K.zeros((batchsize, timesteps - start - length))
                return K.concatenate([before, ones, after], axis = 1)
            
            binaries = K.map_fn(sample_binary, starts, dtype = K.floatx())
            # (samples, batchsize, timesteps)
            binaries = K.permute_dimensions(binaries, [1,0,2])
            # (batchsize, samples, timesteps)
            binaries_by_size.append(binaries)

            if not mask is None:
                def sample_submask(start):
                    return mask_padded[:,start:start+length]

                masks = K.map_fn(sample_submask, starts, dtype = K.dtype(mask))
                # (samples, batchsize, length)
                masks = K.permute_dimensions(masks, [1,0,2])
                masks_by_size.append(masks)
        
        return substrings_by_size, binaries_by_size, masks_by_size
Esempio n. 43
0
 def compute_similarity(self, tensor_1, tensor_2):
     combined_tensors = self._combine_tensors(tensor_1, tensor_2)
     dot_product = K.squeeze(K.dot(combined_tensors, self.weight_vector),
                             axis=-1)
     return self.activation(dot_product + self.bias)
Esempio n. 44
0
    def step(self, inputs, states):
        h_tm1 = states[0]
        c_tm1 = states[1]
        dp_mask = states[2]
        rec_dp_mask = states[3]
        x_input = states[4]

        # alignment model
        h_att = K.repeat(h_tm1, self.timestep_dim)
        att = _time_distributed_dense(x_input,
                                      self.attention_weights,
                                      self.attention_bias,
                                      output_dim=K.int_shape(
                                          self.attention_weights)[1])
        attention_ = self.attention_activation(
            K.dot(h_att, self.attention_recurrent_weights) + att)  # energy
        attention_ = K.squeeze(K.dot(attention_,
                                     self.attention_recurrent_bias),
                               2)  # energy

        alpha = K.exp(attention_)

        if dp_mask is not None:
            alpha *= dp_mask[0]

        alpha /= K.sum(alpha, axis=1, keepdims=True)
        alpha_r = K.repeat(alpha, self.input_dim)
        alpha_r = K.permute_dimensions(alpha_r, (0, 2, 1))

        # make context vector (soft attention after Bahdanau et al.)
        z_hat = x_input * alpha_r
        context_sequence = z_hat
        z_hat = K.sum(z_hat, axis=1)

        if self.implementation == 2:
            z = K.dot(inputs * dp_mask[0], self.kernel)
            z += K.dot(h_tm1 * rec_dp_mask[0], self.recurrent_kernel)
            z += K.dot(z_hat, self.attention_kernel)

            if self.use_bias:
                z = K.bias_add(z, self.bias)

            z0 = z[:, :self.units]
            z1 = z[:, self.units:2 * self.units]
            z2 = z[:, 2 * self.units:3 * self.units]
            z3 = z[:, 3 * self.units:]

            i = self.recurrent_activation(z0)
            f = self.recurrent_activation(z1)
            c = f * c_tm1 + i * self.activation(z2)
            o = self.recurrent_activation(z3)
        else:
            if self.implementation == 0:
                x_i = inputs[:, :self.units]
                x_f = inputs[:, self.units:2 * self.units]
                x_c = inputs[:, 2 * self.units:3 * self.units]
                x_o = inputs[:, 3 * self.units:]
            elif self.implementation == 1:
                x_i = K.dot(inputs * dp_mask[0], self.kernel_i) + self.bias_i
                x_f = K.dot(inputs * dp_mask[1], self.kernel_f) + self.bias_f
                x_c = K.dot(inputs * dp_mask[2], self.kernel_c) + self.bias_c
                x_o = K.dot(inputs * dp_mask[3], self.kernel_o) + self.bias_o
            else:
                raise ValueError('Unknown `implementation` mode.')

            i = self.recurrent_activation(
                x_i + K.dot(h_tm1 * rec_dp_mask[0], self.recurrent_kernel_i) +
                K.dot(z_hat, self.attention_i))
            f = self.recurrent_activation(
                x_f + K.dot(h_tm1 * rec_dp_mask[1], self.recurrent_kernel_f) +
                K.dot(z_hat, self.attention_f))
            c = f * c_tm1 + i * self.activation(
                x_c + K.dot(h_tm1 * rec_dp_mask[2], self.recurrent_kernel_c) +
                K.dot(z_hat, self.attention_c))
            o = self.recurrent_activation(
                x_o + K.dot(h_tm1 * rec_dp_mask[3], self.recurrent_kernel_o) +
                K.dot(z_hat, self.attention_o))
        h = o * self.activation(c)
        if 0 < self.dropout + self.recurrent_dropout:
            h._uses_learning_phase = True

        if self.return_attention:
            return context_sequence, [h, c]
        else:
            return h, [h, c]
plt.plot(cnn_data[num, :], linewidth=4, color='black')
plt.show()
# 获取窗口数据
X, timestep = slide_window(X, window, step)
X = np.expand_dims(X, axis=-1)
# 构建模型
input0 = Input(shape=(X.shape[1], X.shape[2], 1), name='input0')
conv2d = Conv2D(filters=1,
                kernel_size=[1, 4],
                strides=1,
                padding='valid',
                use_bias=0,
                name='conv2d',
                trainable=True)
tensor = conv2d(input0)
tensor = Lambda(lambda x: K.squeeze(x, axis=-1), name='squeeze_0')(tensor)
tensor = Lambda(lambda x: tf.nn.top_k(x, x.shape[2])[0],
                name='sequential_pooling_drop')(tensor)
tensor = Lambda(lambda x: K.reverse(x, axes=2),
                name='sequential_pooling_asend')(tensor)
tensor = LSTM(units=40, name='lstm')(tensor)
tensor = Dense(20, activation='tanh', name='FC0')(tensor)
prediction = Dense(1, activation='sigmoid', name='output',
                   use_bias='False')(tensor)
model = Model(inputs=input0, outputs=prediction)
# 加载训练好的模型权重
model.load_weights(r'.\model\CNN_SP_LSTM\\' + str(signal_len) +
                   's-train_2-fold.h5py')

layer_name = 'sequential_pooling_asend'
intermediate_layer_model = Model(input=model.input,
Esempio n. 46
0
    def get_model(self, num_classes, activation='sigmoid'):
        max_len = opt.max_len
        voca_size = opt.unigram_hash_size + 1

        with tf.device('/gpu:0'):
            embd = Embedding(voca_size, opt.embd_size, name='word_embd')

            word = Input((max_len, ), name="word")
            word_embed = embd(word)
            img = Input((2048, ), name="image")

            permute_layer = Lambda(
                lambda x: K.permute_dimensions(x, (0, 2, 1)))
            dot_product = Lambda(
                lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))
            softmax = Activation('softmax')
            squeeze = Lambda(lambda x: K.squeeze(x, axis=2))

            num_attns = 4
            char_attns = []
            for i in range(num_attns):
                e_char = Dense(1, activation='softmax')(word_embed)
                alpha_char = permute_layer(e_char)
                char_attn = dot_product([word_embed, alpha_char])
                char_attn = squeeze(char_attn)
                char_attns.append(char_attn)

            # concatenate context vector
            char_context = concatenate(char_attns, axis=1)

            h_char = Dense(1)(char_context)
            h_img = Dense(1)(img)
            h_w = concatenate([h_char, h_img], axis=1)
            h_w = softmax(h_w)

            # allocate weights
            h_w_charb = Lambda(lambda x: x[:, 0], output_shape=(1, ))(h_w)
            h_w_img = Lambda(lambda x: x[:, 1], output_shape=(1, ))(h_w)

            h_w_charb = Reshape((1, ))(h_w_charb)
            h_w_img = Reshape((1, ))(h_w_img)

            h_w_charb = RepeatVector(128 * num_attns)(h_w_charb)
            h_w_img = RepeatVector(2048)(h_w_img)

            h_w_char = Reshape((128 * num_attns, ))(h_w_charb)
            h_w_img = Reshape((2048, ))(h_w_img)

            h1_char = multiply([char_context, h_w_char])
            h1_img = multiply([img, h_w_img])

            concat_size = 128 * num_attns + 2048
            h1 = concatenate([h1_char, h1_img], axis=1)
            h1 = Reshape((concat_size, ))(h1)
            h1 = Dropout(0.5)(h1)

            res1 = self.residual(h1, h1, concat_size)
            res2 = self.residual(h1, res1, concat_size)
            outputs = Dense(num_classes, activation=activation)(res2)

            model = Model(inputs=[word, img], outputs=outputs)
            optm = keras.optimizers.Nadam(opt.lr)
            model.compile(loss='binary_crossentropy',
                          optimizer=optm,
                          metrics=[top1_acc])
            model.summary(print_fn=lambda x: self.logger.info(x))

        return model
Esempio n. 47
0
 def content_layer(passage_modeling):
     relu_encoding = Dense(Params.embedding_dim, activation="relu")(passage_modeling) #Dense(100, name="content_activation", activation="relu")(passage_context)
     indice_probability = Dense(1, activation="sigmoid")(relu_encoding) #Dense(100, name="content_indice", activation="sigmoid")(relu_encoding)
     indice_probability = K.squeeze(indice_probability, axis=-1)
     return indice_probability
Esempio n. 48
0
 def _backward_step(gamma_t, states):
     y_tm1 = K.squeeze(states[0], 0)
     y_t = batch_gather(gamma_t, y_tm1)
     return y_t, [K.expand_dims(y_t, 0)]
Esempio n. 49
0
def lovasz_loss(y_true, y_pred):
    y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
    #logits = K.log(y_pred / (1. - y_pred))
    logits = y_pred #Jiaxin
    loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
    return loss
Esempio n. 50
0
 def sparse_loss(self, y_true, y_pred):
     # Linear Chain Conditional Random Field loss function with sparse tag sequences.
     y_true = K.cast(y_true, 'int32')
     y_true = K.squeeze(y_true, 2)
     mask = self._fetch_mask()
     return sparse_chain_crf_loss(y_true, y_pred, self.U, self.b_start, self.b_end, mask)
Esempio n. 51
0
def squeeze(inputs, squeeze_aixs):
    return K.squeeze(inputs,squeeze_aixs)
    def construt_self_attention_policy_output(self):
        tiled_pose = self.get_tiled_input(self.pose_input)
        concat_input = Concatenate(axis=2)([
            self.key_config_input, self.goal_flag_input, self.collision_input,
            tiled_pose
        ])
        dim_input = concat_input.shape[2]._value

        # The query matrix
        query = self.create_conv_layers(concat_input,
                                        dim_input,
                                        use_pooling=False,
                                        use_flatten=False)
        query = Conv2D(filters=1,
                       kernel_size=(1, 1),
                       strides=(1, 1),
                       activation='linear',
                       kernel_initializer=self.kernel_initializer,
                       bias_initializer=self.bias_initializer)(query)

        # The key matrix
        """
        key = self.create_conv_layers(concat_input, dim_input, use_pooling=False, use_flatten=False)
        key = Conv2D(filters=256,
                     kernel_size=(1, 1),
                     strides=(1, 1),
                     activation='linear',
                     kernel_initializer=self.kernel_initializer,
                     bias_initializer=self.bias_initializer)(key)

        def compute_W(x):
            qvals = x[0]
            kvals = x[1]
            qvals = tf.transpose(qvals, perm=[0, 1, 3, 2])
            dotted = tf.keras.backend.batch_dot(kvals, qvals) /tf.sqrt(tf.dtypes.cast(qvals.shape[2]._value,tf.float32))
            dotted = tf.squeeze(dotted, axis=-1)
            dotted = tf.squeeze(dotted, axis=-1)
            return K.softmax(dotted, axis=-1)

        W = Lambda(compute_W, name='softmax')([query, key])
        """
        def compute_W(x):
            x = K.squeeze(x, axis=-1)
            x = K.squeeze(x, axis=-1)
            return K.softmax(x * 100, axis=-1)

        W = Lambda(compute_W, name='softmax')(query)

        self.w_model = Model(inputs=[
            self.goal_flag_input, self.key_config_input, self.collision_input,
            self.pose_input
        ],
                             outputs=W,
                             name='w_model')

        # The value matrix
        value = self.create_conv_layers(concat_input,
                                        dim_input,
                                        use_pooling=False,
                                        use_flatten=False)
        value = Conv2D(
            filters=4,
            kernel_size=(1, 1),
            strides=(1, 1),
            activation='linear',
            kernel_initializer=self.kernel_initializer,
            bias_initializer=self.bias_initializer,
        )(value)

        # value = self.key_config_input
        """
        value = Lambda(lambda x: K.squeeze(x, axis=-1))(self.key_config_input)
        """

        value = Lambda(lambda x: K.squeeze(x, axis=2),
                       name='key_config_transformation')(value)
        self.value_model = Model(inputs=[
            self.goal_flag_input, self.key_config_input, self.collision_input,
            self.pose_input
        ],
                                 outputs=value,
                                 name='value_model')
        output = Lambda(lambda x: K.batch_dot(x[0], x[1]))([W, value])
        return output
Esempio n. 53
0
 def call(self, inputs, **kwargs):
     if inputs.get_shape().ndims == 5:
         assert inputs.get_shape(
         )[-2].value == 1, 'Error: Must have num_capsules = 1 going into Length'
         inputs = K.squeeze(inputs, axis=-2)
     return K.expand_dims(tf.norm(inputs, axis=-1), axis=-1)
 def compute_W(x):
     x = K.squeeze(x, axis=-1)
     x = K.squeeze(x, axis=-1)
     return K.softmax(x * 100, axis=-1)
def pad_backend(inputs, in_channels, out_channels):
    pad_dim = (out_channels - in_channels) // 2
    inputs = K.expand_dims(inputs, -1)
    inputs = K.spatial_3d_padding(inputs, ((0, 0), (0, 0), (pad_dim, pad_dim)),
                                  'channels_last')
    return K.squeeze(inputs, -1)
Esempio n. 56
0
def main():
    import os

    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"  # see issue #152
    os.environ["CUDA_VISIBLE_DEVICES"] = "0"

    input1 = Input(shape=(2, 3), name="input1")  # Context
    input2 = Input(shape=(4, 3), name="input2")  # Query

    # sim = Similarity()([input1, input2])

    sim = Input(shape=(2, 4), name="sim")

    # Context-to-query attention.

    # Softmax on each line.
    col_softmax = Lambda(lambda x: K.softmax(x, axis=-1))(sim)

    # Product between sofmax prob and each query vector.
    UT = Lambda(
        lambda x: K.batch_dot(x[0], x[1], axes=(2, 1)),
        output_shape=lambda x: x[0][:2] + x[1][2:])([col_softmax, input2])

    # Query-to-context attention.

    # Max per line then softmax.
    line_softmax = Lambda(lambda x: K.max(x, axis=-1),
                          output_shape=lambda x: (x[0], x[1]))(sim)
    line_softmax = Lambda(lambda x: K.softmax(x, axis=-1))(line_softmax)

    # Make @line_softmax a matrix with 1 row.
    line_softmax = Lambda(lambda x: K.expand_dims(x, axis=1),
                          output_shape=lambda x: (x[0], 1, x[1]))(line_softmax)

    # Matrix multiplication.
    HT = Lambda(
        lambda x: K.batch_dot(x[0], x[1], axes=(2, 1)),
        output_shape=lambda x: x[0][:2] + x[1][2:])([line_softmax, input1])

    # Remove one extra row.
    HT = Lambda(lambda x: K.squeeze(x, axis=1),
                output_shape=lambda x: (x[0], x[2]))(HT)

    HT = RepeatVector(2)(HT)  # CONTEXT_LEN TIMES

    G = Concatenate(axis=-1)(
        [input1, UT,
         Multiply()([input1, UT]),
         Multiply()([input1, HT])])
    model = Model(inputs=[input1, input2, sim], outputs=[G])
    model.compile(loss=categorical_crossentropy,
                  optimizer='adam',
                  metrics=['accuracy'])
    model.summary()
    '''
    y = model.predict({
        'input1': np.array([
                    [[1, 2, 3], [4, 5, 6]],
                    [[-0.5, -1, -1.5], [-2, -2.5, -3]]
                ]),
        'input2': np.array([
                    [[7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]],
                    [[-13, -14, -15], [-7, -8, -9], [-10, -11, 0], [-16, -17, -18]]
                ])
        }, batch_size=2)
    '''
    y = model.predict(
        {
            'input1':
            np.array([[[1, 2, 3], [4, 5, 6]], [[-0.5, -1, -1.5],
                                               [-2, -2.5, -3]]]),
            'input2':
            np.array([[[7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]],
                      [[-13, -14, -15], [-7, -8, -9], [-10, -11, 0],
                       [-16, -17, -18]]]),
            'sim':
            np.array([[[-2, -2.1, -1.9, -2], [3, 2.8, 3.2, 0]],
                      [[1, 1.5, 0.5, 0.9], [6, 5, 7, 5.5]]])
        },
        batch_size=2)
    import time
    time.sleep(60 * 50)
    print(y)
    '''
Esempio n. 57
0
def create_network(input_shape, prior, L1=0, L2=0, dropout=0, KERNEL_NUM = 64):
    inp = Input(shape=input_shape)
    depth = Lambda(lambda x: x[:,:,:,2],  name='extract_depth')(inp)
    depth = expand_dims(axis=-1)(depth)
    color = Lambda(lambda x: x[:,:,:,:2], name='extract_color')(inp)

    half_kernel = KERNEL_NUM//2
    # Depth Path
    x = Conv2D(half_kernel, 7, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(depth)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    # First Residual Stage
    x = residual_stage(x, half_kernel, iter=2, L1=L1, L2=L2, dropout=dropout)

    # Downsampling
    x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    # Second Residual Stage
    x = residual_stage(x, 2*half_kernel, iter=3, L1=L1, L2=L2, dropout=dropout)

    # Downsampling
    x = Conv2D(2*half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    depth = x

    # Color Path
    x = Conv2D(half_kernel, 7, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(color)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    # First Residual Stage
    x = residual_stage(x, half_kernel, iter=2, L1=L1, L2=L2, dropout=dropout)

    # Downsampling
    x = Conv2D(half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    # Second Residual Stage
    x = residual_stage(x, 2*half_kernel, iter=3, L1=L1, L2=L2, dropout=dropout)

    # Downsampling
    x = Conv2D(2*half_kernel, 3, padding='same', strides=2, kernel_regularizer=l1_l2(l1=L1, l2=L2))(x)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)
    x = Dropout(dropout)(x)

    color = x

    # Combine
    combined = concatenate([color, depth], axis=-1)

    # Third Residual Stage
    features_at_depth = []
    for i in range(8):
        x = residual_stage(combined, half_kernel, iter=5, L1=L1, L2=L2, dropout=dropout)
        x = bilinear_resize((8,8))(x)
        x = expand_dims(axis=1)(x)
        features_at_depth.append(x)
    x = concatenate(features_at_depth, axis=1)

    # Trainable residual 3D blocks
    x = residual_block(x, bottleneck_kernels=2*KERNEL_NUM,
                                out_kernels=8*KERNEL_NUM,
                                kernel_size=3,
                                identity=False,
                                conv=Conv3D,
                                L1=L1,
                                L2=L2)
    x = Dropout(dropout)(x)

    for i in range(2): # Originally 2
        x = residual_block(x, bottleneck_kernels=2*KERNEL_NUM,
                                   out_kernels=8*KERNEL_NUM,
                                   kernel_size=3,
                                   identity=True,
                                   conv=Conv3D,
                                   L1=L1,
                                   L2=L2)
    x = Dropout(dropout)(x)

    x = Conv3D(1, 3, padding='same', activation='tanh')(x)
    x = Lambda(lambda x: K.squeeze(x, axis=-1))(x)

    def add_prior(x, prior=prior):
        prior = K.constant(K.cast_to_floatx(prior))
        prior = expand_dims(axis=0)(prior)
        prior = Lambda( lambda prior: K.tile( prior, (K.shape(x)[0], 1, 1, 1) ))(prior)
        return x + prior

    x = Lambda( lambda x: add_prior(x) )(x)
    return Model(inp, x)
def squeeze_layer(arr, axis=1):
    return K.squeeze(arr, axis)
Esempio n. 59
0
def get_alls(y_true, y_pred):
    """ aka TP + TP + FP + FN, assuming binary images that were removed the background """

    return K.sum(K.sum(K.squeeze(y_true + y_pred, axis=3), axis=2), axis=1)
 def _traspose(x):
     x = K.squeeze(x, axis=-2)
     x = K.expand_dims(x, axis=-1)
     return x