def __DenseNet(
            inp,  #input_shape=None,
            dense_blocks=3,
            dense_layers=-1,
            growth_rate=12,
            nb_classes=None,
            dropout_rate=None,
            bottleneck=False,
            compression=1.0,
            weight_decay=1e-4,
            depth=40):
        """
        Creating a DenseNet

        Arguments:
            input_shape  : shape of the input images. E.g. (28,28,1) for MNIST
            dense_blocks : amount of dense blocks that will be created (default: 3)
            dense_layers : number of layers in each dense block. You can also use a list for numbers of layers [2,4,3]
                           or define only 2 to add 2 layers at all dense blocks. -1 means that dense_layers will be calculated
                           by the given depth (default: -1)
            growth_rate  : number of filters to add per dense block (default: 12)
            nb_classes   : number of classes
            dropout_rate : defines the dropout rate that is accomplished after each conv layer (except the first one).
                           In the paper the authors recommend a dropout of 0.2 (default: None)
            bottleneck   : (True / False) if true it will be added in convolution block (default: False)
            compression  : reduce the number of feature-maps at transition layer. In the paper the authors recomment a compression
                           of 0.5 (default: 1.0 - will have no compression effect)
            weight_decay : weight decay of L2 regularization on weights (default: 1e-4)
            depth        : number or layers (default: 40)

        Returns:
            Model        : A Keras model instance
        """

        if nb_classes == None:
            raise Exception(
                'Please define number of classes (e.g. num_classes=10). This is required for final softmax.'
            )

        if compression <= 0.0 or compression > 1.0:
            raise Exception(
                'Compression have to be a value between 0.0 and 1.0.')

        if type(dense_layers) is list:
            if len(dense_layers) != dense_blocks:
                raise AssertionError(
                    'Number of dense blocks have to be same length to specified layers'
                )
        elif dense_layers == -1:
            dense_layers = int((depth - 4) / 3)
            if bottleneck:
                dense_layers = int(dense_layers / 2)
            dense_layers = [dense_layers for _ in range(dense_blocks)]
        else:
            dense_layers = [dense_layers for _ in range(dense_blocks)]

        #img_input = Input(shape=input_shape)
        nb_channels = growth_rate

        #print('Creating DenseNet %s' % __version__)
        #print('#############################################')
        #print('Dense blocks: %s' % dense_blocks)
        #print('Layers per dense block: %s' % dense_layers)
        #print('#############################################')

        # Initial convolution layer
        #x = Convolution2D(filters=2 * growth_rate, kernel_size=(3,3), padding='same',strides=(1,1),
        #                  use_bias=False, kernel_regularizer=l2(weight_decay))(img_input)

        x = Convolution1D(filters=growth_rate,
                          kernel_size=32,
                          padding='same',
                          strides=1,
                          use_bias=False,
                          kernel_regularizer=l2(weight_decay))(
                              inp)  #(img_input)

        # Building dense blocks
        for block in range(dense_blocks - 1):
            #print("block:::",block)
            # Add dense block
            x, nb_channels = Net.__dense_block(x, dense_layers[block],
                                               nb_channels, growth_rate,
                                               dropout_rate, bottleneck,
                                               weight_decay)

            # Add transition_block
            x = Net.__transition_layer(x, nb_channels, dropout_rate,
                                       compression, weight_decay)
            nb_channels = int(nb_channels * compression)

        # Add last dense block without transition but for that with global average pooling

        x, nb_channels = Net.__dense_block(x, dense_layers[-1], nb_channels,
                                           growth_rate, dropout_rate,
                                           weight_decay)

        x = BatchNormalization()(x)
        x = Activation('relu')(
            x)  #LeakyReLU(alpha=0.2)(x)#Activation('relu')(x)
        #x = GlobalAveragePooling2D()(x)
        #x = GlobalAveragePooling1D()(x)
        x = AveragePooling1D(int(x.shape[1]), int(x.shape[1]))(x)
        #x = Dense(nb_classes, activation='softmax')(x)

        return x  #Model(img_input, x, name='densenet')
model2.add(TimeDistributed(Dense(dim, activation='relu')))
model2.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(dim, )))
#===============

model3 = Sequential()
model3.add(
    Embedding(len(word_index) + 1,
              dim,
              weights=[embedding_matrix],
              input_length=max_len,
              trainable=False))
model3.add(
    Convolution1D(filters=nb_filter,
                  kernel_size=filter_length,
                  padding='valid',
                  activation='relu',
                  strides=1))
model3.add(Dropout(0.2))

model3.add(
    Convolution1D(filters=nb_filter,
                  kernel_size=filter_length,
                  padding='valid',
                  activation='relu',
                  strides=1))

model3.add(GlobalMaxPooling1D())
model3.add(Dropout(0.2))

model3.add(Dense(dim))
Пример #3
0
def trian_cnn():
    # set parameters:
    max_features = 20000
    maxlen = 8
    batch_size = 32
    embedding_dims = 50
    nb_filter = 250
    filter_length = 3
    hidden_dims = 250
    nb_epoch = 2
    print('Loading data...')
    X_train, y_train, X_test, y_test = load_data()
    print(len(X_train), 'train sequences')
    print(len(X_test), 'test sequences')

    print('Pad sequences (samples x time)')
    X_train = sequence.pad_sequences(X_train, maxlen=maxlen, value=3)
    X_test = sequence.pad_sequences(X_test, maxlen=maxlen, value=3)
    print('X_train shape:', X_train.shape)
    print('X_test shape:', X_test.shape)

    y_train = np.array(y_train)
    print('y_train shape', y_train.shape)
    y_test = np.array(y_test)
    print('y_train shape', y_test.shape)

    print('Build model...')
    model = Sequential()

    model.add(
        Embedding(max_features,
                  embedding_dims,
                  input_length=maxlen,
                  dropout=0.2))

    # we add a Convolution1D, which will learn nb_filter
    # word group filters of size filter_length:
    model.add(
        Convolution1D(nb_filter=nb_filter,
                      filter_length=filter_length,
                      border_mode='valid',
                      activation='relu',
                      subsample_length=1))
    # we use max pooling:
    model.add(GlobalMaxPooling1D())

    # We add a vanilla hidden layer:
    model.add(Dense(hidden_dims))
    model.add(Dropout(0.2))
    model.add(Activation('relu'))

    # We project onto a single unit output layer, and squash it with a sigmoid:
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    model.fit(X_train,
              y_train,
              batch_size=batch_size,
              nb_epoch=nb_epoch,
              validation_data=(X_test, y_test))

    score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)

    #save the model &serialize model to JSON
    model_json = model.to_json()
    with open("./dict/model_cnn.json", "w") as json_file:
        json_file.write(model_json)
    # serialize weights to HDF5
    model.save_weights("./dict/model_cnn_weights.h5")
    print("Saved model to disk")

    del model

    print('Test score:', score)
    print('Test accuracy:', acc)
Пример #4
0
def default_model(input_dim, useBatchNorm):

    reg = l2(0.0001)

    model = Sequential()
    model.add(
        InputLayer(input_shape=(input_dim[0], input_dim[1], input_dim[2])))
    model.add(Reshape((input_dim[0], input_dim[2])))

    model.add(
        Convolution1D(filters=8,
                      kernel_size=7,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=16,
                      kernel_size=5,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=32,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=32,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=64,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=64,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=128,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=256,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(Flatten())
    #

    #
    model.add(Dense(units=512, kernel_regularizer=reg, bias_regularizer=reg))
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(Dense(units=128, kernel_regularizer=reg, bias_regularizer=reg))
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    return model
Пример #5
0
scaler = Normalizer().fit(T)
testT = scaler.transform(T)

y_train = np.array(Y)
y_test = np.array(C)

X_train = np.reshape(trainX, (trainX.shape[0], trainX.shape[1], 1))
X_test = np.reshape(testT, (testT.shape[0], testT.shape[1], 1))

lstm_output_size = 128

cnn = Sequential()
cnn.add(
    Convolution1D(64,
                  3,
                  border_mode="same",
                  activation="relu",
                  input_shape=(43, 1)))
cnn.add(Convolution1D(64, 3, border_mode="same", activation="relu"))
cnn.add(MaxPooling1D(pool_length=(2)))
cnn.add(Convolution1D(128, 3, border_mode="same", activation="relu"))
cnn.add(Convolution1D(128, 3, border_mode="same", activation="relu"))
cnn.add(MaxPooling1D(pool_length=(2)))
cnn.add(Flatten())
cnn.add(Dense(128, activation="relu"))
cnn.add(Dropout(0.5))
cnn.add(Dense(1, activation="sigmoid"))

# define optimizer and objective, compile cnn

cnn.compile(loss="binary_crossentropy", optimizer="adam", metrics=['accuracy'])
Пример #6
0
    def build(self):
        assert self.config['question_len'] == self.config['answer_len']

        question = self.question
        answer = self.get_answer()

        # add embedding layers
        embedding = Embedding(self.config['n_words'],
                              self.model_params.get('n_embed_dims', 100))
        question_embedding = embedding(question)
        answer_embedding = embedding(answer)

        # turn off layer updating
        embedding.params = []
        embedding.updates = []

        # dropout
        dropout = Dropout(0.25)
        question_dropout = dropout(question_embedding)
        answer_dropout = dropout(answer_embedding)

        # dense
        dense = TimeDistributed(
            Dense(self.model_params.get('n_hidden', 200), activation='tanh'))
        question_dense = dense(question_dropout)
        answer_dense = dense(answer_dropout)

        # regularization
        question_dense = ActivityRegularization(l2=0.0001)(question_dense)
        answer_dense = ActivityRegularization(l2=0.0001)(answer_dense)

        # dropout
        question_dropout = dropout(question_dense)
        answer_dropout = dropout(answer_dense)

        # cnn
        cnns = [
            Convolution1D(filter_length=filter_length,
                          nb_filter=self.model_params.get('nb_filters', 1000),
                          activation=self.model_params.get(
                              'conv_activation', 'relu'),
                          border_mode='same')
            for filter_length in [2, 3, 5, 7]
        ]
        question_cnn = merge([cnn(question_dropout) for cnn in cnns],
                             mode='concat')
        answer_cnn = merge([cnn(answer_dropout) for cnn in cnns],
                           mode='concat')

        # regularization
        question_cnn = ActivityRegularization(l2=0.0001)(question_cnn)
        answer_cnn = ActivityRegularization(l2=0.0001)(answer_cnn)

        # dropout
        question_dropout = dropout(question_cnn)
        answer_dropout = dropout(answer_cnn)

        # maxpooling
        maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False),
                         output_shape=lambda x: (x[0], x[2]))
        question_pool = maxpool(question_dropout)
        answer_pool = maxpool(answer_dropout)

        # activation
        activation = Activation('tanh')
        question_output = activation(question_pool)
        answer_output = activation(answer_pool)

        return question_output, answer_output
Пример #7
0
def seq_attentionmodel(enhancer_length=600,
                       promoter_length=400,
                       n_kernels=256,
                       filter_length=8,
                       dense_layer_size=800):
    opt = Adam(lr=1e-5)
    input_enh_sequence = Input(shape=(enhancer_length, 4))
    enh_seq = Convolution1D(input_dim=4,
                            input_length=enhancer_length,
                            nb_filter=n_kernels,
                            filter_length=filter_length,
                            border_mode="valid",
                            subsample_length=1,
                            kernel_regularizer=l2(2e-5))(input_enh_sequence)
    enh_seq = relu(enh_seq)
    enh_seq = MaxPooling1D(pool_length=int(filter_length / 2),
                           stride=int(filter_length / 2))(enh_seq)
    input_prom_sequence = Input(shape=(promoter_length, 4))
    prom_seq = Convolution1D(input_dim=4,
                             input_length=promoter_length,
                             nb_filter=n_kernels,
                             filter_length=filter_length,
                             border_mode="valid",
                             subsample_length=1,
                             kernel_regularizer=l2(2e-5))(input_prom_sequence)
    #prom_seq = BatchNormalization(momentum=0.997)(prom_seq)
    prom_seq = relu(prom_seq)
    prom_seq = MaxPooling1D(pool_length=int(filter_length / 2),
                            stride=int(filter_length / 2))(prom_seq)
    seq_mixed = Concatenate(axis=1)([enh_seq, prom_seq])
    seq_mixed = BatchNormalization(momentum=0.997)(seq_mixed)
    seq_mixed = Dropout(0.2)(seq_mixed)
    seq_mixed = Flatten()(seq_mixed)
    seq_mixed = Dense(output_dim=dense_layer_size,
                      init="glorot_uniform",
                      activity_regularizer=l2(1e-6))(seq_mixed)
    seq_mixed = BatchNormalization(momentum=0.997)(seq_mixed)
    seq_mixed = relu(seq_mixed)
    a_prob = Dense(dense_layer_size,
                   activation='softmax',
                   name='attention_vec',
                   kernel_regularizer=l2(weightDecay))(seq_mixed)
    attention_mul = multiply([seq_mixed, a_prob], name='attention_mul')
    seq_mixed = Dense(256, kernel_regularizer=l2(weightDecay))(attention_mul)
    a_prob = Dense(256,
                   activation='softmax',
                   name='attention_vec1',
                   kernel_regularizer=l2(weightDecay / 10))(seq_mixed)
    attention_mul = multiply([seq_mixed, a_prob], name='attention_mul1')
    attention_mul = Dense(128, kernel_regularizer=l2(weightDecay /
                                                     10))(attention_mul)
    attention_mul = BatchNormalization(momentum=0.997)(attention_mul)
    attention_mul = relu(attention_mul)
    #seq_mixed = Dropout(0.5)(seq_mixed)
    seq_mixed = Dense(1)(attention_mul)
    seq_mixed = BatchNormalization(momentum=0.997, scale=False)(seq_mixed)
    seq_mixed = Activation('sigmoid')(seq_mixed)
    model = Model([input_enh_sequence, input_prom_sequence], seq_mixed)
    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[f1])
    model.summary()
    return model
Пример #8
0
        return (input_shape[0], (input_shape[2] * self.k))

    def call(self, inputs):
        # swap last two dimensions since top_k will be applied along the last dimension
        shifted_input = tf.transpose(inputs, [0, 2, 1])

        # extract top_k, returns two tensors [values, indices]
        top_k = tf.nn.top_k(shifted_input, k=self.k, sorted=True, name=None)[0]

        # return flattened output
        return Flatten()(top_k)


main_input = Input(shape=(MAX_LENGTH, ), dtype='float64')
embed = Embedding(353717 + 1, 378, input_length=MAX_LENGTH)(main_input)
cnn = Convolution1D(256, 3, padding='same', strides=1,
                    activation='relu')(embed)
cnn = MaxPool1D(pool_size=4)(cnn)
cnn = Flatten()(cnn)
cnn = Dense(256)(cnn)
rnn = Bidirectional(GRU(256, dropout=0.2, recurrent_dropout=0.1))(embed)
#rnn = Bidirectional(keras.layers.CuDNNGRU(256))(embed)
rnn = Dense(256)(rnn)
con = concatenate([cnn, rnn], axis=-1)
main_output = Dense(125, activation='softmax')(con)
model = Model(inputs=main_input, outputs=main_output)

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[f1])

history = model.fit(x=x_train,
                    y=y_train,
                    batch_size=32,
Пример #9
0
def CLSTMWordEmbed(nb_labels,
                   nb_filters=1200,
                   n_gram=2,
                   maxlen=15,
                   vecsize=300,
                   cnn_dropout=0.0,
                   nb_rnnoutdim=1200,
                   rnn_dropout=0.2,
                   final_activation='softmax',
                   dense_wl2reg=0.0,
                   optimizer='adam'):
    """ Returns the C-LSTM neural networks for word-embedded vectors.

    Reference: Chunting Zhou, Chonglin Sun, Zhiyuan Liu, Francis Lau,
    "A C-LSTM Neural Network for Text Classification,"
    (arXiv:1511.08630). [`arXiv
    <https://arxiv.org/abs/1511.08630>`_]

    :param nb_labels: number of class labels
    :param nb_filters: number of filters (Default: 1200)
    :param n_gram: n-gram, or window size of CNN/ConvNet (Default: 2)
    :param maxlen: maximum number of words in a sentence (Default: 15)
    :param vecsize: length of the embedded vectors in the model (Default: 300)
    :param cnn_dropout: dropout rate for CNN/ConvNet (Default: 0.0)
    :param nb_rnnoutdim: output dimension for the LSTM networks (Default: 1200)
    :param rnn_dropout: dropout rate for LSTM (Default: 0.2)
    :param final_activation: activation function. Options: softplus, softsign, relu, tanh, sigmoid, hard_sigmoid, linear. (Default: 'softmax')
    :param dense_wl2reg: L2 regularization coefficient (Default: 0.0)
    :param optimizer: optimizer for gradient descent. Options: sgd, rmsprop, adagrad, adadelta, adam, adamax, nadam. (Default: adam)
    :return: keras sequantial model for CNN/ConvNet for Word-Embeddings
    :type nb_labels: int
    :type nb_filters: int
    :type n_gram: int
    :type maxlen: int
    :type vecsize: int
    :type cnn_dropout: float
    :type nb_rnnoutdim: int
    :type rnn_dropout: float
    :type final_activation: str
    :type dense_wl2reg: float
    :type optimizer: str
    :rtype: keras.model.Sequential
    """
    model = Sequential()
    model.add(Convolution1D(nb_filter=nb_filters,
                            filter_length=n_gram,
                            border_mode='valid',
                            activation='relu',
                            input_shape=(maxlen, vecsize)))
    if cnn_dropout > 0.0:
        model.add(Dropout(cnn_dropout))
    model.add(MaxPooling1D(pool_length=maxlen - n_gram + 1))
    model.add(LSTM(nb_rnnoutdim))
    if rnn_dropout > 0.0:
        model.add(Dropout(rnn_dropout))
    model.add(Dense(nb_labels,
                    activation=final_activation,
                    W_regularizer=l2(dense_wl2reg)
                    )
              )
    model.compile(loss='categorical_crossentropy', optimizer=optimizer)

    return model
Пример #10
0
    def _model_constructor(self):
        ########################################
        ## sample train/validation data
        ########################################
        data_1, data_2, labels, _, _, _ = self._preprocess_data()

        perm = np.random.permutation(len(data_1))

        idx_train = perm[:int(len(data_1) * (1 - self.VALIDATION_SPLIT))]
        idx_val = perm[int(len(data_1) * (1 - self.VALIDATION_SPLIT)):]

        # data_1_train = np.vstack((data_1[idx_train], data_2[idx_train]))
        # data_2_train = np.vstack((data_2[idx_train], data_1[idx_train]))
        data_1_train = np.vstack((data_1[idx_train], data_2[idx_train]))
        data_2_train = np.vstack((data_2[idx_train], data_1[idx_train]))

        self.train_custom_features = self.train_custom_features.as_matrix()

        features_train = np.vstack((self.train_custom_features[idx_train],
                                    self.train_custom_features[idx_train]))

        labels_train = np.concatenate((labels[idx_train], labels[idx_train]))

        data_1_val = np.vstack((data_1[idx_val], data_2[idx_val]))
        data_2_val = np.vstack((data_2[idx_val], data_1[idx_val]))

        features_val = np.vstack((self.train_custom_features[idx_val],
                                  self.train_custom_features[idx_val]))

        labels_val = np.concatenate((labels[idx_val], labels[idx_val]))

        weight_val = np.ones(len(labels_val))
        if self.REWEIGH:
            weight_val *= 0.472001959
            weight_val[labels_val == 0] = 1.309028344

        ########################################
        ## define the model structure
        ########################################

        embedding_layer = self._create_embedding_layer()
        # lstm_layer = LSTM(self.NUM_LSTM,
        #                   dropout=self.RATE_DROP_LSTM,
        #                   recurrent_dropout=self.RATE_DROP_LSTM)
        convolution_layer = Convolution1D(nb_filter=NB_FILTER,
                                          filter_length=FILTER_LENGTH,
                                          border_mode='valid',
                                          subsample_length=1)

        custom_dim = int(self.train_custom_features.shape[1])
        custom_input = Input(shape=(custom_dim, ), dtype='float32')
        #custom_features = Dropout(self.RATE_DROP_DENSE)(custom_input)
        custom_features = Dense(self.NUM_DENSE,
                                kernel_initializer='normal')(custom_input)
        custom_features = PReLU()(custom_features)
        custom_features = Dropout(self.RATE_DROP_DENSE)(custom_features)
        custom_features = BatchNormalization()(custom_features)

        sequence_1_input = Input(shape=(self.MAX_SEQUENCE_LENGTH, ),
                                 dtype='int32')
        embedded_sequences_1 = embedding_layer(sequence_1_input)
        # Q1 = lstm_layer(embedded_sequences_1)
        Q1 = convolution_layer(embedded_sequences_1)
        Q1 = PReLU()(Q1)
        Q1 = GlobalMaxPooling1D()(Q1)
        Q1 = Dropout(self.RATE_DROP_DENSE)(Q1)

        sequence_2_input = Input(shape=(self.MAX_SEQUENCE_LENGTH, ),
                                 dtype='int32')
        embedded_sequences_2 = embedding_layer(sequence_2_input)
        #Q2 = lstm_layer(embedded_sequences_2)
        Q2 = convolution_layer(embedded_sequences_2)
        Q2 = PReLU()(Q2)
        Q2 = GlobalMaxPooling1D()(Q2)
        Q2 = Dropout(self.RATE_DROP_DENSE)(Q2)

        # Q2 = convolution_layer(embedded_sequences_2)
        # # Q2 = Dropout(self.RATE_DROP_DENSE)(Q2)
        # # Q2 = convolution_layer(Q2)
        # Q2 = GlobalMaxPooling1D()(Q2)
        # Q2 = Dropout(self.RATE_DROP_DENSE)(Q2)

        merged = concatenate([Q1, Q2, custom_features])
        merged = BatchNormalization()(merged)

        merged = Dense(self.NUM_DENSE, kernel_initializer='normal')(merged)
        merged = PReLU()(merged)
        merged = Dropout(self.RATE_DROP_DENSE)(merged)
        merged = BatchNormalization()(merged)

        merged = Dense(round(0.9 * self.NUM_DENSE),
                       kernel_initializer='normal')(merged)
        merged = PReLU()(merged)
        merged = Dropout(self.RATE_DROP_DENSE)(merged)
        merged = BatchNormalization()(merged)

        merged = Dense(round(0.7 * self.NUM_DENSE),
                       kernel_initializer='normal')(merged)
        merged = PReLU()(merged)
        merged = Dropout(self.RATE_DROP_DENSE)(merged)
        merged = BatchNormalization()(merged)

        merged = Dense(round(0.5 * self.NUM_DENSE),
                       kernel_initializer='normal')(merged)
        merged = PReLU()(merged)
        merged = Dropout(self.RATE_DROP_DENSE)(merged)
        merged = BatchNormalization()(merged)

        preds = Dense(1, activation='sigmoid')(merged)

        ########################################
        ## construct the model
        ########################################
        model = Model(inputs=[sequence_1_input,
                              sequence_2_input,
                              custom_input], \
                      outputs=preds)

        adam = optimizers.Adam(clipnorm=1.)

        model.compile(loss='binary_crossentropy',
                      optimizer=adam,
                      metrics=['acc'])
        #model.summary()
        print("The model {} is built.".format(self.STAMP))

        ########################################
        ## add class weight
        ########################################
        if self.REWEIGH:
            class_weight = {0: 1.309028344, 1: 0.472001959}
        else:
            class_weight = None

        early_stopping = EarlyStopping(monitor='val_loss', patience=3)
        bst_model_path = self.STAMP + '.h5'
        model_checkpoint = ModelCheckpoint(bst_model_path,
                                           save_best_only=True,
                                           save_weights_only=True)

        hist = model.fit(
            [data_1_train, data_2_train, features_train],
            labels_train,
            validation_data=([data_1_val, data_2_val,
                              features_val], labels_val, weight_val),
            epochs=10,
            batch_size=512,
            shuffle=True,
            class_weight=class_weight,
            callbacks=[early_stopping, model_checkpoint])

        model.load_weights(bst_model_path)
        bst_val_score = min(hist.history['val_loss'])
        return (model, hist, bst_val_score)
model = Sequential()

# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
# 先从一个高效的嵌入层开始,它将词汇的索引值映射为 embedding_dims 维度的词向量
model.add(Embedding(len(abc), embedding_dims, input_length=maxlen,
                    dropout=0.5))

# we add a Convolution1D, which will learn nb_filter
# word group filters of size filter_length:
# 添加一个 1D 卷积层,它将学习 nb_filter 个 filter_length 大小的词组卷积核
model.add(
    Convolution1D(nb_filter=nb_filter,
                  filter_length=filter_length,
                  border_mode='valid',
                  activation='relu',
                  subsample_length=1))
# we use max pooling:
# 使用最大池化
model.add(GlobalMaxPooling1D())

# We add a vanilla hidden layer:
# 添加一个原始隐藏层
model.add(Dense(hidden_dims))
model.add(Activation('relu'))
model.add(Dropout(0.5))

# We project onto a single unit output layer, and squash it with a sigmoid:
# 投影到一个单神经元的输出层,并且使用 sigmoid 压缩它
model.add(Dense(1))
                              factor=0.2,
                              verbose=1,
                              patience=5,
                              min_lr=0.001)
checkpoint = ModelCheckpoint(model_weight_file,
                             monitor='val_loss',
                             verbose=1,
                             save_best_only=True,
                             save_weights_only=True,
                             mode='min',
                             period=1)
data_vis = DataVisualized()
model = Sequential([
    Convolution1D(88,
                  4,
                  border_mode='same',
                  input_shape=(timesteps, len(features)),
                  name="conv_1"),
    BatchNormalization(),
    Activation('tanh'),
    Dropout(0.3),
    Convolution1D(60, 4, border_mode='same', name="conv_2"),
    BatchNormalization(),
    Activation('tanh'),
    Dropout(0.3),
    Convolution1D(20, 4, border_mode='same', name="conv_4"),
    BatchNormalization(),
    Activation('tanh'),
    Dropout(0.3),
    Convolution1D(8, 4, border_mode='same', name="conv_5"),
    BatchNormalization(),
Пример #13
0
    def lstm(self):

        for seq_len in [30]:

            print(self.seq_length)
            keyer = self.solve_data()

            toperr = 0.0
            topfar = 0.0
            topfrr = 0.0

            for nflod in range(1):
                train_list = []
                test_list = []
                for j in range(self.usernum):
                    one = keyer[j]
                    random.shuffle(one)  # 随机打乱list
                    train, test = self.split_list(nflod, one)
                    train_list.append(train)
                    test_list.append(test)
                totalerr = 0.0
                totalfar = 0.0
                totalfrr = 0.0
                for userid in range(self.usernum - 1, self.usernum):
                    print("第%d个用户" % (userid), "固定长度:", self.seq_length,
                          "开始训练。。。")
                    x_train_1 = train_list[userid]
                    nb_word = 0

                    y_train_1 = [1] * len(x_train_1)
                    x_test_1 = test_list[userid]
                    y_test_1 = [1] * len(x_test_1)

                    leave_train = []

                    samplenum = len(y_train_1) * 2

                    print(samplenum)
                    if samplenum % 10 >= 5:
                        samplenum = (int(samplenum / 10) + 1) * 10
                    else:
                        samplenum = int(samplenum / 10) * 10
                    len1 = samplenum - len(y_train_1)

                    for k in range(self.usernum):
                        if k != userid:
                            leave_train = leave_train + train_list[k]
                    print(len1)
                    x_train_1 = x_train_1 + random.sample(leave_train, len1)
                    y_train_1 = y_train_1 + [0] * len1
                    leave_test = []

                    realnum = len(y_test_1)
                    testnum = len(y_test_1) * 2
                    print(realnum)

                    if testnum % 10 >= 5:
                        testnum = (int(testnum / 10) + 1) * 10
                    else:
                        testnum = int(testnum / 10) * 10

                    len2 = testnum - len(y_test_1)

                    for k in range(self.usernum):
                        if k != userid:
                            leave_test = leave_test + test_list[k]
                    print(len2)

                    x_test_1 = x_test_1 + random.sample(leave_test, len2)
                    y_test_1 = y_test_1 + [0] * len2
                    y_test_1_tmp = y_test_1[:]

                    x_train_1 = numpy.array(x_train_1)
                    x_test_1 = numpy.array(x_test_1)
                    y_train_1 = numpy.array(y_train_1)
                    # print(y_train_1)
                    model = Sequential()
                    # model.add(Embedding(nb_word + 1,128,dropout=0.2))

                    model.add(
                        Convolution1D(batch_input_shape=(self.batch_size,
                                                         x_train_1.shape[1],
                                                         x_train_1.shape[2]),
                                      filters=self.nb_filter,
                                      kernel_size=self.filter_length,
                                      padding='valid',
                                      activation='relu'))
                    model.add(Dropout(0.5))
                    model.add(MaxPooling1D(pool_size=self.pool_length))

                    model.add(GRU(32, return_sequences=True, stateful=True))
                    model.add(Dropout(0.5))

                    model.add(GRU(32, return_sequences=False))

                    # model.add(Bidirectional(LSTM(32,return_sequences=True,batch_input_shape=(batch_size, x_train_1.shape[1], x_train_1.shape[2]))))
                    # model.add(Dropout(0.5))

                    model.add(Dense(256, activation='relu'))
                    model.add(Dropout(0.5))
                    # model.add(Flatten())
                    model.add(Dense(1, activation="sigmoid"))
                    model.compile(loss="binary_crossentropy",
                                  optimizer='adam',
                                  metrics=['accuracy'])
                    # model.fit(x_train_1, y_train_1,epochs=epochs, batch_size=batch_size,verbose=2,shuffle=False,validation_data=(x_test_1, y_test_1))
                    minerr = 1.0

                    # class_weight={0:0.33,1:0.67}
                    # model.fit(x_train_1, y_train_1, epochs=1, batch_size=batch_size, verbose=2, shuffle=False)
                    '''
                    for nums in range(200):
                        model.fit(x_train_1, y_train_1, epochs=1, batch_size=batch_size, verbose=2, shuffle=False)
                        model.reset_states()

                    setThr(model, x_test_1, realnum)
                    #model.save('model_2.h5')
                   '''
                    model.fit(x_train_1,
                              y_train_1,
                              batch_size=10,
                              epochs=200,
                              validation_data=(x_test_1, y_test_1),
                              verbose=1,
                              shuffle=True)
                    model.fit(x_test_1,
                              y_test_1,
                              batch_size=10,
                              epochs=200,
                              verbose=1,
                              shuffle=True)
                    m = 'model_' + str(self.usernum) + '.h5'
                    model.save(m)


#k=keystroke_lstm(78)
#k.lstm()
    def build(self):
        # Build model
        number_of_class = 4

        if self.model_type == "CNN-non-static":
            vocabulary_size = len(self.embed_matrix)  # include 0 empty
            input_shape = (self.sequence_length, )
            model_input = Input(shape=input_shape)
            z = Embedding(vocabulary_size,
                          self.embedding_dim,
                          input_length=self.sequence_length,
                          weights=[self.embed_matrix])(model_input)
        elif self.model_type == "CNN-static":
            input_shape = (self.sequence_length, self.embedding_dim)
            model_input = Input(shape=input_shape)
            z = model_input

        # MaxPooling1D


#
#         z = Dropout(self.dropout_prob)(z)
#         # Convolutional block
#         conv_blocks = []
#         for sz in self.filter_sizes:
#             conv = Convolution1D(filters=self.num_filters,
#                                  kernel_size=sz,
#                                  padding="valid",
#                                  activation="relu",
#                                  strides=1)(z)
#             conv = MaxPooling1D(pool_size=2)(conv)
#             conv = Flatten()(conv)
#             conv_blocks.append(conv)
#         z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
#
#         z = Dropout(self.dropout_prob)(z)
#         z = Dense(self.hidden_dims, activation="relu")(z)
#         model_output = Dense(number_of_class, activation="sigmoid")(z)
#
#         model = Model(model_input, model_output)
#         model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
#
# GlobalMaxPooling1D

        convolution_output = []
        for sz in self.filter_sizes:
            conv = Convolution1D(filters=self.num_filters,
                                 kernel_size=sz,
                                 activation='tanh',
                                 name='Conv1D_{}_{}'.format(
                                     self.num_filters, sz))(z)
            pool = GlobalMaxPooling1D(name='MaxPoolingOverTime_{}_{}'.format(
                self.num_filters, sz))(conv)
            convolution_output.append(pool)
        x = Concatenate()(convolution_output)
        # Fully connected layers
        for fl in [self.hidden_dims]:
            x = Dense(fl, activation='selu',
                      kernel_initializer='lecun_normal')(x)
            x = AlphaDropout(self.dropout_prob)(x)
        # Output layer
        predictions = Dense(number_of_class, activation='softmax')(x)
        # Build and compile model
        model = Model(inputs=model_input, outputs=predictions)
        model.compile(loss="categorical_crossentropy",
                      optimizer="adam",
                      metrics=["accuracy"])

        model.summary()
        return model
    text_seq_input = Input(shape=(max_seq_len, ))

    embeds = Embedding(embedding_matrix.shape[0],
                       embedding_matrix.shape[1],
                       weights=[embedding_matrix],
                       trainable=False)(text_seq_input)

    xx = SpatialDropout1D(0.08500492158254568)(embeds)

    xx = Bidirectional(CuDNNLSTM(64, return_sequences=True))(xx)
    xx = SpatialDropout1D(0.1733455182727272)(xx)

    cnn1 = Convolution1D(256,
                         5,
                         padding='valid',
                         strides=3,
                         activation='elu',
                         kernel_initializer="glorot_uniform")(xx)
    cnn1 = SpatialDropout1D(0.1231232556)(cnn1)
    cnn1_max = GlobalMaxPooling1D()(cnn1)
    cnn1_ave = GlobalAveragePooling1D()(cnn1)

    cnn2 = Convolution1D(256,
                         5,
                         padding='valid',
                         strides=3,
                         activation='elu',
                         kernel_initializer="glorot_uniform")(xx)
    cnn2 = SpatialDropout1D(0.10348347384)(cnn2)
    cnn2_max = GlobalMaxPooling1D()(cnn2)
    cnn2_ave = GlobalAveragePooling1D()(cnn2)
Пример #16
0
def DoubleCNNWordEmbed(nb_labels,
                       nb_filters_1=1200,
                       nb_filters_2=600,
                       n_gram=2,
                       filter_length_2=10,
                       maxlen=15,
                       vecsize=300,
                       cnn_dropout_1=0.0,
                       cnn_dropout_2=0.0,
                       final_activation='softmax',
                       dense_wl2reg=0.0,
                       optimizer='adam'):
    """ Returns the double-layered convolutional neural network (CNN/ConvNet) for word-embedded vectors.

    :param nb_labels: number of class labels
    :param nb_filters_1: number of filters for the first CNN/ConvNet layer (Default: 1200)
    :param nb_filters_2: number of filters for the second CNN/ConvNet layer (Default: 600)
    :param n_gram: n-gram, or window size of first CNN/ConvNet (Default: 2)
    :param filter_length_2: window size for second CNN/ConvNet layer (Default: 10)
    :param maxlen: maximum number of words in a sentence (Default: 15)
    :param vecsize: length of the embedded vectors in the model (Default: 300)
    :param cnn_dropout_1: dropout rate for the first CNN/ConvNet layer (Default: 0.0)
    :param cnn_dropout_2: dropout rate for the second CNN/ConvNet layer (Default: 0.0)
    :param final_activation: activation function. Options: softplus, softsign, relu, tanh, sigmoid, hard_sigmoid, linear. (Default: 'softmax')
    :param dense_wl2reg: L2 regularization coefficient (Default: 0.0)
    :param optimizer: optimizer for gradient descent. Options: sgd, rmsprop, adagrad, adadelta, adam, adamax, nadam. (Default: adam)
    :return: keras sequantial model for CNN/ConvNet for Word-Embeddings
    :type nb_labels: int
    :type nb_filters_1: int
    :type nb_filters_2: int
    :type n_gram: int
    :type filter_length_2: int
    :type maxlen: int
    :type vecsize: int
    :type cnn_dropout_1: float
    :type cnn_dropout_2: float
    :type final_activation: str
    :type dense_wl2reg: float
    :type optimizer: str
    :rtype: keras.model.Sequential
    """
    model = Sequential()
    model.add(Convolution1D(nb_filter=nb_filters_1,
                            filter_length=n_gram,
                            border_mode='valid',
                            activation='relu',
                            input_shape=(maxlen, vecsize)))
    if cnn_dropout_1 > 0.0:
        model.add(Dropout(cnn_dropout_1))
    model.add(Convolution1D(nb_filter=nb_filters_2,
                            filter_length=filter_length_2,
                            border_mode='valid',
                            activation='relu'))
    if cnn_dropout_2 > 0.0:
        model.add(Dropout(cnn_dropout_2))
    model.add(MaxPooling1D(pool_length=maxlen - n_gram -filter_length_2 + 1))
    model.add(Flatten())
    model.add(Dense(nb_labels,
                    activation=final_activation,
                    W_regularizer=l2(dense_wl2reg)))

    model.compile(loss='categorical_crossentropy', optimizer=optimizer)

    return model
Пример #17
0
    def create_deepnet(self):
        input_query_1 = Input(shape=(None,), name='input_query_1', dtype='int32')
        input_query_2 = Input(shape=(None,), name='input_query_2', dtype='int32')

        nb_words = len(self.embedding_matrix)
        shared_embedding = Embedding(nb_words, EMBEDDING_DIM, weights=[self.embedding_matrix],
                                     input_length=MAX_LEN, trainable=False)

        q1_embedding = shared_embedding(input_query_1)
        q1_dense = (Dense(EMBEDDING_DIM, activation='relu'))(q1_embedding)
        q1_lambda = Lambda(lambda x: k.sum(x, axis=1), output_shape=(EMBEDDING_DIM,))(q1_dense)

        q2_embedding = shared_embedding(input_query_2)
        q2_dense = (Dense(EMBEDDING_DIM, activation='relu'))(q2_embedding)
        q2_lambda = Lambda(lambda x: k.sum(x, axis=1), output_shape=(EMBEDDING_DIM,))(q2_dense)

        q3_embedding = shared_embedding(input_query_1)
        q3_conv_1d_1 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
                                     activation='relu', subsample_length=1)(q3_embedding)
        q3_dropout_1 = Dropout(0.2)(q3_conv_1d_1)
        q3_conv_1d_2 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
                                     activation='relu', subsample_length=1)(q3_dropout_1)
        q3_global_pooling_1d_1 = GlobalMaxPooling1D()(q3_conv_1d_2)
        q3_dropout_2 = Dropout(0.2)(q3_global_pooling_1d_1)
        q3_dense = Dense(EMBEDDING_DIM)(q3_dropout_2)
        q3_dropout_3 = Dropout(0.2)(q3_dense)
        q3_bn = BatchNormalization()(q3_dropout_3)

        q4_embedding = shared_embedding(input_query_2)
        q4_conv_1d_1 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
                                     activation='relu', subsample_length=1)(q4_embedding)
        q4_dropout_1 = Dropout(0.2)(q4_conv_1d_1)
        q4_conv_1d_2 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
                                     activation='relu', subsample_length=1)(q4_dropout_1)
        q4_global_pooling_1d_1 = GlobalMaxPooling1D()(q4_conv_1d_2)
        q4_dropout_2 = Dropout(0.2)(q4_global_pooling_1d_1)
        q4_dense = Dense(EMBEDDING_DIM)(q4_dropout_2)
        q4_dropout_3 = Dropout(0.2)(q4_dense)
        q4_bn = BatchNormalization()(q4_dropout_3)

        q5_embedding = shared_embedding(input_query_1)
        q5_lstm = LSTM(EMBEDDING_DIM, dropout_W=0.2, dropout_U=0.2)(q5_embedding)

        q6_embedding = shared_embedding(input_query_2)
        q6_lstm = LSTM(EMBEDDING_DIM, dropout_W=0.2, dropout_U=0.2)(q6_embedding)

        merge_layer = Merge(mode='concat')([q1_lambda, q2_lambda, q3_bn, q4_bn, q5_lstm, q6_lstm])
        bn_layer_1 = BatchNormalization()(merge_layer)
        dense_layer_1 = Dense(EMBEDDING_DIM)(bn_layer_1)
        prelu_layer_1 = PReLU()(dense_layer_1)
        dropout_layer_1 = Dropout(0.2)(prelu_layer_1)

        bn_layer_2 = BatchNormalization()(dropout_layer_1)
        dense_layer_2 = Dense(EMBEDDING_DIM)(bn_layer_2)
        prelu_layer_2 = PReLU()(dense_layer_2)
        dropout_layer_2 = Dropout(0.2)(prelu_layer_2)

        bn_layer_3 = BatchNormalization()(dropout_layer_2)
        dense_layer_3 = Dense(EMBEDDING_DIM)(bn_layer_3)
        prelu_layer_3 = PReLU()(dense_layer_3)
        dropout_layer_3 = Dropout(0.2)(prelu_layer_3)

        bn_layer_4 = BatchNormalization()(dropout_layer_3)
        dense_layer_4 = Dense(EMBEDDING_DIM)(bn_layer_4)
        prelu_layer_4 = PReLU()(dense_layer_4)
        dropout_layer_4 = Dropout(0.2)(prelu_layer_4)

        bn_layer_5 = BatchNormalization()(dropout_layer_4)
        dense_layer_5 = Dense(EMBEDDING_DIM)(bn_layer_5)
        prelu_layer_5 = PReLU()(dense_layer_5)
        dropout_layer_5 = Dropout(0.2)(prelu_layer_5)

        output = Dense(1, activation='sigmoid')(dropout_layer_5)

        self.model = Model(input=[input_query_1, input_query_2], output=output)
        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

        return self.model
Пример #18
0
    def _build_model(self):
        # Neural Net for Deep-Q learning Model
        model = Sequential()
        # model.add(Dense(24, input_dim=self.state_size, activation='relu'))
        # model.add(Dropout(0.5))
        # model.add(Dense(32, activation='relu'))
        # model.add(Dropout(0.5))
        # model.add(Dense(10, activation='relu'))
        # model.add(Dense(self.action_size, activation='linear'))
        # model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
        #
        if self.config['network'] == "CNNRNN":
            model.add(LSTM(10, return_sequences=True, input_shape=(30, 8)))
            model.add(Dropout(0.5))
            model.add(LSTM(50, return_sequences=True))
            model.add(BatchNormalization())
            model.add(Activation('relu'))
            model.add(Dropout(0.5))
            model.add(Dense(50))
            model.add(BatchNormalization())
            model.add(Activation('relu'))
            model.add(Dropout(0.5))
            model.add(Dense(10, activation='relu'))
            model.add(Dense(self.action_size, activation='linear'))
            model.compile(loss=self.config['loss'],
                          optimizer=Adam(lr=self.learning_rate))
        else:
            PERIODS_PER_X = 30
            # model.add(LSTM(32,  return_sequences=True,input_shape=(30,8)))
            # model.add(Dropout(0.5))
            # model.add(LSTM(32))
            # model.add(Dropout(0.5))
            # model.add(Dense(32, activation='relu'))
            # model.add(Dropout(0.5))
            # model.add(Dense(10, activation='relu'))
            # model.add(Dense(self.action_size, activation='linear'))
            # model.compile(loss=self.config['loss'], optimizer=Adam(lr=self.learning_rate))
            # model.add(Reshape((1, PERIODS_PER_X, self.state_size), input_shape=(PERIODS_PER_X, self.state_size)))
            # model.add(Input((PERIODS_PER_X, self.state_size)))
            model.add(
                Convolution1D(128,
                              3,
                              padding='same',
                              activation='relu',
                              input_shape=(PERIODS_PER_X, 8)))
            model.add(MaxPooling1D(pool_size=2))
            model.add(Convolution1D(64, 3, activation='relu', padding="same"))
            model.add(MaxPooling1D(pool_size=2))
            # model.add(Reshape((PERIODS_PER_X, self.state_size)))
            model.add(LSTM(64, return_sequences=True))
            model.add(Dropout(0.25))
            model.add(LSTM(32, return_sequences=True))
            model.add(Dropout(0.5))
            model.add(Dense(16, activation='relu'))
            model.add(Dense(self.action_size, activation='softmax'))
            # if doesn't fit as well as other one, try adding more layers, or try using the old model with only price information (KISS)

            model.compile(loss=self.config['loss'],
                          optimizer=Adam(lr=self.learning_rate))
        # Create the model based on the information above
        #model.compile(loss='mean_squared_error', optimizer='adam')
        if os.path.isfile(self.weight_backup):
            #load_model(self.weight_backup)
            model.load_weights(self.weight_backup, True)
            self.exploration_rate = self.exploration_min
        return model
Пример #19
0
model_input = Input(shape=input_shape)

# Static model does not have embedding layer
if model_type == "CNN-static":
    z = model_input
else:
    z = Embedding(len(vocabulary_inv), embedding_dim, input_length=sequence_length, name="embedding")(model_input)

z = Dropout(dropout_prob[0])(z)

# Convolutional block
conv_blocks = []
for sz in filter_sizes:
    conv = Convolution1D(filters=num_filters,
                         kernel_size=sz,
                         padding="valid",
                         activation="relu",
                         strides=1)(z)
    conv = MaxPooling1D(pool_size=2)(conv)
    conv = Flatten()(conv)
    conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]

z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)

model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])

Пример #20
0
    num_seq_per_batch=num_seq_per_batch,
    input_processing=[DivideBy(input_std),
                      IndependentlyCenter()],
    target_processing=[DivideBy(target_std)])

# get the shape of X_train
(X_train, Y_train) = pipeline.train_generator().next()

starting_time = time.time()
nb_epoch = 1
# define the network architecture = Conv Net
model = Sequential()
model.add(
    Convolution1D(64,
                  3,
                  border_mode='same',
                  input_shape=(1, X_train.shape[2]),
                  init='normal',
                  activation='relu'))
model.add(Flatten())
model.add(Dense(32))
model.add(Activation('relu'))
model.add(Dense(3, activation='sigmoid'))
# compile the model
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer=sgd)
compiling_time = time.time() - starting_time
print('compiling time = ', compiling_time)
model.fit_generator(pipeline.train_generator(fold = ' train', source_id = 0), \
                    samples_per_epoch = 64064, \
                    nb_epoch = 2)
print('run time = ', time.time() - compiling_time)
Пример #21
0
def CRNN_1d(input_dim, useBatchNorm):

    reg = l2(0.0001)

    model = Sequential()
    model.add(
        InputLayer(input_shape=(input_dim[0], input_dim[1], input_dim[2])))
    model.add(Reshape((input_dim[0], input_dim[2])))

    model.add(
        Convolution1D(filters=16,
                      kernel_size=7,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    #model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=16,
                      kernel_size=5,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    #model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=32,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=32,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    #model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    #
    model.add(
        Convolution1D(filters=64,
                      kernel_size=3,
                      padding='same',
                      kernel_regularizer=reg,
                      bias_regularizer=reg))
    #model.add(MaxPooling1D())
    if useBatchNorm:
        model.add(BatchNormalization(scale=False))
        model.add(Activation('relu'))
    else:
        model.add(PReLU(shared_axes=[1]))
    model.add(Dropout(0.4))

    model.add(
        Bidirectional(
            LSTM(units=64,
                 return_sequences=True,
                 dropout=0.4,
                 kernel_regularizer=reg,
                 recurrent_regularizer=reg)))
    model.add(
        Bidirectional(
            LSTM(units=128,
                 return_sequences=True,
                 dropout=0.4,
                 kernel_regularizer=reg,
                 recurrent_regularizer=reg)))
    model.add(
        Bidirectional(
            LSTM(units=64,
                 return_sequences=False,
                 dropout=0.4,
                 kernel_regularizer=reg,
                 recurrent_regularizer=reg)))

    return model
Пример #22
0
def block_deepFlavourConvolutions(charged,
                                  neutrals,
                                  vertices,
                                  dropoutRate,
                                  active=True,
                                  batchnorm=False,
                                  batchmomentum=0.6):
    '''
    deep Flavour convolution part. 
    '''
    cpf = charged
    if active:
        cpf = Convolution1D(64,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='cpf_conv0')(cpf)
        if batchnorm:
            cpf = BatchNormalization(momentum=batchmomentum,
                                     name='cpf_batchnorm0')(cpf)
        cpf = Dropout(dropoutRate, name='cpf_dropout0')(cpf)
        cpf = Convolution1D(32,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='cpf_conv1')(cpf)
        if batchnorm:
            cpf = BatchNormalization(momentum=batchmomentum,
                                     name='cpf_batchnorm1')(cpf)
        cpf = Dropout(dropoutRate, name='cpf_dropout1')(cpf)
        cpf = Convolution1D(32,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='cpf_conv2')(cpf)
        if batchnorm:
            cpf = BatchNormalization(momentum=batchmomentum,
                                     name='cpf_batchnorm2')(cpf)
        cpf = Dropout(dropoutRate, name='cpf_dropout2')(cpf)
        cpf = Convolution1D(8,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='cpf_conv3')(cpf)
    else:
        cpf = Convolution1D(1, 1, kernel_initializer='zeros',
                            trainable=False)(cpf)

    npf = neutrals
    if active:
        npf = Convolution1D(32,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='npf_conv0')(npf)
        if batchnorm:
            npf = BatchNormalization(momentum=batchmomentum,
                                     name='npf_batchnorm0')(npf)
        npf = Dropout(dropoutRate, name='npf_dropout0')(npf)
        npf = Convolution1D(16,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='npf_conv1')(npf)
        if batchnorm:
            npf = BatchNormalization(momentum=batchmomentum,
                                     name='npf_batchnorm1')(npf)
        npf = Dropout(dropoutRate, name='npf_dropout1')(npf)
        npf = Convolution1D(4,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='npf_conv2')(npf)
    else:
        npf = Convolution1D(1, 1, kernel_initializer='zeros',
                            trainable=False)(npf)

    vtx = vertices
    if active:
        vtx = Convolution1D(64,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='vtx_conv0')(vtx)
        if batchnorm:
            vtx = BatchNormalization(momentum=batchmomentum,
                                     name='vtx_batchnorm0')(vtx)
        vtx = Dropout(dropoutRate, name='vtx_dropout0')(vtx)
        vtx = Convolution1D(32,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='vtx_conv1')(vtx)
        if batchnorm:
            vtx = BatchNormalization(momentum=batchmomentum,
                                     name='vtx_batchnorm1')(vtx)
        vtx = Dropout(dropoutRate, name='vtx_dropout1')(vtx)
        vtx = Convolution1D(32,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='vtx_conv2')(vtx)
        if batchnorm:
            vtx = BatchNormalization(momentum=batchmomentum,
                                     name='vtx_batchnorm2')(vtx)
        vtx = Dropout(dropoutRate, name='vtx_dropout2')(vtx)
        vtx = Convolution1D(8,
                            1,
                            kernel_initializer='lecun_uniform',
                            activation='relu',
                            name='vtx_conv3')(vtx)
    else:
        vtx = Convolution1D(1, 1, kernel_initializer='zeros',
                            trainable=False)(vtx)

    return cpf, npf, vtx
Пример #23
0
def shallow_model(input_dim, useBatchNorm, convFilters, convKernels, convPool,
                  convDO, denseUnits, denseDO, rnnUnits, rnnDO, convModelDim):

    reg = l2(0.00001)
    model = Sequential()
    model.add(
        InputLayer(input_shape=(input_dim[0], input_dim[1], input_dim[2])))
    if convModelDim == 1:
        model.add(Reshape((input_dim[0], input_dim[2])))

    #Convolutional layers block:
    for i in range(len(convFilters)):
        if convModelDim == 1:
            model.add(
                Convolution1D(filters=convFilters[i],
                              kernel_size=convKernels[i],
                              padding='same',
                              kernel_regularizer=reg,
                              bias_regularizer=reg))
        else:
            model.add(
                Convolution2D(filters=convFilters[i],
                              kernel_size=convKernels[i],
                              padding='same',
                              kernel_regularizer=reg,
                              bias_regularizer=reg))
        if convPool[i]:
            if convModelDim == 1:
                model.add(MaxPooling1D())
            else:
                model.add(MaxPooling2D())
        if useBatchNorm:
            model.add(BatchNormalization(scale=False))
            model.add(Activation('relu'))
        else:
            model.add(PReLU(shared_axes=[1]))
        model.add(Dropout(convDO))

    ret = True
    for i in range(len(rnnUnits)):
        if i == (len(rnnUnits) - 1):
            ret = False
        model.add(
            Bidirectional(
                LSTM(units=rnnUnits[i],
                     return_sequences=ret,
                     dropout=rnnDO,
                     kernel_regularizer=reg,
                     recurrent_regularizer=reg)))

    #flattening up filters
    if ret:
        model.add(Flatten())

    #
    for i in range(len(denseUnits)):
        model.add(
            Dense(units=denseUnits[i],
                  kernel_regularizer=reg,
                  bias_regularizer=reg))
        if useBatchNorm:
            model.add(BatchNormalization(scale=False))
            model.add(Activation('relu'))
        else:
            model.add(PReLU(shared_axes=[1]))
        model.add(Dropout(denseDO[min(i, len(denseDO) - 1)]))

    return model
Пример #24
0
def build_model(
    input_size,
    alphabet_size,
    conv_layers,
    fully_connected_layers,
    embedding_size,
    threshold,
    dropout_p,
    num_of_classes,
    optimizer='adam',
    loss='categorical_crossentropy',  # binary
    # loss='sparse_categorical_crossentropy', # integers
    metrics=['accuracy'],
    concat=False,
):
    """
    Build and compile the Character Level CNN model
    Returns: None
    """
    # Input layer
    inputs = Input(shape=(input_size, ), name='sent_input', dtype='int64')
    # Embedding layers
    x = Embedding(alphabet_size + 1,
                  embedding_size,
                  input_length=input_size,
                  trainable=True)(inputs)
    # Convolution layers
    if concat:
        channels = []

    for cl in conv_layers:

        x1 = Convolution1D(cl[0],
                           cl[1],
                           activation='tahn',
                           padding='same',
                           kernel_regularizer=regularizers.l2(0.03))(x)
        # x = ThresholdedReLU(threshold)(x)
        if cl[2] != -1:
            x1 = MaxPooling1D(cl[2], padding='valid')(x1)

        if concat:
            channels.append(x1)
        else:
            x = x1

    if concat:
        x1 = concatenate(channels, axis=1)

    x = Flatten()(x1)
    # Fully connected layers
    for fl in fully_connected_layers:
        x = Dense(fl, activation='relu')(x)
        #x = ThresholdedReLU(threshold)(x)
        x = Dropout(dropout_p)(x)
    # Output layer
    predictions = Dense(num_of_classes, activation='softmax')(x)
    # Build and compile model
    model = Model(inputs=inputs, outputs=predictions)
    model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
    model = model
    print("CharCNNZhang model built: ")
    model.summary()
    return model
Пример #25
0
distance1_input = Input(shape=(max_sentence_len, ),
                        dtype='int32',
                        name='distance1_input')
distance1 = Embedding(max_position, position_dims)(distance1_input)

distance2_input = Input(shape=(max_sentence_len, ),
                        dtype='int32',
                        name='distance2_input')
distance2 = Embedding(max_position, position_dims)(distance2_input)

output = concatenate([words, distance1, distance2])

output = Convolution1D(filters=nb_filter,
                       kernel_size=filter_length,
                       padding='same',
                       activation='tanh',
                       strides=1)(output)
# we use standard max over time pooling
output = GlobalMaxPooling1D()(output)
#output = Flatten()(output)
#output = Dropout(0.25)(output)
#output = Dense(512)(output)
#output = LeakyReLU()(output)
output = Dense(n_out, activation='softmax')(output)

model = Model(inputs=[words_input, distance1_input, distance2_input],
              outputs=[output])
model.compile(loss='sparse_categorical_crossentropy',
              optimizer='Adam',
              metrics=['accuracy'])
Пример #26
0
for i in range(20, 1090):
    X_train.append(training_set[i - 20:i, 0])
    y_train.append(training_set[i, 0])
X_train = np.asarray(X_train)
y_train = np.asarray(y_train)

X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
from keras.models import Sequential
from keras.layers import Convolution1D
from keras.layers import MaxPooling1D
from keras.layers import Flatten
from keras.layers import Dense
model = Sequential()
model.add(
    Convolution1D(filters=16,
                  kernel_size=3,
                  activation='relu',
                  input_shape=(20, 1)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(20, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mean_absolute_error', optimizer='adam')
# fit network
history = model.fit(X_train,
                    y_train,
                    epochs=60,
                    batch_size=8,
                    validation_split=0.02)

test_set = dataset.iloc[1100:1130, 1:2].values
X_test = []
Пример #27
0
def train_cnn_lstm():
    # Embedding
    max_features = 20000
    maxlen = 10
    embedding_size = 40

    # Convolution
    filter_length = 5
    nb_filter = 64
    pool_length = 4

    # LSTM
    lstm_output_size = 70

    # Training
    batch_size = 35
    nb_epoch = 2
    '''
    Note:
    batch_size is highly sensitive.
    Only 2 epochs are needed as the dataset is very small.
    '''

    print('Loading data...')
    #(X_train, y_train), (X_test, y_test) = load_data()
    X_train, y_train, X_test, y_test = load_data()
    print(len(X_train), 'train sequences')
    print(len(X_test), 'test sequences')

    print('Pad sequences (samples x time)')
    X_train = sequence.pad_sequences(X_train, maxlen=maxlen, value=3)
    X_test = sequence.pad_sequences(X_test, maxlen=maxlen, value=3)
    print('X_train shape:', X_train.shape)
    print('X_test shape:', X_test.shape)

    y_train = np.array(y_train)
    print('y_train shape', y_train.shape)
    y_test = np.array(y_test)
    print('y_train shape', y_test.shape)

    print('Build model...')

    model = Sequential()
    model.add(Embedding(max_features, embedding_size, input_length=maxlen))
    model.add(Dropout(0.25))
    model.add(
        Convolution1D(nb_filter=nb_filter,
                      filter_length=filter_length,
                      border_mode='valid',
                      activation='relu',
                      subsample_length=1))
    model.add(MaxPooling1D(pool_length=pool_length))
    model.add(LSTM(lstm_output_size))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])

    print('Train...')
    model.fit(X_train,
              y_train,
              batch_size=batch_size,
              nb_epoch=nb_epoch,
              validation_data=(X_test, y_test))
    score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)

    #save the model &serialize model to JSON
    model_json = model.to_json()
    with open("./dict/model_cnn_lstm.json", "w") as json_file:
        json_file.write(model_json)
    # serialize weights to HDF5
    model.save_weights("./dict/model_cnn_lstm_weights.h5")
    print("Saved model to disk")

    del model
    print('Test score:', score)
    print('Test accuracy:', acc)
Пример #28
0
y = data['y']
(x_train, x_val, y_train, y_val) = train_test_split(x,
                                                    y,
                                                    test_size=0.4,
                                                    random_state=SEED)

#print('Building model...')

n_features = x_train.shape[2]
input_shape = (None, n_features)
model_input = Input(input_shape, name='input')
layer = model_input
for i in range(N_LAYERS):
    # convolutional layer names are used by extract_filters.py
    layer = Convolution1D(nb_filter=CONV_FILTER_COUNT,
                          filter_length=FILTER_LENGTH,
                          name='convolution_' + str(i + 1))(layer)
    layer = Activation('relu')(layer)
    layer = MaxPooling1D(2)(layer)

layer = Dropout(0.5)(layer)
layer = LSTM(LSTM_COUNT, return_sequences=True)(layer)
layer = Dropout(0.5)(layer)
layer = TimeDistributed(Dense(len(GENRES)))(layer)
layer = Activation('softmax', name='output_realtime')(layer)
time_distributed_merge_layer = Lambda(function=lambda x: K.mean(x, axis=1),
                                      output_shape=lambda shape:
                                      (shape[0], ) + shape[2:],
                                      name='output_merged')
model_output = time_distributed_merge_layer(layer)
model = Model(model_input, model_output)
Пример #29
0
def inceptionBlock(x):
    _drop_rate_ = .1
    x = BatchNormalization()(x)
    conv1_1 = Convolution1D(100,
                            1,
                            activation='relu',
                            padding='same',
                            kernel_regularizer=l2(0.001))(x)
    conv1_1 = Dropout(_drop_rate_)(
        conv1_1
    )  #https://www.quora.com/Can-l-combine-dropout-and-l2-regularization
    conv1_1 = BatchNormalization()(conv1_1)

    conv2_1 = Convolution1D(100,
                            1,
                            activation='relu',
                            padding='same',
                            kernel_regularizer=l2(0.001))(x)
    conv2_1 = Dropout(_drop_rate_)(conv2_1)
    conv2_1 = BatchNormalization()(conv2_1)
    conv2_2 = Convolution1D(100,
                            3,
                            activation='relu',
                            padding='same',
                            kernel_regularizer=l2(0.001))(conv2_1)
    conv2_2 = Dropout(_drop_rate_)(conv2_2)
    conv2_2 = BatchNormalization()(conv2_2)

    conv3_1 = Convolution1D(100,
                            1,
                            activation='relu',
                            padding='same',
                            kernel_regularizer=l2(0.001))(x)
    conv3_1 = Dropout(_drop_rate_)(conv3_1)
    conv3_1 = BatchNormalization()(conv3_1)
    conv3_2 = Convolution1D(100,
                            3,
                            activation='relu',
                            padding='same',
                            kernel_regularizer=l2(0.001))(conv3_1)
    conv3_2 = Dropout(_drop_rate_)(conv3_2)
    conv3_2 = BatchNormalization()(conv3_2)
    conv3_3 = Convolution1D(100,
                            3,
                            activation='relu',
                            padding='same',
                            kernel_regularizer=l2(0.001))(conv3_2)
    conv3_3 = Dropout(_drop_rate_)(conv3_3)
    conv3_3 = BatchNormalization()(conv3_3)
    conv3_4 = Convolution1D(100,
                            3,
                            activation='relu',
                            padding='same',
                            kernel_regularizer=l2(0.001))(conv3_3)
    conv3_4 = Dropout(_drop_rate_)(conv3_4)
    conv3_4 = BatchNormalization()(conv3_4)

    concat = concatenate([conv1_1, conv2_2, conv3_4])
    concat = BatchNormalization()(concat)

    return concat
Пример #30
0
    except Exception as e:
        break

    X.append(x_i)
    Y.append(y_i)

X, Y = np.array(X), np.array(Y)
X_train, X_test, Y_train, Y_test = create_Xt_Yt(X, Y)

X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], EMB_SIZE))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], EMB_SIZE))

model = Sequential()
model.add(
    Convolution1D(input_shape=(WINDOW, EMB_SIZE),
                  nb_filter=16,
                  filter_length=4,
                  border_mode='same'))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))

model.add(Convolution1D(nb_filter=8, filter_length=4, border_mode='same'))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))

model.add(Flatten())

model.add(Dense(64))
model.add(BatchNormalization())
model.add(LeakyReLU())