def Model_BiLSTM_CnnDecoder(sourcevocabsize, targetvocabsize, source_W, input_seq_lenth, output_seq_lenth, hidden_dim, emd_dim, sourcecharsize, character_W, input_word_length, char_emd_dim, sourcepossize, pos_W, pos_emd_dim, batch_size=32, loss='categorical_crossentropy', optimizer='rmsprop'): # 0.8349149507609669--attention,lstm*2decoder # pos_input = Input(shape=(input_seq_lenth,), dtype='int32') # pos_embeding = Embedding(input_dim=sourcepossize + 1, # output_dim=pos_emd_dim, # input_length=input_seq_lenth, # mask_zero=False, # trainable=True, # weights=[pos_W])(pos_input) word_input = Input(shape=(input_seq_lenth, ), dtype='int32') char_input = Input(shape=( input_seq_lenth, input_word_length, ), dtype='int32') char_embedding = Embedding(input_dim=sourcecharsize, output_dim=char_emd_dim, batch_input_shape=(batch_size, input_seq_lenth, input_word_length), mask_zero=False, trainable=True, weights=[character_W]) char_embedding2 = TimeDistributed(char_embedding)(char_input) char_cnn = TimeDistributed( Conv1D(50, 3, activation='relu', border_mode='valid'))(char_embedding2) char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn) # char_macpool = Dropout(0.5)(char_macpool) pos_input = Input(shape=( input_seq_lenth, 3, ), dtype='int32') pos_embedding = Embedding(input_dim=sourcepossize + 1, output_dim=pos_emd_dim, batch_input_shape=(batch_size, input_seq_lenth, 3), mask_zero=False, trainable=True, weights=[pos_W]) pos_embedding2 = TimeDistributed(pos_embedding)(pos_input) pos_cnn = TimeDistributed( Conv1D(20, 2, activation='relu', border_mode='valid'))(pos_embedding2) pos_macpool = TimeDistributed(GlobalMaxPooling1D())(pos_cnn) word_embedding_RNN = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth, mask_zero=False, trainable=False, weights=[source_W])(word_input) # word_embedding_RNN = Dropout(0.5)(word_embedding_RNN) embedding = concatenate([word_embedding_RNN, char_macpool, pos_macpool], axis=-1) embedding = Dropout(0.5)(embedding) BiLSTM = Bidirectional(LSTM(int(hidden_dim / 2), return_sequences=True), merge_mode='concat')(embedding) BiLSTM = BatchNormalization()(BiLSTM) # BiLSTM = Dropout(0.3)(BiLSTM) # decodelayer1 = LSTM(50, return_sequences=False, go_backwards=True)(concat_LC_d)#!!!!! # repeat_decodelayer1 = RepeatVector(input_seq_lenth)(decodelayer1) # concat_decoder = concatenate([concat_LC_d, repeat_decodelayer1], axis=-1)#!!!! # decodelayer2 = LSTM(hidden_dim, return_sequences=True)(concat_decoder) # decodelayer = Dropout(0.5)(decodelayer2) # decoderlayer1 = LSTM(50, return_sequences=True, go_backwards=False)(BiLSTM) decoderlayer5 = Conv1D(50, 5, activation='relu', strides=1, padding='same')(BiLSTM) decoderlayer2 = Conv1D(50, 2, activation='relu', strides=1, padding='same')(BiLSTM) decoderlayer3 = Conv1D(50, 3, activation='relu', strides=1, padding='same')(BiLSTM) decoderlayer4 = Conv1D(50, 4, activation='relu', strides=1, padding='same')(BiLSTM) # 0.8868111121100423 decodelayer = concatenate( [decoderlayer2, decoderlayer3, decoderlayer4, decoderlayer5], axis=-1) decodelayer = BatchNormalization()(decodelayer) decodelayer = Dropout(0.5)(decodelayer) TimeD = TimeDistributed(Dense(targetvocabsize + 1))(decodelayer) # TimeD = Dropout(0.5)(TimeD) model = Activation('softmax')(TimeD) # 0.8769744561783556 # crf = CRF(targetvocabsize + 1, sparse_target=False) # model = crf(TimeD) Models = Model([word_input, char_input, pos_input], model) # Models.compile(loss=my_cross_entropy_Weight, optimizer='adam', metrics=['acc']) Models.compile(loss=loss, optimizer='adam', metrics=['acc']) # Models.compile(loss=loss, optimizer='adam', metrics=['acc'], sample_weight_mode="temporal") # Models.compile(loss=loss, optimizer=optimizers.RMSprop(lr=0.01), metrics=['acc']) # Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy]) # Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.005), metrics=[crf.accuracy]) return Models
def compile(self, tokenizer, glove_dir='./data/', embedding_dim=50, dropout_fraction=0.0, kernal_size=5, n_filters=128): """Compile network model for classifier Args: glove_file (str): Location of GloVe file embedding_dim (int): Size of embedding vector tokenizer (WordTokenizer): Object used to tokenize orginal texts dropout_fraction (float): Fraction of randomly zeroed weights in dropout layer kernal_size (int): Size of sliding window for convolution n_filters (int): Number of filters to produce from convolution """ # Load embedding layer print('Loading GloVe embedding...') embeddings_index = {} f = open( os.path.join(glove_dir, 'glove.6B.' + str(embedding_dim) + 'd.txt'), 'rb') for line in f: values = line.split() word = values[0] coefs = np.asarray(values[1:], dtype='float32') embeddings_index[word] = coefs f.close() print(('Found %s word vectors.' % len(embeddings_index))) # Create embedding layer print('Creating embedding layer...') embedding_matrix = np.zeros( (len(tokenizer.tokenizer.word_index) + 1, embedding_dim)) for word, i in list(tokenizer.tokenizer.word_index.items()): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: # words not found in embedding index will be all-zeros. embedding_matrix[i] = embedding_vector embedding_layer = Embedding(len(tokenizer.tokenizer.word_index) + 1, embedding_dim, weights=[embedding_matrix], input_length=tokenizer.max_sequence_length, trainable=False) # Create network print('Creating network...') sequence_input = Input(shape=(tokenizer.max_sequence_length, ), dtype='int32') embedded_sequences = embedding_layer(sequence_input) x = Dropout(dropout_fraction)(embedded_sequences) x = Conv1D(n_filters, kernal_size, activation='relu')(x) x = MaxPooling1D(kernal_size)(x) x = Conv1D(n_filters, kernal_size, activation='relu')(x) x = MaxPooling1D(kernal_size)(x) x = Conv1D(n_filters, kernal_size, activation='relu')(x) x = MaxPooling1D(int(x.shape[1]))(x) # global max pooling x = Flatten()(x) x = Dense(n_filters, activation='relu')(x) preds = Dense(len(self.category_map), activation='softmax')(x) # Compile model print('Compiling network...') self.model = Model(sequence_input, preds) self.model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
def get_model(embedding_layer,RNN,embed_size,Feature_dic,Para_dic): MAX_SEQUENCE_LENGTH=Para_dic['MAX_SEQUENCE_LENGTH'] comment_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32') embedded_sequences_raw= embedding_layer(comment_input) embedded_sequences = SpatialDropout1D(Para_dic['spatial_dropout'])(embedded_sequences_raw) ### RNN if RNN=='LSTM': RNN_x = Bidirectional(CuDNNLSTM(Para_dic['num_lstm'],return_sequences=True))(embedded_sequences) elif RNN=='GRU': RNN_x = Bidirectional(CuDNNGRU(Para_dic['num_lstm'],return_sequences=True))(embedded_sequences) Feature=[] ######## RNN Features ##### Attention if Feature_dic['Attention']==1: Feature.append(Attention(MAX_SEQUENCE_LENGTH)(RNN_x)) if Feature_dic['RNN_maxpool']==1: Feature.append(GlobalMaxPooling1D()(RNN_x)) ##### Capsule if Feature_dic['Capsule']==1: capsule = Capsule(share_weights=True)(RNN_x) capsule = Flatten()(capsule) Feature.append(capsule) ##### RNN_CNNN1d if Feature_dic['RNN_CNN_conv1d']==1: Cx = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform")(RNN_x) avg_pool = GlobalAveragePooling1D()(Cx) max_pool = GlobalMaxPooling1D()(Cx) Feature.append(avg_pool) Feature.append(max_pool) ######## CNN Features ### CNN2d if Feature_dic['CNN2d']==1: CNN2d=get_CNN2d(embedded_sequences,embed_size,MAX_SEQUENCE_LENGTH,Para_dic) Feature.append(CNN2d) ### DPCNN if Feature_dic['DPCNN']==1: DPCNN=get_DPCNN(embedded_sequences,Para_dic) Feature.append(DPCNN) ### Concatnation merged = Concatenate()(Feature) ### dense, add L1 reg to enable sparsity merged = Dense(Para_dic['dense_num'], \ activation=Para_dic['dense_act'],\ kernel_regularizer=regularizers.l1(Para_dic['L1_reg']))(merged) merged = Dropout(Para_dic['dense_dropout'])(merged) preds = Dense(6, activation='sigmoid')(merged) model = Model(inputs=[comment_input], outputs=preds) model.compile(loss='binary_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) print(model.summary()) return model
print( 'Accuracy is %2.2f%%, Precision is %2.2f%%, Recall is %2.2f%%, F-measure is %f\n' % (acc * 100, prec * 100, rec * 100, fmes)) #%% main if __name__ == "__main__": X_train, Y_train, X_val, Y_val = val_split(*get_data('ExoTrain.csv'), 0.7) X_test = get_data('Final Test.csv', test=True) #%% create model model = Sequential() model.add( Conv1D(filters=32, kernel_size=5, activation='relu', input_shape=X_train.shape[1:])) model.add(MaxPool1D(pool_size=2, strides=2)) model.add(Dropout(0.25)) model.add(Conv1D(filters=32, kernel_size=5, activation='relu')) model.add(MaxPool1D(pool_size=2, strides=2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(64, activation='sigmoid')) model.add(Dense(1, activation='sigmoid')) #%% compile the model model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
from keras.models import Sequential from keras.layers import Dense from keras.layers import Activation from keras.layers import LSTM from keras.layers import Input from keras.layers import Bidirectional from keras.layers import TimeDistributed from keras.layers import RepeatVector from keras.models import Model import numpy as np from sklearn.model_selection import train_test_split avg_pt = 208 no_classes = 7 inputs = Input(shape=(avg_pt, 2)) conv1 = Conv1D(8, 2, activation='relu', padding='same')(inputs) conv2 = Conv1D(16, 2, activation='relu', padding='same')(conv1) lstm1 = LSTM(50)(conv2) lstm2 = RepeatVector(avg_pt)(lstm1) lstm3 = LSTM(50)(lstm2) dense = Dense(no_classes, activation='softmax')(lstm3) model = Model(inputs, dense) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) model.load_weights("model/Animals/model_convLstm.h5") app = Flask(__name__)
def get_resNet_model(input_shape, output_shape): def resnet_v1(input_shape, depth, num_classes=10, input_tensor=None, local_conv=False): if (depth - 2) % 6 != 0: raise ValueError('depth should be 6n+2 (eg 20, 32, 44 in [a])') # Start model definition. num_filters = 16 num_res_blocks = int((depth - 2) / 6) if (input_tensor == None): inputs = Input(shape=input_shape) else: inputs = input_tensor x = resnet_layer_naive(inputs=inputs) # Instantiate the stack of residual units for stack in range(3): for res_block in range(num_res_blocks): strides = 1 # if stack > 0 and res_block == 0: # first layer but not first stack # strides = 2 # downsample y = resnet_layer_local(inputs=x, kernel_size=8, num_filters=num_filters, strides=strides) y = resnet_layer_local(inputs=y, kernel_size=16, num_filters=num_filters, activation=None) if stack > 0 and res_block == 0: # first layer but not first stack # linear projection residual shortcut connection to match # changed dims x = resnet_layer_naive(inputs=x, num_filters=num_filters, kernel_size=16, strides=strides, activation=None, batch_normalization=True) x = keras.layers.add([x, y]) x = Activation(default_activation)(x) num_filters *= 2 return x inputs = Input(shape=input_shape) xxx = inputs xxx = Conv1D(filters=xl_filter_num, kernel_size=m_filter_num, padding='same', activation=None, strides=1)(xxx) xxx = BatchNormalization()(xxx) xxx = Activation('relu')(xxx) xxx = MaxPooling1D(pool_size=2, padding='same', strides=2)(xxx) xxx = resnet_v1(input_shape, num_classes=output_shape, depth=3 * 6 + 2, input_tensor=xxx, local_conv=False) xxx = LocallyConnected1D(filters=l_filter_num, kernel_size=m_filter_num, padding='valid', activation=default_activation, strides=1)(xxx) xxx = BatchNormalization()(xxx) xxx = GlobalMaxPooling1D()(xxx) xxx = Dense(output_shape, activation='softmax', kernel_initializer='he_normal')(xxx) model = Model(inputs=inputs, outputs=xxx) return model
from keras.preprocessing.sequence import pad_sequences pad_x = pad_sequences(x, padding='pre', value=0) # Defalut (pre와 0 : 그만큼 이 조합이 많이 쓰인 다는 것) print(pad_x) print(pad_x.shape) # (12, 5) # pad_x = pad_x.reshape(-1, 5, 1) print(pad_x.shape) # (12, 5, 1) word_size = len(token.word_index) + 1 print('전체 토큰 사이즈 :', word_size) # 25 (전체 단어의 개수) ### 모델 from keras.models import Sequential from keras.layers import Dense, Embedding, Flatten, LSTM, Conv1D model = Sequential() model.add(Embedding(25, 10, input_length=5)) model.add(Conv1D(5, 2)) model.add(Flatten()) model.add(Dense(1, activation='sigmoid')) model.summary() ### 실행, 훈련, 평가 model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc']) model.fit(pad_x, labels, epochs=30) acc = model.evaluate(pad_x, labels)[1] # loss가 아닌 metrics 값을 빼겠다. print('\n acc : %.4f' % acc)
def get_model_rnn_cnn(embedding_matrix, cell_size=80, cell_type_GRU=True, maxlen=180, max_features=100000, embed_size=300, prob_dropout=0.2, emb_train=False, filter_size=128, kernel_size=2, stride=1): inp_pre = Input(shape=(maxlen, ), name='input_pre') inp_post = Input(shape=(maxlen, ), name='input_post') ##pre x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=emb_train)(inp_pre) x1 = SpatialDropout1D(prob_dropout)(x1) if cell_type_GRU: x1 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1) else: x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1) x1 = Conv1D(filter_size, kernel_size=kernel_size, strides=stride, padding="valid", kernel_initializer="he_uniform")(x1) avg_pool1 = GlobalAveragePooling1D()(x1) max_pool1 = GlobalMaxPooling1D()(x1) ##post x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=emb_train)(inp_post) x2 = SpatialDropout1D(prob_dropout)(x2) if cell_type_GRU: x2 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2) else: x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2) x2 = Conv1D(filter_size, kernel_size=kernel_size, strides=stride, padding="valid", kernel_initializer="he_uniform")(x2) avg_pool2 = GlobalAveragePooling1D()(x2) max_pool2 = GlobalMaxPooling1D()(x2) ##merge conc = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2]) outp = Dense(6, activation="sigmoid")(conc) model = Model(inputs=[inp_pre, inp_post], outputs=outp) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_crossentropy', 'accuracy']) return model
def get_model_2rnn_cnn(embedding_matrix, cell_size=80, cell_type_GRU=True, maxlen=180, max_features=100000, embed_size=300, prob_dropout=0.2, emb_train=False, filter_size=128, kernel_size=2, stride=1): inp_pre = Input(shape=(maxlen, ), name='input_pre') inp_post = Input(shape=(maxlen, ), name='input_post') ##pre x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=emb_train)(inp_pre) x1 = SpatialDropout1D(prob_dropout)(x1) if cell_type_GRU: x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1) x1 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1) else: x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1) x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1) x1 = Conv1D(filter_size, kernel_size=kernel_size, strides=stride, padding="valid", kernel_initializer="he_uniform")(x1) avg_pool1 = GlobalAveragePooling1D()(x1) max_pool1 = GlobalMaxPooling1D()(x1) ##post x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=emb_train)(inp_post) x2 = SpatialDropout1D(prob_dropout)(x2) if cell_type_GRU: x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2) x2 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2) else: x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2) x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2) x2 = Conv1D(filter_size, kernel_size=kernel_size, strides=stride, padding="valid", kernel_initializer="he_uniform")(x2) avg_pool2 = GlobalAveragePooling1D()(x2) max_pool2 = GlobalMaxPooling1D()(x2) ##merge conc = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2]) outp = Dense(6, activation="sigmoid")(conc) model = Model(inputs=[inp_pre, inp_post], outputs=outp) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_crossentropy', 'accuracy']) return model # def get_model_2rnn_cnn_sp( # embedding_matrix, cell_size = 80, cell_type_GRU = True, # maxlen = 180, max_features = 100000, embed_size = 300, # prob_dropout = 0.2, emb_train = False, # filter_size=128, kernel_size = 2, stride = 1 # ): # inp_pre = Input(shape=(maxlen, ), name='input_pre') # inp_post = Input(shape=(maxlen, ), name='input_post') # ##pre # x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_pre) # x1 = SpatialDropout1D(prob_dropout)(x1) # if cell_type_GRU: # x1_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1) # x1 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1_) # else : # x1_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1) # x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1_) # x1_ = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_) # avg_pool1_ = GlobalAveragePooling1D()(x1_) # max_pool1_ = GlobalMaxPooling1D()(x1_) # x1 = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1) # avg_pool1 = GlobalAveragePooling1D()(x1) # max_pool1 = GlobalMaxPooling1D()(x1) # ##post # x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_post) # x2 = SpatialDropout1D(prob_dropout)(x2) # if cell_type_GRU: # x2_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2) # x2 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2_) # else : # x2_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2) # x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2_) # x2_ = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_) # avg_pool2_ = GlobalAveragePooling1D()(x2_) # max_pool2_ = GlobalMaxPooling1D()(x2_) # x2 = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2) # avg_pool2 = GlobalAveragePooling1D()(x2) # max_pool2 = GlobalMaxPooling1D()(x2) # ##merge # conc = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2, avg_pool1_, max_pool1_, avg_pool2_, max_pool2_]) # outp = Dense(6, activation="sigmoid")(conc) # model = Model(inputs=[inp_pre, inp_post], outputs=outp) # model.compile(loss='binary_crossentropy', # optimizer='adam', # metrics=['binary_crossentropy', 'accuracy']) # return model # def get_model_dual_2rnn_cnn_sp( # embedding_matrix, cell_size = 80, cell_type_GRU = True, # maxlen = 180, max_features = 100000, embed_size = 300, # prob_dropout = 0.2, emb_train = False, # filter_size=128, kernel_size = 2, stride = 1 # ): # inp_pre = Input(shape=(maxlen, ), name='input_pre') # inp_post = Input(shape=(maxlen, ), name='input_post') # ##pre # x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_pre) # x1g = SpatialDropout1D(prob_dropout)(x1) # x1l = SpatialDropout1D(prob_dropout)(x1) # x1_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1g) # x1g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1_g) # x1_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1l) # x1l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1_l) # x1_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_g) # x1_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_l) # avg_pool1_g = GlobalAveragePooling1D()(x1_g) # max_pool1_g = GlobalMaxPooling1D()(x1_g) # avg_pool1_l = GlobalAveragePooling1D()(x1_l) # max_pool1_l = GlobalMaxPooling1D()(x1_l) # x1g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1g) # x1l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1l) # avg_pool1g = GlobalAveragePooling1D()(x1g) # max_pool1g = GlobalMaxPooling1D()(x1g) # avg_pool1l = GlobalAveragePooling1D()(x1l) # max_pool1l = GlobalMaxPooling1D()(x1l) # ##post # x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_post) # x2g = SpatialDropout1D(prob_dropout)(x2) # x2l = SpatialDropout1D(prob_dropout)(x2) # x2_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2g) # x2g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2_g) # x2_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2l) # x2l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2_l) # x2_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_g) # x2_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_l) # avg_pool2_g = GlobalAveragePooling1D()(x2_g) # max_pool2_g = GlobalMaxPooling1D()(x2_g) # avg_pool2_l = GlobalAveragePooling1D()(x2_l) # max_pool2_l = GlobalMaxPooling1D()(x2_l) # x2g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2g) # x2l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2l) # avg_pool2g = GlobalAveragePooling1D()(x2g) # max_pool2g = GlobalMaxPooling1D()(x2g) # avg_pool2l = GlobalAveragePooling1D()(x2l) # max_pool2l = GlobalMaxPooling1D()(x2l) # ##merge # conc = concatenate([avg_pool1g, max_pool1g, avg_pool1l, max_pool1l, avg_pool1_g, max_pool1_g, avg_pool1_l, max_pool1_l, # avg_pool2g, max_pool2g, avg_pool2l, max_pool2l, avg_pool2_g, max_pool2_g, avg_pool2_l, max_pool2_l]) # outp = Dense(6, activation="sigmoid")(conc) # model = Model(inputs=[inp_pre, inp_post], outputs=outp) # model.compile(loss='binary_crossentropy', # optimizer='adam', # metrics=['binary_crossentropy', 'accuracy']) # return model # def get_model_dual_2rnn_cnn_sp_drop( # embedding_matrix, cell_size = 80, cell_type_GRU = True, # maxlen = 180, max_features = 100000, embed_size = 300, # prob_dropout = 0.2, emb_train = False, # filter_size=128, kernel_size = 2, stride = 1 # ): # inp_pre = Input(shape=(maxlen, ), name='input_pre') # inp_post = Input(shape=(maxlen, ), name='input_post') # ##pre # x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_pre) # x1g = SpatialDropout1D(prob_dropout)(x1) # x1l = SpatialDropout1D(prob_dropout)(x1) # x1_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1g) # x1g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1_g) # x1_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1l) # x1l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1_l) # x1_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_g) # x1_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_l) # avg_pool1_g = GlobalAveragePooling1D()(x1_g) # max_pool1_g = GlobalMaxPooling1D()(x1_g) # avg_pool1_l = GlobalAveragePooling1D()(x1_l) # max_pool1_l = GlobalMaxPooling1D()(x1_l) # x1g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1g) # x1l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1l) # avg_pool1g = GlobalAveragePooling1D()(x1g) # max_pool1g = GlobalMaxPooling1D()(x1g) # avg_pool1l = GlobalAveragePooling1D()(x1l) # max_pool1l = GlobalMaxPooling1D()(x1l) # ##post # x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_post) # x2g = SpatialDropout1D(prob_dropout)(x2) # x2l = SpatialDropout1D(prob_dropout)(x2) # x2_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2g) # x2g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2_g) # x2_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2l) # x2l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2_l) # x2_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_g) # x2_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_l) # avg_pool2_g = GlobalAveragePooling1D()(x2_g) # max_pool2_g = GlobalMaxPooling1D()(x2_g) # avg_pool2_l = GlobalAveragePooling1D()(x2_l) # max_pool2_l = GlobalMaxPooling1D()(x2_l) # x2g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2g) # x2l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2l) # avg_pool2g = GlobalAveragePooling1D()(x2g) # max_pool2g = GlobalMaxPooling1D()(x2g) # avg_pool2l = GlobalAveragePooling1D()(x2l) # max_pool2l = GlobalMaxPooling1D()(x2l) # ##merge # conc = concatenate([avg_pool1g, max_pool1g, avg_pool1l, max_pool1l, avg_pool1_g, max_pool1_g, avg_pool1_l, max_pool1_l, # avg_pool2g, max_pool2g, avg_pool2l, max_pool2l, avg_pool2_g, max_pool2_g, avg_pool2_l, max_pool2_l]) # conc = SpatialDropout1D(prob_dropout)(conc) # outp = Dense(6, activation="sigmoid")(conc) # model = Model(inputs=[inp_pre, inp_post], outputs=outp) # model.compile(loss='binary_crossentropy', # optimizer='adam', # metrics=['binary_crossentropy', 'accuracy']) # return model # def get_model_dpcnn( # embedding_matrix, cell_size = 80, cell_type_GRU = True, # maxlen = 180, max_features = 100000, embed_size = 300, # prob_dropout = 0.2, emb_train = False, # filter_nr=128, filter_size = 2, stride = 1, # max_pool_size = 3, max_pool_strides = 2, dense_nr = 256, # spatial_dropout = 0.2, dense_dropout = 0.5, # conv_kern_reg = regularizers.l2(0.00001), conv_bias_reg = regularizers.l2(0.00001) # ): # comment = Input(shape=(maxlen,)) # emb_comment = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=emb_train)(comment) # emb_comment = SpatialDropout1D(spatial_dropout)(emb_comment) # block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(emb_comment) # block1 = BatchNormalization()(block1) # block1 = PReLU()(block1) # block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block1) # block1 = BatchNormalization()(block1) # block1 = PReLU()(block1) # #we pass embedded comment through conv1d with filter size 1 because it needs to have the same shape as block output # #if you choose filter_nr = embed_size (300 in this case) you don't have to do this part and can add emb_comment directly to block1_output # resize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(emb_comment) # resize_emb = PReLU()(resize_emb) # block1_output = add([block1, resize_emb]) # block1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block1_output) # block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block1_output) # block2 = BatchNormalization()(block2) # block2 = PReLU()(block2) # block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block2) # block2 = BatchNormalization()(block2) # block2 = PReLU()(block2) # block2_output = add([block2, block1_output]) # block2_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block2_output) # block3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block2_output) # block3 = BatchNormalization()(block3) # block3 = PReLU()(block3) # block3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block3) # block3 = BatchNormalization()(block3) # block3 = PReLU()(block3) # block3_output = add([block3, block2_output]) # block3_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block3_output) # block4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block3_output) # block4 = BatchNormalization()(block4) # block4 = PReLU()(block4) # block4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block4) # block4 = BatchNormalization()(block4) # block4 = PReLU()(block4) # block4_output = add([block4, block3_output]) # block4_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block4_output) # block5 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block4_output) # block5 = BatchNormalization()(block5) # block5 = PReLU()(block5) # block5 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block5) # block5 = BatchNormalization()(block5) # block5 = PReLU()(block5) # block5_output = add([block5, block4_output]) # block5_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block5_output) # block6 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block5_output) # block6 = BatchNormalization()(block6) # block6 = PReLU()(block6) # block6 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block6) # block6 = BatchNormalization()(block6) # block6 = PReLU()(block6) # block6_output = add([block6, block5_output]) # block6_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block6_output) # block7 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block6_output) # block7 = BatchNormalization()(block7) # block7 = PReLU()(block7) # block7 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', # kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block7) # block7 = BatchNormalization()(block7) # block7 = PReLU()(block7) # block7_output = add([block7, block6_output]) # output = GlobalMaxPooling1D()(block7_output) # output = Dense(dense_nr, activation='linear')(output) # output = BatchNormalization()(output) # output = PReLU()(output) # output = Dropout(dense_dropout)(output) # output = Dense(6, activation='sigmoid')(output) # model = Model(comment, output) # model.compile(loss='binary_crossentropy', # optimizer='adam', # metrics=['accuracy']) # return model
y = encode(Label_1) ################################ Main #################################### cvscores = [] cnt = 0 kfold = StratifiedKFold(n_splits=K, shuffle=True, random_state=1) for train, test in kfold.split(x, decode(y)): cnt = cnt + 1 print(cnt) cnn = Sequential() print(x.shape[1], 1) print('x_train shape:', x[train].shape) cnn = Sequential() cnn.add(Reshape((x.shape[1], 1), input_shape=(x.shape[1], ))) cnn.add(Conv1D(32, 3, activation='relu', padding='same')) cnn.add(MaxPooling1D(3)) cnn.add(Conv1D(64, 3, activation='relu', padding='same')) cnn.add(MaxPooling1D(3)) cnn.add(Conv1D(128, 3, activation='relu', padding='same')) cnn.add(MaxPooling1D(3)) cnn.add(Conv1D(128, 3, activation='relu', padding='same')) cnn.add(MaxPooling1D(3)) cnn.add(Dropout(0.25)) cnn.add(Flatten()) cnn.add(Dense(256, activation=inner_activation_fun))
def produce_model(self): input_tweet = Input(shape=(self.num_tweets, 30, 300), name='in_tweet') hashtag = Input(shape=(self.num_tweets, 200), name='hashtags') handle = Input(shape=(self.num_tweets, 200), name='handles') prev_info = Input(shape=(500, 2), name='history') # Convolution Architecture for tweets k1_raw = TimeDistributed(Conv1D(filters=self.shapes[3], kernel_size=1, padding='same', name='K1'))(input_tweet) k2_raw = TimeDistributed(Conv1D(filters=self.shapes[3], kernel_size=2, padding='same', name='K2'))(input_tweet) k3_raw = TimeDistributed(Conv1D(filters=self.shapes[3], kernel_size=3, padding='same', name='K3'))(input_tweet) k4_raw = TimeDistributed(Conv1D(filters=self.shapes[4], kernel_size=4, padding='same', name='K4'))(input_tweet) k5_raw = TimeDistributed(Conv1D(filters=self.shapes[5], kernel_size=5, padding='same', name='K5'))(input_tweet) k6_raw = TimeDistributed(Conv1D(filters=self.shapes[6], kernel_size=6, padding='same', name='K6'))(input_tweet) k8_raw = TimeDistributed(Conv1D(filters=self.shapes[8], kernel_size=8, padding='same', name='K8'))(input_tweet) k1_max = TimeDistributed(GlobalMaxPool1D())(k1_raw) k2_max = TimeDistributed(GlobalMaxPool1D())(k2_raw) k3_max = TimeDistributed(GlobalMaxPool1D())(k3_raw) k4_max = TimeDistributed(GlobalMaxPool1D())(k4_raw) k5_max = TimeDistributed(GlobalMaxPool1D())(k5_raw) k6_max = TimeDistributed(GlobalMaxPool1D())(k6_raw) k8_max = TimeDistributed(GlobalMaxPool1D())(k8_raw) # Handle and hashtag processing handle_10_vec = TimeDistributed(Dense(units=10, activation='relu'))(handle) hashtag_10_vec = TimeDistributed(Dense(units=10, activation='relu'))(hashtag) # Concatenation out_conv = keras.layers.concatenate( [k1_max, k2_max, k3_max, k4_max, k5_max, k6_max, k8_max, handle_10_vec, hashtag_10_vec], name='Combining_Layers') # Main layer int_layer = TimeDistributed(Dense(units=150, activation='relu'))(out_conv) dropout_int_layer = TimeDistributed(Dropout(rate=0.2))(int_layer) in_lstm = TimeDistributed(Dense(units=100, activation='relu'))(dropout_int_layer) b2_raw = Conv1D(filters=self.shapes[3]*2, kernel_size=4, padding='same', name='b2')(in_lstm) b3_raw = Conv1D(filters=self.shapes[3]*2, kernel_size=8, padding='same', name='b3')(in_lstm) b4_raw = Conv1D(filters=self.shapes[4]*2, kernel_size=16, padding='same', name='b4')(in_lstm) b5_raw = Conv1D(filters=self.shapes[5]*2, kernel_size=32, padding='same', name='b5')(in_lstm) b6_raw = Conv1D(filters=self.shapes[6]*2, kernel_size=64, padding='same', name='b6')(in_lstm) b8_raw = Conv1D(filters=self.shapes[8]*2, kernel_size=128, padding='same', name='b8')(in_lstm) b2_max = GlobalMaxPool1D()(b2_raw) b3_max = GlobalMaxPool1D()(b3_raw) b4_max = GlobalMaxPool1D()(b4_raw) b5_max = GlobalMaxPool1D()(b5_raw) b6_max = GlobalMaxPool1D()(b6_raw) b8_max = GlobalMaxPool1D()(b8_raw) price_lstm = LSTM(units=100)(prev_info) input_lin = keras.layers.concatenate([b2_max, b3_max, b4_max, b5_max, b6_max, b8_max, price_lstm], name='Combining_Layers_Final') out_linear1 = Dense(units=300, activation='relu', name='Linear_1a')(input_lin) out_linear1_d = Dropout(rate=0.1)(out_linear1) out_linear3 = Dense(units=80, activation='relu', name='Linear_1')(out_linear1_d) out_linear4 = Dense(units=40, name='Linear_2')(out_linear3) out = Dense(units=1, name='Final_Layer', activation='sigmoid')(out_linear4) model = keras.Model(inputs=[input_tweet, hashtag, handle, prev_info], outputs=out) model.compile(loss='binary_crossentropy', optimizer=self.optim) return model
#vec+=self.model[word].reshape((1,size)) w2v_model = gensim.models.Word2Vec.load( '/home/liyt/code/text_classification/preproccess/demo0713.model') embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM)) for word, i in word_index.items(): if word in w2v_model: embedding_matrix[i] = np.asarray( w2v_model[word]) #, dtype='float32' embedding_layer = Embedding(len(word_index) + 1, EMBEDDING_DIM, weights=[embedding_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=False) # 搭建模型 model = Sequential() model.add(embedding_layer) model.add(Dropout(0.2)) model.add(Conv1D(250, 3, padding='valid', activation='relu', strides=1)) model.add(MaxPooling1D(3)) model.add(Flatten()) model.add(Dense(EMBEDDING_DIM, activation='relu')) model.add(Dense(labels.shape[1], activation='softmax')) model.summary() # plot_model(model, to_file='model.png',show_shapes=True) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc']) model.fit(x_train, y_train, epochs=2, batch_size=128) model.save('word_vector_cnn.h5') print model.evaluate(x_test, y_test)
def Model_BiLSTM_X2_CRF(sourcevocabsize, targetvocabsize, source_W, input_seq_lenth, output_seq_lenth, hidden_dim, emd_dim, sourcecharsize, character_W, input_word_length, char_emd_dim, batch_size=32, loss='categorical_crossentropy', optimizer='rmsprop'): word_input = Input(shape=(input_seq_lenth, ), dtype='int32') char_input = Input(shape=( input_seq_lenth, input_word_length, ), dtype='int32') char_embedding = Embedding(input_dim=sourcecharsize, output_dim=char_emd_dim, batch_input_shape=(batch_size, input_seq_lenth, input_word_length), mask_zero=False, trainable=True, weights=[character_W]) char_embedding2 = TimeDistributed(char_embedding)(char_input) char_cnn = TimeDistributed(Conv1D(50, 3, activation='relu', padding='same'))(char_embedding2) char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn) # char_macpool = Dropout(0.5)(char_macpool) # !!!!!!!!!!!!!! char_macpool = Dropout(0.25)(char_macpool) word_embedding = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth, mask_zero=True, trainable=True, weights=[source_W])(word_input) word_embedding_dropout = Dropout(0.5)(word_embedding) embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1) BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True), merge_mode='concat')(embedding) BiLSTM = BatchNormalization(axis=1)(BiLSTM) BiLSTM_dropout = Dropout(0.5)(BiLSTM) BiLSTM2 = Bidirectional(LSTM(hidden_dim // 2, return_sequences=True), merge_mode='concat')(BiLSTM_dropout) BiLSTM_dropout2 = Dropout(0.5)(BiLSTM2) TimeD = TimeDistributed(Dense(targetvocabsize + 1))(BiLSTM_dropout2) # TimeD = TimeDistributed(Dense(int(hidden_dim / 2)))(BiLSTM_dropout) # TimeD = Dropout(0.5)(TimeD) # !!!!!!!!!!!!!!!delete dropout # model = Activation('softmax')(TimeD) crflayer = CRF(targetvocabsize + 1, sparse_target=False) model = crflayer(TimeD) #0.8746633147782367 # # model = crf(BiLSTM_dropout)#0.870420501714492 Models = Model([word_input, char_input], [model]) # Models.compile(loss=loss, optimizer='adam', metrics=['acc']) # Models.compile(loss=crflayer.loss_function, optimizer='adam', metrics=[crflayer.accuracy]) Models.compile(loss=crflayer.loss_function, optimizer=optimizers.RMSprop(lr=0.001), metrics=[crflayer.accuracy]) return Models
def Model_Dense_Softmax(sourcevocabsize, targetvocabsize, source_W, input_seq_lenth, output_seq_lenth, hidden_dim, emd_dim, sourcecharsize, character_W, input_word_length, char_emd_dim, sourcepossize, pos_W, pos_emd_dim, batch_size=32, loss='categorical_crossentropy', optimizer='rmsprop'): word_input = Input(shape=(input_seq_lenth, ), dtype='int32') char_input = Input(shape=( input_seq_lenth, input_word_length, ), dtype='int32') char_embedding = Embedding(input_dim=sourcecharsize, output_dim=char_emd_dim, batch_input_shape=(batch_size, input_seq_lenth, input_word_length), mask_zero=False, trainable=True, weights=[character_W]) char_embedding2 = TimeDistributed(char_embedding)(char_input) char_cnn = TimeDistributed(Conv1D(50, 3, activation='relu', padding='same'))(char_embedding2) char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn) # char_macpool = Dropout(0.5)(char_macpool) # !!!!!!!!!!!!!! char_macpool = Dropout(0.25)(char_macpool) word_embedding = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth, mask_zero=False, trainable=True, weights=[source_W])(word_input) word_embedding_dropout = Dropout(0.5)(word_embedding) embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1) Dense1 = TimeDistributed(Dense(400, activation='tanh'))(embedding) Dense1 = Dropout(0.5)(Dense1) Dense2 = TimeDistributed(Dense(200, activation='tanh'))(Dense1) Dense2 = Dropout(0.3)(Dense2) Dense3 = TimeDistributed(Dense(100, activation='tanh'))(Dense2) Dense3 = Dropout(0.2)(Dense3) TimeD = TimeDistributed(Dense(targetvocabsize + 1))(Dense3) # TimeD = Dropout(0.5)(TimeD) # !!!!!!!!!!!!!!!delete dropout model = Activation('softmax')(TimeD) # crflayer = CRF(targetvocabsize+1, sparse_target=False) # model = crflayer(TimeD) Models = Model([word_input, char_input], [model]) Models.compile(loss=loss, optimizer=optimizers.RMSprop(lr=0.001), metrics=['acc']) # Models.compile(loss=crflayer.loss_function, optimizer=optimizers.RMSprop(lr=0.001), metrics=[crflayer.accuracy]) return Models
from keras.layers import Conv1D, MaxPooling1D, Activation, Dropout, Flatten, Dense from scipy.signal import butter, lfilter, boxcar from glob import glob from sklearn.preprocessing import StandardScaler from keras.layers import SimpleRNN import matplotlib.pyplot as plt from keras import layers from random import shuffle ############################################################## model = Sequential() model.add( Conv1D(10, kernel_size=4, strides=2, activation='tanh', input_shape=(42, 1))) model.add(Conv1D(10, kernel_size=4, strides=2, activation='tanh')) model.add(Flatten()) model.add(Dense(60, activation='tanh')) model.add(Dropout(0.5)) model.add(Dense(6, activation='sigmoid')) Adam = optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model.compile(optimizer='Adam', loss='binary_crossentropy',
def preact_block(x, filters, act_fn="relu"): x = BatchNormalization()(x) x = Activation(act_fn)(x) x = Conv1D(filters, 1, strides=1)(x) return x
callbacks=[metrics_lstm], verbose=0) lstm_model.summary() f1_lstm.append(max(metrics_lstm.val_f1s)) acc_lstm.append(history.history["val_acc"][metrics_lstm.ind]) f1_cnn = [] acc_cnn = [] for i in range(iterations): metrics_cnn = misc.Metrics("CNN", i) cnn_model = Sequential() cnn_model.add(Embedding(num_words + 2, EMBEDDING_SIZE)) cnn_model.add( Conv1D(HIDDEN_SIZE * 2, 6, activation='relu', use_bias=False)) cnn_model.add(MaxPooling1D()) cnn_model.add(Dropout(dropout)) cnn_model.add(Conv1D(200, 3, activation='relu', use_bias=False)) cnn_model.add(GlobalMaxPooling1D()) cnn_model.add(Dropout(dropout)) cnn_model.add(Dense(128, activation='relu')) cnn_model.add(Dropout(dropout)) cnn_model.add(Dense(1, activation='sigmoid')) cnn_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) history = cnn_model.fit(X_train, y_train, batch_size=BATCH_SIZE,
ytrain_rho_log_centered = np.log(ytrain_rho) - mean_train plt.hist(ytrain_rho_log_centered) plt.show() ytest_rho_log_centered = np.log(ytest_rho) - mean_test plt.hist(ytest_rho_log_centered) plt.show() print(mean_test, mean_train) #mean train is important, it was 4.78757605959 batch_size = 32 b1 = Sequential() b1.add( Conv1D(1250, kernel_size=2, activation='relu', input_shape=(xtest.shape[1], xtest.shape[2]))) b1.add(Conv1D(256, kernel_size=2, activation='relu')) b1.add(AveragePooling1D(pool_size=2)) b1.add(Dropout(0.25)) b1.add(Conv1D(256, kernel_size=2, activation='relu')) b1.add(AveragePooling1D(pool_size=2)) b1.add(Dropout(0.25)) b1.add(Flatten()) b2 = Sequential() b2.add(Dense(64, input_shape=(418, ), activation='relu')) b2.add(Dropout(0.1)) model = Sequential() #model.add([b1, b2])
def ds2_gru_model(input_dim=161, fc_size=1024, rnn_size=512, output_dim=29, initialization='glorot_uniform', conv_layers=1, gru_layers=1, use_conv=True): K.set_learning_phase(1) input_data = Input(shape=(None, input_dim), name='the_input') x = BatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)(input_data) if use_conv: #conv = ZeroPadding1D(padding=(0, 2048))(x) for l in range(conv_layers): x = Conv1D(filters=fc_size, name='conv_{}'.format(l + 1), kernel_size=11, padding='valid', activation='relu', strides=2)(x) else: for l in range(conv_layers): x = TimeDistributed( Dense(fc_size, name='fc_{}'.format(l + 1), activation='relu'))(x) # >>(?, time, fc_size) x = BatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)(x) for l in range(gru_layers): x = Bidirectional(GRU(rnn_size, name='fc_{}'.format(l + 1), return_sequences=True, activation='relu', kernel_initializer=initialization), merge_mode='sum')(x) x = BatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)(x) # Last Layer 5+6 Time Dist Dense Layer & Softmax x = TimeDistributed(Dense(fc_size, activation='relu'))(x) y_pred = TimeDistributed( Dense(output_dim, name="y_pred", activation="softmax"))(x) # labels = K.placeholder(name='the_labels', ndim=1, dtype='int32') labels = Input(name='the_labels', shape=[ None, ], dtype='int32') input_length = Input(name='input_length', shape=[1], dtype='int32') label_length = Input(name='label_length', shape=[1], dtype='int32') # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')([y_pred, labels, input_length, label_length]) model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out) return model
def create_fusionnet_layers(self, fusionnet, layers=None): """ Extend the FusionNet with more layers :param fusionnet: The FusionNet model :param layers: The layer structure for the added layers :return: The FusionNet model with the added layers """ # stack a deep densely-connected network on top if layers is None: return fusionnet # cnn # layers = [('dense_large', None, None), # ('reshape_expand_dim_4', None, None), # ('conv2d', 32, 'sigmoid'), # ('conv_max_pool_large', None, None), # ('dropout', 0.25, None), # ('flatten', None, None)] # lstm/gru # layers = [('dense_large', None, None), # ('reshape_expand_dim_3', None, None), # ('rnn', 32, 'sigmoid'), # ('dropout', 0.25, None)] for layer, size, activation in layers: # this needs to be preceeded by a dense layer being a multiple of 300 to work if layer == 'reshape_expand_dim_4': # fusionnet = Reshape((-1,keras.backend.get_variable_shape(fusionnet)[1]/700, 700))(fusionnet) fusionnet = Reshape((30, 10, -1))(fusionnet) if layer == 'reshape_expand_dim_3': # fusionnet = Reshape((-1,keras.backend.get_variable_shape(fusionnet)[1]/700, 700))(fusionnet) fusionnet = Reshape((30, -1))(fusionnet) elif layer == 'dense_large': fusionnet = Dense(900, activation='sigmoid', kernel_initializer=keras.initializers.glorot_uniform())(fusionnet) elif layer == 'dense_small': fusionnet = Dense(64, activation='sigmoid', kernel_initializer=keras.initializers.glorot_uniform())(fusionnet) elif layer == 'dense': fusionnet = Dense(size, activation=activation, kernel_initializer=keras.initializers.glorot_uniform())(fusionnet) elif layer == 'conv2d_large': fusionnet = Conv2D(64, (6, 6), activation='relu', kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'conv2d_small': fusionnet = Conv2D(32, (3, 3), activation='relu', kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'conv2d': fusionnet = Conv2D(size, (3, 3), activation=activation, kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'conv1d': fusionnet = Conv1D(size, 3, activation=activation, kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'conv2d_max_pool_large': fusionnet = MaxPooling2D((2, 2))(fusionnet) elif layer == 'conv2d_max_pool_small': fusionnet = MaxPooling2D((1, 1))(fusionnet) elif layer == 'conv1d_max_pool_large': fusionnet = MaxPooling1D(2)(fusionnet) elif layer == 'conv1d_max_pool_small': fusionnet = MaxPooling1D(1)(fusionnet) elif layer == 'conv2d_avg_pool_large': fusionnet = AveragePooling2D((2, 2))(fusionnet) elif layer == 'conv2d_avg_pool_small': fusionnet = AveragePooling2D((1, 1))(fusionnet) elif layer == 'conv1d_avg_pool_large': fusionnet = AveragePooling1D(2)(fusionnet) elif layer == 'conv1d_avg_pool_small': fusionnet = AveragePooling1D(1)(fusionnet) elif layer == 'lstm_large': fusionnet = LSTM(128, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'lstm_small': fusionnet = LSTM(32, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'lstm': fusionnet = LSTM(size, activation=activation, kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'lstm_sequence': fusionnet = LSTM(size, activation=activation, kernel_initializer='glorot_uniform', return_sequences=True)(fusionnet) elif layer == 'gru_large': fusionnet = GRU(128, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'gru_small': fusionnet = GRU(32, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'gru': fusionnet = GRU(size, activation=activation, kernel_initializer='glorot_uniform')(fusionnet) elif layer == 'dropout': fusionnet = Dropout(size)(fusionnet) # always flatten after conv elif layer == 'flatten': fusionnet = Flatten()(fusionnet) return fusionnet
y, test_size=0.2, shuffle=True, random_state=666) print(x_train.shape) print(x_test.shape) print(y_train) print(y_test) #2. 모델 # 2. 모델 model = Sequential() # model.add(LSTM(140, input_shape= (4, 1))) model.add(Conv1D(140, 2, padding='same', input_shape=(4, 1))) model.add(MaxPooling1D()) model.add(Conv1D(140, 2, padding='same')) model.add(Flatten()) model.add(Dense(60)) model.add(Dense(25)) model.add(Dense(1)) model.summary() # #3. 훈련, 컴파일 # from keras.callbacks import EarlyStopping # early_stopping = EarlyStopping(monitor = 'loss', patience=5, mode = 'auto') # model.compile(optimizer='adam', loss='mse', metrics= ['mse']) # model.fit(x_train, y_train, epochs=120, batch_size=1, verbose=1,
embedding_layer = Embedding(MAX_NB_WORDS+1,EMBEDDING_DIM,weights=[word_embedding_matrix],input_length=25,trainable=False) #词嵌入 sequence_1_input = Input(shape=(25,), dtype='int32') embed_1 = embedding_layer(sequence_1_input) sequence_2_input = Input(shape=(25,), dtype='int32') embed_2 = embedding_layer(sequence_2_input) #lstm lstm_layer_1 = LSTM(256,return_sequences=True) lstm_layer_2 = LSTM(256,return_sequences=True) q1 = lstm_layer(embed_1,lstm_layer_1,lstm_layer_2) q2 = lstm_layer(embed_2,lstm_layer_1,lstm_layer_2) #用类似TextCNN的思路构建不同卷积核的特征,两个句子共用同样的卷积层 kernel_size = [2,3,4,5] conv_concat = [] for kernel in kernel_size: conv = Conv1D(64,kernel_size=kernel,activation='relu',padding='same') q1_maxp,q1_meanp = conv_pool(conv,q1) q2_maxp,q2_meanp = conv_pool(conv,q2) mix = mix_layer(q1_maxp,q1_meanp,q2_maxp,q2_meanp) conv_concat.append(mix) conv = Concatenate()(conv_concat) #全连接层 merged = Dropout(0.3)(conv) merged = BatchNormalization()(merged) merged = Dense(512, activation='relu',name='dense_output')(merged) merged = Dropout(0.3)(merged) merged = BatchNormalization()(merged) merged = Dense(256, activation='relu',name='dense_output2')(merged) merged = Dropout(0.3)(merged) merged = BatchNormalization(name='bn_output')(merged) preds = Dense(1, activation='sigmoid')(merged)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) print('Build model...') model = Sequential() # we start off with an efficient embedding layer which maps # our vocab indices into embedding_dims dimensions model.add(Embedding(max_features, embedding_dims, input_length=maxlen)) model.add(Dropout(0.2)) # we add a Convolution1D, which will learn filters # word group filters of size filter_length: model.add( Conv1D(filters, kernel_size, padding='valid', activation='relu', strides=1)) # we use max pooling: model.add(GlobalMaxPooling1D()) # We add a vanilla hidden layer: model.add(Dense(hidden_dims)) model.add(Dropout(0.2)) model.add(Activation('relu')) # We project onto a single unit output layer, and squash it with a sigmoid: model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(train_inputs.shape) # Modeling train_targets = np.array(train_targets) lsvc = LinearSVC(C=0.01, penalty="l1", dual=False).fit(train_inputs, train_targets) sel_model = SelectFromModel(lsvc, prefit=True) train_inputs = sel_model.transform(train_inputs) print(train_inputs.shape) train_inputs = train_inputs.reshape(train_inputs.shape[0], train_inputs.shape[1], 1) #train_inputs = BasicDataProcess.TransposeElements(train_inputs) train_targets = to_categorical(train_targets) #create model model = Sequential() #add model layers model.add(Conv1D(128, kernel_size=4, activation='relu')) model.add(Conv1D(64, kernel_size=4, activation='relu')) model.add(Conv1D(32, kernel_size=4, activation='relu')) model.add(Conv1D(16, kernel_size=4, activation='relu')) model.add(Flatten()) model.add(Dense(2, activation='softmax')) #compile model using accuracy to measure model performance model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.fit(train_inputs, train_targets, epochs=30) # Prediction test_events = BasicDataProcess.LoadDataFromFile( data_dir + "/Test/testEvents.txt") test_data = BasicDataProcess.LoadEEGFromFile(data_dir, False)
MAX_VOCAB_SIZE + 1, VECTOR_DIM, weights=[embeddings], # trainable=False ) input_ = Input(shape=(max_review_length, )) cnn1 = Sequential() cnn1.add( Embedding(MAX_VOCAB_SIZE + 1, VECTOR_DIM, weights=[embeddings], input_length=max_review_length)) cnn1.add(Conv1D(50, 5)) cnn1.add(Flatten()) cnn1.add(Dense(100, activation='relu')) cnn1.add(Dropout(0.2)) cnn1.add(Dense(max(varietal_list_o) + 1, activation='softmax')) cnn1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) history = cnn1.fit(train_x, train_y, epochs=1000, validation_split=0.1, batch_size=64) y_score = cnn1.predict(test_x)
y_test_binary = keras.utils.to_categorical(y_test, num_classes) x_train = x_train.reshape(75, 1000,1) x_test = x_test.reshape(25, 1000,1) from keras.models import Sequential import keras from keras.models import Sequential from keras.layers import Dense, Flatten, Conv1D from keras.callbacks import ModelCheckpoint from keras.models import model_from_json from keras import backend as K model = Sequential() model.add(Conv1D(32, (3), input_shape=(1000,1), activation='relu')) model.add(Flatten()) model.add(Dense(64, activation='softmax')) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) #model.summary() batch_size = 16 epochs = 10 model.fit(x_train, y_train_binary, batch_size=batch_size,
embedding_matrix[i] = embedding_vector # load pre-trained word embeddings into an Embedding layer # note that we set trainable = False so as to keep the embeddings fixed embedding_layer = Embedding(num_words, EMBEDDING_DIM, embeddings_initializer=Constant(embedding_matrix), input_length=MAX_SEQUENCE_LENGTH, trainable=False) print('Training model.') # train a 1D convnet with global maxpooling sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32') embedded_sequences = embedding_layer(sequence_input) x = Conv1D(128, 5, activation='relu')(embedded_sequences) x = MaxPooling1D(5)(x) x = Conv1D(128, 5, activation='relu')(x) x = MaxPooling1D(5)(x) x = Conv1D(128, 5, activation='relu')(x) x = GlobalMaxPooling1D()(x) x = Dense(128, activation='relu')(x) preds = Dense(len(labels_index), activation='softmax')(x) model = Model(sequence_input, preds) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc']) model.fit(x_train, y_train,
if word in word2emb.keys(): embedding_weights[index] = word2emb[word] else: embedding_weights[index] = unk_vec # train X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3) model = Sequential() model.add( Embedding(embedding_weights.shape[0], embedding_weights.shape[1], weights=[embedding_weights], input_length=max_len, trainable=True)) model.add(SpatialDropout1D(0.2)) model.add(Conv1D(256, 3, activation='relu')) model.add(GlobalAveragePooling1D()) model.add(Dense(2, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() early_stopping = EarlyStopping(monitor='val_loss', patience=20, verbose=0, mode='auto') checkpoint = ModelCheckpoint(filepath='model/glove_w2v.model', save_best_only=True) # tensorboard --logdir=*/log/ tensorboard = TensorBoard(log_dir='log/', write_graph=False)
i = 0 y_valid = np.zeros((len(label_valid), max(label_valid) + 1)) for x in label_valid: y_valid[i][x] = 1 i = i + 1 filter_sizes = [3, 4, 5] sequence_length = count embedding_vecor_length = 300 visible = Input(shape=(sequence_length, 300), dtype='float32') print(visible.shape) e = visible conv_0 = Conv1D(64, kernel_size, padding='valid', strides=1, activation=newacti)(e) maxpool_0 = MaxPooling1D(pool_size=pool_size)(conv_0) gru = Reshape((sequence_length, embedding_vecor_length))(e) gru = GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru) gru = GRU(gru_output_size)(gru) gru = Reshape((1, 1, gru_output_size))(gru) merge = maximum([maxpool_0, gru]) flatten = Flatten()(merge) dropout = Dropout(0.5)(flatten)
from keras.models import Sequential from keras.layers import Dense, Dropout from keras.layers import Embedding from keras.layers import Conv1D, GlobalAveragePooling1D, MaxPooling1D model = Sequential() model.add(Conv1D(64, 3, activation='relu', input_shape=(seq_length, 100))) model.add(Conv1D(64, 3, activation='relu')) model.add(MaxPooling1D(3)) model.add(Conv1D(128, 3, activation='relu')) model.add(Conv1D(128, 3, activation='relu')) model.add(GlobalAveragePooling1D()) model.add(Dropout(0.5)) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) model.fit(x_train, y_train, batch_size=16, epochs=10) score = model.evaluate(x_test, y_test, batch_size=16)