def Model_BiLSTM_CnnDecoder(sourcevocabsize,
targetvocabsize,
source_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim,
emd_dim,
sourcecharsize,
character_W,
input_word_length,
char_emd_dim,
sourcepossize,
pos_W,
pos_emd_dim,
batch_size=32,
loss='categorical_crossentropy',
optimizer='rmsprop'):
# 0.8349149507609669--attention,lstm*2decoder
# pos_input = Input(shape=(input_seq_lenth,), dtype='int32')
# pos_embeding = Embedding(input_dim=sourcepossize + 1,
# output_dim=pos_emd_dim,
# input_length=input_seq_lenth,
# mask_zero=False,
# trainable=True,
# weights=[pos_W])(pos_input)
word_input = Input(shape=(input_seq_lenth, ), dtype='int32')
char_input = Input(shape=(
input_seq_lenth,
input_word_length,
),
dtype='int32')
char_embedding = Embedding(input_dim=sourcecharsize,
output_dim=char_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
input_word_length),
mask_zero=False,
trainable=True,
weights=[character_W])
char_embedding2 = TimeDistributed(char_embedding)(char_input)
char_cnn = TimeDistributed(
Conv1D(50, 3, activation='relu', border_mode='valid'))(char_embedding2)
char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn)
# char_macpool = Dropout(0.5)(char_macpool)
pos_input = Input(shape=(
input_seq_lenth,
3,
), dtype='int32')
pos_embedding = Embedding(input_dim=sourcepossize + 1,
output_dim=pos_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
3),
mask_zero=False,
trainable=True,
weights=[pos_W])
pos_embedding2 = TimeDistributed(pos_embedding)(pos_input)
pos_cnn = TimeDistributed(
Conv1D(20, 2, activation='relu', border_mode='valid'))(pos_embedding2)
pos_macpool = TimeDistributed(GlobalMaxPooling1D())(pos_cnn)
word_embedding_RNN = Embedding(input_dim=sourcevocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=False,
weights=[source_W])(word_input)
# word_embedding_RNN = Dropout(0.5)(word_embedding_RNN)
embedding = concatenate([word_embedding_RNN, char_macpool, pos_macpool],
axis=-1)
embedding = Dropout(0.5)(embedding)
BiLSTM = Bidirectional(LSTM(int(hidden_dim / 2), return_sequences=True),
merge_mode='concat')(embedding)
BiLSTM = BatchNormalization()(BiLSTM)
# BiLSTM = Dropout(0.3)(BiLSTM)
# decodelayer1 = LSTM(50, return_sequences=False, go_backwards=True)(concat_LC_d)#!!!!!
# repeat_decodelayer1 = RepeatVector(input_seq_lenth)(decodelayer1)
# concat_decoder = concatenate([concat_LC_d, repeat_decodelayer1], axis=-1)#!!!!
# decodelayer2 = LSTM(hidden_dim, return_sequences=True)(concat_decoder)
# decodelayer = Dropout(0.5)(decodelayer2)
# decoderlayer1 = LSTM(50, return_sequences=True, go_backwards=False)(BiLSTM)
decoderlayer5 = Conv1D(50, 5, activation='relu', strides=1,
padding='same')(BiLSTM)
decoderlayer2 = Conv1D(50, 2, activation='relu', strides=1,
padding='same')(BiLSTM)
decoderlayer3 = Conv1D(50, 3, activation='relu', strides=1,
padding='same')(BiLSTM)
decoderlayer4 = Conv1D(50, 4, activation='relu', strides=1,
padding='same')(BiLSTM)
# 0.8868111121100423
decodelayer = concatenate(
[decoderlayer2, decoderlayer3, decoderlayer4, decoderlayer5], axis=-1)
decodelayer = BatchNormalization()(decodelayer)
decodelayer = Dropout(0.5)(decodelayer)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(decodelayer)
# TimeD = Dropout(0.5)(TimeD)
model = Activation('softmax')(TimeD) # 0.8769744561783556
# crf = CRF(targetvocabsize + 1, sparse_target=False)
# model = crf(TimeD)
Models = Model([word_input, char_input, pos_input], model)
# Models.compile(loss=my_cross_entropy_Weight, optimizer='adam', metrics=['acc'])
Models.compile(loss=loss, optimizer='adam', metrics=['acc'])
# Models.compile(loss=loss, optimizer='adam', metrics=['acc'], sample_weight_mode="temporal")
# Models.compile(loss=loss, optimizer=optimizers.RMSprop(lr=0.01), metrics=['acc'])
# Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])
# Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.005), metrics=[crf.accuracy])
return Models
def compile(self,
tokenizer,
glove_dir='./data/',
embedding_dim=50,
dropout_fraction=0.0,
kernal_size=5,
n_filters=128):
"""Compile network model for classifier
Args:
glove_file (str): Location of GloVe file
embedding_dim (int): Size of embedding vector
tokenizer (WordTokenizer): Object used to tokenize orginal texts
dropout_fraction (float): Fraction of randomly zeroed weights in dropout layer
kernal_size (int): Size of sliding window for convolution
n_filters (int): Number of filters to produce from convolution
"""
# Load embedding layer
print('Loading GloVe embedding...')
embeddings_index = {}
f = open(
os.path.join(glove_dir,
'glove.6B.' + str(embedding_dim) + 'd.txt'), 'rb')
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
print(('Found %s word vectors.' % len(embeddings_index)))
# Create embedding layer
print('Creating embedding layer...')
embedding_matrix = np.zeros(
(len(tokenizer.tokenizer.word_index) + 1, embedding_dim))
for word, i in list(tokenizer.tokenizer.word_index.items()):
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(len(tokenizer.tokenizer.word_index) + 1,
embedding_dim,
weights=[embedding_matrix],
input_length=tokenizer.max_sequence_length,
trainable=False)
# Create network
print('Creating network...')
sequence_input = Input(shape=(tokenizer.max_sequence_length, ),
dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Dropout(dropout_fraction)(embedded_sequences)
x = Conv1D(n_filters, kernal_size, activation='relu')(x)
x = MaxPooling1D(kernal_size)(x)
x = Conv1D(n_filters, kernal_size, activation='relu')(x)
x = MaxPooling1D(kernal_size)(x)
x = Conv1D(n_filters, kernal_size, activation='relu')(x)
x = MaxPooling1D(int(x.shape[1]))(x) # global max pooling
x = Flatten()(x)
x = Dense(n_filters, activation='relu')(x)
preds = Dense(len(self.category_map), activation='softmax')(x)
# Compile model
print('Compiling network...')
self.model = Model(sequence_input, preds)
self.model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
def get_model(embedding_layer,RNN,embed_size,Feature_dic,Para_dic):
MAX_SEQUENCE_LENGTH=Para_dic['MAX_SEQUENCE_LENGTH']
comment_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences_raw= embedding_layer(comment_input)
embedded_sequences = SpatialDropout1D(Para_dic['spatial_dropout'])(embedded_sequences_raw)
### RNN
if RNN=='LSTM':
RNN_x = Bidirectional(CuDNNLSTM(Para_dic['num_lstm'],return_sequences=True))(embedded_sequences)
elif RNN=='GRU':
RNN_x = Bidirectional(CuDNNGRU(Para_dic['num_lstm'],return_sequences=True))(embedded_sequences)
Feature=[]
######## RNN Features
##### Attention
if Feature_dic['Attention']==1:
Feature.append(Attention(MAX_SEQUENCE_LENGTH)(RNN_x))
if Feature_dic['RNN_maxpool']==1:
Feature.append(GlobalMaxPooling1D()(RNN_x))
##### Capsule
if Feature_dic['Capsule']==1:
capsule = Capsule(share_weights=True)(RNN_x)
capsule = Flatten()(capsule)
Feature.append(capsule)
##### RNN_CNNN1d
if Feature_dic['RNN_CNN_conv1d']==1:
Cx = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform")(RNN_x)
avg_pool = GlobalAveragePooling1D()(Cx)
max_pool = GlobalMaxPooling1D()(Cx)
Feature.append(avg_pool)
Feature.append(max_pool)
######## CNN Features
### CNN2d
if Feature_dic['CNN2d']==1:
CNN2d=get_CNN2d(embedded_sequences,embed_size,MAX_SEQUENCE_LENGTH,Para_dic)
Feature.append(CNN2d)
### DPCNN
if Feature_dic['DPCNN']==1:
DPCNN=get_DPCNN(embedded_sequences,Para_dic)
Feature.append(DPCNN)
### Concatnation
merged = Concatenate()(Feature)
### dense, add L1 reg to enable sparsity
merged = Dense(Para_dic['dense_num'], \
activation=Para_dic['dense_act'],\
kernel_regularizer=regularizers.l1(Para_dic['L1_reg']))(merged)
merged = Dropout(Para_dic['dense_dropout'])(merged)
preds = Dense(6, activation='sigmoid')(merged)
model = Model(inputs=[comment_input], outputs=preds)
model.compile(loss='binary_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
print(model.summary())
return model
print(
'Accuracy is %2.2f%%, Precision is %2.2f%%, Recall is %2.2f%%, F-measure is %f\n'
% (acc * 100, prec * 100, rec * 100, fmes))
#%% main
if __name__ == "__main__":
X_train, Y_train, X_val, Y_val = val_split(*get_data('ExoTrain.csv'), 0.7)
X_test = get_data('Final Test.csv', test=True)
#%% create model
model = Sequential()
model.add(
Conv1D(filters=32,
kernel_size=5,
activation='relu',
input_shape=X_train.shape[1:]))
model.add(MaxPool1D(pool_size=2, strides=2))
model.add(Dropout(0.25))
model.add(Conv1D(filters=32, kernel_size=5, activation='relu'))
model.add(MaxPool1D(pool_size=2, strides=2))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(64, activation='sigmoid'))
model.add(Dense(1, activation='sigmoid'))
#%% compile the model
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Activation
from keras.layers import LSTM
from keras.layers import Input
from keras.layers import Bidirectional
from keras.layers import TimeDistributed
from keras.layers import RepeatVector
from keras.models import Model
import numpy as np
from sklearn.model_selection import train_test_split
avg_pt = 208
no_classes = 7
inputs = Input(shape=(avg_pt, 2))
conv1 = Conv1D(8, 2, activation='relu', padding='same')(inputs)
conv2 = Conv1D(16, 2, activation='relu', padding='same')(conv1)
lstm1 = LSTM(50)(conv2)
lstm2 = RepeatVector(avg_pt)(lstm1)
lstm3 = LSTM(50)(lstm2)
dense = Dense(no_classes, activation='softmax')(lstm3)
model = Model(inputs, dense)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
model.load_weights("model/Animals/model_convLstm.h5")
app = Flask(__name__)
def get_resNet_model(input_shape, output_shape):
def resnet_v1(input_shape,
depth,
num_classes=10,
input_tensor=None,
local_conv=False):
if (depth - 2) % 6 != 0:
raise ValueError('depth should be 6n+2 (eg 20, 32, 44 in [a])')
# Start model definition.
num_filters = 16
num_res_blocks = int((depth - 2) / 6)
if (input_tensor == None):
inputs = Input(shape=input_shape)
else:
inputs = input_tensor
x = resnet_layer_naive(inputs=inputs)
# Instantiate the stack of residual units
for stack in range(3):
for res_block in range(num_res_blocks):
strides = 1
# if stack > 0 and res_block == 0: # first layer but not first stack
# strides = 2 # downsample
y = resnet_layer_local(inputs=x,
kernel_size=8,
num_filters=num_filters,
strides=strides)
y = resnet_layer_local(inputs=y,
kernel_size=16,
num_filters=num_filters,
activation=None)
if stack > 0 and res_block == 0: # first layer but not first stack
# linear projection residual shortcut connection to match
# changed dims
x = resnet_layer_naive(inputs=x,
num_filters=num_filters,
kernel_size=16,
strides=strides,
activation=None,
batch_normalization=True)
x = keras.layers.add([x, y])
x = Activation(default_activation)(x)
num_filters *= 2
return x
inputs = Input(shape=input_shape)
xxx = inputs
xxx = Conv1D(filters=xl_filter_num,
kernel_size=m_filter_num,
padding='same',
activation=None,
strides=1)(xxx)
xxx = BatchNormalization()(xxx)
xxx = Activation('relu')(xxx)
xxx = MaxPooling1D(pool_size=2, padding='same', strides=2)(xxx)
xxx = resnet_v1(input_shape,
num_classes=output_shape,
depth=3 * 6 + 2,
input_tensor=xxx,
local_conv=False)
xxx = LocallyConnected1D(filters=l_filter_num,
kernel_size=m_filter_num,
padding='valid',
activation=default_activation,
strides=1)(xxx)
xxx = BatchNormalization()(xxx)
xxx = GlobalMaxPooling1D()(xxx)
xxx = Dense(output_shape,
activation='softmax',
kernel_initializer='he_normal')(xxx)
model = Model(inputs=inputs, outputs=xxx)
return model
from keras.preprocessing.sequence import pad_sequences
pad_x = pad_sequences(x, padding='pre',
value=0) # Defalut (pre와 0 : 그만큼 이 조합이 많이 쓰인 다는 것)
print(pad_x)
print(pad_x.shape) # (12, 5)
# pad_x = pad_x.reshape(-1, 5, 1)
print(pad_x.shape) # (12, 5, 1)
word_size = len(token.word_index) + 1
print('전체 토큰 사이즈 :', word_size) # 25 (전체 단어의 개수)
### 모델
from keras.models import Sequential
from keras.layers import Dense, Embedding, Flatten, LSTM, Conv1D
model = Sequential()
model.add(Embedding(25, 10, input_length=5))
model.add(Conv1D(5, 2))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))
model.summary()
### 실행, 훈련, 평가
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
model.fit(pad_x, labels, epochs=30)
acc = model.evaluate(pad_x, labels)[1] # loss가 아닌 metrics 값을 빼겠다.
print('\n acc : %.4f' % acc)
def get_model_rnn_cnn(embedding_matrix,
cell_size=80,
cell_type_GRU=True,
maxlen=180,
max_features=100000,
embed_size=300,
prob_dropout=0.2,
emb_train=False,
filter_size=128,
kernel_size=2,
stride=1):
inp_pre = Input(shape=(maxlen, ), name='input_pre')
inp_post = Input(shape=(maxlen, ), name='input_post')
##pre
x1 = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=emb_train)(inp_pre)
x1 = SpatialDropout1D(prob_dropout)(x1)
if cell_type_GRU:
x1 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1)
else:
x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1)
x1 = Conv1D(filter_size,
kernel_size=kernel_size,
strides=stride,
padding="valid",
kernel_initializer="he_uniform")(x1)
avg_pool1 = GlobalAveragePooling1D()(x1)
max_pool1 = GlobalMaxPooling1D()(x1)
##post
x2 = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=emb_train)(inp_post)
x2 = SpatialDropout1D(prob_dropout)(x2)
if cell_type_GRU:
x2 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2)
else:
x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2)
x2 = Conv1D(filter_size,
kernel_size=kernel_size,
strides=stride,
padding="valid",
kernel_initializer="he_uniform")(x2)
avg_pool2 = GlobalAveragePooling1D()(x2)
max_pool2 = GlobalMaxPooling1D()(x2)
##merge
conc = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
outp = Dense(6, activation="sigmoid")(conc)
model = Model(inputs=[inp_pre, inp_post], outputs=outp)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['binary_crossentropy', 'accuracy'])
return model
def get_model_2rnn_cnn(embedding_matrix,
cell_size=80,
cell_type_GRU=True,
maxlen=180,
max_features=100000,
embed_size=300,
prob_dropout=0.2,
emb_train=False,
filter_size=128,
kernel_size=2,
stride=1):
inp_pre = Input(shape=(maxlen, ), name='input_pre')
inp_post = Input(shape=(maxlen, ), name='input_post')
##pre
x1 = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=emb_train)(inp_pre)
x1 = SpatialDropout1D(prob_dropout)(x1)
if cell_type_GRU:
x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1)
x1 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1)
else:
x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1)
x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1)
x1 = Conv1D(filter_size,
kernel_size=kernel_size,
strides=stride,
padding="valid",
kernel_initializer="he_uniform")(x1)
avg_pool1 = GlobalAveragePooling1D()(x1)
max_pool1 = GlobalMaxPooling1D()(x1)
##post
x2 = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=emb_train)(inp_post)
x2 = SpatialDropout1D(prob_dropout)(x2)
if cell_type_GRU:
x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2)
x2 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2)
else:
x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2)
x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2)
x2 = Conv1D(filter_size,
kernel_size=kernel_size,
strides=stride,
padding="valid",
kernel_initializer="he_uniform")(x2)
avg_pool2 = GlobalAveragePooling1D()(x2)
max_pool2 = GlobalMaxPooling1D()(x2)
##merge
conc = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
outp = Dense(6, activation="sigmoid")(conc)
model = Model(inputs=[inp_pre, inp_post], outputs=outp)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['binary_crossentropy', 'accuracy'])
return model
# def get_model_2rnn_cnn_sp(
# embedding_matrix, cell_size = 80, cell_type_GRU = True,
# maxlen = 180, max_features = 100000, embed_size = 300,
# prob_dropout = 0.2, emb_train = False,
# filter_size=128, kernel_size = 2, stride = 1
# ):
# inp_pre = Input(shape=(maxlen, ), name='input_pre')
# inp_post = Input(shape=(maxlen, ), name='input_post')
# ##pre
# x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_pre)
# x1 = SpatialDropout1D(prob_dropout)(x1)
# if cell_type_GRU:
# x1_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1)
# x1 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1_)
# else :
# x1_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1)
# x1 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1_)
# x1_ = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_)
# avg_pool1_ = GlobalAveragePooling1D()(x1_)
# max_pool1_ = GlobalMaxPooling1D()(x1_)
# x1 = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1)
# avg_pool1 = GlobalAveragePooling1D()(x1)
# max_pool1 = GlobalMaxPooling1D()(x1)
# ##post
# x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_post)
# x2 = SpatialDropout1D(prob_dropout)(x2)
# if cell_type_GRU:
# x2_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2)
# x2 = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2_)
# else :
# x2_ = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2)
# x2 = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2_)
# x2_ = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_)
# avg_pool2_ = GlobalAveragePooling1D()(x2_)
# max_pool2_ = GlobalMaxPooling1D()(x2_)
# x2 = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2)
# avg_pool2 = GlobalAveragePooling1D()(x2)
# max_pool2 = GlobalMaxPooling1D()(x2)
# ##merge
# conc = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2, avg_pool1_, max_pool1_, avg_pool2_, max_pool2_])
# outp = Dense(6, activation="sigmoid")(conc)
# model = Model(inputs=[inp_pre, inp_post], outputs=outp)
# model.compile(loss='binary_crossentropy',
# optimizer='adam',
# metrics=['binary_crossentropy', 'accuracy'])
# return model
# def get_model_dual_2rnn_cnn_sp(
# embedding_matrix, cell_size = 80, cell_type_GRU = True,
# maxlen = 180, max_features = 100000, embed_size = 300,
# prob_dropout = 0.2, emb_train = False,
# filter_size=128, kernel_size = 2, stride = 1
# ):
# inp_pre = Input(shape=(maxlen, ), name='input_pre')
# inp_post = Input(shape=(maxlen, ), name='input_post')
# ##pre
# x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_pre)
# x1g = SpatialDropout1D(prob_dropout)(x1)
# x1l = SpatialDropout1D(prob_dropout)(x1)
# x1_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1g)
# x1g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1_g)
# x1_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1l)
# x1l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1_l)
# x1_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_g)
# x1_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_l)
# avg_pool1_g = GlobalAveragePooling1D()(x1_g)
# max_pool1_g = GlobalMaxPooling1D()(x1_g)
# avg_pool1_l = GlobalAveragePooling1D()(x1_l)
# max_pool1_l = GlobalMaxPooling1D()(x1_l)
# x1g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1g)
# x1l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1l)
# avg_pool1g = GlobalAveragePooling1D()(x1g)
# max_pool1g = GlobalMaxPooling1D()(x1g)
# avg_pool1l = GlobalAveragePooling1D()(x1l)
# max_pool1l = GlobalMaxPooling1D()(x1l)
# ##post
# x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_post)
# x2g = SpatialDropout1D(prob_dropout)(x2)
# x2l = SpatialDropout1D(prob_dropout)(x2)
# x2_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2g)
# x2g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2_g)
# x2_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2l)
# x2l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2_l)
# x2_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_g)
# x2_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_l)
# avg_pool2_g = GlobalAveragePooling1D()(x2_g)
# max_pool2_g = GlobalMaxPooling1D()(x2_g)
# avg_pool2_l = GlobalAveragePooling1D()(x2_l)
# max_pool2_l = GlobalMaxPooling1D()(x2_l)
# x2g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2g)
# x2l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2l)
# avg_pool2g = GlobalAveragePooling1D()(x2g)
# max_pool2g = GlobalMaxPooling1D()(x2g)
# avg_pool2l = GlobalAveragePooling1D()(x2l)
# max_pool2l = GlobalMaxPooling1D()(x2l)
# ##merge
# conc = concatenate([avg_pool1g, max_pool1g, avg_pool1l, max_pool1l, avg_pool1_g, max_pool1_g, avg_pool1_l, max_pool1_l,
# avg_pool2g, max_pool2g, avg_pool2l, max_pool2l, avg_pool2_g, max_pool2_g, avg_pool2_l, max_pool2_l])
# outp = Dense(6, activation="sigmoid")(conc)
# model = Model(inputs=[inp_pre, inp_post], outputs=outp)
# model.compile(loss='binary_crossentropy',
# optimizer='adam',
# metrics=['binary_crossentropy', 'accuracy'])
# return model
# def get_model_dual_2rnn_cnn_sp_drop(
# embedding_matrix, cell_size = 80, cell_type_GRU = True,
# maxlen = 180, max_features = 100000, embed_size = 300,
# prob_dropout = 0.2, emb_train = False,
# filter_size=128, kernel_size = 2, stride = 1
# ):
# inp_pre = Input(shape=(maxlen, ), name='input_pre')
# inp_post = Input(shape=(maxlen, ), name='input_post')
# ##pre
# x1 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_pre)
# x1g = SpatialDropout1D(prob_dropout)(x1)
# x1l = SpatialDropout1D(prob_dropout)(x1)
# x1_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1g)
# x1g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x1_g)
# x1_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1l)
# x1l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x1_l)
# x1_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_g)
# x1_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1_l)
# avg_pool1_g = GlobalAveragePooling1D()(x1_g)
# max_pool1_g = GlobalMaxPooling1D()(x1_g)
# avg_pool1_l = GlobalAveragePooling1D()(x1_l)
# max_pool1_l = GlobalMaxPooling1D()(x1_l)
# x1g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1g)
# x1l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x1l)
# avg_pool1g = GlobalAveragePooling1D()(x1g)
# max_pool1g = GlobalMaxPooling1D()(x1g)
# avg_pool1l = GlobalAveragePooling1D()(x1l)
# max_pool1l = GlobalMaxPooling1D()(x1l)
# ##post
# x2 = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable = emb_train)(inp_post)
# x2g = SpatialDropout1D(prob_dropout)(x2)
# x2l = SpatialDropout1D(prob_dropout)(x2)
# x2_g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2g)
# x2g = Bidirectional(CuDNNGRU(cell_size, return_sequences=True))(x2_g)
# x2_l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2l)
# x2l = Bidirectional(CuDNNLSTM(cell_size, return_sequences=True))(x2_l)
# x2_g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_g)
# x2_l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2_l)
# avg_pool2_g = GlobalAveragePooling1D()(x2_g)
# max_pool2_g = GlobalMaxPooling1D()(x2_g)
# avg_pool2_l = GlobalAveragePooling1D()(x2_l)
# max_pool2_l = GlobalMaxPooling1D()(x2_l)
# x2g = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2g)
# x2l = Conv1D(filter_size, kernel_size = kernel_size, strides=stride, padding = "valid", kernel_initializer = "he_uniform")(x2l)
# avg_pool2g = GlobalAveragePooling1D()(x2g)
# max_pool2g = GlobalMaxPooling1D()(x2g)
# avg_pool2l = GlobalAveragePooling1D()(x2l)
# max_pool2l = GlobalMaxPooling1D()(x2l)
# ##merge
# conc = concatenate([avg_pool1g, max_pool1g, avg_pool1l, max_pool1l, avg_pool1_g, max_pool1_g, avg_pool1_l, max_pool1_l,
# avg_pool2g, max_pool2g, avg_pool2l, max_pool2l, avg_pool2_g, max_pool2_g, avg_pool2_l, max_pool2_l])
# conc = SpatialDropout1D(prob_dropout)(conc)
# outp = Dense(6, activation="sigmoid")(conc)
# model = Model(inputs=[inp_pre, inp_post], outputs=outp)
# model.compile(loss='binary_crossentropy',
# optimizer='adam',
# metrics=['binary_crossentropy', 'accuracy'])
# return model
# def get_model_dpcnn(
# embedding_matrix, cell_size = 80, cell_type_GRU = True,
# maxlen = 180, max_features = 100000, embed_size = 300,
# prob_dropout = 0.2, emb_train = False,
# filter_nr=128, filter_size = 2, stride = 1,
# max_pool_size = 3, max_pool_strides = 2, dense_nr = 256,
# spatial_dropout = 0.2, dense_dropout = 0.5,
# conv_kern_reg = regularizers.l2(0.00001), conv_bias_reg = regularizers.l2(0.00001)
# ):
# comment = Input(shape=(maxlen,))
# emb_comment = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=emb_train)(comment)
# emb_comment = SpatialDropout1D(spatial_dropout)(emb_comment)
# block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(emb_comment)
# block1 = BatchNormalization()(block1)
# block1 = PReLU()(block1)
# block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block1)
# block1 = BatchNormalization()(block1)
# block1 = PReLU()(block1)
# #we pass embedded comment through conv1d with filter size 1 because it needs to have the same shape as block output
# #if you choose filter_nr = embed_size (300 in this case) you don't have to do this part and can add emb_comment directly to block1_output
# resize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(emb_comment)
# resize_emb = PReLU()(resize_emb)
# block1_output = add([block1, resize_emb])
# block1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block1_output)
# block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block1_output)
# block2 = BatchNormalization()(block2)
# block2 = PReLU()(block2)
# block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block2)
# block2 = BatchNormalization()(block2)
# block2 = PReLU()(block2)
# block2_output = add([block2, block1_output])
# block2_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block2_output)
# block3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block2_output)
# block3 = BatchNormalization()(block3)
# block3 = PReLU()(block3)
# block3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block3)
# block3 = BatchNormalization()(block3)
# block3 = PReLU()(block3)
# block3_output = add([block3, block2_output])
# block3_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block3_output)
# block4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block3_output)
# block4 = BatchNormalization()(block4)
# block4 = PReLU()(block4)
# block4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block4)
# block4 = BatchNormalization()(block4)
# block4 = PReLU()(block4)
# block4_output = add([block4, block3_output])
# block4_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block4_output)
# block5 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block4_output)
# block5 = BatchNormalization()(block5)
# block5 = PReLU()(block5)
# block5 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block5)
# block5 = BatchNormalization()(block5)
# block5 = PReLU()(block5)
# block5_output = add([block5, block4_output])
# block5_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block5_output)
# block6 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block5_output)
# block6 = BatchNormalization()(block6)
# block6 = PReLU()(block6)
# block6 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block6)
# block6 = BatchNormalization()(block6)
# block6 = PReLU()(block6)
# block6_output = add([block6, block5_output])
# block6_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block6_output)
# block7 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block6_output)
# block7 = BatchNormalization()(block7)
# block7 = PReLU()(block7)
# block7 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear',
# kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block7)
# block7 = BatchNormalization()(block7)
# block7 = PReLU()(block7)
# block7_output = add([block7, block6_output])
# output = GlobalMaxPooling1D()(block7_output)
# output = Dense(dense_nr, activation='linear')(output)
# output = BatchNormalization()(output)
# output = PReLU()(output)
# output = Dropout(dense_dropout)(output)
# output = Dense(6, activation='sigmoid')(output)
# model = Model(comment, output)
# model.compile(loss='binary_crossentropy',
# optimizer='adam',
# metrics=['accuracy'])
# return model
y = encode(Label_1)
################################ Main ####################################
cvscores = []
cnt = 0
kfold = StratifiedKFold(n_splits=K, shuffle=True, random_state=1)
for train, test in kfold.split(x, decode(y)):
cnt = cnt + 1
print(cnt)
cnn = Sequential()
print(x.shape[1], 1)
print('x_train shape:', x[train].shape)
cnn = Sequential()
cnn.add(Reshape((x.shape[1], 1), input_shape=(x.shape[1], )))
cnn.add(Conv1D(32, 3, activation='relu', padding='same'))
cnn.add(MaxPooling1D(3))
cnn.add(Conv1D(64, 3, activation='relu', padding='same'))
cnn.add(MaxPooling1D(3))
cnn.add(Conv1D(128, 3, activation='relu', padding='same'))
cnn.add(MaxPooling1D(3))
cnn.add(Conv1D(128, 3, activation='relu', padding='same'))
cnn.add(MaxPooling1D(3))
cnn.add(Dropout(0.25))
cnn.add(Flatten())
cnn.add(Dense(256, activation=inner_activation_fun))
def produce_model(self):
input_tweet = Input(shape=(self.num_tweets, 30, 300), name='in_tweet')
hashtag = Input(shape=(self.num_tweets, 200), name='hashtags')
handle = Input(shape=(self.num_tweets, 200), name='handles')
prev_info = Input(shape=(500, 2), name='history')
# Convolution Architecture for tweets
k1_raw = TimeDistributed(Conv1D(filters=self.shapes[3], kernel_size=1, padding='same', name='K1'))(input_tweet)
k2_raw = TimeDistributed(Conv1D(filters=self.shapes[3], kernel_size=2, padding='same', name='K2'))(input_tweet)
k3_raw = TimeDistributed(Conv1D(filters=self.shapes[3], kernel_size=3, padding='same', name='K3'))(input_tweet)
k4_raw = TimeDistributed(Conv1D(filters=self.shapes[4], kernel_size=4, padding='same', name='K4'))(input_tweet)
k5_raw = TimeDistributed(Conv1D(filters=self.shapes[5], kernel_size=5, padding='same', name='K5'))(input_tweet)
k6_raw = TimeDistributed(Conv1D(filters=self.shapes[6], kernel_size=6, padding='same', name='K6'))(input_tweet)
k8_raw = TimeDistributed(Conv1D(filters=self.shapes[8], kernel_size=8, padding='same', name='K8'))(input_tweet)
k1_max = TimeDistributed(GlobalMaxPool1D())(k1_raw)
k2_max = TimeDistributed(GlobalMaxPool1D())(k2_raw)
k3_max = TimeDistributed(GlobalMaxPool1D())(k3_raw)
k4_max = TimeDistributed(GlobalMaxPool1D())(k4_raw)
k5_max = TimeDistributed(GlobalMaxPool1D())(k5_raw)
k6_max = TimeDistributed(GlobalMaxPool1D())(k6_raw)
k8_max = TimeDistributed(GlobalMaxPool1D())(k8_raw)
# Handle and hashtag processing
handle_10_vec = TimeDistributed(Dense(units=10, activation='relu'))(handle)
hashtag_10_vec = TimeDistributed(Dense(units=10, activation='relu'))(hashtag)
# Concatenation
out_conv = keras.layers.concatenate(
[k1_max, k2_max, k3_max, k4_max, k5_max, k6_max, k8_max, handle_10_vec, hashtag_10_vec],
name='Combining_Layers')
# Main layer
int_layer = TimeDistributed(Dense(units=150, activation='relu'))(out_conv)
dropout_int_layer = TimeDistributed(Dropout(rate=0.2))(int_layer)
in_lstm = TimeDistributed(Dense(units=100, activation='relu'))(dropout_int_layer)
b2_raw = Conv1D(filters=self.shapes[3]*2, kernel_size=4, padding='same', name='b2')(in_lstm)
b3_raw = Conv1D(filters=self.shapes[3]*2, kernel_size=8, padding='same', name='b3')(in_lstm)
b4_raw = Conv1D(filters=self.shapes[4]*2, kernel_size=16, padding='same', name='b4')(in_lstm)
b5_raw = Conv1D(filters=self.shapes[5]*2, kernel_size=32, padding='same', name='b5')(in_lstm)
b6_raw = Conv1D(filters=self.shapes[6]*2, kernel_size=64, padding='same', name='b6')(in_lstm)
b8_raw = Conv1D(filters=self.shapes[8]*2, kernel_size=128, padding='same', name='b8')(in_lstm)
b2_max = GlobalMaxPool1D()(b2_raw)
b3_max = GlobalMaxPool1D()(b3_raw)
b4_max = GlobalMaxPool1D()(b4_raw)
b5_max = GlobalMaxPool1D()(b5_raw)
b6_max = GlobalMaxPool1D()(b6_raw)
b8_max = GlobalMaxPool1D()(b8_raw)
price_lstm = LSTM(units=100)(prev_info)
input_lin = keras.layers.concatenate([b2_max, b3_max, b4_max, b5_max, b6_max, b8_max, price_lstm],
name='Combining_Layers_Final')
out_linear1 = Dense(units=300, activation='relu', name='Linear_1a')(input_lin)
out_linear1_d = Dropout(rate=0.1)(out_linear1)
out_linear3 = Dense(units=80, activation='relu', name='Linear_1')(out_linear1_d)
out_linear4 = Dense(units=40, name='Linear_2')(out_linear3)
out = Dense(units=1, name='Final_Layer', activation='sigmoid')(out_linear4)
model = keras.Model(inputs=[input_tweet, hashtag, handle, prev_info], outputs=out)
model.compile(loss='binary_crossentropy', optimizer=self.optim)
return model
#vec+=self.model[word].reshape((1,size))
w2v_model = gensim.models.Word2Vec.load(
'/home/liyt/code/text_classification/preproccess/demo0713.model')
embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))
for word, i in word_index.items():
if word in w2v_model:
embedding_matrix[i] = np.asarray(
w2v_model[word]) #, dtype='float32'
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
# 搭建模型
model = Sequential()
model.add(embedding_layer)
model.add(Dropout(0.2))
model.add(Conv1D(250, 3, padding='valid', activation='relu', strides=1))
model.add(MaxPooling1D(3))
model.add(Flatten())
model.add(Dense(EMBEDDING_DIM, activation='relu'))
model.add(Dense(labels.shape[1], activation='softmax'))
model.summary() # plot_model(model, to_file='model.png',show_shapes=True)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
model.fit(x_train, y_train, epochs=2, batch_size=128)
model.save('word_vector_cnn.h5')
print model.evaluate(x_test, y_test)
def Model_BiLSTM_X2_CRF(sourcevocabsize,
targetvocabsize,
source_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim,
emd_dim,
sourcecharsize,
character_W,
input_word_length,
char_emd_dim,
batch_size=32,
loss='categorical_crossentropy',
optimizer='rmsprop'):
word_input = Input(shape=(input_seq_lenth, ), dtype='int32')
char_input = Input(shape=(
input_seq_lenth,
input_word_length,
),
dtype='int32')
char_embedding = Embedding(input_dim=sourcecharsize,
output_dim=char_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
input_word_length),
mask_zero=False,
trainable=True,
weights=[character_W])
char_embedding2 = TimeDistributed(char_embedding)(char_input)
char_cnn = TimeDistributed(Conv1D(50, 3, activation='relu',
padding='same'))(char_embedding2)
char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn)
# char_macpool = Dropout(0.5)(char_macpool)
# !!!!!!!!!!!!!!
char_macpool = Dropout(0.25)(char_macpool)
word_embedding = Embedding(input_dim=sourcevocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=True,
trainable=True,
weights=[source_W])(word_input)
word_embedding_dropout = Dropout(0.5)(word_embedding)
embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1)
BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True),
merge_mode='concat')(embedding)
BiLSTM = BatchNormalization(axis=1)(BiLSTM)
BiLSTM_dropout = Dropout(0.5)(BiLSTM)
BiLSTM2 = Bidirectional(LSTM(hidden_dim // 2, return_sequences=True),
merge_mode='concat')(BiLSTM_dropout)
BiLSTM_dropout2 = Dropout(0.5)(BiLSTM2)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(BiLSTM_dropout2)
# TimeD = TimeDistributed(Dense(int(hidden_dim / 2)))(BiLSTM_dropout)
# TimeD = Dropout(0.5)(TimeD)
# !!!!!!!!!!!!!!!delete dropout
# model = Activation('softmax')(TimeD)
crflayer = CRF(targetvocabsize + 1, sparse_target=False)
model = crflayer(TimeD) #0.8746633147782367
# # model = crf(BiLSTM_dropout)#0.870420501714492
Models = Model([word_input, char_input], [model])
# Models.compile(loss=loss, optimizer='adam', metrics=['acc'])
# Models.compile(loss=crflayer.loss_function, optimizer='adam', metrics=[crflayer.accuracy])
Models.compile(loss=crflayer.loss_function,
optimizer=optimizers.RMSprop(lr=0.001),
metrics=[crflayer.accuracy])
return Models
def Model_Dense_Softmax(sourcevocabsize,
targetvocabsize,
source_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim,
emd_dim,
sourcecharsize,
character_W,
input_word_length,
char_emd_dim,
sourcepossize,
pos_W,
pos_emd_dim,
batch_size=32,
loss='categorical_crossentropy',
optimizer='rmsprop'):
word_input = Input(shape=(input_seq_lenth, ), dtype='int32')
char_input = Input(shape=(
input_seq_lenth,
input_word_length,
),
dtype='int32')
char_embedding = Embedding(input_dim=sourcecharsize,
output_dim=char_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
input_word_length),
mask_zero=False,
trainable=True,
weights=[character_W])
char_embedding2 = TimeDistributed(char_embedding)(char_input)
char_cnn = TimeDistributed(Conv1D(50, 3, activation='relu',
padding='same'))(char_embedding2)
char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn)
# char_macpool = Dropout(0.5)(char_macpool)
# !!!!!!!!!!!!!!
char_macpool = Dropout(0.25)(char_macpool)
word_embedding = Embedding(input_dim=sourcevocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=True,
weights=[source_W])(word_input)
word_embedding_dropout = Dropout(0.5)(word_embedding)
embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1)
Dense1 = TimeDistributed(Dense(400, activation='tanh'))(embedding)
Dense1 = Dropout(0.5)(Dense1)
Dense2 = TimeDistributed(Dense(200, activation='tanh'))(Dense1)
Dense2 = Dropout(0.3)(Dense2)
Dense3 = TimeDistributed(Dense(100, activation='tanh'))(Dense2)
Dense3 = Dropout(0.2)(Dense3)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(Dense3)
# TimeD = Dropout(0.5)(TimeD)
# !!!!!!!!!!!!!!!delete dropout
model = Activation('softmax')(TimeD)
# crflayer = CRF(targetvocabsize+1, sparse_target=False)
# model = crflayer(TimeD)
Models = Model([word_input, char_input], [model])
Models.compile(loss=loss,
optimizer=optimizers.RMSprop(lr=0.001),
metrics=['acc'])
# Models.compile(loss=crflayer.loss_function, optimizer=optimizers.RMSprop(lr=0.001), metrics=[crflayer.accuracy])
return Models
from keras.layers import Conv1D, MaxPooling1D, Activation, Dropout, Flatten, Dense
from scipy.signal import butter, lfilter, boxcar
from glob import glob
from sklearn.preprocessing import StandardScaler
from keras.layers import SimpleRNN
import matplotlib.pyplot as plt
from keras import layers
from random import shuffle
##############################################################
model = Sequential()
model.add(
Conv1D(10,
kernel_size=4,
strides=2,
activation='tanh',
input_shape=(42, 1)))
model.add(Conv1D(10, kernel_size=4, strides=2, activation='tanh'))
model.add(Flatten())
model.add(Dense(60, activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(6, activation='sigmoid'))
Adam = optimizers.Adam(lr=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
model.compile(optimizer='Adam',
loss='binary_crossentropy',
def preact_block(x, filters, act_fn="relu"):
x = BatchNormalization()(x)
x = Activation(act_fn)(x)
x = Conv1D(filters, 1, strides=1)(x)
return x
callbacks=[metrics_lstm],
verbose=0)
lstm_model.summary()
f1_lstm.append(max(metrics_lstm.val_f1s))
acc_lstm.append(history.history["val_acc"][metrics_lstm.ind])
f1_cnn = []
acc_cnn = []
for i in range(iterations):
metrics_cnn = misc.Metrics("CNN", i)
cnn_model = Sequential()
cnn_model.add(Embedding(num_words + 2, EMBEDDING_SIZE))
cnn_model.add(
Conv1D(HIDDEN_SIZE * 2, 6, activation='relu', use_bias=False))
cnn_model.add(MaxPooling1D())
cnn_model.add(Dropout(dropout))
cnn_model.add(Conv1D(200, 3, activation='relu', use_bias=False))
cnn_model.add(GlobalMaxPooling1D())
cnn_model.add(Dropout(dropout))
cnn_model.add(Dense(128, activation='relu'))
cnn_model.add(Dropout(dropout))
cnn_model.add(Dense(1, activation='sigmoid'))
cnn_model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = cnn_model.fit(X_train,
y_train,
batch_size=BATCH_SIZE,
ytrain_rho_log_centered = np.log(ytrain_rho) - mean_train
plt.hist(ytrain_rho_log_centered)
plt.show()
ytest_rho_log_centered = np.log(ytest_rho) - mean_test
plt.hist(ytest_rho_log_centered)
plt.show()
print(mean_test, mean_train) #mean train is important, it was 4.78757605959
batch_size = 32
b1 = Sequential()
b1.add(
Conv1D(1250,
kernel_size=2,
activation='relu',
input_shape=(xtest.shape[1], xtest.shape[2])))
b1.add(Conv1D(256, kernel_size=2, activation='relu'))
b1.add(AveragePooling1D(pool_size=2))
b1.add(Dropout(0.25))
b1.add(Conv1D(256, kernel_size=2, activation='relu'))
b1.add(AveragePooling1D(pool_size=2))
b1.add(Dropout(0.25))
b1.add(Flatten())
b2 = Sequential()
b2.add(Dense(64, input_shape=(418, ), activation='relu'))
b2.add(Dropout(0.1))
model = Sequential()
#model.add([b1, b2])
def ds2_gru_model(input_dim=161,
fc_size=1024,
rnn_size=512,
output_dim=29,
initialization='glorot_uniform',
conv_layers=1,
gru_layers=1,
use_conv=True):
K.set_learning_phase(1)
input_data = Input(shape=(None, input_dim), name='the_input')
x = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(input_data)
if use_conv:
#conv = ZeroPadding1D(padding=(0, 2048))(x)
for l in range(conv_layers):
x = Conv1D(filters=fc_size,
name='conv_{}'.format(l + 1),
kernel_size=11,
padding='valid',
activation='relu',
strides=2)(x)
else:
for l in range(conv_layers):
x = TimeDistributed(
Dense(fc_size, name='fc_{}'.format(l + 1),
activation='relu'))(x) # >>(?, time, fc_size)
x = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(x)
for l in range(gru_layers):
x = Bidirectional(GRU(rnn_size,
name='fc_{}'.format(l + 1),
return_sequences=True,
activation='relu',
kernel_initializer=initialization),
merge_mode='sum')(x)
x = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(x)
# Last Layer 5+6 Time Dist Dense Layer & Softmax
x = TimeDistributed(Dense(fc_size, activation='relu'))(x)
y_pred = TimeDistributed(
Dense(output_dim, name="y_pred", activation="softmax"))(x)
# labels = K.placeholder(name='the_labels', ndim=1, dtype='int32')
labels = Input(name='the_labels', shape=[
None,
], dtype='int32')
input_length = Input(name='input_length', shape=[1], dtype='int32')
label_length = Input(name='label_length', shape=[1], dtype='int32')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length, label_length])
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
return model
def create_fusionnet_layers(self, fusionnet, layers=None):
"""
Extend the FusionNet with more layers
:param fusionnet: The FusionNet model
:param layers: The layer structure for the added layers
:return: The FusionNet model with the added layers
"""
# stack a deep densely-connected network on top
if layers is None:
return fusionnet
# cnn
# layers = [('dense_large', None, None),
# ('reshape_expand_dim_4', None, None),
# ('conv2d', 32, 'sigmoid'),
# ('conv_max_pool_large', None, None),
# ('dropout', 0.25, None),
# ('flatten', None, None)]
# lstm/gru
# layers = [('dense_large', None, None),
# ('reshape_expand_dim_3', None, None),
# ('rnn', 32, 'sigmoid'),
# ('dropout', 0.25, None)]
for layer, size, activation in layers:
# this needs to be preceeded by a dense layer being a multiple of 300 to work
if layer == 'reshape_expand_dim_4':
# fusionnet = Reshape((-1,keras.backend.get_variable_shape(fusionnet)[1]/700, 700))(fusionnet)
fusionnet = Reshape((30, 10, -1))(fusionnet)
if layer == 'reshape_expand_dim_3':
# fusionnet = Reshape((-1,keras.backend.get_variable_shape(fusionnet)[1]/700, 700))(fusionnet)
fusionnet = Reshape((30, -1))(fusionnet)
elif layer == 'dense_large':
fusionnet = Dense(900, activation='sigmoid',
kernel_initializer=keras.initializers.glorot_uniform())(fusionnet)
elif layer == 'dense_small':
fusionnet = Dense(64, activation='sigmoid',
kernel_initializer=keras.initializers.glorot_uniform())(fusionnet)
elif layer == 'dense':
fusionnet = Dense(size, activation=activation,
kernel_initializer=keras.initializers.glorot_uniform())(fusionnet)
elif layer == 'conv2d_large':
fusionnet = Conv2D(64, (6, 6), activation='relu', kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'conv2d_small':
fusionnet = Conv2D(32, (3, 3), activation='relu', kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'conv2d':
fusionnet = Conv2D(size, (3, 3), activation=activation, kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'conv1d':
fusionnet = Conv1D(size, 3, activation=activation, kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'conv2d_max_pool_large':
fusionnet = MaxPooling2D((2, 2))(fusionnet)
elif layer == 'conv2d_max_pool_small':
fusionnet = MaxPooling2D((1, 1))(fusionnet)
elif layer == 'conv1d_max_pool_large':
fusionnet = MaxPooling1D(2)(fusionnet)
elif layer == 'conv1d_max_pool_small':
fusionnet = MaxPooling1D(1)(fusionnet)
elif layer == 'conv2d_avg_pool_large':
fusionnet = AveragePooling2D((2, 2))(fusionnet)
elif layer == 'conv2d_avg_pool_small':
fusionnet = AveragePooling2D((1, 1))(fusionnet)
elif layer == 'conv1d_avg_pool_large':
fusionnet = AveragePooling1D(2)(fusionnet)
elif layer == 'conv1d_avg_pool_small':
fusionnet = AveragePooling1D(1)(fusionnet)
elif layer == 'lstm_large':
fusionnet = LSTM(128, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'lstm_small':
fusionnet = LSTM(32, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'lstm':
fusionnet = LSTM(size, activation=activation, kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'lstm_sequence':
fusionnet = LSTM(size, activation=activation, kernel_initializer='glorot_uniform', return_sequences=True)(fusionnet)
elif layer == 'gru_large':
fusionnet = GRU(128, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'gru_small':
fusionnet = GRU(32, activation='tanh', kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'gru':
fusionnet = GRU(size, activation=activation, kernel_initializer='glorot_uniform')(fusionnet)
elif layer == 'dropout':
fusionnet = Dropout(size)(fusionnet)
# always flatten after conv
elif layer == 'flatten':
fusionnet = Flatten()(fusionnet)
return fusionnet
y,
test_size=0.2,
shuffle=True,
random_state=666)
print(x_train.shape)
print(x_test.shape)
print(y_train)
print(y_test)
#2. 모델
# 2. 모델
model = Sequential()
# model.add(LSTM(140, input_shape= (4, 1)))
model.add(Conv1D(140, 2, padding='same', input_shape=(4, 1)))
model.add(MaxPooling1D())
model.add(Conv1D(140, 2, padding='same'))
model.add(Flatten())
model.add(Dense(60))
model.add(Dense(25))
model.add(Dense(1))
model.summary()
# #3. 훈련, 컴파일
# from keras.callbacks import EarlyStopping
# early_stopping = EarlyStopping(monitor = 'loss', patience=5, mode = 'auto')
# model.compile(optimizer='adam', loss='mse', metrics= ['mse'])
# model.fit(x_train, y_train, epochs=120, batch_size=1, verbose=1,
embedding_layer = Embedding(MAX_NB_WORDS+1,EMBEDDING_DIM,weights=[word_embedding_matrix],input_length=25,trainable=False)
#词嵌入
sequence_1_input = Input(shape=(25,), dtype='int32')
embed_1 = embedding_layer(sequence_1_input)
sequence_2_input = Input(shape=(25,), dtype='int32')
embed_2 = embedding_layer(sequence_2_input)
#lstm
lstm_layer_1 = LSTM(256,return_sequences=True)
lstm_layer_2 = LSTM(256,return_sequences=True)
q1 = lstm_layer(embed_1,lstm_layer_1,lstm_layer_2)
q2 = lstm_layer(embed_2,lstm_layer_1,lstm_layer_2)
#用类似TextCNN的思路构建不同卷积核的特征，两个句子共用同样的卷积层
kernel_size = [2,3,4,5]
conv_concat = []
for kernel in kernel_size:
conv = Conv1D(64,kernel_size=kernel,activation='relu',padding='same')
q1_maxp,q1_meanp = conv_pool(conv,q1)
q2_maxp,q2_meanp = conv_pool(conv,q2)
mix = mix_layer(q1_maxp,q1_meanp,q2_maxp,q2_meanp)
conv_concat.append(mix)
conv = Concatenate()(conv_concat)
#全连接层
merged = Dropout(0.3)(conv)
merged = BatchNormalization()(merged)
merged = Dense(512, activation='relu',name='dense_output')(merged)
merged = Dropout(0.3)(merged)
merged = BatchNormalization()(merged)
merged = Dense(256, activation='relu',name='dense_output2')(merged)
merged = Dropout(0.3)(merged)
merged = BatchNormalization(name='bn_output')(merged)
preds = Dense(1, activation='sigmoid')(merged)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('Build model...')
model = Sequential()
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(max_features, embedding_dims, input_length=maxlen))
model.add(Dropout(0.2))
# we add a Convolution1D, which will learn filters
# word group filters of size filter_length:
model.add(
Conv1D(filters, kernel_size, padding='valid', activation='relu',
strides=1))
# we use max pooling:
model.add(GlobalMaxPooling1D())
# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(0.2))
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(train_inputs.shape)
# Modeling
train_targets = np.array(train_targets)
lsvc = LinearSVC(C=0.01, penalty="l1",
dual=False).fit(train_inputs, train_targets)
sel_model = SelectFromModel(lsvc, prefit=True)
train_inputs = sel_model.transform(train_inputs)
print(train_inputs.shape)
train_inputs = train_inputs.reshape(train_inputs.shape[0],
train_inputs.shape[1], 1)
#train_inputs = BasicDataProcess.TransposeElements(train_inputs)
train_targets = to_categorical(train_targets)
#create model
model = Sequential()
#add model layers
model.add(Conv1D(128, kernel_size=4, activation='relu'))
model.add(Conv1D(64, kernel_size=4, activation='relu'))
model.add(Conv1D(32, kernel_size=4, activation='relu'))
model.add(Conv1D(16, kernel_size=4, activation='relu'))
model.add(Flatten())
model.add(Dense(2, activation='softmax'))
#compile model using accuracy to measure model performance
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_inputs, train_targets, epochs=30)
# Prediction
test_events = BasicDataProcess.LoadDataFromFile(
data_dir + "/Test/testEvents.txt")
test_data = BasicDataProcess.LoadEEGFromFile(data_dir, False)
MAX_VOCAB_SIZE + 1,
VECTOR_DIM,
weights=[embeddings],
# trainable=False
)
input_ = Input(shape=(max_review_length, ))
cnn1 = Sequential()
cnn1.add(
Embedding(MAX_VOCAB_SIZE + 1,
VECTOR_DIM,
weights=[embeddings],
input_length=max_review_length))
cnn1.add(Conv1D(50, 5))
cnn1.add(Flatten())
cnn1.add(Dense(100, activation='relu'))
cnn1.add(Dropout(0.2))
cnn1.add(Dense(max(varietal_list_o) + 1, activation='softmax'))
cnn1.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = cnn1.fit(train_x,
train_y,
epochs=1000,
validation_split=0.1,
batch_size=64)
y_score = cnn1.predict(test_x)
y_test_binary = keras.utils.to_categorical(y_test, num_classes)
x_train = x_train.reshape(75, 1000,1)
x_test = x_test.reshape(25, 1000,1)
from keras.models import Sequential
import keras
from keras.models import Sequential
from keras.layers import Dense, Flatten, Conv1D
from keras.callbacks import ModelCheckpoint
from keras.models import model_from_json
from keras import backend as K
model = Sequential()
model.add(Conv1D(32, (3), input_shape=(1000,1), activation='relu'))
model.add(Flatten())
model.add(Dense(64, activation='softmax'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
#model.summary()
batch_size = 16
epochs = 10
model.fit(x_train, y_train_binary,
batch_size=batch_size,
embedding_matrix[i] = embedding_vector
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
print('Training model.')
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
model.fit(x_train,
y_train,
if word in word2emb.keys():
embedding_weights[index] = word2emb[word]
else:
embedding_weights[index] = unk_vec
# train
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
model = Sequential()
model.add(
Embedding(embedding_weights.shape[0],
embedding_weights.shape[1],
weights=[embedding_weights],
input_length=max_len,
trainable=True))
model.add(SpatialDropout1D(0.2))
model.add(Conv1D(256, 3, activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dense(2, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
early_stopping = EarlyStopping(monitor='val_loss',
patience=20,
verbose=0,
mode='auto')
checkpoint = ModelCheckpoint(filepath='model/glove_w2v.model',
save_best_only=True)
# tensorboard --logdir=*/log/
tensorboard = TensorBoard(log_dir='log/', write_graph=False)
i = 0
y_valid = np.zeros((len(label_valid), max(label_valid) + 1))
for x in label_valid:
y_valid[i][x] = 1
i = i + 1
filter_sizes = [3, 4, 5]
sequence_length = count
embedding_vecor_length = 300
visible = Input(shape=(sequence_length, 300), dtype='float32')
print(visible.shape)
e = visible
conv_0 = Conv1D(64,
kernel_size,
padding='valid',
strides=1,
activation=newacti)(e)
maxpool_0 = MaxPooling1D(pool_size=pool_size)(conv_0)
gru = Reshape((sequence_length, embedding_vecor_length))(e)
gru = GRU(gru_output_size,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)(gru)
gru = GRU(gru_output_size)(gru)
gru = Reshape((1, 1, gru_output_size))(gru)
merge = maximum([maxpool_0, gru])
flatten = Flatten()(merge)
dropout = Dropout(0.5)(flatten)
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.layers import Embedding
from keras.layers import Conv1D, GlobalAveragePooling1D, MaxPooling1D
model = Sequential()
model.add(Conv1D(64, 3, activation='relu', input_shape=(seq_length, 100)))
model.add(Conv1D(64, 3, activation='relu'))
model.add(MaxPooling1D(3))
model.add(Conv1D(128, 3, activation='relu'))
model.add(Conv1D(128, 3, activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=16, epochs=10)
score = model.evaluate(x_test, y_test, batch_size=16)
