def build_model_1(embedding_matrix, one_hot_shape):
words = Input(shape=(MAX_LEN, ))
x = Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
trainable=False)(words)
x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True),
merge_mode='concat')(x)
x = SpatialDropout1D(rate=0.3)(x)
#x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True), merge_mode='ave')(x)
#x = SpatialDropout1D(rate=0.3)(x)
#x = GlobalAveragePooling1D()(x) # this layer average each output from the Bidirectional layer
x = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
summary = Input(shape=(MAX_LEN, ))
x_aux = Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
trainable=False)(summary)
x_aux = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True),
merge_mode='concat')(x_aux)
x_aux = SpatialDropout1D(rate=0.3)(x_aux)
#x_aux = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True), merge_mode='ave')(x_aux)
#x_aux = SpatialDropout1D(rate=0.3)(x_aux)
# x_aux = GlobalAveragePooling1D()(x_aux)
x_aux = concatenate([
GlobalMaxPooling1D()(x_aux),
GlobalAveragePooling1D()(x_aux),
])
one_hot = Input(shape=(one_hot_shape, ))
hidden = concatenate([x, x_aux, one_hot])
hidden = Dense(400, activation='relu')(hidden)
hidden = Dropout(0.4)(hidden)
hidden = Dense(400, activation='relu')(hidden)
hidden = Dropout(0.4)(hidden)
hidden = Dense(300, activation='relu')(hidden)
hidden = Dropout(0.4)(hidden)
hidden = Dense(300, activation='relu')(hidden)
hidden = Dropout(0.4)(hidden)
hidden = Dense(100, activation='relu')(hidden)
result = Dense(1, activation='linear')(hidden)
model = Model(inputs=[words, summary, one_hot], outputs=[result])
# adam = keras.optimizers.Adam(lr=0.0001, clipnorm=1.0, clipvalue=0.5)
model.compile(loss='mse', optimizer='adam')
return model
def get_model(self):
input_current = Input((self.maxlen, ))
input_left = Input((self.maxlen, ))
input_right = Input((self.maxlen, ))
embedder = Embedding(self.max_features,
self.embedding_dims,
input_length=self.maxlen)
embedding_current = embedder(input_current)
embedding_left = embedder(input_left)
embedding_right = embedder(input_right)
x_left = SimpleRNN(128, return_sequences=True)(embedding_left)
x_right = SimpleRNN(128, return_sequences=True,
go_backwards=True)(embedding_right)
x_right = Lambda(lambda x: K.reverse(x, axes=1))(x_right)
x = Concatenate(axis=2)([x_left, embedding_current, x_right])
x = Conv1D(64, kernel_size=1, activation='tanh')(x)
x = GlobalMaxPooling1D()(x)
output = Dense(self.class_num, activation=self.last_activation)(x)
model = Model(inputs=[input_current, input_left, input_right],
outputs=output)
return model
def make_cnn_model(vocab_size=10000, embed_dim=8, input_seq_length=20):
"""
I am the builder function for the CNN Model.
:param vocab_size: size of the vocabulary of the embedding, should be size of vocab of the vectorizer
:param embed_dim: how many dimensions to use for the vector embedding
:param input_seq_length: how long the sequence of inputs will be
:return: Keras Model
"""
x = inp = Input(shape=(None, ), dtype="int64")
x = Embedding(
input_dim=vocab_size,
output_dim=embed_dim,
input_length=input_seq_length,
)(x)
x = Conv1D(filters=64, kernel_size=3, strides=2, activation="linear")(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Conv1D(filters=64, kernel_size=3, strides=2, activation="linear")(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Conv1D(filters=64, kernel_size=3, strides=2, activation="linear")(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = GlobalMaxPooling1D()(x)
x = Dense(units=128, activation="linear")(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
out = Dense(1, activation="sigmoid")(x)
return Model(inputs=[inp], outputs=[out], name="cnn_model")
def pooling_blend(self, input):
avg_pool = GlobalAveragePooling1D()(input)
if self.top_k > 1:
max_pool = Lambda(self._top_k)(input)
else:
max_pool = GlobalMaxPooling1D()(input)
conc = Concatenate()([avg_pool, max_pool])
return conc
def init_model(self, config):
input_shape = config['max_len']
num_classes = config['num_classes']
inputs = Input(shape=(input_shape, 96))
x = inputs
cnn1 = Conv1D(50,
kernel_size=1,
strides=1,
padding='same',
kernel_initializer='he_normal')(x)
cnn1 = BatchNormalization(axis=-1)(cnn1)
cnn1 = LeakyReLU()(cnn1)
cnn1 = GlobalMaxPooling1D()(
cnn1) # CNN_Dynamic_MaxPooling(cnn1,50,2,2)
cnn2 = Conv1D(50,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer='he_normal')(x)
cnn2 = BatchNormalization(axis=-1)(cnn2)
cnn2 = LeakyReLU()(cnn2)
cnn2 = GlobalMaxPooling1D()(cnn2)
cnn3 = Conv1D(50,
kernel_size=5,
strides=1,
padding='same',
kernel_initializer='he_normal')(x)
cnn3 = BatchNormalization(axis=-1)(cnn3)
cnn3 = LeakyReLU()(cnn3)
cnn3 = GlobalMaxPooling1D()(cnn3)
x = concatenate([cnn1, cnn2, cnn3], axis=-1)
x = Dense(units=num_classes, activation='softmax')(x)
model = TFModel(inputs=inputs, outputs=x)
opt = optimizers.rmsprop(lr=0.0001, decay=1e-6)
model.compile(optimizer=opt,
loss="sparse_categorical_crossentropy",
metrics=['acc'])
model.summary()
self._model = model
self.is_init = True
def best_model():
epochs = [2, 5, 10, 15, 20]
dropout_rate = [0.1, 0.2, 0.3, 0.4]
# learning_rates = [0.01, 0.05, 0.1]
list_of_all_scores = list()
list_of_scores = list()
list_of_dropout = list()
list_of_all_dropouts = list()
list_of_epochs = list()
for i in dropout_rate:
model = Sequential()
model.add(
Embedding(input_dim=vocab_size,
output_dim=embedding_dim,
input_length=max_length))
model.add(
Conv1D(filters=max_length,
kernel_size=5,
padding='same',
activation='relu'))
model.add(GlobalMaxPooling1D())
model.add(Dropout(i))
model.add(
Dense(1,
kernel_regularizer=regularizers.l2(0.01),
activation='sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
list_of_dropout.append(i)
for e in epochs:
list_of_all_dropouts.append(i)
list_of_epochs.append(e)
model.fit(X_train,
y_train,
epochs=e,
batch_size=128,
verbose=1,
validation_split=0.2)
score = model.evaluate(X_test, y_test, verbose=1)
list_of_all_scores.append(score)
if score not in list_of_scores:
list_of_scores.append(score)
#print('Dropout:', i, '\n', 'Epoch:', e, '\n', 'Score:', float(score))
lowest = min(list_of_all_scores)
num = list_of_scores.index(lowest)
epoch = list_of_epochs[num]
dropout = list_of_all_dropouts[num]
print('Lowest score:', lowest, 'Epoch:', epoch, 'Dropout', dropout)
return epoch, dropout
def __init__(self, vocab_size, embedding_dim=20):
super().__init__(self)
self.embedding = Embedding(vocab_size + 1, embedding_dim)
self.conv1d_32 = Conv1D(32, 3, activation='relu')
self.conv1d_64 = Conv1D(64, 3, activation='relu')
self.conv1d_128 = Conv1D(128, 3, activation='relu')
self.max_pooling = MaxPooling1D(4)
self.global_max_pooling = GlobalMaxPooling1D()
self.dense = Dense(4, activation='softmax')
def recurrent_layers(cls):
num_filters = 20
filter_size = 5
return [
Conv1D(num_filters, filter_size, activation='relu'),
GlobalMaxPooling1D(),
# Dropout(0.1),
Dense(20, activation='relu'),
Dropout(0.05),
]
def shallow_and_wide_cnn(inputs, filters, kernel_sizes):
outputs = []
for kernel_size in kernel_sizes:
conv = tf.layers.conv1d(inputs, filters, kernel_size, padding="same",
kernel_regularizer=regularizer)
conv = tf.layers.batch_normalization(conv, training=is_training)
conv = tf.nn.relu(conv)
conv = GlobalMaxPooling1D()(conv)
outputs.append(conv)
output = tf.concat(outputs, 1)
return dropout(output, dropout_keep_prob)
def build_cnn_1d_model(maxlen=500):
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen))
model.add(Conv1D(32, 7, activation='relu'))
model.add(MaxPooling1D(5))
model.add(Conv1D(32, 7, activation='relu'))
model.add(GlobalMaxPooling1D())
model.add(Dense(1))
model.summary()
model.compile(optimizer=RMSprop(lr=1e-4),
loss='binary_crossentropy',
metrics=['acc'])
return model
def build_model(embedding_weights,
embedding_dim,
num_words,
input_length,
num_classes=20):
""" Builds a Keras model. It sets embeddings layer trainable to False
to keep the embeddings fixed
Parameters
----------
embedding_weights: np.ndarray
A numpy array contains embedding weights
embedding_dim: int
Embeddings dimension
num_words: int
Number of words in the dataset
input_length: int
Maximum sequence length
num_classes: int
Number of classes in the dataset
Returns
-------
model: Model
A keras compiled model instance
"""
embedding = Embedding(num_words,
embedding_dim,
embeddings_initializer=Constant(embedding_weights),
input_length=input_length,
trainable=False)
seq_input = Input(shape=(input_length, ), dtype='int32')
embedded_sequences = embedding(seq_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(num_classes, activation='softmax')(x)
model = Model(seq_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
return model
def recurrent_layers(cls):
#num_filters = 20
num_filters = 5
weight_decay = 1e-4
return [
Conv1D(num_filters, 7, activation='relu', padding='same'),
MaxPooling1D(2),
Conv1D(num_filters, 7, activation='relu', padding='same'),
GlobalMaxPooling1D(),
Dropout(0.05),
Dense(32,
activation='relu',
kernel_regularizer=regularizers.l2(weight_decay))
]
def get_model(self, input_length: int, numpy_matrix_embeddings: numpy.ndarray, **kwargs):
# get the number of topics
lda_vector_size = kwargs.get('lda_vector_size', 0)
assert lda_vector_size, "lda_vector_size not specified"
# this is the placeholder tensor for the input sequences
# input sequence is a numpy array of word indices
input_sequence = Input(shape=(input_length,), name="input_sequence", dtype='int32')
input_lda = Input(shape=(lda_vector_size,), name="input_lda", dtype='float32')
# input_sequence = Input(shape=(input_length,), dtype='int32')
# input_lda = Input(shape=(lda_vector_size,), dtype='float32')
# this embedding layer will transform the sequences of integers
# into vectors of size dimension of embeddings
# Conv1D does not support masking
embedded = Embedding(numpy_matrix_embeddings.shape[0], numpy_matrix_embeddings.shape[1],
input_length=input_length, weights=[numpy_matrix_embeddings], mask_zero=False)(
input_sequence)
# 4 convolution layers (500 filters each)
nb_filters = 500
kernel_sizes = [3, 5, 7, 9]
cnn = [Conv1D(filters=nb_filters, kernel_size=kernel_size, padding="same")(embedded) for kernel_size in
kernel_sizes]
# concatenate
concatenated_cnn_outputs = Concatenate()([c for c in cnn])
# max pooling
pooled = GlobalMaxPooling1D()(concatenated_cnn_outputs)
# after_dropout = Dropout(dropout)(pooled)
# just for fun - no regularization... and it worked!
after_dropout = Dropout(0)(pooled)
# batch normalization? No, worse performance... 0.43321227654613681
# batch_normalization = BatchNormalization()(after_dropout)
# now concatenate the LDA vector with output from CNN
cnn_and_lda = Concatenate()([after_dropout, input_lda])
output_layer = Dense(1, activation='linear')(cnn_and_lda)
model = Model(inputs=[input_sequence, input_lda], outputs=output_layer)
print("Compiling model")
model.compile('rmsprop', mean_squared_error)
return model
def tnet(inputs, num_features):
bias = Constant(np.eye(num_features).flatten())
reg = OrthogonalRegularizer(num_features)
x = conv_bn(inputs, 32)
x = conv_bn(x, 64)
x = conv_bn(x, 512)
x = GlobalMaxPooling1D()(x)
x = dense_bn(x, 256)
x = dense_bn(x, 128)
x = Dense(num_features * num_features,
kernel_initializer='zeros',
bias_initializer=bias,
activity_regularizer=reg)(x)
feat_T = Reshape((num_features, num_features))(x)
return Dot(axes=(2, 1))([inputs, feat_T])
def get_model(self):
input = Input((self.maxlen, )) # 表示输入是maxlen维的向量
# input_dim: 词汇表大小 output_dim：词向量的维度 input_length: 输入序列的长度
embedding = Embedding(self.max_features,
self.embedding_dims,
input_length=self.maxlen)(input)
convs = []
for kernel_size in [3, 4, 5]:
c = Conv1D(128, kernel_size, activation='relu')(embedding)
c = GlobalMaxPooling1D()(c)
convs.append(c)
x = Concatenate()(convs)
x = Dropout(0.3)(x)
output = Dense(self.class_num, activation=self.last_activation)(x)
model = Model(inputs=input, outputs=output)
return model
def create_channel(self, x, filter_size, feature_map):
"""
Creates a layer, working channel wise
Arguments:
x : Input for convoltuional channel
filter_size : Filter size for creating Conv1D
feature_map : Feature map
Returns:
x : Channel including (Conv1D + GlobalMaxPooling + Dense + Dropout)
"""
x = SeparableConv1D(feature_map, kernel_size=filter_size, activation='relu', strides=1, padding='same',
depth_multiplier=4)(x)
x = GlobalMaxPooling1D()(x)
x = Dense(self.hidden_units)(x)
x = Dropout(self.dropout_rate)(x)
return x
def get_rnn_model(MAX_NB_WORDS, embedding_matrix_2):
"""
Creating the RNN model
:param MAX_NB_WORDS: maximum length of the seuquence
:param embedding_matrix_2: embedding matrix
"""
# defining input shape of the data
inp = Input(shape=(50, ))
# defining input shape of the metadata
# the data predictions from the first network
meta_input = Input(shape=(1,))
# layers:
# ------------------------------------------------
x = Embedding(MAX_NB_WORDS, 300, input_length=50, weights=[
embedding_matrix_2], trainable=False)(inp)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(GRU(100, return_sequences=True))(x)
x = Bidirectional(GRU(100, return_sequences=True))(x)
x = Conv1D(512, kernel_size=1, padding="valid",
kernel_initializer="he_uniform")(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
conc = concatenate([avg_pool, max_pool])
conc = BatchNormalization()(conc)
# on this layer, the input and the metadata
# from the first model are concatenated
conc = concatenate([conc, meta_input])
conc = Dense(512)(conc)
conc = BatchNormalization()(conc)
conc = LeakyReLU()(conc)
outp = Dense(90, activation='softmax')(conc)
# input here is an array (main input and meta-input)
model = Model(inputs=[inp, meta_input], outputs=outp)
# Compiling the model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def classification_model():
inputs = Input(shape=(NUM_POINTS, 3))
x = tnet(inputs, 3)
x = conv_bn(x, 32)
x = conv_bn(x, 32)
x = tnet(x, 32)
x = conv_bn(x, 32)
x = conv_bn(x, 64)
x = conv_bn(x, 512)
x = GlobalMaxPooling1D()(x)
x = dense_bn(x, 256)
x = Dropout(0.3)(x)
x = dense_bn(x, 128)
x = Dropout(0.3)(x)
outputs = Dense(NUM_CLASSES, activation='softmax')(x)
model = Model(inputs=inputs, outputs=outputs, name='pointnet')
return model
def get_model(self):
input = Input((self.maxlen,))
embedding = Embedding(self.max_tokens, self.embedding_size, input_length=self.maxlen)(input)
x_context = Bidirectional(CuDNNLSTM(128, return_sequences=True))(embedding)
x = Concatenate()([embedding, x_context])
convs = []
for kernel_size in self.kernel_size_list:
conv = Conv1D(128, kernel_size, activation='relu')(x)
convs.append(GlobalMaxPooling1D()(conv))
convs.append(GlobalAveragePooling1D()(conv))
# poolings = [GlobalAveragePooling1D()(conv) for conv in convs] + \
# [GlobalMaxPooling1D()(conv) for conv in convs]
x = Concatenate()(convs)
output = Dense(self.num_class, activation=self.last_activation)(x)
model = Model(inputs=input, outputs=output)
return model
def get_model(self, numpy_matrix_embeddings: numpy.ndarray, **kwargs) -> Model:
# get default parameters
dropout = kwargs.get('dropout', 0.9)
assert isinstance(dropout, float)
# this is the placeholder tensor for the input sequences
# input sequence is a numpy array of word indices
input_sequence = Input(shape=(None,), dtype='int32', name='input_sequence')
# this embedding layer will transform the sequences of integers
# into vectors of size dimension of embeddings
# Conv1D does not support masking
embedded = Embedding(numpy_matrix_embeddings.shape[0], numpy_matrix_embeddings.shape[1],
weights=[numpy_matrix_embeddings], mask_zero=False)(input_sequence)
# 4 convolution layers (500 filters each)
nb_filters = 500
kernel_sizes = [5, 7, 9, 11]
cnn = [Conv1D(filters=nb_filters, kernel_size=kernel_size, padding="same")(embedded) for kernel_size in
kernel_sizes]
# concatenate
concatenated_cnn_outputs = Concatenate()([c for c in cnn])
# max pooling
pooled = GlobalMaxPooling1D()(concatenated_cnn_outputs)
# after_dropout = Dropout(dropout)(pooled)
# just for fun - no regularization... and it worked!
after_dropout = Dropout(dropout)(pooled)
# batch normalization? No, worse performance... 0.43321227654613681
# batch_normalization = BatchNormalization()(after_dropout)
# classification
output = Dense(2, activation='softmax')(after_dropout)
model = Model(inputs=[input_sequence], outputs=output)
print("Compiling model")
model.compile('adam', loss=categorical_crossentropy)
return model
def build_model(p_tokenizer, p_embedding_matrix, p_max_tokens,
number_of_classes, number_of_filters, p_weight_decay):
cnn_model = Sequential()
cnn_model.add(
Embedding(
input_dim=len(list(p_tokenizer.word_index)) + 1,
output_dim=p_embedding_matrix.shape[1],
weights=[p_embedding_matrix],
input_length=p_max_tokens,
trainable=True, # the layer is trained
name='embedding_layer'))
cnn_model.add(
Conv1D(number_of_filters, 7, activation='relu', padding='same'))
cnn_model.add(MaxPooling1D(2))
cnn_model.add(
Conv1D(number_of_filters, 7, activation='relu', padding='same'))
cnn_model.add(GlobalMaxPooling1D())
cnn_model.add(Dropout(0.5))
cnn_model.add(
Dense(32,
activation='relu',
kernel_regularizer=regularizers.l2(p_weight_decay)))
cnn_model.add(
Dense(number_of_classes,
activation='softmax')) # multi-label (k-hot encoding)
adam = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
cnn_model.compile(loss='sparse_categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
cnn_model.summary()
# define callbacks
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0.01,
patience=4,
verbose=1)
return cnn_model, [early_stopping]
def segmentation_model(num_classes):
inputs = Input(shape=(None, 3))
x = tnet(inputs, 3)
x = conv_bn(x, 32)
x = conv_bn(x, 32)
concat = tnet(x, 32)
x = conv_bn(concat, 32)
x = conv_bn(x, 64)
x = conv_bn(x, 512)
global_vector = GlobalMaxPooling1D()(x)
global_repeat = tf.tile(global_vector[:, tf.newaxis, ...],
(1, NUM_POINTS, 1))
x = tf.keras.layers.Concatenate()([concat, global_repeat])
x = conv_bn(x, 256)
x = conv_bn(x, 128)
x = conv_bn(x, 64)
x = conv_bn(x, 64)
outputs = conv_bn(x, num_classes)
model = Model(inputs=inputs, outputs=outputs, name='pointnet_seg')
return model
def buildprotSeq_embedding_model(num_filters_list,
filter_length_list,
prot_seq_dim,
embed_size,
max_seq_length,
name=None):
assert len(num_filters_list) == len(
filter_length_list), "incompatible hyper parameter."
num_conv_layers = len(num_filters_list)
protSeq = Input(shape=(max_seq_length, ))
seq_embed = Embedding(input_dim=prot_seq_dim + 1,
output_dim=embed_size,
input_length=max_seq_length)(protSeq)
for i in range(num_conv_layers):
seq_embed = Conv1D(filters=num_filters_list[i],
kernel_size=filter_length_list[i],
activation='relu',
padding='valid',
strides=1)(seq_embed)
seq_embed = GlobalMaxPooling1D()(seq_embed)
model = Model(inputs=protSeq, outputs=seq_embed, name=name)
return model
def construct_model():
input_creative_id = Input(shape=(None, ),
dtype='int32',
name='creative_id')
embedded_creative_id = Embedding(
creative_id_window, embedding_size,
name='embedded_creative_id')(input_creative_id)
encoded_creative_id = GlobalMaxPooling1D(
name='encoded_creative_id')(embedded_creative_id)
input_product_id = Input(shape=(None, ), dtype='int32', name='product_id')
embedded_product_id = Embedding(
product_id_max, 32, name='embedded_product_id')(input_product_id)
encoded_product_id = GlobalMaxPooling1D(
name='encoded_product_id')(embedded_product_id)
input_category = Input(shape=(None, ), dtype='int32', name='category')
embedded_category = Embedding(category_max, 2,
name='embedded_category')(input_category)
encoded_category = GlobalMaxPooling1D(
name='encoded_category')(embedded_category)
# encoded_category = Bidirectional(
# LSTM(32, dropout = 0.2, recurrent_dropout = 0.2), name = 'encoded_category')(embedded_category)
input_advertiser_id = Input(shape=(None, ),
dtype='int32',
name='advertiser_id')
embedded_advertiser_id = Embedding(
advertiser_id_max, 32,
name='embedded_advertiser_id')(input_advertiser_id)
encoded_advertiser_id = GlobalMaxPooling1D(
name='encoded_advertiser_id')(embedded_advertiser_id)
input_industry = Input(shape=(None, ), dtype='int32', name='industry')
embedded_industry = Embedding(industry_max, 16,
name='embedded_industry')(input_industry)
encoded_industry = GlobalMaxPooling1D(
name='encoded_industry')(embedded_industry)
# encoded_industry = Bidirectional(
# LSTM(32, dropout = 0.2, recurrent_dropout = 0.2, name = 'encoded_industry'))(embedded_industry)
# LSTM(14) : 是因为 91 天正好是 14 个星期
# LSTM(32) : 方便计算
input_click_times = Input(shape=(None, 1),
dtype='float32',
name='click_times')
encoded_click_times = Bidirectional(
LSTM(32, dropout=0.2, recurrent_dropout=0.2),
name='encoded_click_times')(input_click_times)
concatenated = concatenate([
encoded_creative_id, encoded_click_times, encoded_product_id,
encoded_category, encoded_advertiser_id, encoded_industry
],
axis=-1)
x = Dropout(0.5, name='Dropout_0101')(concatenated)
x = Dense(embedding_size, kernel_regularizer=l2(0.001),
name='Dense_0101')(x)
x = BatchNormalization(name='BN_0101')(x)
x = Activation('relu', name='relu_0101')(x)
x = Dropout(0.5, name='Dropout_0102')(x)
x = Dense(embedding_size, kernel_regularizer=l2(0.001),
name='Dense_0102')(x)
x = BatchNormalization(name='BN_0102')(x)
x = Activation('relu', name='relu_0102')(x)
x = Dropout(0.5, name='Dropout_0103')(x)
x = Dense(embedding_size, kernel_regularizer=l2(0.001),
name='Dense_0103')(x)
x = BatchNormalization(name='BN_0103')(x)
x = Activation('relu', name='relu_0103')(x)
x = Dropout(0.5, name='Dropout_0201')(x)
x = Dense(embedding_size // 2,
kernel_regularizer=l2(0.001),
name='Dense_0201')(x)
x = BatchNormalization(name='BN_0201')(x)
x = Activation('relu', name='relu_0201')(x)
x = Dropout(0.5, name='Dropout_0202')(x)
x = Dense(embedding_size // 2,
kernel_regularizer=l2(0.001),
name='Dense_0202')(x)
x = BatchNormalization(name='BN_0202')(x)
x = Activation('relu', name='relu_0202')(x)
x = Dropout(0.5, name='Dropout_0203')(x)
x = Dense(embedding_size // 2,
kernel_regularizer=l2(0.001),
name='Dense_0203')(x)
x = BatchNormalization(name='BN_0203')(x)
x = Activation('relu', name='relu_0203')(x)
if label_name == "age" and age_sigmoid == -1:
x = Dropout(0.5)(x)
x = Dense(10, kernel_regularizer=l2(0.001), name='output')(x)
x = BatchNormalization()(x)
output_tensor = Activation('softmax')(x)
model = Model([
input_creative_id, input_click_times, input_product_id,
input_category, input_advertiser_id, input_industry
], output_tensor)
print('-' * 5 + ' ' * 3 + "编译模型" + ' ' * 3 + '-' * 5)
model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.sparse_categorical_crossentropy,
metrics=[metrics.sparse_categorical_accuracy])
elif label_name == 'gender' or age_sigmoid != -1:
x = Dropout(0.5)(x)
x = Dense(1, kernel_regularizer=l2(0.001), name='output')(x)
x = BatchNormalization()(x)
output_tensor = Activation('sigmoid')(x)
model = Model([
input_creative_id, input_click_times, input_product_id,
input_category, input_advertiser_id, input_industry
], output_tensor)
print('-' * 5 + ' ' * 3 + "编译模型" + ' ' * 3 + '-' * 5)
model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.binary_crossentropy,
metrics=[metrics.binary_accuracy])
else:
raise Exception("错误的标签类型！")
return model
def SpamDectectionCNN():
def prepData(text, size):
# Convert to array
textDataArray = [text]
# Convert into list with word ids
Features = tokenizer.texts_to_sequences(textDataArray)
Features = pad_sequences(Features, size, padding='post')
return Features
results = []
predictions = []
# read ds
df = pd.read_csv('data/spam.csv', encoding='ISO-8859-1')
# print(df.head()) # theres some missing data here
# drop garbage columns
df = df.drop(["Unnamed: 2", "Unnamed: 3", "Unnamed: 4"], axis=1)
# rename columns
df.columns = ['labels', 'data']
# create binary labels (ham/spam)
df['b_labels'] = df['labels'].map({'ham': 0, 'spam': 1})
y = df['b_labels'].values
# split dataset
X_train, X_test, y_train, y_test = train_test_split(df['data'], y, test_size=0.1)
# convert sentences to sequences (embedding technique: bag of words)
# : every unique character will be given one index position
tokenizer = Tokenizer(num_words=20000)
tokenizer.fit_on_texts(X_train)
seq_train = tokenizer.texts_to_sequences(X_train) # each number is unique and corresponds to a unique word
seq_test = tokenizer.texts_to_sequences(X_test)
# create one big matrix to pass to the CNN
# --> we have to pad it because the CNN accepts a FIXED length but the above process produces a dynamic length due to
# each sentence having a different length.
data_train = pad_sequences(seq_train)
T = data_train.shape[1]
word2idx = tokenizer.word_index
V = len(word2idx) # total number of unique words
# pad the test set
data_test = pad_sequences(seq_test, maxlen=T) # we dont know the maxlen
# build the model
D = 20 # this is a hyper parameter -> word vector size
input_layer = Input(shape=T)
x = Embedding(V + 1, D)(input_layer) # N * T * D array. Returns sequence of word vectors
x = Conv1D(32, 3, activation='relu', padding='same')(x)
x = MaxPool1D(3)(x)
x = Conv1D(64, 3, activation='relu', padding='same')(x)
x = MaxPool1D(3)(x)
x = Conv1D(128, 3, activation='relu', padding='same')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(input_layer, x)
# compile model & train
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) # ...binary problem
hist = model.fit(x=data_train, y=y_train, epochs=5)
# model.summary()
accuracy = hist.history['accuracy'][-1]
# test model
pd.options.display.max_colwidth = 80 # show a part of the message
# make prediction on your custom txt file with text
for i in range(0, len(lines)):
results.append(lines[i])
textTokenizedTest = prepData(results[i], T)
predictions.append(model.predict(textTokenizedTest).item())
return accuracy, results, predictions
print(batch_size)
print(pool_len1)
print(learningrates)
def get_output(input_layer, hidden_layers):
output = input_layer
for hidden_layer in hidden_layers:
output = hidden_layer(output)
return output
forward_input = Input(shape=(w, channel_num))
reverse_input = Input(shape=(w, channel_num))
hidden_layers = [
Conv1D(filters = nb_filter1, kernel_size = filter_len1, activation='relu', kernel_regularizer=regularizers.l2(0.01)),
GlobalMaxPooling1D(),
Dropout(dropout_pool),
Dense(nb_dense, activation='relu', kernel_regularizer=regularizers.l2(0.01)),
Dropout(dropout_dense),
Dense(1, activation='sigmoid')
]
forward_output = get_output(forward_input, hidden_layers)
reverse_output = get_output(reverse_input, hidden_layers)
output = Average()([forward_output, reverse_output])
model = Model(inputs=[forward_input, reverse_input], outputs=output)
print(model.summary())
history = []
for i in range(len(learningrates)):
print("Testing with learning rate = " + str(learningrates[i]))
model_dir = '/content/drive/My Drive/cs230_metagenomics/BugNet/bas_test/saved_models'
def construct_model(creative_id_num, embedding_size, max_len, RMSProp_lr,
model_type = "MLP", label_name = "gender"):
'''
构建与编译模型
:param creative_id_num: 字典容量
:param embedding_size: 嵌入维度
:param max_len: 序列长度
:param RMSProp_lr: 学习步长
:param model_type: 模型的类型
MLP：多层感知机
Conv1D：1维卷积神经网络
GlobalMaxPooling1D：1维全局池化层
GlobalMaxPooling1D+MLP：1维全局池化层+多层感知机
Conv1D+LSTM：1维卷积神经网络+LSTM
Bidirectional+LSTM：双向 LSTM
:param label_name: 标签的类型
age ： 根据年龄进行的多分类问题
gender : 根据性别进行的二分类问题
:return: 返回构建的模型
'''
print("* 构建网络")
model = Sequential()
model.add(Embedding(creative_id_num, embedding_size, input_length = max_len))
if model_type == 'MLP':
model.add(Flatten())
model.add(Dense(8, activation = 'relu', kernel_regularizer = l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(4, activation = 'relu', kernel_regularizer = l2(0.001)))
model.add(Dropout(0.5))
elif model_type == 'Conv1D':
model.add(Conv1D(32, 7, activation = 'relu', kernel_regularizer = l2(0.001)))
model.add(Conv1D(32, 7, activation = 'relu', kernel_regularizer = l2(0.001)))
model.add(GlobalMaxPooling1D())
elif model_type == 'GlobalMaxPooling1D':
model.add(GlobalMaxPooling1D())
elif model_type == 'GlobalMaxPooling1D+MLP':
model.add(GlobalMaxPooling1D())
model.add(Dense(64, activation = 'relu', kernel_regularizer = l2(0.001)))
model.add(Dense(32, activation = 'relu', kernel_regularizer = l2(0.001)))
elif model_type == 'LSTM':
# model.add(LSTM(128, dropout = 0.5, recurrent_dropout = 0.5))
model.add(LSTM(128))
elif model_type == 'Conv1D+LSTM':
model.add(Conv1D(32, 5, activation = 'relu', kernel_regularizer = l2(0.001)))
model.add(Conv1D(32, 5, activation = 'relu', kernel_regularizer = l2(0.001)))
model.add(LSTM(16, dropout = 0.5, recurrent_dropout = 0.5))
elif model_type == 'Bidirectional-LSTM':
model.add(Bidirectional(LSTM(embedding_size, dropout = 0.2, recurrent_dropout = 0.2)))
else:
raise Exception("错误的网络模型类型")
if label_name == "age":
model.add(Dense(10, activation = 'softmax'))
print("%s——模型构建完成！" % model_type)
print("* 编译模型")
model.compile(optimizer = optimizers.RMSprop(lr = RMSProp_lr),
loss = losses.sparse_categorical_crossentropy,
metrics = [metrics.sparse_categorical_accuracy])
elif label_name == 'gender':
model.add(Dense(1, activation = 'sigmoid'))
print("%s——模型构建完成！" % model_type)
print("* 编译模型")
model.compile(optimizer = optimizers.RMSprop(lr = RMSProp_lr),
loss = losses.binary_crossentropy,
metrics = [metrics.binary_accuracy])
else:
raise Exception("错误的标签类型！")
print(model.summary())
return model
def get_test_model_full():
"""Returns a maximally complex test model,
using all supported layer types with different parameter combination.
"""
input_shapes = [
(26, 28, 3),
(4, 4, 3),
(4, 4, 3),
(4, ),
(2, 3),
(27, 29, 1),
(17, 1),
(17, 4),
]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
for inp in inputs[6:8]:
for padding in ['valid', 'same']:
for s in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4):
outputs.append(
Conv1D(out_channels,
s,
padding=padding,
dilation_rate=d)(inp))
for padding_size in range(0, 5):
outputs.append(ZeroPadding1D(padding_size)(inp))
for crop_left in range(0, 2):
for crop_right in range(0, 2):
outputs.append(Cropping1D((crop_left, crop_right))(inp))
for upsampling_factor in range(1, 5):
outputs.append(UpSampling1D(upsampling_factor)(inp))
for padding in ['valid', 'same']:
for pool_factor in range(1, 6):
for s in range(1, 4):
outputs.append(
MaxPooling1D(pool_factor, strides=s,
padding=padding)(inp))
outputs.append(
AveragePooling1D(pool_factor,
strides=s,
padding=padding)(inp))
outputs.append(GlobalMaxPooling1D()(inp))
outputs.append(GlobalAveragePooling1D()(inp))
for inp in [inputs[0], inputs[5]]:
for padding in ['valid', 'same']:
for h in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4):
outputs.append(
Conv2D(out_channels, (h, 1),
padding=padding,
dilation_rate=(d, 1))(inp))
outputs.append(
SeparableConv2D(out_channels, (h, 1),
padding=padding,
dilation_rate=(d, 1))(inp))
for sy in range(1, 4):
outputs.append(
Conv2D(out_channels, (h, 1),
strides=(1, sy),
padding=padding)(inp))
outputs.append(
SeparableConv2D(out_channels, (h, 1),
strides=(sy, sy),
padding=padding)(inp))
for sy in range(1, 4):
outputs.append(
MaxPooling2D((h, 1), strides=(1, sy),
padding=padding)(inp))
for w in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4) if sy == 1 else [1]:
outputs.append(
Conv2D(out_channels, (1, w),
padding=padding,
dilation_rate=(1, d))(inp))
outputs.append(
SeparableConv2D(out_channels, (1, w),
padding=padding,
dilation_rate=(1, d))(inp))
for sx in range(1, 4):
outputs.append(
Conv2D(out_channels, (1, w),
strides=(sx, 1),
padding=padding)(inp))
outputs.append(
SeparableConv2D(out_channels, (1, w),
strides=(sx, sx),
padding=padding)(inp))
for sx in range(1, 4):
outputs.append(
MaxPooling2D((1, w), strides=(1, sx),
padding=padding)(inp))
outputs.append(ZeroPadding2D(2)(inputs[0]))
outputs.append(ZeroPadding2D((2, 3))(inputs[0]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[0]))
outputs.append(Cropping2D(2)(inputs[0]))
outputs.append(Cropping2D((2, 3))(inputs[0]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[0]))
for y in range(1, 3):
for x in range(1, 3):
outputs.append(UpSampling2D(size=(y, x))(inputs[0]))
outputs.append(GlobalAveragePooling2D()(inputs[0]))
outputs.append(GlobalMaxPooling2D()(inputs[0]))
outputs.append(AveragePooling2D((2, 2))(inputs[0]))
outputs.append(MaxPooling2D((2, 2))(inputs[0]))
outputs.append(UpSampling2D((2, 2))(inputs[0]))
outputs.append(keras.layers.concatenate([inputs[0], inputs[0]]))
outputs.append(Dropout(0.5)(inputs[0]))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(center=False)(inputs[0]))
outputs.append(BatchNormalization(scale=False)(inputs[0]))
outputs.append(Conv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(Conv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(SeparableConv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(SeparableConv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(Dense(2, use_bias=True)(inputs[3]))
outputs.append(Dense(2, use_bias=False)(inputs[3]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[1])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[2])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1), padding='valid')(up_scale_2(inputs[2])) # (1, 8, 8)
x = keras.layers.concatenate([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = keras.layers.concatenate(
[MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[3]),
Activation('hard_sigmoid')(inputs[3]),
Activation('selu')(inputs[3]),
Activation('sigmoid')(inputs[3]),
Activation('softplus')(inputs[3]),
Activation('softmax')(inputs[3]),
Activation('relu')(inputs[3]),
LeakyReLU()(inputs[3]),
ELU()(inputs[3]),
shared_activation(inputs[3]),
inputs[4],
inputs[1],
x,
shared_activation(x),
]
print('Model has {} outputs.'.format(len(outputs)))
model = Model(inputs=inputs, outputs=outputs, name='test_model_full')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 1
batch_size = 1
epochs = 10
data_in = generate_input_data(training_data_size, input_shapes)
data_out = generate_output_data(training_data_size, outputs)
model.fit(data_in, data_out, epochs=epochs, batch_size=batch_size)
return model
def get_rnn_model(MAX_NB_WORDS, embedding_matrix_2):
"""
Creating the RNN model
:param MAX_NB_WORDS: maximum length of the seuquence
:param embedding_matrix_2: embedding matrix
"""
# -----------------------------------------------------
"""
both previous models are used here. predictions from both models are used
as metadata in the network
"""
# defining input shape of the data
inp = Input(shape=(50, ))
# defining input shape of the level_1 predictions
meta_input_1 = Input(shape=(1,))
# defining input shape of the level_2 predictions
meta_input_2 = Input(shape=(1,))
# -----------------------------------------------------
# defining Embedding layer
x = Embedding(MAX_NB_WORDS, 300, input_length=50, weights=[
embedding_matrix_2], trainable=False)(inp)
# -----------------------------------------------
# defining spatial dropout
x = SpatialDropout1D(0.2)(x)
# ----------------------------------------------------------
# defining RRN part
x = Bidirectional(GRU(100, return_sequences=True))(x)
x = Bidirectional(GRU(100, return_sequences=True))(x)
# defining the convolutional layer
x = Conv1D(512, kernel_size=1, padding="valid",
kernel_initializer="he_uniform")(x)
# --------------------------------------------------------
# defing two pooling layers average and maximum
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
# concating the two pooling layers
conc = concatenate([avg_pool, max_pool])
# -----------------------------------------------------
# applying batch normalization to speed the weights learning
conc = BatchNormalization()(conc)
# ----------------------------------------------------
"""
both predictions are concatenated with the new input data
and fed into a dense layer
"""
# concating the numerical and embedding feature
conc = concatenate([conc, meta_input_1, meta_input_2])
# applying dense layer on the concatenation
conc = Dense(512)(conc)
# ---------------------------------------------------
# increases the learning speed
conc = BatchNormalization()(conc)
# ---------------------------------
# applying leakyRelu
conc = LeakyReLU()(conc)
# --------------------------------------------------
# defining the ouput layer
outp = Dense(477, activation='softmax')(conc)
# --------------------------------------------------
# 3 inputs
model = Model(inputs=[inp, meta_input_1, meta_input_2], outputs=outp)
# if you to load pre-trained weights
# .load_weights("weights-improvement-01-0.69.hdf5")
# Compiling the model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
y_train
model = Sequential()
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(max_features, embedding_dims, input_length=maxlen))
model.add(Dropout(0.2))
# we add a Convolution1D, which will learn filters
# word group filters of size filter_length:
model.add(
Conv1D(filters, kernel_size, padding='valid', activation='relu',
strides=1))
# we use max pooling:
model.add(GlobalMaxPooling1D())
# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(0.2))
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(x_train,
y_train,
