def HDP_feature_extracter_from_texts(self,mashup_api):
if self.gd is None:
self.gd = get_default_gd(tag_times=0,mashup_only=True,strict_train=True) # 用gensim处理文本,文本中不加tag
self.mashup_features ,self.api_features =self.gd.model_pcs(new_Para.param.text_extracter_mode) #
self.feature_dim = len(self.mashup_features[0])
ID_input = Input(shape=(1,), dtype='int32')
if mashup_api=='mashup':
if self.mashup_text_feature_extracter is None: # 没求过
mashup_text_embedding_layer = Embedding(self.all_mashup_num,self.feature_dim,
embeddings_initializer=Constant(self.mashup_features),
mask_zero=False, input_length=1,
trainable=False, name = 'mashup_text_embedding_layer')
x = mashup_text_embedding_layer(ID_input)
self.mashup_text_feature_extracter = Model(ID_input, x, name='mashup_text_feature_extracter')
return self.mashup_text_feature_extracter
elif mashup_api=='api':
if self.api_text_feature_extracter is None: # 没求过
api_text_feature_extracter = Embedding(self.all_api_num,self.feature_dim,
embeddings_initializer=Constant(self.api_features),
mask_zero=False, input_length=1,
trainable=False, name='api_text_embedding_layer')
x = api_text_feature_extracter(ID_input)
self.api_text_feature_extracter = Model(ID_input, x, name='api_text_feature_extracter')
return self.api_text_feature_extracter
def get_tag_embedding_layer(self, nonstatic=True):
""""
同text，处理tags,得到定制的embedding层
"""
if self.tag_embedding_layer is None:
if self.encoded_tags is None:
self.process_tags()
# 得到词典中每个词对应的embedding
num_words = min(MAX_NUM_WORDS, len(self.encoded_tags.word2index)) + 1 # 实际词典大小 +1 因为0代表0的填充向量
self.tag_embedding_matrix = get_embedding_matrix(self.encoded_tags.word2index, new_Para.param.embedding_name,
dimension=self.word_embedding_dim)
print('built tag embedding matrix, done!')
self.tag_embedding_layer = Embedding(num_words,
self.word_embedding_dim,
embeddings_initializer=Constant(self.tag_embedding_matrix),
embeddings_regularizer=regularizers.l2(
new_Para.param.embeddings_regularizer),
input_length=MAX_SEQUENCE_LENGTH,
mask_zero=True,
trainable=new_Para.param.embedding_train,
name='tag_embedding_layer') #
print('built tag embedding layer, done!')
return self.tag_embedding_layer
def __init__(self,
num_layers,
d_model,
num_heads,
dff,
vocab_size,
input_seq_len,
output_seq_len,
rate=0.1):
super(AbstractiveSummarization, self).__init__()
self.input_seq_len = input_seq_len
self.output_seq_len = output_seq_len
self.vocab_size = vocab_size
self.bert = BertLayer(d_embedding=d_model, trainable=False)
embedding_matrix = _embedding_from_bert()
self.embedding = tf.keras.layers.Embedding(
vocab_size,
d_model,
trainable=False,
embeddings_initializer=Constant(embedding_matrix))
self.decoder = Decoder(num_layers, d_model, num_heads, dff, vocab_size,
rate)
self.final_layer = tf.keras.layers.Dense(vocab_size)
def seq_2_seq_att_LSTM(X_embedding, MAX_LEN, num_words,
EMBEDDING_DIM, LSTM_units, LSTM_dropout):
# Encoder
# Encoder input shape is (batch size, max length)
encoder_inputs = Input(shape=(MAX_LEN,))
encoder_embedding = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM,
input_length = MAX_LEN, embeddings_initializer=Constant(X_embedding),
trainable=False)(encoder_inputs)
# LSTM
encoder_lstm = LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout)
encoder_outputs, state_h, state_c = encoder_lstm(encoder_embedding)
encoder_states = [state_h, state_c]
# Decoder
decoder_inputs = Input(shape=(None,))
decoder_embedding_layer = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, trainable=True)
decoder_embedding = decoder_embedding_layer(decoder_inputs)
decoder_lstm = LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
# Attention
attention_weight = dot([decoder_outputs, encoder_outputs], axes=[2, 2], normalize=True) # cosine similarity
attention = Activation('softmax')(attention_weight)
context = dot([attention, encoder_outputs], axes=[2,1])
decoder_combined_context = concatenate([context, decoder_outputs])
att_output = TimeDistributed(Dense(64, activation="tanh"))(decoder_combined_context)
output = TimeDistributed(Dense(num_words, activation="softmax"))(att_output)
model = Model(inputs=[encoder_inputs,decoder_inputs], outputs=output)
return model
def get_initializer(initializer_params):
if initializer_params["function"] == "truncated_normal":
return TruncatedNormal(stddev=initializer_params["stddev"])
elif initializer_params["function"] == "constant":
return Constant(value=initializer_params["value"])
else:
return initializer_params["function"]
def get_text_embedding_layer(self, nonstatic=True):
""""
得到定制的embedding层
paras:
data_dirs: 存放mashup api 信息的文件夹
embedding_name：使用哪种pre-trained embedding，google_news or glove
embedding_path:embedding 文件存放路径
EMBEDDING_DIM：维度
nonstatic：基于pre-trained embedding是否微调？
"""
if self.text_embedding_layer is None:
if self.encoded_texts is None:
self.process_texts()
# 得到词典中每个词对应的embedding
num_words = min(MAX_NUM_WORDS, len(self.encoded_texts.word2index))+ 1 # 实际词典大小 +1 因为0代表0的填充向量
self.text_embedding_matrix = get_embedding_matrix(self.encoded_texts.word2index, new_Para.param.embedding_name,
dimension=self.word_embedding_dim)
print('built embedding matrix, done!')
self.text_embedding_layer = Embedding(num_words,
self.word_embedding_dim,
embeddings_initializer=Constant(self.text_embedding_matrix),
embeddings_regularizer= regularizers.l2(new_Para.param.embeddings_regularizer),
input_length=MAX_SEQUENCE_LENGTH,
mask_zero=True,
trainable=new_Para.param.embedding_train, name = 'text_embedding_layer') # 定义一层 # mask_zero=True, !!!
print('built text embedding layer, done!')
return self.text_embedding_layer
def set_embedding_layers(self):
self.api_implict_emb_layer = Embedding(
self.all_api_num + 1,
new_Para.param.num_feat,
embeddings_initializer=Constant(self.i_factors_matrix),
mask_zero=False,
trainable=False,
name='api_implict_embedding_layer')
def kdd_create_embedding_dict(sparse_feature_columns,
varlen_sparse_feature_columns,
init_std,
seed,
l2_reg,
prefix='sparse_',
seq_mask_zero=True):
global user_embed_np, item_embed_np
sparse_embedding = {}
for feat in sparse_feature_columns:
embed_initializer = RandomNormal(mean=0.0, stddev=init_std, seed=seed)
if feat.embedding_name == 'user_id':
print('init user embed')
embed_initializer = Constant(user_embed_np)
if feat.embedding_name == 'item_id':
print('init item embed')
embed_initializer = Constant(item_embed_np)
sparse_embedding[feat.embedding_name] = Embedding(
feat.vocabulary_size,
feat.embedding_dim,
embeddings_initializer=embed_initializer,
name=prefix + '_emb_' + feat.embedding_name)
if varlen_sparse_feature_columns and len(
varlen_sparse_feature_columns) > 0:
for feat in varlen_sparse_feature_columns:
embed_initializer = RandomNormal(mean=0.0,
stddev=init_std,
seed=seed)
if feat.embedding_name == 'user_id':
print('init user embed')
embed_initializer = Constant(user_embed_np)
if feat.embedding_name == 'item_id':
print('init item embed')
embed_initializer = Constant(item_embed_np)
sparse_embedding[feat.embedding_name] = Embedding(
feat.vocabulary_size,
feat.embedding_dim,
embeddings_initializer=embed_initializer,
name=prefix + '_seq_emb_' + feat.name,
mask_zero=seq_mask_zero)
return sparse_embedding
def sep_cnn_model(input_shape,
num_classes,
num_features,
embedding_matrix,
blocks=1,
filters=64,
kernel_size=4,
dropout_rate=0.5):
op_units, op_activation = _get_last_layer_units_and_activation(num_classes)
model = models.Sequential()
model.add(
Embedding(input_dim=num_features,
output_dim=300,
input_length=input_shape,
embeddings_initializer=Constant(embedding_matrix)))
for _ in range(blocks - 1):
model.add(Dropout(rate=dropout_rate))
model.add(
SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(
SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(MaxPooling1D(pool_size=3))
model.add(
SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(
SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(GlobalAveragePooling1D())
# model.add(MaxPooling1D())
model.add(Dropout(rate=0.5))
model.add(Dense(op_units, activation=op_activation))
return model
def set_embedding_layers(self):
self.mid2text_fea_layer = Embedding(self.all_mashup_num,
self.text_fea_dim,
embeddings_initializer=Constant(
self.mashup_texts_features),
mask_zero=False,
input_length=1,
trainable=False,
name='mashup_text_fea_layer')
self.aid2text_fea_layer = Embedding(self.all_api_num + 1,
self.tag_fea_dim,
embeddings_initializer=Constant(
self.api_texts_features),
mask_zero=False,
trainable=False,
name='api_text_fea_layer')
self.mid2tag_fea_layer = Embedding(self.all_mashup_num,
self.text_fea_dim,
embeddings_initializer=Constant(
self.mashup_tag_features),
mask_zero=False,
input_length=1,
trainable=False,
name='mashup_tag_fea_layer')
self.aid2tag_fea_layer = Embedding(self.all_api_num + 1,
self.tag_fea_dim,
embeddings_initializer=Constant(
self.api_tag_features),
mask_zero=False,
trainable=False,
name='api_tag_fea_layer')
self.api_implict_emb_layer = Embedding(self.all_api_num + 1,
new_Para.param.num_feat,
embeddings_initializer=Constant(
self.i_factors_matrix),
mask_zero=False,
trainable=False,
name='api_implict_emb_layer')
def text_cnn():
"""
构建text_cnn模型
:return:
"""
# Inputs
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
# Embeddings layers
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embeddings_index = get_embeddings_index()
word_index = get_word_index()
num_words = min(MAX_NUM_WORDS, len(word_index) + 1)
embedding_matrix = get_embedding_matrix(embeddings_index, word_index)
embedding_layer = Embedding(
num_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
# embeddings_index = get_embeddings_index()
# word_index = get_word_index()
# embedding_matrix = get_embedding_matrix(embeddings_index, word_index)
# embedding_layer = Embedding(len(word_index) + 1,
# EMBEDDING_DIM,
# weights=[embedding_matrix],
# input_length=MAX_SEQUENCE_LENGTH,
# trainable=False)
embedded_sequences = embedding_layer(sequence_input)
# conv layers
convs = []
for fs in FILTER_SIZES:
l_conv = Conv1D(filters=100, kernel_size=fs,
activation='relu')(embedded_sequences)
l_pool = MaxPooling1D(MAX_SEQUENCE_LENGTH - fs + 1)(l_conv)
l_pool = Flatten()(l_pool)
convs.append(l_pool)
merge = concatenate(convs, axis=1)
x = Dropout(DROPOUT_RATE)(merge)
x = Dense(32, activation='relu')(x)
preds = Dense(units=1, activation='sigmoid')(x)
model = Model(sequence_input, preds)
model.compile(loss="categorical_crossentropy",
optimizer="rmsprop",
metrics=['acc'])
return model
def set_embedding_layers(self):
# embedding
# using the embedding layer instead of packing in the instance, to save memory
self.mid2text_embedding_layer = Embedding(
self.all_mashup_num,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.mid2text_wordindex),
mask_zero=False,
input_length=1,
trainable=False,
name='mashup_text_encoding_embedding_layer')
self.aid2text_embedding_layer = Embedding(
self.all_api_num + 1,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.aid2text_wordindex),
mask_zero=False,
trainable=False,
name='api_text_encoding_embedding_layer')
self.mid2tag_embedding_layer = Embedding(
self.all_mashup_num,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.mid2tag_wordindex),
mask_zero=False,
input_length=1,
trainable=False,
name='mashup_tag_encoding_embedding_layer')
self.aid2tag_embedding_layer = Embedding(
self.all_api_num + 1,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.aid2tag_wordindex),
mask_zero=False,
trainable=False,
name='api_tag_encoding_embedding_layer')
self.user_text_feature_extracter = None
self.item_text_feature_extracter = None
self.user_tag_feature_extracter = None
self.item_tag_feature_extracter = None
def biLSTM_baseline(embedding_matrix, MAX_LEN, num_words, EMBEDDING_DIM, LSTM_units, LSTM_dropout): input_dimen = Input(shape=(MAX_LEN,)) model = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, input_length=MAX_LEN, embeddings_initializer=Constant(embedding_matrix), trainable=False)(input_dimen) model = Bidirectional(LSTM(units=LSTM_units, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout))(model) model = Bidirectional(LSTM(units=LSTM_units, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout))(model) out = TimeDistributed(Dense(1, activation='sigmoid'))(model) model = Model(input_dimen, out) return model
def build(self, input_shape):
# Sanity Check
if isinstance(input_shape, list):
raise ValueError('ConstraintEnforcementLayer has only 1 input.')
assert input_shape[-1] == self.A.shape[1]
# Create computation graph matrices and set values
self.A_graph = self.add_weight(shape=self.A.shape,
initializer=Constant(self.A),
name='A',
trainable=False)
self.b_graph = self.add_weight(shape=self.b.shape,
initializer=Constant(self.b),
name='b',
trainable=False)
self.c_graph = self.add_weight(shape=self.c.shape,
initializer=Constant(self.c),
name='c',
trainable=False)
# Done building
super(ConstraintEnforcementLayer, self).build(input_shape)
def build_model(embedding_weights,
embedding_dim,
num_words,
input_length,
num_classes=20):
""" Builds a Keras model. It sets embeddings layer trainable to False
to keep the embeddings fixed
Parameters
----------
embedding_weights: np.ndarray
A numpy array contains embedding weights
embedding_dim: int
Embeddings dimension
num_words: int
Number of words in the dataset
input_length: int
Maximum sequence length
num_classes: int
Number of classes in the dataset
Returns
-------
model: Model
A keras compiled model instance
"""
embedding = Embedding(num_words,
embedding_dim,
embeddings_initializer=Constant(embedding_weights),
input_length=input_length,
trainable=False)
seq_input = Input(shape=(input_length, ), dtype='int32')
embedded_sequences = embedding(seq_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(num_classes, activation='softmax')(x)
model = Model(seq_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
return model
def create_actor_model(self,
state_input,
root_net,
action_dim,
lr=0.001,
dropout=0.3):
"""
net = Dense(64, kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01), activation="relu")(root_net)
"""
net = Dense(40, activation="relu")(root_net)
#net = Dropout(dropout)(net)
if self.use_batch_norm:
net = BatchNormalization()(net)
"""
net = Dense(64, input_dim=64,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01), activation="relu")(net)
"""
net = Flatten()(net)
net = Dense(action_dim * 2, activation="sigmoid")(net)
#net = Dropout(dropout)(net)
if self.use_batch_norm:
net = BatchNormalization()(net)
# Final layer weights are init to Uniform[-3e-3, 3e-3]
random_initializer = RandomUniform(minval=0.01, maxval=0.1, seed=None)
# add input
# @TODO add the previous action
actor_out = Dense(
action_dim,
#kernel_initializer=random_initializer,
#kernel_initializer="glorot_uniform", # for softmax
#kernel_initializer="he_uniform", # for relu
activation="softmax",
# activation="relu",
kernel_initializer=Constant(1.0 / action_dim),
name="actor_out")(net)
# actor_out = CustomActivation()(actor_out)
#actor_out = Lambda(lambda x: tf.sigmoid(x) / (1e-5+ tf.norm(tf.sigmoid(x), axis=0, ord=1, keep_dims=True)))(actor_out)
actor_model = Model(inputs=state_input, outputs=actor_out)
if DEBUG:
print("ACTOR MODEL :", actor_model.summary())
return actor_model
def seq_2_seq_biLSTM_att(X_embedding, MAX_LEN, num_words,
EMBEDDING_DIM, LSTM_units, LSTM_dropout):
# Encoder
# [?, 100]
encoder_inputs = Input(shape=(MAX_LEN,))
# [?, 100, 300]
encoder_embedding = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM,
input_length = MAX_LEN, embeddings_initializer=Constant(X_embedding),
trainable=False)(encoder_inputs)
# LSTM
encoder_lstm = Bidirectional(LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout))
# [?, 100, 300]
encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder_lstm(encoder_embedding)
# [?, 300]
state_h = concatenate([forward_h, backward_h])
state_c = concatenate([forward_c, backward_c])
encoder_states = [state_h, state_c]
# Decoder
# [?, 30]
decoder_inputs = Input(shape=(None,))
decoder_embedding_layer = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, trainable=True)
# [?, 30, 300]
decoder_embedding = decoder_embedding_layer(decoder_inputs)
decoder_lstm = LSTM(units=2*LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout)
# [?, 30, 300]
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
# [?, 30, 100]
attention_weight = dot([decoder_outputs, encoder_outputs], axes=[2, 2])
attention = Activation('softmax')(attention_weight)
# [?, 30, 300]
context = dot([attention, encoder_outputs], axes=[2,1]) #[?, 100, 300] = dot([?,?,100] , [?, 100, 300])
# [?, 30, 600]
decoder_combined_context = concatenate([context, decoder_outputs])
# [?, 30, 64]
att_output = TimeDistributed(Dense(128, activation="tanh"))(decoder_combined_context)
# [?, 30, 39093]
output = TimeDistributed(Dense(num_words, activation="softmax"))(att_output)
model = Model(inputs=[encoder_inputs,decoder_inputs], outputs=output)
return model
def cnn_model(
input_shape, # input list shape (word index list)
num_classes, # output shape
num_features, # number of word + empty
embedding_matrix,
filters,
kernel_sizes,
dropout_rate,
embedding_trainable,
l2_lambda):
embedding_layer = Embedding(
input_dim=num_features,
output_dim=300, # hard code
embeddings_initializer=Constant(embedding_matrix),
input_length=input_shape,
trainable=embedding_trainable)
# word index list, not map to embedding yet
sequence_input = Input(shape=(input_shape, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
nn_layers = list()
for kernel_size in kernel_sizes:
conv_layer_0 = Convolution1D(filters, kernel_size,
padding='valid')(embedded_sequences)
conv_layer_1 = BatchNormalization(axis=1)(conv_layer_0)
conv_layer_2 = Activation('relu')(conv_layer_1)
pool_layer_0 = MaxPooling1D(input_shape - kernel_size +
1)(conv_layer_2)
pool_layer_1 = Dropout(dropout_rate)(pool_layer_0)
nn_layers.append(pool_layer_1)
# merge diff kernal size generated output
line_merge_layer = concatenate(nn_layers)
line_flat_layer = Flatten()(line_merge_layer)
norm_layer = BatchNormalization(axis=1)(line_flat_layer)
drop_layer = Dropout(dropout_rate)(norm_layer)
preds = Dense(num_classes,
kernel_regularizer=regularizers.l2(l2_lambda),
activation='softmax')(drop_layer)
cnn_model = Model(inputs=sequence_input, outputs=preds)
return cnn_model
def seq_2_seq(X_embedding, MAX_LEN, num_words, EMBEDDING_DIM, LSTM_units,
LSTM_dropout):
# Encoder
encoder_inputs = Input(shape=(MAX_LEN, ))
encoder_embedding = Embedding(input_dim=num_words,
output_dim=EMBEDDING_DIM,
input_length=MAX_LEN,
embeddings_initializer=Constant(X_embedding),
trainable=False)(encoder_inputs)
# LSTM
encoder_lstm = LSTM(units=LSTM_units,
return_state=True,
return_sequences=True,
recurrent_dropout=LSTM_dropout,
dropout=LSTM_dropout)
encoder_output, state_h, state_c = encoder_lstm(encoder_embedding)
# Decoder
decoder_inputs = Input(shape=(None, ))
decoder_embedding_layer = Embedding(input_dim=num_words,
output_dim=EMBEDDING_DIM,
trainable=True)
decoder_embedding = decoder_embedding_layer(decoder_inputs)
decoder_lstm = LSTM(units=LSTM_units,
return_state=True,
return_sequences=True,
recurrent_dropout=LSTM_dropout,
dropout=LSTM_dropout)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding,
initial_state=[state_h, state_c])
output = TimeDistributed(Dense(num_words,
activation='softmax'))(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], output)
return model
def build(self, input_shape):
self.gamma = self.add_weight(name=self.name + '_gamma',
shape=(input_shape[-1], ),
initializer=Constant(self.scale),
trainable=True)
super(Normalize, self).build(input_shape)
def build(self, input_shape):
self.multiplier = self.add_weight(
name="multiplier",
shape=[],
dtype=tf.float32,
initializer=Constant(value=self.initial_value))
def sepCNN(blocks,
filters,
kernel_size,
embedding_dim,
dropout_rate,
pool_size,
input_shape,
num_features,
pretrained_embedding=False,
embedding_trainable=False,
embedding_weights=None,
learning_rate=1e-3):
""" Creates an instance of a separable CNN model.
Parameters
----------
blocks: int
Number of pairs of sepCNN and pooling blocks in the model. One block
contains [DropOut, Conv1D, Conv1D, MaxPool]
filters: int
Output dimension of the layers.
kernel_size: int
Length of the convolution window.
embedding_dim: int
Dimension of the embedding vectors.
dropout_rate: float
Percentage of input to drop at Dropout layers.
pool_size: int
Factor by which to downscale input at MaxPooling layer.
input_shape: tuple
Shape of input to the model.
num_features: int
Number of words (embedding input dimension).
pretrained_embedding: bool
True if pre-trained embedding is on.
embedding_trainable: bool
True if embedding layer is trainable.
embedding_weights: np.ndarray
Dictionary with embedding coefficients.
learning_rate: float
Learning rate parameter for the model
Returns
-------
model:
A compiled sepCNN keras model instance.
"""
model = Sequential()
if pretrained_embedding:
model.add(
Embedding(num_features,
embedding_dim,
input_length=input_shape[0],
embeddings_initializer=Constant(embedding_weights),
trainable=embedding_trainable))
else:
model.add(
Embedding(num_features, embedding_dim,
input_length=input_shape[0]))
for _ in range(blocks - 1):
model.add(Dropout(dropout_rate))
model.add(
SeparableConv1D(filters,
kernel_size,
activation='relu',
padding='same'))
model.add(
SeparableConv1D(filters,
kernel_size,
activation='relu',
padding='same'))
model.add(MaxPooling1D(pool_size))
model.add(
SeparableConv1D(filters * 2,
kernel_size,
activation='relu',
padding='same'))
model.add(
SeparableConv1D(filters * 2,
kernel_size,
activation='relu',
padding='same'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(dropout_rate))
model.add(Dense(1, activation='sigmoid'))
optimizer = Adam(lr=learning_rate)
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['acc'])
return model
eval_data = data.test.images
eval_labels = data.test.labels
eval_labels = np.asarray(data.test.labels,dtype=np.int32)
eval_data, eval_labels = shuffle(eval_data, eval_labels)
return (train_data, train_labels, eval_data, eval_labels)
# Laden der Daten
train_data, train_labels, eval_data, eval_labels = load_fashion_data()
train_data = train_data.reshape(-1, 28, 28, 1)
train_labels = np_utils.to_categorical(train_labels, 10)
print(train_data.shape)
# Model mit Keras
model.add(InputLayer(input_shape=(28, 28,1),name="1_Eingabe"))
model.add(Conv2D(32,(2, 2),padding='same',bias_initializer=Constant(0.01),kernel_initializer='random_uniform',name="2_Conv2D"))
model.add(Activation(activation='relu',name="3_ReLu"))
model.add(MaxPool2D(padding='same',name="4_MaxPooling2D"))
model.add(Conv2D(32,(2, 2),padding='same',bias_initializer=Constant(0.01),kernel_initializer='random_uniform',name="5_Conv2D"))
model.add(Activation(activation='relu',name="6_ReLu"))
model.add(MaxPool2D(padding='same',name="7_MaxPooling2D"))
model.add(Flatten())
model.add(Dense(1024,activation='relu',bias_initializer=Constant(0.01),kernel_initializer='random_uniform',name="8_Dense"))
model.add(Dropout(0.4,name="9_Dense"))
model.add(Dense(10, activation='softmax',name="10_Ausgabe"))
model.compile(loss=losses.categorical_crossentropy, optimizer=optimizers.Adadelta(), metrics = ["accuracy","mse",metrics.categorical_accuracy])
#keras.backend.set_session(tf_debug.TensorBoardDebugWrapperSession(tf.Session(), "localhost:12345"))
K.set_session(tf_debug.TensorBoardDebugWrapperSession(tf.Session(), "localhost:12345"))
if embedding_vector is not None:
#words not found in the embedding index will be all-zeros
embedding_matrix[i] = embedding_vector
print(num_words)
# In[22]:
from tensorflow.python.keras.initializers import Constant
#Defining model and using the embeddding matrix as input to the embedding layer
model = Sequential()
embedding_layer = Embedding(
num_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=max_length,
trainable=False
) #Setting trainable as false since the embedding is already learned
model.add(embedding_layer)
model.add(GRU(units=32, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='relu'))
#Setting model parameters
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# In[25]:
#Model summary with pre-trained embedding
for word, i in tokenizer.word_index.items():
if i > vocabSize:
continue
if word in word2vecModel.wv.vocab.keys():
embeddingWeights[i] = word2vecModel.wv.get_vector(word)
XTrainTokens = tokenizer.texts_to_sequences(X_train)
XTrainPad = pad_sequences(XTrainTokens, maxlen=maxLength, padding='post')
XTestTokens = tokenizer.texts_to_sequences(X_test)
XTestPad = pad_sequences(XTestTokens, maxlen=maxLength, padding='post')
biGRU = Sequential()
biGRU.add(
Embedding(vocabSize,
embDim,
embeddings_initializer=Constant(embeddingWeights),
input_length=maxLength,
mask_zero=True))
biGRU.add(Bidirectional(GRU(units=20, dropout=0.3)))
biGRU.add(Dense(1))
optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)
lossFunction = tf.keras.losses.MeanSquaredError()
biGRU.compile(optimizer=optimizer, loss=lossFunction)
print('\nTraining Deep Learning Model\n')
biGRU.fit(XTrainPad, y_train, batch_size=256, epochs=20)
# model.save('my_model.h5')
preds = biGRU.predict(XTestPad)
