def build_model(embeddings_path, output_dim, stored_model=None):
model = None
index_dict, vector_dict = read_embeddings(embeddings_path, EMBEDDING_DIM)
if stored_model is None:
model = Sequential()
embedding_weights = np.zeros((len(index_dict), EMBEDDING_DIM))
for word, index in index_dict.items():
embedding_weights[index, :] = vector_dict[word]
# define inputs here
embedding = Embedding(
output_dim=EMBEDDING_DIM, input_dim=len(index_dict),
input_length=MAX_SEQUENCE_LENGTH, trainable=False
)
embedding.build((None,))
embedding.set_weights([embedding_weights])
# add layers
model.add(embedding)
model.add(Dropout(0.4))
model.add(Conv1D(filters=10, kernel_size=5, padding="same"))
model.add(keras.layers.PReLU())
model.add(MaxPooling1D(pool_size=3))
model.add(Bidirectional(LSTM(50, recurrent_dropout=0.4, return_sequences=True)))
model.add(Bidirectional(LSTM(50, recurrent_dropout=0.4, return_sequences=False)))
model.add(Dense(output_dim, activation="softmax"))
model.compile(loss="binary_crossentropy", optimizer="nadam", metrics=[f1])
else:
# load model state
model = keras.models.load_model(stored_model)
return model, index_dict
def pretrained_embedding_layer(emb_matrix, input_length):
"""
Creates a Keras Embedding() layer and loads in pre-trained GloVe 50-dimensional vectors.
Arguments:
word_to_vec_map -- dictionary mapping words to their Word2Vec vector representation.
word_to_index -- dictionary mapping from words to their indices in the vocabulary
Returns:
embedding_layer -- pretrained layer Keras instance
"""
vocab_len = emb_matrix.shape[0]
# vocab_len = vocab_len + 1 # adding 1 to fit Keras embedding (requirement)
emb_dim = emb_matrix.shape[1] # define dimensionality of your word vectors
print("Found word2vec dimensions: ", emb_dim)
# Use Embedding(...). Make sure to set trainable=False.
embedding_layer = Embedding(vocab_len,
emb_dim,
input_length=input_length,
mask_zero=False,
trainable=False)
# Build the embedding layer, it is required before setting the weights of the embedding layer.
# Do not modify the "None".
embedding_layer.build((None, ))
# Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def make_embedding_layer(train_descriptions, embedding_dim=50, glove=True):
if glove == False:
print('Just a zero matrix loaded')
embedding_matrix = np.zeros((vocab_size, embedding_dim)) # just a zero matrix
else:
glove_dir = './glove.6B/'
embeddings_index = {}
f = open(os.path.join(glove_dir, 'glove.6B.'+str(embedding_dim)+'d.txt'), encoding="utf-8")
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
# Get x-dim dense vector for each of the vocab_rocc
vocab_size, ixtoword, wordtoix, vocab = get_vocab_size_and_indexing(train_descriptions)
# max_length = max_length(desc, 90)
embedding_matrix = np.zeros((vocab_size, embedding_dim)) # to import as weights for Keras Embedding layer
for word, i in wordtoix.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# Words not found in the embedding index will be all zeros
embedding_matrix[i] = embedding_vector
print('GloVe loaded!')
embedding_layer = Embedding(vocab_size, embedding_dim, mask_zero=True, trainable=False)
embedding_layer.build((None,))
embedding_layer.set_weights([embedding_matrix])
return embedding_layer
def pretrained_embedding_layer(word_to_vec_map, source_vocab_to_int):
"""
构造Embedding层并加载预训练好的词向量（这里我使用的是100维）
@param word_to_vec_map: 单词到向量的映射
@param word_to_index: 单词到数字编码的映射
"""
vocab_len = len(source_vocab_to_int) + 1 # Keras Embedding的API要求+1
emb_dim = word_to_vec_map["the"].shape[0]
# 初始化embedding矩阵
emb_matrix = np.zeros((vocab_len, emb_dim))
# 用词向量填充embedding矩阵
for word, index in source_vocab_to_int.items():
word_vector = word_to_vec_map.get(word, np.zeros(emb_dim))
emb_matrix[index, :] = word_vector
# 定义Embedding层，并指定不需要训练该层的权重
embedding_layer = Embedding(vocab_len, emb_dim, trainable=False)
# build
embedding_layer.build((None,))
# set weights
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def get_model(num_users,num_items,ratings,layers,l2_param):
input_a = Input(shape=(1,),name='input-a',dtype='int32')
input_b = Input(shape=(1,),name='input-b',dtype='int32')
user_embedding_layer = Embedding(input_dim=num_users,output_dim=num_items,trainable=False,input_length=1,name='user-table')
user_embedding_layer.build((None,))
user_embedding_layer.set_weights([ratings])
user_embedding = user_embedding_layer(input_a)
user_embedding = Flatten()(user_embedding)
item_embedding_layer = Embedding(input_dim=num_items,output_dim=num_users,trainable=False,input_length=1,name='item-table')
item_embedding_layer.build((None,))
item_embedding_layer.set_weights([ratings.T])
item_embedding = item_embedding_layer(input_b)
item_embedding = Flatten()(item_embedding)
user_hidden = Dense(layers[0],activation='relu',kernel_regularizer=regularizers.l2(l2_param),name='user-encoding-layer-{}'.format(0))(user_embedding)
item_hidden = Dense(layers[0],activation='relu',kernel_regularizer=regularizers.l2(l2_param),name='item-encoding-layer-{}'.format(0))(item_embedding)
encoded_a = Dense(layers[1],activation='relu',kernel_regularizer=regularizers.l1(l2_param),name='user-encoding-layer-{}'.format(1))(user_hidden)
encoded_b = Dense(layers[1],activation='relu',kernel_regularizer=regularizers.l1(l2_param),name='item-encoding-layer-{}'.format(1))(item_hidden)
user_out_hidden = Dense(layers[0],activation='relu',kernel_regularizer=regularizers.l2(l2_param),name='user-decoding-layer-{}'.format(0))(encoded_a)
item_out_hidden = Dense(layers[0],activation='relu',kernel_regularizer=regularizers.l2(l2_param),name='item-decoding-layer-{}'.format(0))(encoded_b)
decoded_a = Dense(num_items,activation='sigmoid',kernel_regularizer=regularizers.l2(l2_param),name='user-decoding-layer-{}'.format(1))(user_out_hidden)
decoded_b = Dense(num_users,activation='sigmoid',kernel_regularizer=regularizers.l2(l2_param),name='item-decoding-layer-{}'.format(1))(item_out_hidden)
embedding_diff = Concatenate()([encoded_a,encoded_b])
result_hidden = Dense(layers[1],activation='relu',kernel_regularizer=regularizers.l2(l2_param),name='predict-layer-{}'.format(0))(embedding_diff)
result = Dense(1,activation='sigmoid',kernel_regularizer=regularizers.l2(l2_param),name='predict-layer-{}'.format(1))(result_hidden)
model = Model(inputs=[input_a,input_b],outputs=[decoded_a,decoded_b,result])
# opt = Adam(lr=1e-4)
# model.compile(optimizer=opt,loss=[recostruction_loss,recostruction_loss,edge_wise_loss],loss_weights=[1,1,alpha])
predictor = Model(inputs=[input_a,input_b],outputs=[result])
# predictor.compile(optimizer=opt,loss=[edge_wise_loss],metrics=[edge_wise_loss])
# model.summary()
encoder = Model(input_a,encoded_a)
decoder = Model(input_a,decoded_a)
# decoder.compile(optimizer='adadelta',loss=recostruction_loss)
return model,encoder,decoder,predictor
def pretrained_embedding_layer(word_to_vec_map, word_to_index, length,
embedding_dim):
"""
Creates a Keras Embedding() layer and loads in pre-trained GloVe 50-dimensional vectors.
Arguments:
word_to_vec_map -- dictionary mapping words to their GloVe vector representation.
word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)
Returns:
embedding_layer -- pretrained layer Keras instance
"""
vocab_len = len(
word_to_index) + 1 # adding 1 to fit Keras embedding (requirement)
emb_matrix = np.zeros((vocab_len, embedding_dim))
for word, index in word_to_index.items():
emb_matrix[index, :] = word_to_vec_map[word]
embedding_layer = Embedding(input_dim=vocab_len,
output_dim=embedding_dim,
trainable=False,
input_length=length)
embedding_layer.build((None, ))
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def createEmbeddingLayer(word2index: dict, word2vec: dict) -> Embedding:
""" Set up pretrained Embedding layer
Credits to: https://keras.io/examples/pretrained_word_embeddings/
Args:
word2index: dictionnary mapping words to indices
word2vec: dictionnary mapping words to embedding vectors
Returns:
embedding_layer layer with pretrained GloVe word embeddings
"""
num_words = min(MAX_WORDS, len(word2index)) + 1
emb_dim = word2vec["cucumber"].shape[0]
emb_matrix = np.zeros((num_words, emb_dim))
for word, index in word2index.items():
if index > MAX_WORDS:
break
embedding_vector = word2vec.get(word)
if embedding_vector is not None:
emb_matrix[index, :] = embedding_vector
embedding_layer = Embedding(
num_words, emb_dim, input_length=MAX_SEQUENCE_LEN,
trainable=False) # Do not update word embeddings
embedding_layer.build((None, ))
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def pretrained_embedding_layer(g_vectors, words_to_index):
"""
Creates a Keras Embedding() layer and loads in pre-trained GloVe 50-dimensional vectors.
Arguments:
word_to_vec_map -- dictionary mapping words to their GloVe vector representation.
word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)
Returns:
embedding_layer -- pretrained layer Keras instance
"""
vocab_len = len(words_to_index) + 1 # adding 1 to fit Keras embedding (requirement)
emb_dim = g_vectors["hello"].shape[0] # define dimensionality of your GloVe word vectors (= 50)
# Initialize the embedding matrix as a numpy array of zeros of shape (vocab_len, dimensions of word vectors = emb_dim)
emb_matrix = np.zeros((vocab_len, emb_dim))
# Set each row "index" of the embedding matrix to be the word vector representation of the "index"th word of the vocabulary
for word, index in words_to_index.items():
emb_matrix[index, :] = g_vectors[word]
# Define Keras embedding layer with the correct output/input sizes, make it trainable. Use Embedding(...). Make sure to set trainable=False.
embedding_layer = Embedding(vocab_len, emb_dim, trainable = False)
# Build the embedding layer, it is required before setting the weights of the embedding layer. Do not modify the "None".
embedding_layer.build((None,))
# Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def make_embedding_layer(vocab_size, wordtoix, embedding_dim=50, glove=True):
'''
create a vector matrix for the descriptions, make en embedding layer
INPUT : vocabulary size (int), wordtoix (dict), embeddings dimension (int), glove (T/F)
OUTPUT : embedding layer
'''
if glove == False:
embedding_matrix = np.zeros((vocab_size, embedding_dim))
print('Just a zero matrix loaded')
else:
embeddings_index = load_glove(embedding_dim)
print('GloVe loaded!')
# to import as weights for Keras Embedding layer
embedding_matrix = np.zeros((vocab_size, embedding_dim))
for word, i in wordtoix.items():
# Get x-dim dense vector for each of the vocab_rocc
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# Words not found in the embedding index will be all zeros
embedding_matrix[i] = embedding_vector
# create an embedding layer
embedding_layer = Embedding(vocab_size,
embedding_dim,
mask_zero=True,
trainable=False)
embedding_layer.build((None, ))
embedding_layer.set_weights([embedding_matrix])
return embedding_layer
def create_nn_model(X, embeddings_dim, embeddings_matrix=None):
num_asts = np.amax(X)
max_timestep = np.shape(X)[1]
model = Sequential()
if embeddings_matrix is None:
embedding_layer = Embedding(num_asts + 1, embeddings_dim,
input_length=max_timestep)
else:
embeddings_dim = np.shape(embeddings_matrix)[1]
embedding_layer = Embedding(num_asts + 1, embeddings_dim,
input_length=max_timestep, trainable=False)
embedding_layer.build((None,))
embedding_layer.set_weights([embeddings_matrix])
model.add(embedding_layer)
hidden_dim = 32
model.add(LSTM(hidden_dim, return_sequences=True,
input_shape=(max_timestep, embeddings_dim)))
model.add(TimeDistributed(Dense(1)))
model.add(Activation("sigmoid"))
model.add(Reshape((max_timestep,)))
model.compile(loss="binary_crossentropy", optimizer="adam",
metrics=["accuracy", precision, recall, f1])
model.summary()
return model
def get_model(max_len, embedding_size, vocab_size, embedding, hidden_size,
distance_metric):
seq_1 = Input(shape=(max_len, ), dtype='int32', name='sequence1')
seq_2 = Input(shape=(max_len, ), dtype='int32', name='sequence2')
embed_layer = Embedding(output_dim=embedding_size,
input_dim=vocab_size + 1,
input_length=max_len,
trainable=False)
embed_layer.build((None, ))
embed_layer.set_weights([embedding.embedding_matrix])
input_1 = embed_layer(seq_1)
input_2 = embed_layer(seq_2)
l1 = LSTM(units=hidden_size)
l1_out = l1(input_1)
l2_out = l1(input_2)
concats = concatenate([l1_out, l2_out], axis=-1)
if distance_metric == 'cosine':
main_output = Lambda(exponent_neg_cosine_distance,
output_shape=(1, ))(concats)
else:
main_output = Lambda(exponent_neg_manhattan_distance,
output_shape=(1, ))(concats)
model = Model(inputs=[seq_1, seq_2], outputs=[main_output])
return model
def pretrained_embedding_layer(word_to_vec_map, source_vocab_to_int, emb_dim,trainable = False):
"""
构造Embedding层并加载预训练好的词向量
@param word_to_vec_map: 单词到向量的映射
@param word_to_index: 单词到数字编码的映射
@param emb_dim：embedding维度
"""
vocab_len = len(source_vocab_to_int)
# 初始化embedding矩阵
emb_matrix = np.zeros((vocab_len, emb_dim))
# 用词向量填充embedding矩阵
for word, index in source_vocab_to_int.items():
word_vector = word_to_vec_map.get(word, np.zeros(emb_dim))
emb_matrix[index, :] = word_vector
# 定义Embedding层，并指定不需要训练该层的权重
embedding_layer = Embedding(vocab_len, emb_dim, trainable=trainable)
# build
embedding_layer.build((None,))
# set weights
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def pretrained_embedding_layer( language ):
"""Creates a Keras Embedding() layer and loads in pre-trained word2vec 300-dimensional vectors.
Args:
word_to_vec_map: dictionary mapping words to their word2vec vector representation.
word_to_index: dictionary mapping from words to their indices in the vocabulary
Returns:
embedding_layer: pretrained layer Keras instance
"""
emb_matrix = np.load( word2vec_path + language + "/" + language )
vocab_len = emb_matrix.shape[0] + 1
emb_dim = emb_matrix.shape[1]
emb_matrix = np.append( emb_matrix, np.zeros( ( 1, emb_dim ) ), axis = 0 )
print( "embedding: ", language, emb_matrix.shape )
# Define Keras embedding layer with the correct output/input sizes, make it trainable.
# Use Embedding(...). Make sure to set trainable=False.
embedding_layer = Embedding( vocab_len, emb_dim, trainable = False, mask_zero = True )
# Build the embedding layer, it is required before setting the weights of the embedding layer.
# Do not modify the "None".
embedding_layer.build( ( None, ) )
# Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.
embedding_layer.set_weights( [emb_matrix] )
return embedding_layer
def make_embedding_layer_without_glove(train_descriptions,
embedding_dim=50,
glove=True):
embeddings_index = {}
vocab_size, ixtoword, wordtoix, vocab = get_vocab_size_and_indexing(
train_descriptions)
embedding_matrix = np.zeros(
(vocab_size,
embedding_dim)) # to import as weights for Keras Embedding layer
for word, i in wordtoix.items():
embedding_vector = i
if embedding_vector is not None:
# Words not found in the embedding index will be all zeros
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(vocab_size,
embedding_dim,
mask_zero=True,
trainable=False)
embedding_layer.build((None, ))
embedding_layer.set_weights([embedding_matrix])
return embedding_layer
def keras_embeddings_layer(self):
vocab_len = len(self.model.wv.vocab)
emb_dim = self.model.wv.vector_size
emb_matrix = self.model.wv.syn0
embedding_layer = Embedding(vocab_len, emb_dim, trainable=True)
embedding_layer.build((None, ))
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def build_word_sequence_lstm_model(word_indices, word_vectors, output_dim):
lstm_unit_size = args.lstm_unit_size
learning_rate = args.learning_rate
dropout = 0.6
# from word_indices of all word vocabulary to word_embedded for one sentence of text
word_symbols = len(word_indices) + 1
word_embedding_weights = np.zeros((word_symbols, word_embeddings_size))
for word, index in word_indices.items():
try:
word_embedding_weights[index, :] = word_vectors[word]
except KeyError:
word_embedding_weights[
index, :] = np.ones(word_embeddings_size) * -1
doc_word_input = Input(shape=(max_num_of_sentences,
max_num_of_tokens_per_sentence),
dtype='int64',
name="doc_word_input")
sent_word_input = Input(shape=(max_num_of_tokens_per_sentence, ),
dtype='int64',
name="sent_word_input")
word_embedding_layer = Embedding(output_dim=word_embeddings_size,
input_dim=word_symbols,
mask_zero=True)
word_embedding_layer.build(
(None, )) # if you don't do this, the next step won't work
word_embedding_layer.set_weights([word_embedding_weights])
word_embedded = word_embedding_layer(sent_word_input)
bi_lstm_word_sent = Bidirectional(
GRU(lstm_unit_size, return_sequences=rueT))(word_embedded)
#attention_word = AttentionWithContext()(bi_lstm_word_sent)
word_sent_encode = Dropout(dropout)(bi_lstm_word_sent) #(attention_word)
word_encoder = Model(inputs=sent_word_input, outputs=word_sent_encode)
word_encoded = TimeDistributed(word_encoder)(doc_word_input)
b_lstm_word_doc = Bidirectional(GRU(lstm_unit_size,
return_sequences=False))(word_encoded)
word_output = Dropout(dropout, name='final_layer_word')(
b_lstm_word_doc) #(attention_doc)
word_output = Dense(output_dim,
activation='softmax')(word_output) #(word_output)
model = Model(inputs=doc_word_input, outputs=word_output)
rmsprop = RMSprop(lr=learning_rate)
model.compile(optimizer=rmsprop,
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
final_layer = Model(inputs=model.input,
outputs=model.get_layer('final_layer_word').output)
# final_layer = Model(inputs=model.input, outputs=model.get_layer('final_layer_word').output)
return model, final_layer
def pretrained_embedding_layer(word_to_vec_map, word_to_index):
vocab_len = len(word_to_index) + 1
emb_dim = word_to_vec_map["cucumber"].shape[0]
emb_matrix = np.zeros((vocab_len, emb_dim))
for word, index in word_to_index.items():
emb_matrix[index, :] = word_to_vec_map[word]
embedding_layer = Embedding(vocab_len, emb_dim, trainable=False)
embedding_layer.build((None,))
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def get_model(num_genes, num_diseases):
# Input variables
gene_input = Input(shape=(1, ), dtype='int32', name='gene_input')
disease_input = Input(shape=(1, ), dtype='int32', name='disease_input')
# using gene feature matrix for initialization
gene_feas = h5py.File('gene_features.mat', 'r')
humannet_features = gene_feas['features'][:, :].T
gene_feature_size = humannet_features.shape
assert gene_feature_size[0] + 1 == num_genes
humannet_features = np.insert(humannet_features, 0,
[0] * gene_feature_size[1], 0)
humannet_embedding_layer = Embedding(input_dim=gene_feature_size[0] + 1,
output_dim=gene_feature_size[1],
trainable=True)
humannet_embedding_layer.build((None, ))
humannet_embedding_layer.set_weights([humannet_features])
# using disease feature matrix for initializationmerge
feas = h5py.File('disease_features.mat', 'r')
omim_features = feas['col_features'][:, :].T
disease_feature_size = omim_features.shape
assert disease_feature_size[0] + 1 == num_diseases
omim_features = np.insert(omim_features, 0, [0] * disease_feature_size[1],
0)
omim_embedding_layer = Embedding(input_dim=disease_feature_size[0] + 1,
output_dim=disease_feature_size[1],
trainable=True)
omim_embedding_layer.build((None, ))
omim_embedding_layer.set_weights([omim_features])
# get specific features
gene_feature = Flatten(name='')(humannet_embedding_layer(gene_input))
disease_feature = Flatten()(omim_embedding_layer(disease_input))
# disease_feature = K.transpose(disease_feature)
# projection matrix, using W * H' as initialization
project_disease = Dense(gene_feature_size[1],
trainable=True,
name='disease_gene_projection',
activation='relu',
kernel_initializer=init_weights)(disease_feature)
score = dot([gene_feature, project_disease], 1, name='inner_product')
# calculate score
# score = merge([gene_feature, project_disease], mode='mul', name='score')
# prediction = Dense(1, activation='sigmoid', init='he_uniform', name='prediction')(score)
model = Model(inputs=[gene_input, disease_input], outputs=score)
return model
def build_resnet_model(word_indices, word_vectors,
output_dim): # Currently does not work
learning_rate = args.learning_rate
dropout = 0.5
word_symbols = len(word_indices) + 1
word_embedding_weights = np.zeros((word_symbols, word_embeddings_size))
for word, index in word_indices.items():
word_embedding_weights[index, :] = word_vectors[word]
doc_word_input = Input(shape=(max_num_of_sentences,
max_num_of_tokens_per_sentence),
dtype='int64',
name="doc_word_input")
sent_word_input = Input(shape=(max_num_of_tokens_per_sentence, ),
dtype='int64',
name="sent_word_input")
word_embedding_layer = Embedding(output_dim=word_embeddings_size,
input_dim=word_symbols,
mask_zero=True)
word_embedding_layer.build(
(None, )) # if you don't do this, the next step won't work
word_embedding_layer.set_weights([word_embedding_weights])
word_embedded = word_embedding_layer(sent_word_input)
blocks = [2, 2, 2, 2]
block = keras_resnet.blocks.basic_1d
resnet_word_sent = keras_resnet.models.ResNet50(word_embedded,
blocks,
block,
classes=output_dim)
word_sent_encode = Dropout(dropout)(resnet_word_sent)
word_encoder = Model(inputs=sent_word_input, outputs=word_sent_encode)
word_encoded = TimeDistributed(word_encoder)(doc_word_input)
resnet_word_doc = keras_resnet.models.ResNet(word_encoded,
blocks,
block,
classes=output_dim)
word_output = Dropout(dropout)(resnet_word_doc)
word_output = Dense(output_dim, activation='softmax')(word_output)
model = Model(inputs=doc_word_input, outputs=word_output)
rmsprop = RMSprop(lr=learning_rate)
model.compile(optimizer=rmsprop,
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
return model
def build_char_sequence_lstm_model(char_indices, char_vectors, output_dim):
lstm_unit_size = args.lstm_unit_size
learning_rate = args.learning_rate
lstm_dropout = 0.15
dropout = 0.3
doc_char_input = Input(shape=(max_num_of_sentences, max_num_of_chars), dtype='int64', name="doc_char_input")
sent_char_input = Input(shape=(max_num_of_chars,), dtype='int64', name="sent_char_input")
char_symbols = len(char_indices) + 1
char_embedding_weights = np.zeros((char_symbols, char_embeddings_size))
for char, index in char_indices.items():
char_embedding_weights[index, :] = char_vectors[char]
char_embedding_layer = Embedding(output_dim=char_embeddings_size, input_dim=char_symbols, mask_zero=True)
char_embedding_layer.build((None,)) # if you don't do this, the next step won't work
char_embedding_layer.set_weights([char_embedding_weights])
char_embedded = char_embedding_layer(sent_char_input)
# filter_length = [5, 3, 3]
# nb_filter = [196, 196, 256]
# pool_length = 2
# char_embedded = Lambda(binarize_char, output_shape=binarize_char_outshape)(sent_char_input)
# for i in range(len(nb_filter)):
# char_embedded = Conv1D(filters=nb_filter[i], kernel_size=filter_length[i], padding='valid', activation='relu',
# kernel_initializer='glorot_normal', strides=1)(char_embedded)
#
# char_embedded = Dropout(0.1)(char_embedded)
# char_embedded = MaxPooling1D(pool_size=pool_length)(char_embedded)
bi_lstm_char_sent = Bidirectional(
LSTM(lstm_unit_size, return_sequences=False, dropout=lstm_dropout, recurrent_dropout=lstm_dropout))(
char_embedded)
char_sent_encode = Dropout(dropout)(bi_lstm_char_sent)
char_encoder = Model(inputs=sent_char_input, outputs=char_sent_encode)
char_encoded = TimeDistributed(char_encoder)(doc_char_input)
b_lstm_char_doc = Bidirectional(
LSTM(lstm_unit_size, return_sequences=False, dropout=lstm_dropout, recurrent_dropout=lstm_dropout))(
char_encoded)
char_output = Dropout(dropout, name='final_layer_char')(b_lstm_char_doc)
char_output = Dense(output_dim, activation='softmax')(char_output)
model = Model(inputs=doc_char_input, outputs=char_output)
rmsprop = RMSprop(lr=learning_rate)
model.compile(optimizer=rmsprop, loss='categorical_crossentropy', metrics=['categorical_accuracy'])
final_layer = Model(inputs=model.input, outputs=model.get_layer('final_layer_char').output)
return model, final_layer
class PositionEmbedding(Layer):
def __init__(self, max_time=1000, n_waves=16, d_model=64, name='PositionEmbedding', **kwargs):
"""
Position embedding via sin and cos functions
For incoming ``position`` produces embedding of dimension ``n_waves * 2``
``embedding[2*i] = sin(positions / 10. ** (2. * i / d_model))``
``embedding[2*i+1] = cos(positions / 10. ** (2. * i / d_model))``
:param max_time: maximum time dimension of input sequence
"""
self.max_time = max_time
self.n_waves = n_waves
self.d_model = d_model
emb_weights = pos_encoding(max_time, n_waves,d_model)
self.embedding_layer = Embedding(max_time, n_waves * 2,
weights=[emb_weights],
trainable=False)
super(PositionEmbedding, self).__init__(**kwargs)
self.name = name
def build(self, input_shapes):
self.embedding_layer.build((None, None))
self.built = True
def call(self, x):
samples = K.shape(x)[0]
time = K.shape(x)[1]
pos_enc = self.embedding_layer(
K.reshape(K.arange(time, dtype='int32'), (1, -1)))
return K.tile(pos_enc, (samples, 1, 1))
def compute_output_shape(self, input_shape):
"""
For Keras internal compatability checking
"""
return (None, None, self.n_waves * 2)
def get_config(self):
"""
For rebuilding models on load time.
"""
# TODO: teacher forcing
config = {
'n_waves': self.n_waves,
'max_time': self.max_time,
'd_model': self.d_model
}
base_config = super(PositionEmbedding, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
def create_model(**kwargs):
""" Function creating the model's graph in Keras.
Argument:
input_shape -- shape of the model's input data (using Keras conventions)
embedding_matrix -- matrix to map word index to word embedding vector
vocab_len -- the size of vocaburary
n_d1 -- ouput dimension for 1st GRU layer
n_d2 -- ouput dimension for 2nd GRU layer
n_c -- ouput dimension for output layer
Returns:
model -- Keras model instance
"""
embedding_matrix = kwargs.get('embedding_matrix')
vocab_len = kwargs.get('vocab_len')
n_d1 = kwargs.get('n_d1')
n_d2 = kwargs.get('n_d2')
n_c = kwargs.get('n_c')
#define input
X_input = Input(shape=kwargs.get('input_shape'))
#define and create mbedding layer
embedding_layer = Embedding(vocab_len + 1,
embedding_matrix.shape[1],
trainable=False)
embedding_layer.build((None, ))
embedding_layer.set_weights([embedding_matrix])
#add embedding layer
X = embedding_layer(X_input)
#add bidirectional GRU layer
X = Bidirectional(GRU(n_d1, return_sequences=True))(X)
X = Dropout(0.5)(X)
X = BatchNormalization()(X)
#add another GRU layer (unidirectional)
X = GRU(n_d2, return_sequences=True)(X)
X = Dropout(0.5)(X)
X = BatchNormalization()(X)
#get output for each time slot
outputs = TimeDistributed(Dense(n_c, activation=softmax2))(X)
#create and return keras model instance
return Model(inputs=[X_input], outputs=outputs)
def create_embedding_layer(embeddings_index,
words_to_index,
vocab_length=100,
output_dim=300):
emb_matrix = np.zeros((vocab_length + 1, output_dim))
for word, index in words_to_index.items():
if word in embeddings_index:
emb_matrix[index, :] = embeddings_index[word]
embedding_layer = Embedding(vocab_length + 1, output_dim, trainable=True)
embedding_layer.build((None, ))
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def pretrained_embedding_layer(word_vec, word_index):
vocab_len = cf['MAX_WORDS'] + 1
emb_dim = 300
emb_matrix = np.zeros((vocab_len, emb_dim))
for word, index in word_index.items():
vec = word_vec.get(word, np.zeros(emb_dim))
if (index > cf['MAX_WORDS']):
break
emb_matrix[index, :] = vec
# 定义Embedding层，并指定不需要训练该层的权重
embedding_layer = Embedding(vocab_len, emb_dim, trainable=False)
embedding_layer.build((None, )) # build
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def _create_binary_embedding():
"""
Creates an embedding matrix based on the binary representation
Keras will transform each one byte to a one-hot vector of size 256
This is required for RNN networks
:return: An embedding layer
"""
emb_matrix = np.zeros((256, 256))
for i in range(0, 256):
emb_matrix[i, i] = 1
embedding_layer = Embedding(256, 256, trainable=False)
embedding_layer.build((None, ))
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def word_sequence_GRU_model(word_indices, word_model, unique_labels_count):
data_dim = word_embeddings_size
timesteps = 50
num_classes = unique_labels_count
batch_size = 25
#word_embedding_weights is the (index, vector) of words vocabulary
word_symbols = len(word_indices) + 1
word_embedding_weights = np.zeros((word_symbols, word_embeddings_size))
for word, index in word_indices.items():
try:
word_embedding_weights[index, :] = word_model[word]
except KeyError:
word_embedding_weights[
index, :] = np.ones(word_embeddings_size) * -1
# word_model["some"] return the vector of data "some"
# word_data[0] is the data for text0 with the index of 0.
sent_word_input = Input(shape=(max_num_of_tokens_per_sentence, ),
dtype='int64',
name="sent_word_input")
word_embedding_layer = Embedding(output_dim=word_embeddings_size,
input_dim=word_symbols,
mask_zero=True)
word_embedding_layer.build(
(None, )) # if you don't do this, the next step won't work
word_embedding_layer.set_weights([word_embedding_weights])
word_embedded = word_embedding_layer(sent_word_input)
model = Sequential()(word_embedded)
model.add(
LSTM(lstm_unit_size,
return_sequences=True,
input_shape=(timesteps, word_embeddings_size)))
model.add(
LSTM(lstm_unit_size,
return_sequences=True,
stateful=True,
batch_input_shape=(batch_size, timesteps, lstm_unit_size)))
model.add(Dense(11, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def pre_trained_emb_model(word_to_vec_map, word_to_idx):
vocab_len = len(word_to_index)
emb_len = word_to_vec_map['the'].shape[0]
embed_matrix = np.zeros((vocab_len, emb_len))
for word, index in word_to_idx.items():
embed_matrix[index:] = word_to_vec_map[word]
embedding_layer = Embedding(input_shape=vocab_len, output_dim=emb_len, trainable=False)
embedding_layer.build((None,))
embedding_layer.set_weights([embed_matrix])
return embedding_layer
def get_chebi_common_channel(chebi_input, id_to_index):
e_ancestors = Embedding(len(id_to_index),
chebi_embbed_size,
input_length=max_ancestors_length * 2,
trainable=True)
e_ancestors.build((None, ))
e_ancestors = e_ancestors(chebi_input)
e_ancestors = Dropout(0.5)(e_ancestors)
ancestors_lstm = LSTM(LSTM_units,
input_shape=(max_ancestors_length * 2,
chebi_embbed_size),
return_sequences=True)(e_ancestors)
#kernel_regularizer=regularizers.l2(sigmoid_l2_reg))(e_ancestors)
ancestors_pool = GlobalMaxPooling1D()(ancestors_lstm)
return ancestors_pool
def get_chebi_concat_channel(chebi_input, id_to_index):
e_ancestors_left = Embedding(len(id_to_index),
chebi_embbed_size,
input_length=max_ancestors_length,
trainable=True)
e_ancestors_left.build((None, ))
e_ancestors_left = e_ancestors_left(chebi_input[0])
e_ancestors_left = Dropout(dropout1)(e_ancestors_left)
e_ancestors_right = Embedding(len(id_to_index),
chebi_embbed_size,
input_length=max_ancestors_length,
trainable=True)
e_ancestors_right.build((None, ))
# e_right.set_weights([embedding_matrix])
e_ancestors_right = e_ancestors_right(chebi_input[1])
e_ancestors_right = Dropout(dropout1)(e_ancestors_right)
#ancestors_lstm_left = LSTM(LSTM_units, input_shape=(max_ancestors_length, chebi_embbed_size), return_sequences=True,
# kernel_regularizer=regularizers.l2(sigmoid_l2_reg))(e_ancestors_left)
#ancestors_lstm_right = LSTM(LSTM_units, input_shape=(max_ancestors_length, chebi_embbed_size),
# return_sequences=True,
# kernel_regularizer=regularizers.l2(sigmoid_l2_reg))(e_ancestors_right)
#ancestors_pool_left = GlobalMaxPooling1D()(ancestors_lstm_left)
#ancestors_pool_right = GlobalMaxPooling1D()(ancestors_lstm_right)
attention_rnn_left = LSTM(LSTM_units,
input_shape=(max_ancestors_length,
chebi_embbed_size),
return_sequences=True)(e_ancestors_left)
#kernel_regularizer=regularizers.l2(sigmoid_l2_reg))(e_ancestors_left)
attention_rnn_right = LSTM(LSTM_units,
input_shape=(max_ancestors_length,
chebi_embbed_size),
return_sequences=True)(e_ancestors_right)
# kernel_regularizer=regularizers.l2(sigmoid_l2_reg))(e_ancestors_right)
ancestors_pool_left = GlobalMaxPooling1D()(attention_rnn_left)
ancestors_pool_right = GlobalMaxPooling1D()(attention_rnn_right)
#ancestors_dense_left = Dense(LSTM_units, input_shape=(max_ancestors_length, chebi_embbed_size))(e_ancestors_left)
#ancestors_dense_right = Dense(LSTM_units, input_shape=(max_ancestors_length, chebi_embbed_size))(e_ancestors_right)
#return ancestors_dense_left, ancestors_dense_right
#return ancestors_dense_left, ancestors_dense_right
return ancestors_pool_left, ancestors_pool_right
def pretrained_embedding_layer(word_to_vec_map, source_vocab_to_int):
vocab_len = len(source_vocab_to_int) + 1
emb_dim = word_to_vec_map["you"].shape[0]
emb_matrix = np.zeros((vocab_len, emb_dim))
for word, index in source_vocab_to_int.items():
word_vector = word_to_vec_map.get(word, np.zeros(emb_dim))
emb_matrix[index, :] = word_vector
embedding_layer = Embedding(vocab_len, emb_dim, trainable=False)
embedding_layer.build((None, ))
embedding_layer.set_weights([emb_matrix])
return embedding_layer
def create_model(G=None, max_len=0):
###### CREACION DEL MODELO ######
# Embedding
# create Keras embedding layer
word_to_idx, idx_to_word, word_embeddings = G.read_embedding()
# vocabulary_len = len(word_to_idx) + 1
vocabulary_len = len(word_to_idx)
emb_dimension = G.get_dimensions()
# get the matrix for the sentences
embedding_matrix = word_embeddings
# embedding layer
embedding_layer = Embedding(input_dim=vocabulary_len,
output_dim=emb_dimension,
input_length=max_len,
trainable=False,
name="EMBEDDING")
embedding_layer.build((None, ))
embedding_layer.set_weights([embedding_matrix])
first_layer_units = 128
first_layer_dropout = 0.5
second_layer_units = 128
second_layer_dropout = 0.5
relu_dense_layer = 64
dense_layer_units = 3
#Modelo
model = Sequential()
model.add(embedding_layer)
model.add(
LSTM(first_layer_units,
return_sequences=True,
name='LSTM_1',
recurrent_dropout=0.4))
model.add(Dropout(first_layer_dropout, name="DROPOUT_1"))
model.add(Dense(200, activation='relu'))
model.add(LSTM(second_layer_units, return_sequences=False, name="LSTM_2"))
model.add(Dropout(second_layer_dropout, name="DROPOUT_2"))
model.add(Dense(relu_dense_layer, activation='relu'))
model.add(Dense(dense_layer_units))
model.add(Activation("softmax", name="softmax_final"))
model.compile(optimizer=RMSprop(decay=0.001),
loss="categorical_crossentropy",
metrics=['accuracy'])
return model
