def define_glove_model(length, vocab_size, embedding_matrix, num_classes):
# channel 1
inputs1 = Input(shape=(length, ))
e1 = Embedding(input_dim=vocab_size,
output_dim=300,
embeddings_initializer=Constant(embedding_matrix),
input_length=length)
e1.trainable = False
embedding1 = e1(inputs1)
conv1 = Conv1D(filters=16, kernel_size=2, activation='relu')(embedding1)
drop1 = Dropout(0.5)(conv1)
pool1 = MaxPooling1D(pool_size=2)(drop1)
flat1 = Flatten()(pool1)
# channel 2
inputs2 = Input(shape=(length, ))
e2 = Embedding(input_dim=vocab_size,
output_dim=300,
embeddings_initializer=Constant(embedding_matrix),
input_length=length)
e2.trainable = False
embedding2 = e2(inputs2)
conv2 = Conv1D(filters=16, kernel_size=3, activation='relu')(embedding2)
drop2 = Dropout(0.5)(conv2)
pool2 = MaxPooling1D(pool_size=2)(drop2)
flat2 = Flatten()(pool2)
# channel 3
inputs3 = Input(shape=(length, ))
e3 = Embedding(input_dim=vocab_size,
output_dim=300,
embeddings_initializer=Constant(embedding_matrix),
input_length=length)
e3.trainable = False
embedding3 = e3(inputs3)
conv3 = Conv1D(filters=16, kernel_size=4, activation='relu')(embedding3)
drop3 = Dropout(0.5)(conv3)
pool3 = MaxPooling1D(pool_size=2)(drop3)
flat3 = Flatten()(pool3)
# merge
merged = concatenate([flat1, flat2, flat3])
# interpretation
dense1 = Dense(10, activation='relu')(merged)
# DEFINE the right model based on number of classes and loss function
if num_classes == 2:
outputs = Dense(1, activation='sigmoid')(dense1)
model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)
else: # num_classes=3
outputs = Dense(3, activation='softmax')(dense1)
model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)
return model
def build_model(n_celltypes, n_celltype_factors, n_assays, n_assay_factors,
n_genomic_positions, n_25bp_factors, n_250bp_factors, n_5kbp_factors,
n_layers, n_nodes, freeze_celltypes=False, freeze_assays=False,
freeze_genome_25bp=False, freeze_genome_250bp=False,
freeze_genome_5kbp=False, freeze_network=False):
"""This function builds a multi-scale deep tensor factorization model."""
celltype_input = Input(shape=(1,), name="celltype_input")
celltype_embedding = Embedding(n_celltypes, n_celltype_factors,
input_length=1, name="celltype_embedding")
celltype_embedding.trainable = not freeze_celltypes
celltype = Flatten()(celltype_embedding(celltype_input))
assay_input = Input(shape=(1,), name="assay_input")
assay_embedding = Embedding(n_assays, n_assay_factors,
input_length=1, name="assay_embedding")
assay_embedding.trainable = not freeze_assays
assay = Flatten()(assay_embedding(assay_input))
genome_25bp_input = Input(shape=(1,), name="genome_25bp_input")
genome_25bp_embedding = Embedding(n_genomic_positions, n_25bp_factors,
input_length=1, name="genome_25bp_embedding")
genome_25bp_embedding.trainable = not freeze_genome_25bp
genome_25bp = Flatten()(genome_25bp_embedding(genome_25bp_input))
genome_250bp_input = Input(shape=(1,), name="genome_250bp_input")
genome_250bp_embedding = Embedding(int(n_genomic_positions / 10) + 1,
n_250bp_factors, input_length=1, name="genome_250bp_embedding")
genome_250bp_embedding.trainable = not freeze_genome_250bp
genome_250bp = Flatten()(genome_250bp_embedding(genome_250bp_input))
genome_5kbp_input = Input(shape=(1,), name="genome_5kbp_input")
genome_5kbp_embedding = Embedding(int(n_genomic_positions / 200) + 1,
n_5kbp_factors, input_length=1, name="genome_5kbp_embedding")
genome_5kbp_embedding.trainable = not freeze_genome_5kbp
genome_5kbp = Flatten()(genome_5kbp_embedding(genome_5kbp_input))
layers = [celltype, assay, genome_25bp, genome_250bp, genome_5kbp]
inputs = (celltype_input, assay_input, genome_25bp_input,
genome_250bp_input, genome_5kbp_input)
x = concatenate(layers)
for i in range(n_layers):
layer = Dense(n_nodes, activation='relu', name="dense_{}".format(i))
layer.trainable = not freeze_network
x = layer(x)
layer = Dense(1, name="y_pred")
layer.trainable = not freeze_network
y = layer(x)
model = Model(inputs=inputs, outputs=y)
model.compile(optimizer='adam', loss='mse', metrics=['mse'])
return model
def _generate_model(self, lembedding, num_classes=2, unit='gru', rnn_size=128, train_vectors=True):
model = Sequential()
if lembedding.vector_box.W is None:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
W_constraint=None)
else:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
weights=[lembedding.vector_box.W], W_constraint=None)
emb.trainable = train_vectors
model.add(emb)
if unit == 'gru':
model.add(GRU(rnn_size))
else:
model.add(LSTM(rnn_size))
model.add(Dropout(0.2))
if num_classes == 2:
model.add(Dense(1, activation='sigmoid'))
if self.optimizer is None:
self.optimizer = 'rmsprop'
model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
else:
if self.optimizer is None:
self.optimizer = 'adam'
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
return model
def _generate_model(self, lembedding, num_classes=2, unit='gru', rnn_size=128, train_vectors=True):
input = Input(shape=(lembedding.size,), dtype='int32')
if lembedding.vector_box.W is None:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
W_constraint=None)(input)
else:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
weights=[lembedding.vector_box.W], W_constraint=None, )(input)
emb.trainable = train_vectors
if unit == 'gru':
forward = GRU(rnn_size)(emb)
backward = GRU(rnn_size, go_backwards=True)(emb)
else:
forward = LSTM(rnn_size)(emb)
backward = LSTM(rnn_size, go_backwards=True)(emb)
merged_rnn = merge([forward, backward], mode='concat')
dropped = Dropout(0.5)(merged_rnn)
if num_classes == 2:
out = Dense(1, activation='sigmoid')(dropped)
model = Model(input=input, output=out)
if self.optimizer is None:
self.optimizer = 'rmsprop'
model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
else:
out = Dense(num_classes, activation='softmax')(dropped)
model = Model(input=input, output=out)
if self.optimizer is None:
self.optimizer = 'adam'
model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
return model
def tunable_embed_apply(word_cnt_post, vocab_size, embed_mat, word_feat_name):
input_seq = Input(shape=(word_cnt_post, ), name=word_feat_name + '_t')
embed_layer = Embedding(
vocab_size,
embed_mat.shape[1],
embeddings_initializer=initializers.Constant(embed_mat),
input_length=word_cnt_post,
name=word_feat_name)
embed_layer.trainable = True
embed_l = embed_layer(input_seq)
return input_seq, embed_l
def create_embedding_dict(sparse_feature_columns ,seed, l2_reg,
prefix='sparse_', seq_mask_zero=True):
#将特征进行embedding ,输入维度是某个特征的种类数
sparse_embedding = {}
for feat in sparse_feature_columns:
emb = Embedding(feat.vocabulary_size, feat.embedding_dim,
embeddings_initializer=feat.embeddings_initializer,
embeddings_regularizer=l2(l2_reg),
name=prefix + '_emb_' + feat.embedding_name)
emb.trainable = feat.trainable
sparse_embedding[feat.embedding_name] = emb
return sparse_embedding
def __init__(self, embed, mappings, cats, embed_len):
word_inv = mappings['form']
npos = len(mappings['pos'])
nne = len(mappings['ne'])
nout = len(cats)
width = len(word_inv) + 2
pos = Input(shape=(None, ), dtype='int32', name='POS')
pos_emb = Embedding(npos, npos // 2, name='emb_pos')(pos)
ne = Input(shape=(None, ), dtype='int32', name='NE')
ne_emb = Embedding(nne, nne // 2, name='emb_ne')(ne)
capital = Input(shape=(None, 3), name='Capital')
special = Input(shape=(None, 12), name='Special')
form = Input(shape=(None, ), dtype='int32', name='Form')
emb = Embedding(70000,
embed_len,
embeddings_initializer=emb_mat_init(embed, word_inv),
input_length=None,
name='emb_form')(form)
emb.trainable = True
concat = Concatenate()([emb, pos_emb, ne_emb, capital, special])
drop = Dropout(0.25)(concat)
lstm1 = Bidirectional(CuDNNLSTM(100, return_sequences=True),
input_shape=(None, width))(drop)
skip = Concatenate()([drop, lstm1])
lstm2 = Bidirectional(CuDNNLSTM(100, return_sequences=True),
input_shape=(None, width))(skip)
dense = Dense(nout, activation='softmax')(lstm2)
# crf = CRF(nout, learn_mode='join', activation='softmax')
# out = crf(dense)
out = dense
model = Model(inputs=[form, pos, ne, capital, special], outputs=out)
# model.compile(loss=crf.loss_function, optimizer='nadam', metrics=[crf.accuracy])
model.compile(loss='categorical_crossentropy',
optimizer='nadam',
metrics=['acc'])
#model.summary()
self.model = model
self.mappings = mappings
self.cats = cats
self.embed_len = embed_len
def build_model(self):
# ================================ Input ==============================
user_input = Input(shape=(self.walk_length, ),
dtype='float32',
name="friends_input")
article_input = Input(shape=(self.max_num_sent, self.max_len_sent),
dtype='float32',
name="article_input")
# ======================== GRU-based sequence encoder =================
word_embedding = Embedding(input_dim=self.vocab_size,
output_dim=self.embedding_dim,
weights=[self.embedding_matrix],
mask_zero=False,
name="word_embedding")
embed_article = TimeDistributed(word_embedding)(article_input)
# ============================== Word Level Attention =================
u_w = self.social_attention(user_input)
s_i = self.multi_sequence_encode(self.embedding_dim, embed_article,
u_w)
print("encoded_article:", s_i._keras_shape)
# ============================== Sentence Level Attention =============
s_i_dim = s_i._keras_shape[-1]
s_encoding = self.bi_gru_encode(s_i_dim)
h_i = s_encoding(s_i)
u_s = self.social_attention(user_input)
v = self.personalized_attention(u_s, h_i)
# ============================ Prediction =============================
preds = Dense(1, activation="sigmoid")(v)
adam = Adam(lr=0.0005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-06,
decay=0.0,
clipvalue=5.)
self.model = Model([user_input, article_input], preds)
word_embedding.trainable = False
self.model.compile(optimizer=adam,
loss='binary_crossentropy',
metrics=['accuracy'])
def _generate_model(self, lembedding, num_classes=2, ngrams=[1,2,3,4,5],
nfilters=64, train_vectors=True):
def sub_ngram(n):
return Sequential([
Convolution1D(nfilters, n,
activation='relu',
input_shape=(lembedding.size, lembedding.vector_box.vector_dim)),
Lambda(
lambda x: K.max(x, axis=1),
output_shape=(nfilters,)
)
])
doc = Input(shape=(lembedding.size, ), dtype='int32')
embedded = Embedding(input_dim=lembedding.vector_box.size,
output_dim=lembedding.vector_box.vector_dim,
weights=[lembedding.vector_box.W])(doc)
embedded.trainable = train_vectors
rep = Dropout(0.5)(
merge(
[sub_ngram(n)(embedded) for n in ngrams],
mode='concat',
concat_axis=-1
)
)
if num_classes == 2:
out = Dense(1, activation='sigmoid')(rep)
model = Model(input=doc, output=out)
if self.optimizer is None:
self.optimizer = 'rmsprop'
model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
else:
out = Dense(num_classes, activation='softmax')(rep)
model = Model(input=doc, output=out)
if self.optimizer is None:
self.optimizer = 'adam'
model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
return model
def _generate_model(self, lembedding, num_classes=2, num_features=128, train_vectors=True):
model = Sequential()
if lembedding.vector_box.W is None:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
W_constraint=None,
input_length=lembedding.size)
else:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
weights=[lembedding.vector_box.W], W_constraint=None,
input_length=lembedding.size)
emb.trainable = train_vectors
model.add(emb)
model.add(Convolution1D(num_features, 3, init='uniform'))
model.add(Activation('relu'))
model.add(MaxPooling1D(2))
model.add(Dropout(0.25))
model.add(Convolution1D(num_features, 3, init='uniform'))
model.add(Activation('relu'))
model.add(MaxPooling1D(2))
model.add(Dropout(0.25))
model.add(Flatten())
if num_classes == 2:
model.add(Dense(1, activation='sigmoid'))
if self.optimizer is None:
self.optimizer = 'rmsprop'
model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
else:
if self.optimizer is None:
self.optimizer = 'adam'
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
return model
def cnn_autoencoder(shape=(500, 64),
num_classes=2,
input_dim=93,
n=1,
use_attention=False,
**kwargs):
input_array = keras.Input(shape=(shape[0], ), name='input')
embedding = Embedding(input_dim=input_dim,
output_dim=shape[1],
weights=[np.load("weight.npy")])
embedding.trainable = False
embedding_output = embedding(input_array)
_embed = SpatialDropout1D(0.25)(embedding_output)
kernel_size = 3
dilated_rate = 1
num_filters_in = 64
conv1d = Conv1D(filters=num_filters_in,
kernel_size=kernel_size,
padding="same")(_embed)
b = BatchNormalization()(conv1d)
r = Activation("elu")(b)
x = r
conv1 = Conv1D(filters=num_filters_in, kernel_size=1, padding="same")(x)
b = BatchNormalization()(conv1)
r = Activation("elu")(b)
conv2 = Conv1D(filters=num_filters_in, kernel_size=3, padding="same")(r)
b = BatchNormalization()(conv2)
r = Activation("elu")(b)
x = keras.layers.add([x, r])
x = BatchNormalization()(x)
x = Activation('elu')(x)
x = MaxPool1D(pool_size=128, strides=128, data_format='channels_last')(x)
fc = Activation("sigmoid", name="feature_output")(x)
fc = layers.UpSampling1D(size=128)(fc)
fc = BatchNormalization()(fc)
fc = Activation('elu')(fc) # add later
# mid = Flatten(name="feature_output")(fc)
conv2 = Conv1D(filters=num_filters_in, kernel_size=3, padding="same")(fc)
b = BatchNormalization()(conv2)
r = Activation("elu")(b)
conv1 = Conv1D(filters=num_filters_in, kernel_size=1, padding="same")(r)
b = BatchNormalization()(conv1)
r = Activation("elu")(b)
x = keras.layers.add([fc, r])
conv1d = Conv1D(filters=num_filters_in,
kernel_size=kernel_size,
padding="same")(x)
b = BatchNormalization()(conv1d)
r = Activation("elu")(b)
output = r
output = Lambda(lambda x: keras.losses.mean_squared_error(x[0], x[1]),
name='loss',
output_shape=(1, ))([output, embedding_output])
model = Model(inputs=input_array, outputs=output)
model.compile(loss=loss_first, optimizer='adam')
model.summary()
plot_model(model, "attention.png")
return model
def build_model(p):
""" build a Keras model using the parameters in p """
max_posts = p['max_posts']
max_length = p['max_length']
filters = p['filters']
filtlen = p['filtlen']
poollen = p['poollen']
densed = p['densed']
embed_size = p['embed_size']
batch = p['batch']
random.seed(p['seed'])
np.random.seed(p['seed'])
# https://github.com/fchollet/keras/issues/2280
tf.reset_default_graph()
if len(tf.get_default_graph()._nodes_by_id.keys()) > 0:
raise RuntimeError(
"Seeding is not supported after building part of the graph. "
"Please move set_seed to the beginning of your code.")
tf.set_random_seed(p['seed'])
sess = tf.Session()
K.set_session(sess)
nb_words, genf, tok = datagen(max_posts,
max_length,
stype='training',
batch_size=batch,
randposts=p['randposts'],
mintf=p['mintf'],
mindf=p['mindf'],
noempty=p['noempty'],
prep=p['prep'],
returntok=True)
n_classes = 2
inp = Input(shape=(max_posts, max_length), dtype='int32')
nextin = inp
if p['cosine']:
from keras.constraints import unitnorm
wconstrain = unitnorm()
else:
wconstrain = None
if p['w2v']:
embeddings_index = {}
fn = 'data/w2v_50_sg_export.txt'
with open(fn) as f:
for line in f:
values = line.strip().split()
word = values[0]
if word in tok.word_index:
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
print('Found %s word vectors.' % len(embeddings_index))
embedding_matrix = np.zeros((nb_words, embed_size))
for word, i in tok.word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = np.random.uniform(-0.2,
0.2,
size=embed_size)
emb = Embedding(nb_words,
embed_size,
mask_zero=True,
input_length=max_length,
W_constraint=wconstrain,
weights=[embedding_matrix],
trainable=p['etrain'])
if not p['etrain']:
print("making not trainable")
emb.trainable = False
else:
assert p['etrain'], "must have etrain=True with w2v=False"
emb = Embedding(nb_words,
embed_size,
mask_zero=True,
W_constraint=wconstrain)
embedded = TimeDistributed(emb)(nextin)
if not p['etrain']:
emb.trainable = False
embedded.trainable = False
conv = Sequential()
conv.add(
Convolution1D(nb_filter=filters,
filter_length=filtlen,
border_mode='valid',
W_constraint=wconstrain,
activation='linear',
subsample_length=1,
input_shape=(max_length, embed_size)))
conv.add(Activation(p['af']))
conv.add(GlobalAveragePooling1D())
posts = TimeDistributed(conv)(embedded)
combined = Convolution1D(nb_filter=filters,
filter_length=p['acl'],
border_mode='valid',
activation=p['af'],
subsample_length=p['acl'])(posts)
combined = Flatten()(combined)
if densed != 0:
combined = Dense(densed, activation=p['af'])(combined)
outlayer = Dense(2, activation='softmax')(combined)
model = Model(inp, outlayer)
return model, genf
for k, v in w2v_model.wv.vocab.items():
vocab.append([k, w2v_model.wv[k]])
embedding_matrix = np.zeros((len(idx2word), w2v_model.vector_size))
for word, vec in vocab:
index = word2idx[word]
embedding_matrix[index] = vec
# build model
embedding_layer = Embedding(input_dim=len(embedding_matrix),
output_dim=embed_size,
weights=[embedding_matrix],
input_length=maxlen)
embedding_layer.trainable = False
model = Sequential()
model.add(embedding_layer)
model.add(Bidirectional(LSTM(512, dropout=0.5, recurrent_dropout=0.5, return_sequences=True)))
model.add(LeakyReLU())
model.add(Bidirectional(LSTM(512, dropout=0.5, recurrent_dropout=0.5)))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(Dense(512))
model.add(LeakyReLU())
model.add(Dropout(0.5))
def main(data_cnf, model_cnf, mode):
model_name = os.path.split(model_cnf)[1].split(".")[0]
# 設定log檔案位置
logfile("./logs/logfile_" + model_name + ".log")
yaml = YAML(typ='safe')
data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf))
model, model_name, data_name = None, model_cnf['name'], data_cnf['name']
# model_path = model_cnf['path'] + "/" + model_cnf['name'] + '.h'
model_path = r'E:\\PycharmProject\\CorNet\\' + model_name + '.h5'
emb_init = get_word_emb(data_cnf['embedding']['emb_init'])
logger.info(F'Model Name: {model_name}')
# keras log file
csv_logger = CSVLogger('./logs/' + model_name + '_log.csv', append=True, separator=',')
if mode is None or mode == 'train':
logger.info('Loading Training and Validation Set')
train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels'])
if 'size' in data_cnf['valid']:
random_state = data_cnf['valid'].get('random_state', 1240)
train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels,
test_size=data_cnf['valid']['size'],
random_state=random_state)
else:
valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels'])
mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels)))
train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels)
labels_num = len(mlb.classes_)
logger.info(F'Number of Labels: {labels_num}')
logger.info(F'Size of Training Set: {len(train_x)}')
logger.info(F'Size of Validation Set: {len(valid_x)}')
vocab_size = emb_init.shape[0]
emb_size = emb_init.shape[1]
# 可調參數
data_num = len(train_x)
ks = 3
output_channel = model_cnf['model']['num_filters']
dynamic_pool_length = model_cnf['model']['dynamic_pool_length']
num_bottleneck_hidden = model_cnf['model']['bottleneck_dim']
drop_out = model_cnf['model']['dropout']
cornet_dim = model_cnf['model']['cornet_dim']
nb_cornet_block = model_cnf['model'].get('nb_cornet_block', 0)
nb_epochs = model_cnf['train']['nb_epoch']
batch_size = model_cnf['train']['batch_size']
max_length = 500
input_tensor = Input(batch_shape=(batch_size, max_length), name='input')
emb_data = Embedding(input_dim=vocab_size,
output_dim=emb_size,
input_length=max_length,
weights=[emb_init],
trainable=False,
name='embedding1')(input_tensor)
emb_data.trainable = False
# emd_out_4d = keras.layers.core.RepeatVector(1)(emb_data)
# unsqueeze_emb_data = tf.keras.layers.Reshape((1, 500, 300), input_shape=(500, 300))(emb_data)
# emb_data = tf.expand_dims(emb_data, axis=1)
# emb_data = Lambda(reshape_tensor, arguments={'shape': (1, max_length, 300)}, name='lambda1')(
# emb_data)
conv1_output = Convolution1D(output_channel, 2, padding='same',
kernel_initializer=keras.initializers.glorot_uniform(seed=None),
activation='relu', name='conv1')(emb_data)
# conv1_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)},
# name='conv1_lambda')(
# conv1_output)
conv2_output = Convolution1D(output_channel, 4, padding='same',
kernel_initializer=keras.initializers.glorot_uniform(seed=None),
activation='relu', name='conv2')(emb_data)
# conv2_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)},
# name='conv2_lambda')(
# conv2_output)
conv3_output = Convolution1D(output_channel, 8, padding='same',
kernel_initializer=keras.initializers.glorot_uniform(seed=None),
activation='relu', name='conv3')(emb_data)
# conv3_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)},
# name='conv3_lambda')(
# conv3_output)
# pool1 = adapmaxpooling(conv1_output, dynamic_pool_length)
pool1 = GlobalMaxPooling1D(name='globalmaxpooling1')(conv1_output)
pool2 = GlobalMaxPooling1D(name='globalmaxpooling2')(conv2_output)
pool3 = GlobalMaxPooling1D(name='globalmaxpooling3')(conv3_output)
output = concatenate([pool1, pool2, pool3], axis=-1)
# output = Dense(num_bottleneck_hidden, activation='relu',name='bottleneck')(output)
output = Dropout(drop_out, name='dropout1')(output)
output = Dense(labels_num, activation='softmax', name='dense_final',
kernel_initializer=keras.initializers.glorot_uniform(seed=None))(output)
if nb_cornet_block > 0:
for i in range(nb_cornet_block):
x_shortcut = output
x = keras.layers.Activation('sigmoid', name='cornet_sigmoid_{0}'.format(i + 1))(output)
x = Dense(cornet_dim, kernel_initializer='glorot_uniform', name='cornet_1st_dense_{0}'.format(i + 1))(x)
# x = Dense(cornet_dim, kernel_initializer=keras.initializers.glorot_uniform(seed=None),
# activation='sigmoid', name='cornet_1st_dense_{0}'.format(i + 1))(output)
x = keras.layers.Activation('elu', name='cornet_elu_{0}'.format(i + 1))(x)
x = Dense(labels_num, kernel_initializer='glorot_uniform', name='cornet_2nd_dense_{0}'.format(i + 1))(x)
# x = Dense(labels_num, kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='elu',
# name='cornet_2nd_dense_{0}'.format(i + 1))(x)
output = Add()([x, x_shortcut])
model = Model(input_tensor, output)
model.summary()
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.Precision(top_k=5)])
# model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.top_k_categorical_accuracy(k=5)])
model.fit_generator(steps_per_epoch=data_num / batch_size,
generator=batch_generator(train_x, train_y, batch_size),
validation_data=batch_generator(valid_x, valid_y, batch_size),
validation_steps=valid_x.shape[0] / batch_size,
nb_epoch=nb_epochs, callbacks=[csv_logger])
model.save(model_path)
elif mode is None or mode == 'eval':
logger.info('Loading Training and Validation Set')
train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels'])
if 'size' in data_cnf['valid']: # 如果有設定valid的size 則直接使用train的一部分作為valid
random_state = data_cnf['valid'].get('random_state', 1240)
train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels,
test_size=data_cnf['valid']['size'],
random_state=random_state)
else:
valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels'])
mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels)))
train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels)
labels_num = len(mlb.classes_)
##################################################################################################
logger.info('Loading Test Set')
logger.info('model path: ', model_path)
mlb = get_mlb(data_cnf['labels_binarizer'])
labels_num = len(mlb.classes_)
test_x, test_label = get_data(data_cnf['test']['texts'], data_cnf['test']['labels'])
logger.info(F'Size of Test Set: {len(test_x)}')
test_y = mlb.transform(test_label).toarray()
model = tf.keras.models.load_model(model_path)
score = model.predict(test_x)
print("p5: ", p5(test_y, score))
def _generate_model(self, lembedding, num_classes=2, first_kernel_size=3,
num_features=1024, conv_dropout=False, train_vectors=True):
model = Sequential()
if lembedding.vector_box.W is None:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
W_constraint=None,
input_length=lembedding.size)
else:
emb = Embedding(lembedding.vector_box.size,
lembedding.vector_box.vector_dim,
weights=[lembedding.vector_box.W], W_constraint=None,
input_length=lembedding.size)
emb.trainable = train_vectors
model.add(emb)
# Two conv layers with original kernel size, maxpooling is 2
model.add(Convolution1D(num_features, first_kernel_size, init='uniform'))
model.add(Activation('relu'))
model.add(MaxPooling1D(2))
if conv_dropout:
model.add(Dropout(0.25))
model.add(Convolution1D(num_features, first_kernel_size, init='uniform'))
model.add(Activation('relu'))
model.add(MaxPooling1D(2))
if conv_dropout:
model.add(Dropout(0.25))
# Three conv layers with kernel size = 3, no maxpooling
model.add(Convolution1D(num_features, 3, init='uniform'))
model.add(Activation('relu'))
if conv_dropout:
model.add(Dropout(0.25))
model.add(Convolution1D(num_features, 3, init='uniform'))
model.add(Activation('relu'))
if conv_dropout:
model.add(Dropout(0.25))
model.add(Convolution1D(num_features, 3, init='uniform'))
model.add(Activation('relu'))
if conv_dropout:
model.add(Dropout(0.25))
# One final conv layer with maxpooling
model.add(Convolution1D(num_features, 3, init='uniform'))
model.add(Activation('relu'))
model.add(MaxPooling1D(2))
model.add(Dropout(0.25))
model.add(Flatten())
# Two dense layers with heavy dropout
model.add(Dense(2048))
model.add(Dropout(0.5))
model.add(Dense(2048))
model.add(Dropout(0.5))
if num_classes == 2:
model.add(Dense(1, activation='sigmoid'))
if self.optimizer is None:
self.optimizer = 'rmsprop'
model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
else:
if self.optimizer is None:
self.optimizer = 'rmsprop'
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"])
return model
def __build_graph(self, enable_fine_tune):
# create the base pre-trained model
base_model = Xception(weights='imagenet', include_top=False)
# add a global spatial max pooling layer
x = base_model.output #(?,3,3,2048)
x = GlobalAveragePooling2D()(x) #(?, 2048)
x = Dropout(rate=self.drop_out_rate)(x)
assert K.int_shape(x) == (None, 2048)
## Load pretrained deep weigths
if not os.path.exists(self.model_file) and os.path.exists(
os.path.join(self.deep_model_dir, MODEL_BEST_NAME)):
print("@@@@Load DEEP model weights from %s" %
(self.deep_model_dir))
pred = Dense(self.num_classes, activation='sigmoid')(x)
model = Model(inputs=base_model.input, outputs=pred)
model.load_weights(
os.path.join(self.deep_model_dir, MODEL_BEST_NAME))
## Load DONE
##########################################
# RNN begins
# hidden_size: 20
# embedding_size: 20
# output_size: 20
h0 = Dense(self.gru_hidden_size)(x)
hr = Dense(self.gru_hidden_size)(x)
assert K.int_shape(h0) == (
None, self.gru_hidden_size), 'h0.shape: {}'.format(K.int_shape(h0))
label_ids = Lambda(
lambda h: K.arange(self.num_classes, dtype='int32'))(h0)
def batch_labels(h, label_ids):
# check: https://github.com/keras-team/keras/issues/8343
# check: https://stackoverflow.com/questions/47066635/checkpointing-keras-model-typeerror-cant-pickle-thread-lock-objects
label_id_tf = K.constant(label_ids, dtype='int32')
return K.ones(shape=(K.shape(h)[0], 228),
dtype='int32') * label_id_tf
batch_label_ids = Lambda(
batch_labels,
output_shape=(228, ),
arguments={'label_ids': np.array([i for i in range(228)])})(h0)
assert K.int_shape(batch_label_ids) == (None, 228)
label_emb = Embedding(
self.num_classes,
self.gru_hidden_size,
embeddings_initializer=self.__pretrained_label_embed,
input_length=self.num_classes)(batch_label_ids)
assert K.int_shape(label_emb) == (None, 228, 20)
if not self.label_emb_trainable:
label_emb.trainable = False
# gru_fn = CuDNNGRU if self.use_cudnn else GRU
gru_fn = GRU
label_gru_emb = Bidirectional(
gru_fn(self.gru_hidden_size,
input_shape=(self.num_classes, self.gru_hidden_size),
return_sequences=True))(label_emb, initial_state=[h0, hr])
assert K.int_shape(label_gru_emb) == (None, 228, 40)
# Approach 1: use a giant fully connect
if self.rnn_concat_all:
label_gru_emb = Reshape(
[self.num_classes * self.gru_hidden_size * 2])(label_gru_emb)
x = Concatenate(axis=1)([label_gru_emb, x])
predictions = Dense(self.num_classes, activation='sigmoid')(x)
# Approach 2: separate out prediction for each class
else:
# (batch_size, 228, 40) + (batch_size, 228, 2048)
x = RepeatVector(self.num_classes)(x)
x = Concatenate(axis=2)([label_gru_emb,
x]) # (batch_size, 228, 2088)
predictions = TimeDistributed(Dense(1, activations='sigmoid'),
input_shape=(self.num_classes,
2088))(x)
# RNN ends
##########################################
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
f_metrics = FMetrics()
f_scores = f_metrics.get_fscores()
if not enable_fine_tune:
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
for layer in base_model.layers:
layer.trainable = False
# compile the model (should be done *after* setting layers to non-trainable)
optimizer = RMSprop(lr=5e-4, decay=0.001)
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'] + f_scores)
else:
print("@@@@@Fine tune enabled.@@@@@")
print("Fine tune the last feature flow and the entire exit flow")
for layer in model.layers: # change from 116 to 36 to ALL
layer.trainable = True
layer.kernel_regularizer = regularizers.l2(self.reg)
# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
optimizer = Nadam(lr=5e-4, schedule_decay=0.001)
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'] + f_scores)
print(model.summary())
return model
# Read back data
train_reviews = np.load(path_join(ROOT_PATH, "IMDB_train_fulltext_glove_X.npy"))
train_labels = np.load(path_join(ROOT_PATH, "IMDB_train_fulltext_glove_y.npy"))
test_reviews = np.load(path_join(ROOT_PATH, "IMDB_test_fulltext_glove_X.npy"))
test_labels = np.load(path_join(ROOT_PATH, "IMDB_test_fulltext_glove_y.npy"))
WV_FILE_GLOBAL = path_join(ROOT_PATH, './embeddings/wv/glove.42B.300d.120000-glovebox.pkl')
gb_global = pickle.load(open(WV_FILE_GLOBAL, 'rb'))
wv_size = gb_global.W.shape[1]
model = Sequential()
emb = Embedding(gb_global.W.shape[0], wv_size, weights=[gb_global.W],
input_length=train_reviews.shape[1])
emb.trainable = False
model.add(emb)
#model.add(Permute((2,1)))
model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform'))
def multichannel(nb_classes,
vocab_size,
input_len,
embedding_dim=100,
matrix=None,
activation="sigmoid",
drop=False):
inp = Input(shape=(input_len, ))
if matrix is None:
matrix = np.random.rand(vocab_size + 1, embedding_dim)
assert matrix.shape == (vocab_size + 1, embedding_dim)
embed_layer_1 = Embedding(vocab_size + 1,
embedding_dim,
input_length=input_len)
embed_layer_1.build(input_shape=(input_len, ))
embed_layer_1.set_weights([matrix])
embed_layer_2 = Embedding(vocab_size + 1,
embedding_dim,
input_length=input_len)
embed_layer_2.build(input_shape=(input_len, ))
embed_layer_2.set_weights([matrix])
embed_layer_2.trainable = False
window_sizes = [2, 3, 5]
dic = {}
for size in window_sizes:
#addding all the layer instances to a dic for easy handling
dic[size] = Conv1D(kernel_size=size,
strides=1,
padding="same",
filters=100,
activation="relu")
x1 = embed_layer_1(inp)
x2 = embed_layer_2(inp)
#notice we are sharing the layers , for both x1,x2 we are using the same weights
y1_2 = dic[2](x1)
y1_3 = dic[3](x1)
y1_5 = dic[5](x1)
y2_2 = dic[2](x2)
y2_3 = dic[3](x2)
y2_5 = dic[5](x2)
concat_1 = concatenate([y1_2, y1_3, y1_5])
concat_2 = concatenate([y2_2, y2_3, y2_5])
add_1 = add([concat_1, concat_2])
pool_1 = GlobalAveragePooling1D()(add_1)
if drop == True:
#add dropout
pool_1 = Dropout(0.5)(pool_1)
final_1 = Dense(nb_classes, activation=activation)(pool_1)
model = Model([inp], [final_1])
return model
