def define_glove_model(length, vocab_size, embedding_matrix, num_classes): # channel 1 inputs1 = Input(shape=(length, )) e1 = Embedding(input_dim=vocab_size, output_dim=300, embeddings_initializer=Constant(embedding_matrix), input_length=length) e1.trainable = False embedding1 = e1(inputs1) conv1 = Conv1D(filters=16, kernel_size=2, activation='relu')(embedding1) drop1 = Dropout(0.5)(conv1) pool1 = MaxPooling1D(pool_size=2)(drop1) flat1 = Flatten()(pool1) # channel 2 inputs2 = Input(shape=(length, )) e2 = Embedding(input_dim=vocab_size, output_dim=300, embeddings_initializer=Constant(embedding_matrix), input_length=length) e2.trainable = False embedding2 = e2(inputs2) conv2 = Conv1D(filters=16, kernel_size=3, activation='relu')(embedding2) drop2 = Dropout(0.5)(conv2) pool2 = MaxPooling1D(pool_size=2)(drop2) flat2 = Flatten()(pool2) # channel 3 inputs3 = Input(shape=(length, )) e3 = Embedding(input_dim=vocab_size, output_dim=300, embeddings_initializer=Constant(embedding_matrix), input_length=length) e3.trainable = False embedding3 = e3(inputs3) conv3 = Conv1D(filters=16, kernel_size=4, activation='relu')(embedding3) drop3 = Dropout(0.5)(conv3) pool3 = MaxPooling1D(pool_size=2)(drop3) flat3 = Flatten()(pool3) # merge merged = concatenate([flat1, flat2, flat3]) # interpretation dense1 = Dense(10, activation='relu')(merged) # DEFINE the right model based on number of classes and loss function if num_classes == 2: outputs = Dense(1, activation='sigmoid')(dense1) model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs) else: # num_classes=3 outputs = Dense(3, activation='softmax')(dense1) model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs) return model
def build_model(n_celltypes, n_celltype_factors, n_assays, n_assay_factors, n_genomic_positions, n_25bp_factors, n_250bp_factors, n_5kbp_factors, n_layers, n_nodes, freeze_celltypes=False, freeze_assays=False, freeze_genome_25bp=False, freeze_genome_250bp=False, freeze_genome_5kbp=False, freeze_network=False): """This function builds a multi-scale deep tensor factorization model.""" celltype_input = Input(shape=(1,), name="celltype_input") celltype_embedding = Embedding(n_celltypes, n_celltype_factors, input_length=1, name="celltype_embedding") celltype_embedding.trainable = not freeze_celltypes celltype = Flatten()(celltype_embedding(celltype_input)) assay_input = Input(shape=(1,), name="assay_input") assay_embedding = Embedding(n_assays, n_assay_factors, input_length=1, name="assay_embedding") assay_embedding.trainable = not freeze_assays assay = Flatten()(assay_embedding(assay_input)) genome_25bp_input = Input(shape=(1,), name="genome_25bp_input") genome_25bp_embedding = Embedding(n_genomic_positions, n_25bp_factors, input_length=1, name="genome_25bp_embedding") genome_25bp_embedding.trainable = not freeze_genome_25bp genome_25bp = Flatten()(genome_25bp_embedding(genome_25bp_input)) genome_250bp_input = Input(shape=(1,), name="genome_250bp_input") genome_250bp_embedding = Embedding(int(n_genomic_positions / 10) + 1, n_250bp_factors, input_length=1, name="genome_250bp_embedding") genome_250bp_embedding.trainable = not freeze_genome_250bp genome_250bp = Flatten()(genome_250bp_embedding(genome_250bp_input)) genome_5kbp_input = Input(shape=(1,), name="genome_5kbp_input") genome_5kbp_embedding = Embedding(int(n_genomic_positions / 200) + 1, n_5kbp_factors, input_length=1, name="genome_5kbp_embedding") genome_5kbp_embedding.trainable = not freeze_genome_5kbp genome_5kbp = Flatten()(genome_5kbp_embedding(genome_5kbp_input)) layers = [celltype, assay, genome_25bp, genome_250bp, genome_5kbp] inputs = (celltype_input, assay_input, genome_25bp_input, genome_250bp_input, genome_5kbp_input) x = concatenate(layers) for i in range(n_layers): layer = Dense(n_nodes, activation='relu', name="dense_{}".format(i)) layer.trainable = not freeze_network x = layer(x) layer = Dense(1, name="y_pred") layer.trainable = not freeze_network y = layer(x) model = Model(inputs=inputs, outputs=y) model.compile(optimizer='adam', loss='mse', metrics=['mse']) return model
def _generate_model(self, lembedding, num_classes=2, unit='gru', rnn_size=128, train_vectors=True): model = Sequential() if lembedding.vector_box.W is None: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, W_constraint=None) else: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, weights=[lembedding.vector_box.W], W_constraint=None) emb.trainable = train_vectors model.add(emb) if unit == 'gru': model.add(GRU(rnn_size)) else: model.add(LSTM(rnn_size)) model.add(Dropout(0.2)) if num_classes == 2: model.add(Dense(1, activation='sigmoid')) if self.optimizer is None: self.optimizer = 'rmsprop' model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) else: if self.optimizer is None: self.optimizer = 'adam' model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) return model
def _generate_model(self, lembedding, num_classes=2, unit='gru', rnn_size=128, train_vectors=True): input = Input(shape=(lembedding.size,), dtype='int32') if lembedding.vector_box.W is None: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, W_constraint=None)(input) else: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, weights=[lembedding.vector_box.W], W_constraint=None, )(input) emb.trainable = train_vectors if unit == 'gru': forward = GRU(rnn_size)(emb) backward = GRU(rnn_size, go_backwards=True)(emb) else: forward = LSTM(rnn_size)(emb) backward = LSTM(rnn_size, go_backwards=True)(emb) merged_rnn = merge([forward, backward], mode='concat') dropped = Dropout(0.5)(merged_rnn) if num_classes == 2: out = Dense(1, activation='sigmoid')(dropped) model = Model(input=input, output=out) if self.optimizer is None: self.optimizer = 'rmsprop' model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) else: out = Dense(num_classes, activation='softmax')(dropped) model = Model(input=input, output=out) if self.optimizer is None: self.optimizer = 'adam' model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) return model
def tunable_embed_apply(word_cnt_post, vocab_size, embed_mat, word_feat_name): input_seq = Input(shape=(word_cnt_post, ), name=word_feat_name + '_t') embed_layer = Embedding( vocab_size, embed_mat.shape[1], embeddings_initializer=initializers.Constant(embed_mat), input_length=word_cnt_post, name=word_feat_name) embed_layer.trainable = True embed_l = embed_layer(input_seq) return input_seq, embed_l
def create_embedding_dict(sparse_feature_columns ,seed, l2_reg, prefix='sparse_', seq_mask_zero=True): #将特征进行embedding ,输入维度是某个特征的种类数 sparse_embedding = {} for feat in sparse_feature_columns: emb = Embedding(feat.vocabulary_size, feat.embedding_dim, embeddings_initializer=feat.embeddings_initializer, embeddings_regularizer=l2(l2_reg), name=prefix + '_emb_' + feat.embedding_name) emb.trainable = feat.trainable sparse_embedding[feat.embedding_name] = emb return sparse_embedding
def __init__(self, embed, mappings, cats, embed_len): word_inv = mappings['form'] npos = len(mappings['pos']) nne = len(mappings['ne']) nout = len(cats) width = len(word_inv) + 2 pos = Input(shape=(None, ), dtype='int32', name='POS') pos_emb = Embedding(npos, npos // 2, name='emb_pos')(pos) ne = Input(shape=(None, ), dtype='int32', name='NE') ne_emb = Embedding(nne, nne // 2, name='emb_ne')(ne) capital = Input(shape=(None, 3), name='Capital') special = Input(shape=(None, 12), name='Special') form = Input(shape=(None, ), dtype='int32', name='Form') emb = Embedding(70000, embed_len, embeddings_initializer=emb_mat_init(embed, word_inv), input_length=None, name='emb_form')(form) emb.trainable = True concat = Concatenate()([emb, pos_emb, ne_emb, capital, special]) drop = Dropout(0.25)(concat) lstm1 = Bidirectional(CuDNNLSTM(100, return_sequences=True), input_shape=(None, width))(drop) skip = Concatenate()([drop, lstm1]) lstm2 = Bidirectional(CuDNNLSTM(100, return_sequences=True), input_shape=(None, width))(skip) dense = Dense(nout, activation='softmax')(lstm2) # crf = CRF(nout, learn_mode='join', activation='softmax') # out = crf(dense) out = dense model = Model(inputs=[form, pos, ne, capital, special], outputs=out) # model.compile(loss=crf.loss_function, optimizer='nadam', metrics=[crf.accuracy]) model.compile(loss='categorical_crossentropy', optimizer='nadam', metrics=['acc']) #model.summary() self.model = model self.mappings = mappings self.cats = cats self.embed_len = embed_len
def build_model(self): # ================================ Input ============================== user_input = Input(shape=(self.walk_length, ), dtype='float32', name="friends_input") article_input = Input(shape=(self.max_num_sent, self.max_len_sent), dtype='float32', name="article_input") # ======================== GRU-based sequence encoder ================= word_embedding = Embedding(input_dim=self.vocab_size, output_dim=self.embedding_dim, weights=[self.embedding_matrix], mask_zero=False, name="word_embedding") embed_article = TimeDistributed(word_embedding)(article_input) # ============================== Word Level Attention ================= u_w = self.social_attention(user_input) s_i = self.multi_sequence_encode(self.embedding_dim, embed_article, u_w) print("encoded_article:", s_i._keras_shape) # ============================== Sentence Level Attention ============= s_i_dim = s_i._keras_shape[-1] s_encoding = self.bi_gru_encode(s_i_dim) h_i = s_encoding(s_i) u_s = self.social_attention(user_input) v = self.personalized_attention(u_s, h_i) # ============================ Prediction ============================= preds = Dense(1, activation="sigmoid")(v) adam = Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=1e-06, decay=0.0, clipvalue=5.) self.model = Model([user_input, article_input], preds) word_embedding.trainable = False self.model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
def _generate_model(self, lembedding, num_classes=2, ngrams=[1,2,3,4,5], nfilters=64, train_vectors=True): def sub_ngram(n): return Sequential([ Convolution1D(nfilters, n, activation='relu', input_shape=(lembedding.size, lembedding.vector_box.vector_dim)), Lambda( lambda x: K.max(x, axis=1), output_shape=(nfilters,) ) ]) doc = Input(shape=(lembedding.size, ), dtype='int32') embedded = Embedding(input_dim=lembedding.vector_box.size, output_dim=lembedding.vector_box.vector_dim, weights=[lembedding.vector_box.W])(doc) embedded.trainable = train_vectors rep = Dropout(0.5)( merge( [sub_ngram(n)(embedded) for n in ngrams], mode='concat', concat_axis=-1 ) ) if num_classes == 2: out = Dense(1, activation='sigmoid')(rep) model = Model(input=doc, output=out) if self.optimizer is None: self.optimizer = 'rmsprop' model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) else: out = Dense(num_classes, activation='softmax')(rep) model = Model(input=doc, output=out) if self.optimizer is None: self.optimizer = 'adam' model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) return model
def _generate_model(self, lembedding, num_classes=2, num_features=128, train_vectors=True): model = Sequential() if lembedding.vector_box.W is None: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, W_constraint=None, input_length=lembedding.size) else: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, weights=[lembedding.vector_box.W], W_constraint=None, input_length=lembedding.size) emb.trainable = train_vectors model.add(emb) model.add(Convolution1D(num_features, 3, init='uniform')) model.add(Activation('relu')) model.add(MaxPooling1D(2)) model.add(Dropout(0.25)) model.add(Convolution1D(num_features, 3, init='uniform')) model.add(Activation('relu')) model.add(MaxPooling1D(2)) model.add(Dropout(0.25)) model.add(Flatten()) if num_classes == 2: model.add(Dense(1, activation='sigmoid')) if self.optimizer is None: self.optimizer = 'rmsprop' model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) else: if self.optimizer is None: self.optimizer = 'adam' model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) return model
def cnn_autoencoder(shape=(500, 64), num_classes=2, input_dim=93, n=1, use_attention=False, **kwargs): input_array = keras.Input(shape=(shape[0], ), name='input') embedding = Embedding(input_dim=input_dim, output_dim=shape[1], weights=[np.load("weight.npy")]) embedding.trainable = False embedding_output = embedding(input_array) _embed = SpatialDropout1D(0.25)(embedding_output) kernel_size = 3 dilated_rate = 1 num_filters_in = 64 conv1d = Conv1D(filters=num_filters_in, kernel_size=kernel_size, padding="same")(_embed) b = BatchNormalization()(conv1d) r = Activation("elu")(b) x = r conv1 = Conv1D(filters=num_filters_in, kernel_size=1, padding="same")(x) b = BatchNormalization()(conv1) r = Activation("elu")(b) conv2 = Conv1D(filters=num_filters_in, kernel_size=3, padding="same")(r) b = BatchNormalization()(conv2) r = Activation("elu")(b) x = keras.layers.add([x, r]) x = BatchNormalization()(x) x = Activation('elu')(x) x = MaxPool1D(pool_size=128, strides=128, data_format='channels_last')(x) fc = Activation("sigmoid", name="feature_output")(x) fc = layers.UpSampling1D(size=128)(fc) fc = BatchNormalization()(fc) fc = Activation('elu')(fc) # add later # mid = Flatten(name="feature_output")(fc) conv2 = Conv1D(filters=num_filters_in, kernel_size=3, padding="same")(fc) b = BatchNormalization()(conv2) r = Activation("elu")(b) conv1 = Conv1D(filters=num_filters_in, kernel_size=1, padding="same")(r) b = BatchNormalization()(conv1) r = Activation("elu")(b) x = keras.layers.add([fc, r]) conv1d = Conv1D(filters=num_filters_in, kernel_size=kernel_size, padding="same")(x) b = BatchNormalization()(conv1d) r = Activation("elu")(b) output = r output = Lambda(lambda x: keras.losses.mean_squared_error(x[0], x[1]), name='loss', output_shape=(1, ))([output, embedding_output]) model = Model(inputs=input_array, outputs=output) model.compile(loss=loss_first, optimizer='adam') model.summary() plot_model(model, "attention.png") return model
def build_model(p): """ build a Keras model using the parameters in p """ max_posts = p['max_posts'] max_length = p['max_length'] filters = p['filters'] filtlen = p['filtlen'] poollen = p['poollen'] densed = p['densed'] embed_size = p['embed_size'] batch = p['batch'] random.seed(p['seed']) np.random.seed(p['seed']) # https://github.com/fchollet/keras/issues/2280 tf.reset_default_graph() if len(tf.get_default_graph()._nodes_by_id.keys()) > 0: raise RuntimeError( "Seeding is not supported after building part of the graph. " "Please move set_seed to the beginning of your code.") tf.set_random_seed(p['seed']) sess = tf.Session() K.set_session(sess) nb_words, genf, tok = datagen(max_posts, max_length, stype='training', batch_size=batch, randposts=p['randposts'], mintf=p['mintf'], mindf=p['mindf'], noempty=p['noempty'], prep=p['prep'], returntok=True) n_classes = 2 inp = Input(shape=(max_posts, max_length), dtype='int32') nextin = inp if p['cosine']: from keras.constraints import unitnorm wconstrain = unitnorm() else: wconstrain = None if p['w2v']: embeddings_index = {} fn = 'data/w2v_50_sg_export.txt' with open(fn) as f: for line in f: values = line.strip().split() word = values[0] if word in tok.word_index: coefs = np.asarray(values[1:], dtype='float32') embeddings_index[word] = coefs print('Found %s word vectors.' % len(embeddings_index)) embedding_matrix = np.zeros((nb_words, embed_size)) for word, i in tok.word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector else: embedding_matrix[i] = np.random.uniform(-0.2, 0.2, size=embed_size) emb = Embedding(nb_words, embed_size, mask_zero=True, input_length=max_length, W_constraint=wconstrain, weights=[embedding_matrix], trainable=p['etrain']) if not p['etrain']: print("making not trainable") emb.trainable = False else: assert p['etrain'], "must have etrain=True with w2v=False" emb = Embedding(nb_words, embed_size, mask_zero=True, W_constraint=wconstrain) embedded = TimeDistributed(emb)(nextin) if not p['etrain']: emb.trainable = False embedded.trainable = False conv = Sequential() conv.add( Convolution1D(nb_filter=filters, filter_length=filtlen, border_mode='valid', W_constraint=wconstrain, activation='linear', subsample_length=1, input_shape=(max_length, embed_size))) conv.add(Activation(p['af'])) conv.add(GlobalAveragePooling1D()) posts = TimeDistributed(conv)(embedded) combined = Convolution1D(nb_filter=filters, filter_length=p['acl'], border_mode='valid', activation=p['af'], subsample_length=p['acl'])(posts) combined = Flatten()(combined) if densed != 0: combined = Dense(densed, activation=p['af'])(combined) outlayer = Dense(2, activation='softmax')(combined) model = Model(inp, outlayer) return model, genf
for k, v in w2v_model.wv.vocab.items(): vocab.append([k, w2v_model.wv[k]]) embedding_matrix = np.zeros((len(idx2word), w2v_model.vector_size)) for word, vec in vocab: index = word2idx[word] embedding_matrix[index] = vec # build model embedding_layer = Embedding(input_dim=len(embedding_matrix), output_dim=embed_size, weights=[embedding_matrix], input_length=maxlen) embedding_layer.trainable = False model = Sequential() model.add(embedding_layer) model.add(Bidirectional(LSTM(512, dropout=0.5, recurrent_dropout=0.5, return_sequences=True))) model.add(LeakyReLU()) model.add(Bidirectional(LSTM(512, dropout=0.5, recurrent_dropout=0.5))) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(Dense(512)) model.add(LeakyReLU()) model.add(Dropout(0.5))
def main(data_cnf, model_cnf, mode): model_name = os.path.split(model_cnf)[1].split(".")[0] # 設定log檔案位置 logfile("./logs/logfile_" + model_name + ".log") yaml = YAML(typ='safe') data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf)) model, model_name, data_name = None, model_cnf['name'], data_cnf['name'] # model_path = model_cnf['path'] + "/" + model_cnf['name'] + '.h' model_path = r'E:\\PycharmProject\\CorNet\\' + model_name + '.h5' emb_init = get_word_emb(data_cnf['embedding']['emb_init']) logger.info(F'Model Name: {model_name}') # keras log file csv_logger = CSVLogger('./logs/' + model_name + '_log.csv', append=True, separator=',') if mode is None or mode == 'train': logger.info('Loading Training and Validation Set') train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels']) if 'size' in data_cnf['valid']: random_state = data_cnf['valid'].get('random_state', 1240) train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels, test_size=data_cnf['valid']['size'], random_state=random_state) else: valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels']) mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels))) train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels) labels_num = len(mlb.classes_) logger.info(F'Number of Labels: {labels_num}') logger.info(F'Size of Training Set: {len(train_x)}') logger.info(F'Size of Validation Set: {len(valid_x)}') vocab_size = emb_init.shape[0] emb_size = emb_init.shape[1] # 可調參數 data_num = len(train_x) ks = 3 output_channel = model_cnf['model']['num_filters'] dynamic_pool_length = model_cnf['model']['dynamic_pool_length'] num_bottleneck_hidden = model_cnf['model']['bottleneck_dim'] drop_out = model_cnf['model']['dropout'] cornet_dim = model_cnf['model']['cornet_dim'] nb_cornet_block = model_cnf['model'].get('nb_cornet_block', 0) nb_epochs = model_cnf['train']['nb_epoch'] batch_size = model_cnf['train']['batch_size'] max_length = 500 input_tensor = Input(batch_shape=(batch_size, max_length), name='input') emb_data = Embedding(input_dim=vocab_size, output_dim=emb_size, input_length=max_length, weights=[emb_init], trainable=False, name='embedding1')(input_tensor) emb_data.trainable = False # emd_out_4d = keras.layers.core.RepeatVector(1)(emb_data) # unsqueeze_emb_data = tf.keras.layers.Reshape((1, 500, 300), input_shape=(500, 300))(emb_data) # emb_data = tf.expand_dims(emb_data, axis=1) # emb_data = Lambda(reshape_tensor, arguments={'shape': (1, max_length, 300)}, name='lambda1')( # emb_data) conv1_output = Convolution1D(output_channel, 2, padding='same', kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='relu', name='conv1')(emb_data) # conv1_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)}, # name='conv1_lambda')( # conv1_output) conv2_output = Convolution1D(output_channel, 4, padding='same', kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='relu', name='conv2')(emb_data) # conv2_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)}, # name='conv2_lambda')( # conv2_output) conv3_output = Convolution1D(output_channel, 8, padding='same', kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='relu', name='conv3')(emb_data) # conv3_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)}, # name='conv3_lambda')( # conv3_output) # pool1 = adapmaxpooling(conv1_output, dynamic_pool_length) pool1 = GlobalMaxPooling1D(name='globalmaxpooling1')(conv1_output) pool2 = GlobalMaxPooling1D(name='globalmaxpooling2')(conv2_output) pool3 = GlobalMaxPooling1D(name='globalmaxpooling3')(conv3_output) output = concatenate([pool1, pool2, pool3], axis=-1) # output = Dense(num_bottleneck_hidden, activation='relu',name='bottleneck')(output) output = Dropout(drop_out, name='dropout1')(output) output = Dense(labels_num, activation='softmax', name='dense_final', kernel_initializer=keras.initializers.glorot_uniform(seed=None))(output) if nb_cornet_block > 0: for i in range(nb_cornet_block): x_shortcut = output x = keras.layers.Activation('sigmoid', name='cornet_sigmoid_{0}'.format(i + 1))(output) x = Dense(cornet_dim, kernel_initializer='glorot_uniform', name='cornet_1st_dense_{0}'.format(i + 1))(x) # x = Dense(cornet_dim, kernel_initializer=keras.initializers.glorot_uniform(seed=None), # activation='sigmoid', name='cornet_1st_dense_{0}'.format(i + 1))(output) x = keras.layers.Activation('elu', name='cornet_elu_{0}'.format(i + 1))(x) x = Dense(labels_num, kernel_initializer='glorot_uniform', name='cornet_2nd_dense_{0}'.format(i + 1))(x) # x = Dense(labels_num, kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='elu', # name='cornet_2nd_dense_{0}'.format(i + 1))(x) output = Add()([x, x_shortcut]) model = Model(input_tensor, output) model.summary() model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.Precision(top_k=5)]) # model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.top_k_categorical_accuracy(k=5)]) model.fit_generator(steps_per_epoch=data_num / batch_size, generator=batch_generator(train_x, train_y, batch_size), validation_data=batch_generator(valid_x, valid_y, batch_size), validation_steps=valid_x.shape[0] / batch_size, nb_epoch=nb_epochs, callbacks=[csv_logger]) model.save(model_path) elif mode is None or mode == 'eval': logger.info('Loading Training and Validation Set') train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels']) if 'size' in data_cnf['valid']: # 如果有設定valid的size 則直接使用train的一部分作為valid random_state = data_cnf['valid'].get('random_state', 1240) train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels, test_size=data_cnf['valid']['size'], random_state=random_state) else: valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels']) mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels))) train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels) labels_num = len(mlb.classes_) ################################################################################################## logger.info('Loading Test Set') logger.info('model path: ', model_path) mlb = get_mlb(data_cnf['labels_binarizer']) labels_num = len(mlb.classes_) test_x, test_label = get_data(data_cnf['test']['texts'], data_cnf['test']['labels']) logger.info(F'Size of Test Set: {len(test_x)}') test_y = mlb.transform(test_label).toarray() model = tf.keras.models.load_model(model_path) score = model.predict(test_x) print("p5: ", p5(test_y, score))
def _generate_model(self, lembedding, num_classes=2, first_kernel_size=3, num_features=1024, conv_dropout=False, train_vectors=True): model = Sequential() if lembedding.vector_box.W is None: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, W_constraint=None, input_length=lembedding.size) else: emb = Embedding(lembedding.vector_box.size, lembedding.vector_box.vector_dim, weights=[lembedding.vector_box.W], W_constraint=None, input_length=lembedding.size) emb.trainable = train_vectors model.add(emb) # Two conv layers with original kernel size, maxpooling is 2 model.add(Convolution1D(num_features, first_kernel_size, init='uniform')) model.add(Activation('relu')) model.add(MaxPooling1D(2)) if conv_dropout: model.add(Dropout(0.25)) model.add(Convolution1D(num_features, first_kernel_size, init='uniform')) model.add(Activation('relu')) model.add(MaxPooling1D(2)) if conv_dropout: model.add(Dropout(0.25)) # Three conv layers with kernel size = 3, no maxpooling model.add(Convolution1D(num_features, 3, init='uniform')) model.add(Activation('relu')) if conv_dropout: model.add(Dropout(0.25)) model.add(Convolution1D(num_features, 3, init='uniform')) model.add(Activation('relu')) if conv_dropout: model.add(Dropout(0.25)) model.add(Convolution1D(num_features, 3, init='uniform')) model.add(Activation('relu')) if conv_dropout: model.add(Dropout(0.25)) # One final conv layer with maxpooling model.add(Convolution1D(num_features, 3, init='uniform')) model.add(Activation('relu')) model.add(MaxPooling1D(2)) model.add(Dropout(0.25)) model.add(Flatten()) # Two dense layers with heavy dropout model.add(Dense(2048)) model.add(Dropout(0.5)) model.add(Dense(2048)) model.add(Dropout(0.5)) if num_classes == 2: model.add(Dense(1, activation='sigmoid')) if self.optimizer is None: self.optimizer = 'rmsprop' model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) else: if self.optimizer is None: self.optimizer = 'rmsprop' model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=["accuracy"]) return model
def __build_graph(self, enable_fine_tune): # create the base pre-trained model base_model = Xception(weights='imagenet', include_top=False) # add a global spatial max pooling layer x = base_model.output #(?,3,3,2048) x = GlobalAveragePooling2D()(x) #(?, 2048) x = Dropout(rate=self.drop_out_rate)(x) assert K.int_shape(x) == (None, 2048) ## Load pretrained deep weigths if not os.path.exists(self.model_file) and os.path.exists( os.path.join(self.deep_model_dir, MODEL_BEST_NAME)): print("@@@@Load DEEP model weights from %s" % (self.deep_model_dir)) pred = Dense(self.num_classes, activation='sigmoid')(x) model = Model(inputs=base_model.input, outputs=pred) model.load_weights( os.path.join(self.deep_model_dir, MODEL_BEST_NAME)) ## Load DONE ########################################## # RNN begins # hidden_size: 20 # embedding_size: 20 # output_size: 20 h0 = Dense(self.gru_hidden_size)(x) hr = Dense(self.gru_hidden_size)(x) assert K.int_shape(h0) == ( None, self.gru_hidden_size), 'h0.shape: {}'.format(K.int_shape(h0)) label_ids = Lambda( lambda h: K.arange(self.num_classes, dtype='int32'))(h0) def batch_labels(h, label_ids): # check: https://github.com/keras-team/keras/issues/8343 # check: https://stackoverflow.com/questions/47066635/checkpointing-keras-model-typeerror-cant-pickle-thread-lock-objects label_id_tf = K.constant(label_ids, dtype='int32') return K.ones(shape=(K.shape(h)[0], 228), dtype='int32') * label_id_tf batch_label_ids = Lambda( batch_labels, output_shape=(228, ), arguments={'label_ids': np.array([i for i in range(228)])})(h0) assert K.int_shape(batch_label_ids) == (None, 228) label_emb = Embedding( self.num_classes, self.gru_hidden_size, embeddings_initializer=self.__pretrained_label_embed, input_length=self.num_classes)(batch_label_ids) assert K.int_shape(label_emb) == (None, 228, 20) if not self.label_emb_trainable: label_emb.trainable = False # gru_fn = CuDNNGRU if self.use_cudnn else GRU gru_fn = GRU label_gru_emb = Bidirectional( gru_fn(self.gru_hidden_size, input_shape=(self.num_classes, self.gru_hidden_size), return_sequences=True))(label_emb, initial_state=[h0, hr]) assert K.int_shape(label_gru_emb) == (None, 228, 40) # Approach 1: use a giant fully connect if self.rnn_concat_all: label_gru_emb = Reshape( [self.num_classes * self.gru_hidden_size * 2])(label_gru_emb) x = Concatenate(axis=1)([label_gru_emb, x]) predictions = Dense(self.num_classes, activation='sigmoid')(x) # Approach 2: separate out prediction for each class else: # (batch_size, 228, 40) + (batch_size, 228, 2048) x = RepeatVector(self.num_classes)(x) x = Concatenate(axis=2)([label_gru_emb, x]) # (batch_size, 228, 2088) predictions = TimeDistributed(Dense(1, activations='sigmoid'), input_shape=(self.num_classes, 2088))(x) # RNN ends ########################################## # this is the model we will train model = Model(inputs=base_model.input, outputs=predictions) f_metrics = FMetrics() f_scores = f_metrics.get_fscores() if not enable_fine_tune: # first: train only the top layers (which were randomly initialized) # i.e. freeze all convolutional InceptionV3 layers for layer in base_model.layers: layer.trainable = False # compile the model (should be done *after* setting layers to non-trainable) optimizer = RMSprop(lr=5e-4, decay=0.001) model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'] + f_scores) else: print("@@@@@Fine tune enabled.@@@@@") print("Fine tune the last feature flow and the entire exit flow") for layer in model.layers: # change from 116 to 36 to ALL layer.trainable = True layer.kernel_regularizer = regularizers.l2(self.reg) # compile the model with a SGD/momentum optimizer # and a very slow learning rate. optimizer = Nadam(lr=5e-4, schedule_decay=0.001) model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'] + f_scores) print(model.summary()) return model
# Read back data train_reviews = np.load(path_join(ROOT_PATH, "IMDB_train_fulltext_glove_X.npy")) train_labels = np.load(path_join(ROOT_PATH, "IMDB_train_fulltext_glove_y.npy")) test_reviews = np.load(path_join(ROOT_PATH, "IMDB_test_fulltext_glove_X.npy")) test_labels = np.load(path_join(ROOT_PATH, "IMDB_test_fulltext_glove_y.npy")) WV_FILE_GLOBAL = path_join(ROOT_PATH, './embeddings/wv/glove.42B.300d.120000-glovebox.pkl') gb_global = pickle.load(open(WV_FILE_GLOBAL, 'rb')) wv_size = gb_global.W.shape[1] model = Sequential() emb = Embedding(gb_global.W.shape[0], wv_size, weights=[gb_global.W], input_length=train_reviews.shape[1]) emb.trainable = False model.add(emb) #model.add(Permute((2,1))) model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform')) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform')) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform')) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Convolution1D(128, 3, subsample_length=2, init='he_uniform'))
def multichannel(nb_classes, vocab_size, input_len, embedding_dim=100, matrix=None, activation="sigmoid", drop=False): inp = Input(shape=(input_len, )) if matrix is None: matrix = np.random.rand(vocab_size + 1, embedding_dim) assert matrix.shape == (vocab_size + 1, embedding_dim) embed_layer_1 = Embedding(vocab_size + 1, embedding_dim, input_length=input_len) embed_layer_1.build(input_shape=(input_len, )) embed_layer_1.set_weights([matrix]) embed_layer_2 = Embedding(vocab_size + 1, embedding_dim, input_length=input_len) embed_layer_2.build(input_shape=(input_len, )) embed_layer_2.set_weights([matrix]) embed_layer_2.trainable = False window_sizes = [2, 3, 5] dic = {} for size in window_sizes: #addding all the layer instances to a dic for easy handling dic[size] = Conv1D(kernel_size=size, strides=1, padding="same", filters=100, activation="relu") x1 = embed_layer_1(inp) x2 = embed_layer_2(inp) #notice we are sharing the layers , for both x1,x2 we are using the same weights y1_2 = dic[2](x1) y1_3 = dic[3](x1) y1_5 = dic[5](x1) y2_2 = dic[2](x2) y2_3 = dic[3](x2) y2_5 = dic[5](x2) concat_1 = concatenate([y1_2, y1_3, y1_5]) concat_2 = concatenate([y2_2, y2_3, y2_5]) add_1 = add([concat_1, concat_2]) pool_1 = GlobalAveragePooling1D()(add_1) if drop == True: #add dropout pool_1 = Dropout(0.5)(pool_1) final_1 = Dense(nb_classes, activation=activation)(pool_1) model = Model([inp], [final_1]) return model