def get_model(self): num_words = len(self.dataset.alphabet) num_labels = len(self.dataset.labels) model = Sequential() # MIR Embedding turns positive integers into dense vectors, for 1st layer of model. Why? Paper uses one-hot model.add( Embedding( num_words, # MIR input dimension self.config.embed_size, # MIR dense embedding mask_zero=True)) model.add(Dropout(self.config.input_dropout)) for _ in range(self.config.recurrent_stack_depth): model.add( Bidirectional( LSTM(self.config.num_lstm_units, return_sequences=True))) # MIR return_sequences means return full sequence, not just last output model.add(Dropout(self.config.output_dropout)) # MIR Does the paper have both input- and output-dropout? model.add(TimeDistributed(Dense(num_labels, activation='softmax'))) # TODO Add Viterbi decoder here, see Kuru et al. optimizer = Adam(lr=self.config.learning_rate, clipnorm=1.0) # MIR Add precision, recall, f1 metrics for all labels extra_metrics = { "precision_null": keras_metrics.precision(label=0), "precision_loc": keras_metrics.precision(label=1), "precision_org": keras_metrics.precision(label=2), "precision_per": keras_metrics.precision(label=3), "recall_null": keras_metrics.recall(label=0), "recall_loc": keras_metrics.recall(label=1), "recall_org": keras_metrics.recall(label=2), "recall_per": keras_metrics.recall(label=3), "f1_null": keras_metrics.f1_score(label=0), "f1_loc": keras_metrics.f1_score(label=1), "f1_org": keras_metrics.f1_score(label=2), "f1_per": keras_metrics.f1_score(label=3), } model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=[ 'categorical_accuracy', self.non_null_label_accuracy, ].extend(extra_metrics)) # MIR non_null_label_accuracy is a func return model
def cnn(filters, pooling_size=2, epochs=15, table_folder="/", kernel_size=3, input_dim=34, batch_size=32, nb_filters=34, time_from=32, time_to=8, downsample_ratio=None, oversample=None): timesteps = time_from - time_to X_train, X_test, y_train, y_test, churn_number, total_number, feature_names = import_and_preprocess_table( timesteps, time_from, time_to, filters, table_folder, downsample_ratio, oversample) print("Creating layers...") model = Sequential() model.add( Conv1D(nb_filters, kernel_size=kernel_size, input_shape=(timesteps, input_dim), activation='relu')) model.add(MaxPooling1D(pooling_size)) # model.add(Conv1D(nb_filters, kernel_size=kernel_size, activation='relu')) # model.add(Conv1D(nb_filters*2, kernel_size=3, activation='relu')) # model.add(GlobalAveragePooling1D()) # model.add(Dropout(0.5)) model.add(Flatten()) # model.add(Dense(128, activation='relu')) model.add(Dense(1, activation='sigmoid')) print("Compiling model...") model.compile(loss='mean_squared_error', optimizer='rmsprop', metrics=[ 'accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score() ]) print("Fitting model...") print(model.summary()) callback = [EarlyStopping(monitor='loss', patience=5)] history = model.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=batch_size, epochs=epochs) #, callbacks=callback) score = model.evaluate(X_test, y_test, batch_size=batch_size) y_pred = model.predict(X_test) log_to_csv("cnn", score, history, filters, table_folder, input_dim, batch_size, time_from, time_to, model.to_json(), nb_filters, kernel_size) return [score, history, churn_number, total_number, y_pred]
def test_save_load(self): custom_objects = { "true_positive": keras_metrics.true_positive(sparse=self.sparse), "true_negative": keras_metrics.true_negative(sparse=self.sparse), "false_positive": keras_metrics.false_positive(sparse=self.sparse), "false_negative": keras_metrics.false_negative(sparse=self.sparse), "precision": keras_metrics.precision(sparse=self.sparse), "recall": keras_metrics.recall(sparse=self.sparse), "f1_score": keras_metrics.f1_score(sparse=self.sparse), "sin": keras.backend.sin, "abs": keras.backend.abs, } x, y = self.samples(100) self.model.fit(x, y, epochs=10) with tempfile.NamedTemporaryFile() as file: self.model.save(file.name, overwrite=True) model = keras.models.load_model(file.name, custom_objects=custom_objects) expected = self.model.evaluate(x, y)[1:] received = model.evaluate(x, y)[1:] self.assertEqual(expected, received)
def lstm5(filters, epochs=15, table_folder="/", save_file=None, input_dim=34, batch_size=32, time_from=32, time_to=8, downsample_ratio=None, oversample=None): timesteps = time_from - time_to X_train, X_test, y_train, y_test, churn_number, total_number, feature_names = import_and_preprocess_table( timesteps, time_from, time_to, filters, table_folder, downsample_ratio, oversample) print("Creating layers...") model = Sequential() model.add( LSTM(34, input_length=timesteps, input_dim=34, return_sequences=True)) model.add(Dropout(0.2)) model.add(LSTM(input_dim, return_sequences=True)) model.add(Dropout(0.2)) # model.add(LSTM(input_dim, return_sequences=True)) # model.add(Dropout(0.2)) # model.add(LSTM(input_dim, return_sequences=True)) # model.add(Dropout(0.2)) model.add(LSTM(input_dim)) model.add(Dropout(0.2)) model.add(Dense(1, activation='sigmoid')) print("Compiling model...") model.compile(loss='mean_squared_error', optimizer='rmsprop', metrics=[ 'accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score() ]) print("Fitting model...") print(model.summary()) callback = [EarlyStopping(monitor='val_loss', patience=5)] history = model.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=batch_size, epochs=epochs) #, callbacks=callback) score = model.evaluate(X_test, y_test, batch_size=batch_size) y_pred = model.predict(X_test) log_to_csv("lstm", score, history, filters, table_folder, input_dim, batch_size, time_from, time_to, model.to_json()) return [score, history, churn_number, total_number, y_pred]
def load_predict(pre_list): # 清除session,避免重复调用出错 keras.backend.clear_session() BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) model_path = os.path.join(BASE_DIR, 'recommend', 'model.h5') model = load_model(model_path, custom_objects={ 'binary_precision': km.binary_precision(), 'binary_recall': km.binary_recall(), 'binary_f1_score': km.f1_score() }) predictions = model.predict(pre_list) print(predictions) return predictions
def bert_lstm(train_x, train_y, val_x, val_y, tokenEmb, train_posEmb, target_posEmb, vocab_size=100000, embeding_size=64, hidden_size=64, batch_size=64, dropout=0.1): input = Input(shape=(512, ), batch_size=batch_size, dtype="int32") input2 = Input(shape=(512, 64), batch_size=batch_size, dtype="float32") token_embed = Embedding(vocab_size, embeding_size, weights=[tokenEmb], trainable=True)(input) #pos_embed = PosEmb(posEmb)(input) embeding = tf.add(token_embed, input2) embeding = Dropout(dropout)(embeding) feat = Bidirectional(LSTM(hidden_size, dropout=0.1, return_sequences=True))(embeding) att = Attention(512)(feat) feat1 = GlobalMaxPooling1D()(feat) feat2 = GlobalAveragePooling1D()(feat) feat = concatenate([att, feat1, feat2]) feat = Dropout(0.1)(feat) train_y = to_categorical(train_y, 2) valy = to_categorical(val_y, 2) output = Dense(2, activation="softmax")(feat) model = Model([input, input2], output) model.compile(optimizer="adam", loss="binary_crossentropy", metrics=[km.f1_score(), km.precision(), km.recall()]) model.fit([train_x, train_posEmb], train_y, batch_size=batch_size, validation_data=([val_x, target_posEmb], valy), epochs=5, verbose=1, callbacks=[EarlyStopping(patience=5)]) pred = model.predict([val_x, target_posEmb], batch_size=batch_size).argmax(-1) return pred
def train_lstm(self): # build model and compile embedding_layer = Embedding(len(self.word_index) + 1, self.GLOVE_DIM, weights=[self.embedding_matrix], input_length=self.MAX_PAD, trainable=False) model = Sequential() model.add(embedding_layer) model.add(SpatialDropout1D(self.dropout)) model.add( LSTM(self.units, dropout=self.dropout, recurrent_dropout=self.dropout, activation=self.activation)) model.add(Dense(2, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[ keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score(), 'acc' ]) if self.sampler_name != '': # sample the data train_resample, train_label_resample = self.sampler.fit_resample( self.x_train, self.y_train) train_resample = np.asarray(train_resample) train_label_resample = to_categorical(train_label_resample) # training model.fit(train_resample, train_label_resample, batch_size=self.batch_size, epochs=self.epochs) else: x_train = np.asarray(self.x_train) y_train = to_categorical(self.y_train) model.fit(x_train, y_train, batch_size=self.batch_size, epochs=self.epochs) self.model = model print(model.summary())
def train_cnn(self): # build model and compile embedding_layer = Embedding(len(self.word_index) + 1, self.GLOVE_DIM, weights=[self.embedding_matrix], input_length=self.MAX_PAD, trainable=False) model = Sequential() model.add(embedding_layer) model.add(Dropout(self.dropout)) model.add(Conv1D(self.filters, self.kernel_size, activation='relu')) model.add(GlobalMaxPooling1D()) # model.add(Dropout(self.dropout)) model.add(Dense(10, activation='relu')) model.add(Dense(2, activation='softmax')) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[ keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score(), 'acc' ]) if self.sampler_name != '': # sample the data train_resample, train_label_resample = self.sampler.fit_resample( self.x_train, self.y_train) train_resample = np.asarray(train_resample) train_label_resample = to_categorical(train_label_resample) # training model.fit(train_resample, train_label_resample, batch_size=self.batch_size, epochs=self.epochs) else: x_train = np.asarray(self.x_train) y_train = to_categorical(self.y_train) model.fit(x_train, y_train, batch_size=self.batch_size, epochs=self.epochs) self.model = model print(model.summary())
def Mobile_Classifier(self,params): model = load_model(os.path.join(Base_Video_path,'model.h5'),custom_objects={"binary_precision": keras_metrics.precision(), "binary_recall":keras_metrics.recall(),"binary_f1_score":keras_metrics.f1_score()}) # summarize model. model.summary() url = "" if 'url' in params.keys(): im = self.getimage(params) elif 'photo_name'in params.keys(): im = cv2.imread(os.path.join(Base_Video_path,params['photo_name'])) IMG_SIZE = 100 im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) im = cv2.resize(im, (IMG_SIZE, IMG_SIZE)) im =np.array(im).reshape(-1, IMG_SIZE, IMG_SIZE, 3) predictions = model.predict_proba([im]) predictions = pd.Series([predictions]).to_json(orient='values') return predictions
def bilstm_textcnn(trainX, trainy, valX, valy, embedding, vocab): inputs = Input(shape=(220, ), dtype='int32') # Embedding x = Embedding(len(vocab) + 1, 300, weights=[embedding], trainable=False)(inputs) # BiLSTM and attention cnn1 = Conv1D(32, 2, padding="same", strides=1, activation="relu")(x) cnn1 = MaxPooling1D()(cnn1) cnn2 = Conv1D(32, 3, padding="same", strides=1, activation="relu")(x) cnn2 = MaxPooling1D()(cnn2) cnn3 = Conv1D(32, 4, padding="same", strides=1, activation="relu")(x) cnn3 = MaxPooling1D()(cnn3) features = concatenate([cnn1, cnn2, cnn3], axis=-1) flat = Flatten()(features) x = Bidirectional(LSTM(128, dropout=0.2, return_sequences=True))(x) # Attention x = AttentionLayer(attention_size=128)(x) #x = MaxPooling1D(pool_size=4)(x) x = concatenate([x, flat], axis=-1) x = Dropout(0.2)(x) output = Dense(2, activation='softmax')(x) #output = Dense(2, activation="softmax")(outputs) trainy = to_categorical(trainy, 2) val_y = to_categorical(valy, 2) model = Model(inputs=inputs, outputs=output) model.compile(optimizer=Adam(0.005), loss="binary_crossentropy", metrics=[km.recall(), km.precision(), km.f1_score()]) model.fit(trainX, trainy, batch_size=64, epochs=5) pred = model.predict(valX).argmax(-1) # metrics f1 = f1_score(valy, pred) precision = precision_score(valy, pred) recall = recall_score(valy, pred) return f1, precision, recall
def build_gru_model(vocab_size, pretrained_weights): model = Sequential() model.add( Embedding(input_dim=vocab_size, output_dim=hps['embedding_size'], weights=[pretrained_weights], trainable=False)) model.add( Bidirectional( GRU(units=hps['pad_length'], dropout=hps['dropout'], recurrent_dropout=hps['recurrent_dropout']))) model.add(Dense(1, activation='sigmoid')) model.compile( loss='binary_crossentropy', optimizer=optimizers.Adam(hps['lr']), metrics=['accuracy', km.precision(), km.recall(), km.f1_score()]) return model
def create_model(args): input_shape = (params.img_height, params.img_width, 3) # build the VGG16 network base_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=input_shape) # build a classifier model to put on top of the convolutional model top_model = Sequential() top_model.add(Flatten(input_shape=base_model.output_shape[1:])) top_model.add(Dense(256, activation='relu')) top_model.add(Dropout(params.drop_rate)) top_model.add(Dense(1, activation='sigmoid')) # note that it is necessary to start with a fully-trained # classifier, including the top classifier, # in order to successfully do fine-tuning top_model.load_weights(params.top_path) # add the model on top of the convolutional base model = Model(inputs=base_model.input, outputs=top_model(base_model.output)) # set the first 25 layers (up to the last conv block) # to non-trainable (weights will not be updated) for layer in model.layers[1:15]: layer.trainable = False # compile the model with a SGD/momentum optimizer # and a very slow learning rate. model.compile(loss='binary_crossentropy', optimizer=SGD(lr=params.learning_rate, momentum=0.9), metrics=[ keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score(), 'accuracy' ]) return model
def setUp(self): tp = keras_metrics.true_positive(sparse=self.sparse) tn = keras_metrics.true_negative(sparse=self.sparse) fp = keras_metrics.false_positive(sparse=self.sparse) fn = keras_metrics.false_negative(sparse=self.sparse) precision = keras_metrics.precision(sparse=self.sparse) recall = keras_metrics.recall(sparse=self.sparse) f1 = keras_metrics.f1_score(sparse=self.sparse) self.model = keras.models.Sequential() self.model.add(keras.layers.Activation(keras.backend.sin)) self.model.add(keras.layers.Activation(keras.backend.abs)) if self.sparse: loss = "sparse_categorical_crossentropy" else: loss = "binary_crossentropy" self.model.compile(optimizer="sgd", loss=loss, metrics=[tp, tn, fp, fn, precision, recall, f1])
def textcnn(mode, train_x, train_y, vocab, val_x, val_y, embedding, maxlen, tokenEmb=None, train_posEmb=None, target_posEmb=None, trainable=True): """ Args: train_x: training semantics feature train_sx: training statistical feature val_x: validation semantics feature val_sx: validation statistical feature train_y: label of tr##################################################################################### #tensorflow 2.0 version of bert modelaining set val_y: label of validation set return:[f1 sorce, precision, recall] """ #embedding method, if albert, choose albert, else word2vec if mode == "bert": # Embedding layer input = Input(shape=(512, ), batch_size=64, dtype="int32") input2 = Input(shape=(512, 64), batch_size=64, dtype="float32") token_embed = Embedding(vocab, 64, weights=[tokenEmb], trainable=trainable)(input) # pos_embed = PosEmb(posEmb)(input) embeding = tf.add(token_embed, input2) embed = Dropout(0.1)(embeding) else: input = Input(shape=(maxlen, ), dtype="float32") embed = Embedding(len(vocab) + 1, 300, weights=[embedding], trainable=trainable)(input) #textcnn for text classification in other cnn1 = Conv1D(32, 2, padding="same", strides=1, activation="relu")(embed) cnn1 = MaxPooling1D()(cnn1) cnn2 = Conv1D(32, 3, padding="same", strides=1, activation="relu")(embed) cnn2 = MaxPooling1D()(cnn2) cnn3 = Conv1D(32, 4, padding="same", strides=1, activation="relu")(embed) cnn3 = MaxPooling1D()(cnn3) features = concatenate([cnn1, cnn2, cnn3], axis=-1) flat = Flatten()(features) # statistitical features if mode == "bert": drop = Dropout(0.1)(flat) main_output = Dense(2, activation="softmax")(drop) model = Model(inputs=[input, input2], outputs=main_output) model.compile( loss="categorical_crossentropy", optimizer="adam", metrics=[km.f1_score(), km.binary_recall(), km.binary_precision()]) one_hot_label = keras.utils.to_categorical(train_y, 2) one_hot_label1 = keras.utils.to_categorical(val_y, 2) model.fit([train_x, train_posEmb], one_hot_label, batch_size=64, epochs=5, validation_data=([val_x, target_posEmb], one_hot_label1)) pred = model.predict([val_x, target_posEmb]).argmax(-1) else: drop = Dropout(0.1)(flat) main_output = Dense(2, activation="softmax")(drop) model = Model(inputs=input, outputs=main_output) model.compile( loss="categorical_crossentropy", optimizer="adam", metrics=[km.f1_score(), km.binary_recall(), km.binary_precision()]) one_hot_label = keras.utils.to_categorical(train_y, 2) one_hot_label1 = keras.utils.to_categorical(val_y, 2) from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint early = EarlyStopping() #check_point = ModelCheckpoint("../data/experiment_results/RQ1/model/") model.fit(train_x, one_hot_label, batch_size=64, epochs=5, validation_data=(val_x, one_hot_label1)) pred = model.predict(val_x, batch_size=64).argmax(-1) # metrics calculation f1 = f1_score(val_y, pred) precision = precision_score(val_y, pred) recall = recall_score(val_y, pred) return f1, precision, recall
def experiment_RQ2_B( mode="WPDP", datatype="ast", model="textcnn", ): dataset, dataset_list = dataset_generation(mode=mode, datatype=datatype) res = {} for project_name in dataset_list: if mode == "WPDP": pre_project_name = dataset[project_name][0][0] train_seq_feat, train_stat_feat, train_y = dataset[project_name][1][0], dataset[project_name][1][1], \ dataset[project_name][1][2] target_seq_feat, target_stat_feat, target_y = dataset[project_name][2][0], dataset[project_name][2][1], \ dataset[project_name][2][2] else: train_seq_feat, train_stat_feat, train_y = dataset[project_name][0][0], dataset[project_name][0][1], \ dataset[project_name][0][2] target_seq_feat , target_stat_feat, target_y = dataset[project_name][1][0], dataset[project_name][1][1], \ dataset[project_name][1][2] train_seq_feat, train_stat_feat, train_y = data_oversampling( train_seq_feat, train_stat_feat, train_y) #if augmentation: # new_data, stat_feat = gen_eda(seq_feat.tolist(), stat_feat, y.tolist(), 0.1, argumentation) # seq_feat = new_data["seq"] # y = new_data["bug"] # del new_data # del new_data maxlen = 512 if model != "bert": tokenizer = Tokenizer(num_words=max_feature, lower=False) tokenizer.fit_on_texts( list(train_seq_feat) + list(target_seq_feat)) word_index = tokenizer.word_index train_seq_feat = tokenizer.texts_to_sequences(list(train_seq_feat)) train_seq_feat = pad_sequences(train_seq_feat, maxlen=maxlen) target_seq_feat = tokenizer.texts_to_sequences( list(target_seq_feat)) target_seq_feat = pad_sequences(target_seq_feat, maxlen=maxlen) #load the embedding index with open("./data/embedding_index.pkl", "rb") as f: embedding_index = pickle.load(f) embedding_matrix = build_matrix(word_index, embedding_index) if model == "textcnn": f1, precision, recall = textcnn(train_seq_feat, train_y, word_index, target_seq_feat, target_y, embedding=embedding_matrix, maxlen=maxlen) else: f1, precision, recall = bilstm_att_model( train_seq_feat, train_y, target_seq_feat, target_y, word_index, 64, 2, embedding=embedding_matrix) else: from transformers import TFAlbertForSequenceClassification from tensorflow.keras.layers import Input from tensorflow.keras.models import Model from tensorflow.keras.utils import to_categorical train_seq_feat, train_stat_feat, train_y = data_oversampling( train_seq_feat, train_stat_feat, train_y) target_rate = len(target_y[target_y == 1]) / len(target_y) new_data, train_stat_feat = gen_eda(train_seq_feat.tolist(), train_stat_feat, train_y.tolist(), 0.1, 3) # new_data, statics_feat = gen_eda(seq_feat.tolist(), statics_feat, y.tolist(), 0.1, i) train_seq_feat = new_data["seq"] train_y = new_data["bug"] maxlen = 512 del new_data tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2") train_seq_feat = train_seq_feat.apply( lambda x: tokenizer.encode(x, pad_to_max_length=True)) target_seq_feat = target_seq_feat.apply( lambda x: tokenizer.encode(x, pad_to_max_length=True)) train_seq_feat = list(train_seq_feat) target_seq_feat = list(target_seq_feat) valX = [] for i in range(len(target_seq_feat)): valX.append(target_seq_feat[i][:maxlen]) target_seq_feat = np.array(valX) del valX trainX = [] for i in range(len(train_seq_feat)): trainX.append(train_seq_feat[i][:maxlen]) train_seq_feat = np.array(trainX) del trainX #input = Input(shape=(maxlen,),batch_size=64, dtype="int32") input = Input(shape=(maxlen, ), batch_size=64, dtype="int32") output = TFAlbertForSequenceClassification.from_pretrained( "albert-base-v2")(input)[0] print(output) model = Model(input, output) model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=[ km.f1_score(), km.binary_recall(), km.binary_precision() ]) train_y = to_categorical(train_y, 2) targety = to_categorical(target_y, 2) modelcheck = ModelCheckpoint("../data/experiment_results/" + project_name + ".h5", save_best_only=True) callbacks = [early, modelcheck] model.fit(train_seq_feat, train_y, batch_size=64, validation_data=(target_seq_feat, targety), epochs=5, callbacks=callbacks) pred = model.predict(target_seq_feat, batch_size=64).argmax(-1) f1 = f1_score(target_y, pred) print(f1) print(output) if mode == "WPDP": res[pre_project_name] = [f1, precision, recall] else: res[project_name] = [f1, precision, recall] df = pd.DataFrame(res) df.to_csv("../data/experiment_results/RQ2/b/" + mode + "_" + model + "_" + ".csv", index=False)
# layer=Dense(256,activation='relu')(layer) # layer=Dense(128,activation='relu')(layer) layer = Dense(3, name='out_layer', activation='softmax')(layer) model = Model(inputs=[inp1], outputs=layer) return model model = BidLstm(max_len, max_features=len(tok.word_index) + 1, embed_size=300) # In[ ]: model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['acc', km.f1_score()]) # In[ ]: model.summary() # In[ ]: cp_filepath = root_path + '/checkpoints/bilstm_self_attention.h5' cp_check_point = keras.callbacks.ModelCheckpoint(cp_filepath, monitor='val_f1_score', verbose=0, save_best_only=True, save_weights_only=False, mode='max', period=1)
encoder = load_model('model_encoder.h5') encoded_test_x = encoder.predict(test_x[test_index]) model = Sequential() model.add(Dense(50, activation='relu', input_shape=(ENCODE_DIM, ))) #model.add(Dropout(0.5)) model.add(Dense(20, activation='relu')) #model.add(Dropout(0.5)) model.add(Dense(5, activation='relu')) model.add(Dense(2, activation='softmax')) model.load_weights('new_model_classifier_nodropout.h5') model.compile(optimizer='adadelta', loss='binary_crossentropy', metrics=[km.precision(), km.recall(), km.f1_score()]) log_filepath = "/home/jira/hackdays/test_log" tb_cb = TensorBoard(log_dir=log_filepath, histogram_freq=1) cbks = [tb_cb] #model.fit(encoded_train_x, train_y[train_index], validation_split=0.1, epochs=50, callbacks=cbks) score = model.evaluate(encoded_test_x, test_y) print("eval result") print("[binary_crossentropy, precision, recall, f1_score] = {}".format(score)) #print(model.predict(encoded_test_x)) #print(test_y)
def train_mlp(x, y, num_epoch, batch_size): ### Cross Validation ### # Get and compile model model = mlp() model.summary() optimizer = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()]) # Shuffle data x, y = unison_shuffled_copies(x, y, dual_input=False) x_split, y_split = split_data(x, y, chunks=True) # Train model start_time = time() history = [] for i in range(5): x_train, y_train = collect_train_data(x_split, y_split, i) x_test, y_test = x_split[i], y_split[i] history.append(model.fit(x=np.array(x_train), y=np.array(y_train), epochs=num_epoch, verbose=1, validation_data=(np.array(x_test), np.array(y_test)), shuffle=True, batch_size=batch_size).history) model = mlp() model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()]) end_time = time() time_taken = end_time - start_time print("Took {} to train over {} epochs and with batch size {}".format(end_time - start_time, num_epoch, batch_size)) # Compute mean metrics loss_mean, val_loss_mean, loss_stdev, val_loss_stdev = compute_desc_stats(history, 'loss', 'val_loss') acc_mean, val_acc_mean, acc_stdev, val_acc_stdev = compute_desc_stats(history, 'acc', 'val_acc') recall_mean, val_recall_mean, recall_stdev, val_recall_stdev = compute_desc_stats(history, 'recall', 'val_recall') precision_mean, val_precision_mean, precision_stdev, val_precision_stdev = compute_desc_stats(history, 'precision', 'val_precision') f1_mean, val_f1_mean, f1_stdev, val_f1_stdev = compute_desc_stats(history, 'f1_score', 'val_f1_score') # Plot graphs plot_graph(loss_mean, val_loss_mean, 'loss', 'Loss', "loss_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 100) plot_graph(acc_mean, val_acc_mean, 'accuracy', 'Accuracy', "acc_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 101) plot_graph(recall_mean, val_recall_mean, 'recall', 'Recall', "rec_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 102) plot_graph(precision_mean, val_precision_mean, 'precision', 'Precision', "pre_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 103) plot_graph(f1_mean, val_f1_mean, 'f1', 'F1 Score', "f1_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 104) # Save metrics save_metrics_ext(acc_mean[-1], recall_mean[-1], precision_mean[-1], f1_mean[-1], val_acc_mean[-1], val_recall_mean[-1], val_precision_mean[-1], val_f1_mean[-1], acc_stdev[-1], recall_stdev[-1], precision_stdev[-1], f1_stdev[-1], val_acc_stdev[-1], val_recall_stdev[-1], val_precision_stdev[-1], val_f1_stdev[-1], time_taken, num_epoch, batch_size, "metrics_epoch{}_batch{}_mlp.txt".format(num_epoch, batch_size)) ### Learning Curve ### model = mlp() model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()]) lc_history = [] i = 1 size = int(len(x)/100*i) while i < 101 and size < len(x): lc_history.append(model.fit(x=np.array(x[:size]), y=np.array(y[:size]), epochs=num_epoch, verbose=1, shuffle=True, batch_size=batch_size).history) model = mlp() model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()]) i = i + 1 size = int(len(x)/100*i) save_lc(lc_history, "lc_epoch{}_batch{}_mlp.csv".format(num_epoch, batch_size)) # Save model save_model(model, 'model_epoch{}_batch{}_mlp.h5'.format(num_epoch, batch_size))
def train_cnn(x1, x2, y, num_epoch, batch_size): # Get and compile model model = siamese_net() model.summary() model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()]) # Shuffle data x, y = unison_shuffled_copies(np.array([x1, x2]), y) x_train, x_test, y_train, y_test = split_data(x, y, dual_input=True) # Train model start_time = time() history = model.fit(x=x_train, y=y_train, epochs=num_epoch, verbose=1, validation_data=(x_test, y_test), shuffle=True, batch_size=batch_size, callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=2)]) end_time = time() time_taken = end_time - start_time epochs_trained = len(history.history['loss']) print("Took {} to train over {} epochs with batch size {}".format(end_time - start_time, epochs_trained, batch_size)) # Plot train vs test metric graphs plot_graph(history.history['loss'], history.history['val_loss'], 'loss', 'Loss', "loss_epoch{}_batch{}.png".format(epochs_trained, batch_size), 200) plot_graph(history.history['acc'], history.history['val_acc'], 'accuracy', 'Accuracy', "acc_epoch{}_batch{}.png".format(epochs_trained, batch_size), 201) plot_graph(history.history['recall'], history.history['val_recall'], 'recall', 'Recall', "rec_epoch{}_batch{}.png".format(epochs_trained, batch_size), 202) plot_graph(history.history['precision'], history.history['val_precision'], 'precision', 'Precision', "pre_epoch{}_batch{}.png".format(epochs_trained, batch_size), 203) plot_graph(history.history['f1_score'], history.history['val_f1_score'], 'f1', 'F1 Score', "f1_epoch{}_batch{}.png".format(epochs_trained, batch_size), 204) # Save metrics save_metrics(history, time_taken, num_epoch, batch_size, "metrics_epoch{}_batch{}.txt".format(epochs_trained, batch_size)) # Save model save_model(model, 'model_epoch{}_batch{}.h5'.format(epochs_trained, batch_size))
def main(args): model_name = args.model_name lr = args.learning_rate image_url_cols = [args.image_url_cols] borad_log_dir = str(args.borad_log_dir) text_sim_metrics = int(args.text_sim_metrics) if text_sim_metrics == 1: text_sim_metrics = ['cosine'] if text_sim_metrics == 2: text_sim_metrics = ['cosine', 'inverse_l1'] if text_sim_metrics == 3: #text_sim_metrics = ['learnable_l1'] text_sim_metrics = ['concat'] if text_sim_metrics == 4: text_sim_metrics = ['inverse_l1'] print(text_sim_metrics, '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') text_compositions = [args.text_compositions] dropout = args.dropout image_url_cols = [args.image_url_cols] patience = args.patience batch_size = args.batch_size epochs = args.epoches workers = args.workers max_queue_size = args.max_queue_size pd.options.display.max_colwidth = 1000 # rcParams['font.family'] = 'serif' # rcParams['font.serif'] = 'times new roman' #%config InlineBackend.figure_format = 'retina' #%matplotlib inline # reload(hp) with open('./embedding/fasttext-300.map', 'rb') as f: map = pkl.load(f) print('Data preparation..') data_dir = os.path.join('.', 'set_transfer') source_dir = os.path.join(data_dir) nan_idx = map['word2idx']['NaN'] print('model established... ') log_dir = './Graph/' + borad_log_dir import keras tbCallBack = keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, write_graph=True, write_images=True) histories = dict(acc=list(), val_acc=list(), loss=list(), val_loss=list(), precision=list(), val_precision=list(), f1_score=list(), val_f1_score=list(), auroc=list(), val_auroc=list()) # from #with tf.device('/cpu:0'): if model_name == 'Deeper': print(model_name, '################') model = deeper_generator( embedding_file='./embedding/fasttext-300.matrix.npy', text_columns=['title_clean'], numeric_columns=[ 'price', 'lat', 'lon', 'categoryID', 'locationID' ], text_nan_idx=nan_idx, num_nan_val=0, text_sim_metrics=text_sim_metrics, text_compositions=['average'], numeric_sim_metrics=[ 'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio' ], dense_nodes=[args.dense_node], document_frequencies=None, idf_smoothing=2, make_isna=False, embedding_trainable=True, padding_limit=100, batch_norm=True, dropout=dropout, shared_lstm=True, #lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25), lstm_args=dict(units=25)) if model_name == 'Deeper_img': print(model_name, '################') model = deeper_img_generator( embedding_file='./embedding/fasttext-300.matrix.npy', text_columns=['title_clean'], numeric_columns_1D=['price'], numeric_columns_2D=['lat', 'lon'], category_num_cols=['categoryID', 'locationID'], image_url_cols=['images_array'], text_nan_idx=nan_idx, num_nan_val=0, text_sim_metrics=text_sim_metrics, text_compositions=['average'], image_sim_metrics=['cosine', 'inverse_l1', 'inverse_l2'], numeric_sim_metrics=[ 'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio' ], dense_nodes=[args.dense_node], document_frequencies=None, idf_smoothing=2, make_isna=False, embedding_trainable=True, padding_limit=100, batch_norm=True, dropout=dropout, shared_lstm=True, #lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25), lstm_args=dict(units=25)) if model_name == 'Deeper_img_decom': print(model_name, '################') model = deeper_img_decom_generator( embedding_file='./embedding/fasttext-300.matrix.npy', text_columns=['title_clean'], numeric_columns_1D=['price'], numeric_columns_2D=['lat', 'lon'], category_num_cols=['categoryID', 'locationID'], image_url_cols=['images_array'], text_nan_idx=nan_idx, num_nan_val=0, text_sim_metrics=text_sim_metrics, text_compositions=['decomposable'], numeric_sim_metrics=[ 'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio' ], dense_nodes=[args.dense_node], document_frequencies=None, idf_smoothing=2, make_isna=False, embedding_trainable=True, padding_limit=100, batch_norm=True, dropout=dropout, shared_lstm=True, #lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25), lstm_args=dict(units=25)) if model_name == 'Deeper_img_decom_advanced': print(model_name, '################') model = deeper_img_decom_advance_generator( embedding_file='./embedding/fasttext-300.matrix.npy', text_columns=['title_clean'], numeric_columns_1D=['price'], numeric_columns_2D=['lat', 'lon'], category_num_cols=['categoryID', 'locationID'], image_url_cols=['images_array'], text_nan_idx=nan_idx, num_nan_val=0, text_sim_metrics=text_sim_metrics, text_compositions=['hybrid'], numeric_sim_metrics=[ 'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio' ], dense_nodes=[args.dense_node], document_frequencies=None, idf_smoothing=2, make_isna=False, embedding_trainable=True, padding_limit=100, batch_norm=True, dropout=dropout, shared_lstm=True, #lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25), lstm_args=dict(units=25)) if model_name == 'Deeper_img_decom_vbi': print(model_name, '################') model = deeper_img_decom_vbi_generator( embedding_file='./embedding/fasttext-300.matrix.npy', text_columns=['title_clean'], numeric_columns_1D=['price'], numeric_columns_2D=['lat', 'lon'], category_num_cols=['categoryID', 'locationID'], image_url_cols=['images_array'], text_nan_idx=nan_idx, num_nan_val=0, text_sim_metrics=text_sim_metrics, text_compositions=['vbi'], numeric_sim_metrics=[ 'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio' ], dense_nodes=[args.dense_node], document_frequencies=None, idf_smoothing=2, make_isna=False, embedding_trainable=True, padding_limit=100, batch_norm=True, dropout=dropout, shared_lstm=True, #lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25), lstm_args=dict(units=25)) print('model complie...') #sgd = keras.optimizers.SGD(lr=0.001, momentum=0.0, decay=0.0, nesterov=False) import keras_metrics as km import tensorflow as tf from sklearn.metrics import roc_auc_score from keras.utils import multi_gpu_model as gpu # model = gpu(model, 2) def auroc(y_true, y_pred): return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double) adam = keras.optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc', km.binary_precision(), km.f1_score(), auroc]) print('model fitting...') import numpy as np import time #def load_image(x, y): # print(x[13]) # x[13] = np.load('./imagess'+x[13][1:]+'.npy').reshape(1,64,2048) # x[6] = np.load('./imagess'+x[6][1:]+'.npy').reshape(1,64,2048) # return x,y #print('loading X_train images.. ') #t_train = time.time() #X_train = tf.data.Dataset.from_tensor_slices((X_train, y_train)).map(load_image) #print(time.time()-t_train, 'loading X_test images..') #t_test = time.time() #X_test = tf.data.Dataset.from_tensor_slices((X_test, y_test)).map(load_image) #print(time.time()-t_test, 'loading X_val inages..') #X_val = tf.data.Dataset.from_tensor_slices((X_val, y_val)).map(load_image) #X_train[13] = np.array(list(X_train[13])).reshape(-1,1,64,2048) #X_train[6] = np.array(list(X_train[6])).reshape(-1,1,64,2048) #X_test[13] = np.array(list(X_test[13])).reshape(-1,1,64,2048) #X_test[6] = np.array(list(X_test[6])).reshape(-1,1,64,2048) #X_val[13] = np.array(list(X_val[13])).reshape(-1,1,64,2048) #X_val[6] = np.array(list(X_val[6])).reshape(-1,1,64,2048) with open('./y_fit_trainimg.map', 'rb') as f: y_train = pkl.load(f) with open('./y_fit_valimg.map', 'rb') as f: y_val = pkl.load(f) with open('./dataset_fit_trainimg.map', 'rb') as f: X_train = pkl.load(f) with open('./dataset_fit_valimg.map', 'rb') as f: X_val = pkl.load(f) with open('./dataset_fit_testimg.map', 'rb') as f: X_test = pkl.load(f) with open('./y_fit_testimg.map', 'rb') as f: y_test = pkl.load(f) # import pickle as pkl # with open('./dataset_fit_trainnoimg.map', 'wb') as f: # pkl.dump(X_train, f) # with open('./dataset_fit_testnoimg.map', 'wb') as f: # pkl.dump(X_test, f) # with open('./y_fit_testnoimg.map', 'wb') as f: # pkl.dump(y_test, f) # with open('./y_fit_trainnoimg.map', 'wb') as f: # pkl.dump(y_train, f) # with open('./y_fit_valnoimg.map', 'wb') as f: # pkl.dump(y_val, f) # with open('./dataset_fit_valnoimg.map', 'wb') as f: # pkl.dump(X_val, f) training_generator = DataGenerator(X_train, y_train, batch_size=batch_size) validation_generator = DataGenerator(X_val, y_val, batch_size=batch_size) print('len', len(X_train[1])) import keras earlystop = keras.callbacks.EarlyStopping(monitor='val_auroc', min_delta=0, patience=2, verbose=1, mode='max', baseline=None, restore_best_weights=True) # checkpoint = keras.callbacks.ModelCheckpoint('./checkpoints/'+ model_name + borad_log_dir+'.hdf5', monitor='val_f1_score', verbose=1, save_best_only=True, mode='max') history = model.fit_generator(generator=training_generator, steps_per_epoch=len(X_train[1]) // batch_size, epochs=3, validation_data=validation_generator, validation_steps=len(X_val[1]) // batch_size, workers=workers, use_multiprocessing=True, max_queue_size=max_queue_size, verbose=1, shuffle=True, callbacks=[earlystop]) model_log = './model_log/' + model_name + borad_log_dir + 'h5' # with open( model_log, 'wb') as f: # pkl.dump(model, f) model.save(model_log) #model.load_weights('./model_log/'+ model_name + borad_log_dir +'.tmod') generator = DataGenerator(X_test, y_test, batch_size=100) prediction = model.predict_generator(generator, steps=len(X_test[1]) // 100, max_queue_size=10, workers=7, use_multiprocessing=True, verbose=0) label = y_test dic = {} dic['prediction'] = prediction dic['label'] = label dic['data'] = X_test with open('./data_' + model_name + '.map', 'wb') as f: pkl.dump(dic, f) #from sklearn.metrics import precision_recall_curve, auc, average_precision_score #p, r, th = precision_recall_curve(y_test, prediction) #auc = auc(r, p) #ap = average_precision_score(y_test, prediction) #dic['auc'] = auc #dic['ap'] = ap #with open('./data_'+model_name + text_sim_metrics[-1]+'.map+auc', 'wb') as f: # pkl.dump(dic, f) #history = model.fit(X_train, y_train, epochs=8, batch_size=100, # validation_data=(X_val, y_val), # shuffle=True, callbacks = [tbCallBack]) # len(X_train[1])//batch_size print('history finished.. ') # model_log = './model_log/'+ model_name + borad_log_dir +'.tmod' # with open( model_log, 'wb') as f: # pkl.dump(model, f) # model.save(model_log) histories['acc'].extend(history.history['acc']) histories['val_acc'].extend(history.history['val_acc']) histories['loss'].extend(history.history['loss']) histories['val_loss'].extend(history.history['val_loss']) histories['precision'].extend(history.history['precision']) histories['f1_score'].extend(history.history['f1_score']) histories['val_precision'].extend(history.history['val_precision']) histories['val_f1_score'].extend(history.history['val_f1_score']) histories['auroc'].extend(history.history['auroc']) histories['val_auroc'].extend(history.history['val_auroc']) history_log = './history_log/' + model_name + borad_log_dir + '.tmap' with open(history_log, 'wb') as f: pkl.dump(histories, f) from sklearn.metrics import precision_recall_curve, auc, average_precision_score
def auroc(y_true, y_pred): return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double) adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc', km.binary_precision(), km.f1_score(), auroc]) print('model fitting...') import numpy as np import time #def load_image(x, y): # print(x[13]) # x[13] = np.load('./imagess'+x[13][1:]+'.npy').reshape(1,64,2048) # x[6] = np.load('./imagess'+x[6][1:]+'.npy').reshape(1,64,2048) # return x,y #print('loading X_train images.. ') #t_train = time.time() #X_train = tf.data.Dataset.from_tensor_slices((X_train, y_train)).map(load_image) #print(time.time()-t_train, 'loading X_test images..') #t_test = time.time() #X_test = tf.data.Dataset.from_tensor_slices((X_test, y_test)).map(load_image)
def transformer_models(trainX, trainy, valX, valy, embedding, vocab, maxlen, head_num, encoder_num, hidden_dim, project_name): #import relevant packages from keras instead of tensorflow.keras to avoid the runtime problem in this model. from keras.layers import Input, MaxPooling1D, Flatten, Dense, Embedding, SpatialDropout1D, Dropout, Conv1D from keras.models import Model from keras.utils import to_categorical from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau import keras input = Input(shape=(maxlen, ), dtype="int32") # embedding x = Embedding(len(vocab) + 1, 300, weights=[embedding], trainable=False)(input) #x = Dropout(0.2)(x) print(x.shape) #postition embedding x = PositionEmbedding(120, 300, "add")(x) output = get_encoders(encoder_num=encoder_num, input_layer=x, head_num=head_num, hidden_dim=hidden_dim, attention_activation="relu", dropout_rate=0.1) # three kind of filters size are 2,3,4 cnn1 = Conv1D(32, 2, padding="same", strides=1, activation="relu")(x) cnn1 = MaxPooling1D()(cnn1) cnn2 = Conv1D(32, 3, padding="same", strides=1, activation="relu")(x) cnn2 = MaxPooling1D()(cnn2) cnn3 = Conv1D(32, 4, padding="same", strides=1, activation="relu")(x) cnn3 = MaxPooling1D()(cnn3) features = keras.layers.concatenate([cnn1, cnn2, cnn3], axis=-1) output = Flatten()(features) output = Dropout(0.2)(output) output = Dense(2, activation="softmax")(output) trainy = to_categorical(trainy, 2) val_y = to_categorical(valy, 2) model = Model(inputs=input, outputs=output) path = "../data/experiment_results/RQ5/model/" + project_name + str( hidden_dim) + ".h5" early_stopping = EarlyStopping() callbacks = [early_stopping, ModelCheckpoint(path, save_best_only=True)] model.compile(optimizer=Adam(0.005), loss="binary_crossentropy", metrics=[km.recall(), km.precision(), km.f1_score()]) model.fit(trainX, trainy, batch_size=64, epochs=5, callbacks=callbacks, validation_data=(valX, val_y)) pred = model.predict(valX) pred = pred.argmax(-1) f1 = f1_score(valy, pred) precision = precision_score(valy, pred) recall = recall_score(valy, pred) return f1, precision, recall
encoder = load_model('model_encoder.h5') print(train_x.shape) encoded_train_x = encoder.predict(train_x[train_index]) encoded_test_x = encoder.predict(test_x[test_index]) model = Sequential() model.add(Dense(50, activation='relu', input_shape=(ENCODE_DIM, ))) #model.add(Dropout(0.5)) model.add(Dense(20, activation='relu')) #model.add(Dropout(0.5)) model.add(Dense(5, activation='relu')) model.add(Dense(2, activation='softmax')) model.compile(optimizer='adadelta', loss='binary_crossentropy', metrics=[km.precision(), km.recall(), km.f1_score()]) early_stopping_monitor = EarlyStopping(patience=15) log_filepath = "/home/jira/hackdays/train_log" tb_cb = TensorBoard(log_dir=log_filepath, histogram_freq=1) cbks = []#[tb_cb] model.fit(encoded_train_x, train_y[train_index], validation_split=0.1, epochs=50, callbacks=cbks) model.save('new_model_classifier_nodropout.h5') test_result = model.evaluate(encoded_test_x, test_y[test_index]) print("eval result") print(test_result) #print(model.predict(encoded_test_x))
# XXX benyomok még egy réteget, akkor vajh mi lesz? :) f1=71.5% model.add(Dense(hidden_dims)) model.add(Dropout(0.2)) model.add(Activation('relu')) # We project onto a single unit output layer, and squash it with a sigmoid: model.add(Dense(1)) model.add(Activation('sigmoid')) print(' * Compile...') model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', km.precision(), km.recall(), km.f1_score()]) # https://stackoverflow.com/questions/43076609 # itt kell beírni, hogy milyen mértékeket szeretnék # elvileg nem kell km-et használni, de másképp nekem nem megy... print(model.summary()) print(' * Fit...') model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test)) # Evaluate your performance on test print(' * Evaluate performance on test:')
from keras_metrics import f1_score, precision, recall all_metrics = [f1_score(label=1), precision(label=1), recall(label=1)]
batch_norm=True, dropout=0.75, shared_lstm=True, #lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25), lstm_args=dict(units=25) ) print('model complie...') #sgd = keras.optimizers.SGD(lr=0.001, momentum=0.0, decay=0.0, nesterov=False) import keras_metrics as km def auroc(y_true, y_pred): return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double) adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc', km.binary_precision(),km.f1_score(),auroc]) #adam = keras.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) #import keras_metrics as km #model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc', km.binary_precision(), km.binary_recall()]) print('model fitting...') training_generator = DataGenerator(X_train, y_train, batch_size=100) validation_generator = DataGenerator(X_val, y_val, batch_size = 100) print('len', len(X_train[1])) batch_size = 100 epochs = 7 earlystop = keras.callbacks.EarlyStopping(monitor='val_auroc', min_delta=0, patience=2, verbose=0, mode='max', baseline=None, restore_best_weights=True) #checkpoint = keras.callbacks.ModelCheckpoint('./checkpoints/'+'baseline'+text_sim_metrics[0] +'.hdf5', monitor='val_auroc', verbose=1, save_best_only=True, mode='max')
y_train = keras.utils.to_categorical(y_train, num_classes) # y_test = keras.utils.to_categorical(y_test, num_classes) print("********************") print(y_test.shape) model = Sequential() model.add(Dense(512, activation='relu', input_dim=1024)) # model.add(Dropout(0.2)) model.add(Dense(512, activation='relu')) # model.add(Dropout(0.2)) model.add(Dense(num_classes, activation='softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()]) history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) logging = True if logging: # list all data in history print(history.history.keys()) # summarize history for accuracy plt.plot(history.history['acc']) # plt.plot(history.history['val_acc'])
def bilstm_att_model(mode, train_x, train_y, val_x, val_y, vocab, hidden_size, class_nums, embedding, tokenEmb=None, train_posEmb=None, target_posEmb=None, trainable=True): """ train_x: training semantics feature train_sx: training statistical feature val_x: validation semantics feature val_sx: validation statistical feature train_y: label of training set val_y: label of validation set hidden_size: complete """ # input layer if mode == "bert": # Embedding layer input = Input(shape=(512, ), batch_size=64, dtype="int32") input2 = Input(shape=(512, 64), batch_size=64, dtype="float32") token_embed = Embedding(vocab, 64, weights=[tokenEmb], trainable=trainable)(input) # pos_embed = PosEmb(posEmb)(input) embeding = tf.add(token_embed, input2) x = Dropout(0.1)(embeding) else: input = Input(shape=(512, ), dtype="int32") x = Embedding(len(vocab) + 1, 300, weights=[embedding], trainable=trainable)(input) # BiLSTM layer x = Bidirectional(LSTM(hidden_size, dropout=0.2, return_sequences=True))(x) # Attention layer #x = keras.layers.Attention()([x,x]) #x = Conv1D(16,512,padding="same")(x) #x = Flatten()(x) x = Attention(512)(x) # output layer if mode == "bert": outputs = Dense(class_nums, activation='softmax')(x) # BiLSTM layer model = Model(inputs=[input, input2], outputs=outputs) one_hot_label = keras.utils.to_categorical(train_y, class_nums) one_hot_label1 = keras.utils.to_categorical(val_y, class_nums) model.compile( optimizer=Adam(0.005), loss="binary_crossentropy", metrics=[km.binary_precision(), km.binary_recall(), km.f1_score()]) model.fit([train_x, train_posEmb], one_hot_label, batch_size=64, epochs=5, validation_data=([val_x, target_posEmb], one_hot_label1), callbacks=[EarlyStopping(patience=10)]) pred = model.predict([val_x, target_posEmb]).argmax(-1) else: outputs = Dense(class_nums, activation='softmax')(x) # BiLSTM layer model = Model(inputs=input, outputs=outputs) one_hot_label = keras.utils.to_categorical(train_y, class_nums) one_hot_label1 = keras.utils.to_categorical(val_y, class_nums) model.compile( optimizer=Adam(0.005), loss="binary_crossentropy", metrics=[km.binary_precision(), km.binary_recall(), km.f1_score()]) model.fit(train_x, one_hot_label, batch_size=64, epochs=5, validation_data=(val_x, one_hot_label1)) pred = model.predict(val_x).argmax(-1) # metrics 计算 f1 = f1_score(val_y, pred) precision = precision_score(val_y, pred) recall = recall_score(val_y, pred) return f1, precision, recall
model.add(Dropout(dropout_fc)) # Output layer model.add(Dense(num_classes, activation='softmax')) print(model.summary()) if gpu_count > 1: model = multi_gpu_model(model, gpus=gpu_count) model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer=tf.keras.optimizers.Adam(), metrics=[ 'accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score() ]) # Define callback to save best epoch chk_name = 'fmnist-cnn-{epoch:04d}' checkpointer = ModelCheckpoint(filepath=os.path.join(chk_dir, chk_name), monitor='val_accuracy') # Define callback for early stopping early_stopping = EarlyStopping(monitor='val_accuracy', min_delta=0, patience=25, verbose=1, mode='auto')
def unet(pretrained_weights=None, input_size=(256, 256, 1)): inputs = Input(input_size) conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(inputs) conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv1) pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool1) conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv2) pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool2) conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv3) pool3 = MaxPooling2D(pool_size=(2, 2))(conv3) conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool3) conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv4) drop4 = Dropout(0.5)(conv4) pool4 = MaxPooling2D(pool_size=(2, 2))(drop4) conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool4) conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5) drop5 = Dropout(0.5)(conv5) up6 = Conv2D(512, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(drop5)) merge6 = concatenate([drop4, up6], axis=3) conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6) conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6) up7 = Conv2D(256, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv6)) merge7 = concatenate([conv3, up7], axis=3) conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7) conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7) up8 = Conv2D(128, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv7)) merge8 = concatenate([conv2, up8], axis=3) conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge8) conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8) up9 = Conv2D(64, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv8)) merge9 = concatenate([conv1, up9], axis=3) conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge9) conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9) conv9 = Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9) conv10 = Conv2D(1, 1, activation='sigmoid')(conv9) model = Model(input=inputs, output=conv10) model.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy', metrics=[ keras_metrics.true_positive(), keras_metrics.true_negative(), keras_metrics.false_positive(), keras_metrics.false_negative(), 'accuracy', keras_metrics.f1_score(), keras_metrics.precision(), keras_metrics.recall() ]) #model.summary() if (pretrained_weights): model.load_weights(pretrained_weights) return model