def get_model(self):
num_words = len(self.dataset.alphabet)
num_labels = len(self.dataset.labels)
model = Sequential()
# MIR Embedding turns positive integers into dense vectors, for 1st layer of model. Why? Paper uses one-hot
model.add(
Embedding(
num_words, # MIR input dimension
self.config.embed_size, # MIR dense embedding
mask_zero=True))
model.add(Dropout(self.config.input_dropout))
for _ in range(self.config.recurrent_stack_depth):
model.add(
Bidirectional(
LSTM(self.config.num_lstm_units, return_sequences=True)))
# MIR return_sequences means return full sequence, not just last output
model.add(Dropout(self.config.output_dropout))
# MIR Does the paper have both input- and output-dropout?
model.add(TimeDistributed(Dense(num_labels, activation='softmax')))
# TODO Add Viterbi decoder here, see Kuru et al.
optimizer = Adam(lr=self.config.learning_rate, clipnorm=1.0)
# MIR Add precision, recall, f1 metrics for all labels
extra_metrics = {
"precision_null": keras_metrics.precision(label=0),
"precision_loc": keras_metrics.precision(label=1),
"precision_org": keras_metrics.precision(label=2),
"precision_per": keras_metrics.precision(label=3),
"recall_null": keras_metrics.recall(label=0),
"recall_loc": keras_metrics.recall(label=1),
"recall_org": keras_metrics.recall(label=2),
"recall_per": keras_metrics.recall(label=3),
"f1_null": keras_metrics.f1_score(label=0),
"f1_loc": keras_metrics.f1_score(label=1),
"f1_org": keras_metrics.f1_score(label=2),
"f1_per": keras_metrics.f1_score(label=3),
}
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=[
'categorical_accuracy',
self.non_null_label_accuracy,
].extend(extra_metrics))
# MIR non_null_label_accuracy is a func
return model
def cnn(filters,
pooling_size=2,
epochs=15,
table_folder="/",
kernel_size=3,
input_dim=34,
batch_size=32,
nb_filters=34,
time_from=32,
time_to=8,
downsample_ratio=None,
oversample=None):
timesteps = time_from - time_to
X_train, X_test, y_train, y_test, churn_number, total_number, feature_names = import_and_preprocess_table(
timesteps, time_from, time_to, filters, table_folder, downsample_ratio,
oversample)
print("Creating layers...")
model = Sequential()
model.add(
Conv1D(nb_filters,
kernel_size=kernel_size,
input_shape=(timesteps, input_dim),
activation='relu'))
model.add(MaxPooling1D(pooling_size))
# model.add(Conv1D(nb_filters, kernel_size=kernel_size, activation='relu'))
# model.add(Conv1D(nb_filters*2, kernel_size=3, activation='relu'))
# model.add(GlobalAveragePooling1D())
# model.add(Dropout(0.5))
model.add(Flatten())
# model.add(Dense(128, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
print("Compiling model...")
model.compile(loss='mean_squared_error',
optimizer='rmsprop',
metrics=[
'accuracy',
keras_metrics.precision(),
keras_metrics.recall(),
keras_metrics.f1_score()
])
print("Fitting model...")
print(model.summary())
callback = [EarlyStopping(monitor='loss', patience=5)]
history = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
batch_size=batch_size,
epochs=epochs) #, callbacks=callback)
score = model.evaluate(X_test, y_test, batch_size=batch_size)
y_pred = model.predict(X_test)
log_to_csv("cnn", score, history, filters,
table_folder, input_dim, batch_size, time_from, time_to,
model.to_json(), nb_filters, kernel_size)
return [score, history, churn_number, total_number, y_pred]
def test_save_load(self):
custom_objects = {
"true_positive": keras_metrics.true_positive(sparse=self.sparse),
"true_negative": keras_metrics.true_negative(sparse=self.sparse),
"false_positive": keras_metrics.false_positive(sparse=self.sparse),
"false_negative": keras_metrics.false_negative(sparse=self.sparse),
"precision": keras_metrics.precision(sparse=self.sparse),
"recall": keras_metrics.recall(sparse=self.sparse),
"f1_score": keras_metrics.f1_score(sparse=self.sparse),
"sin": keras.backend.sin,
"abs": keras.backend.abs,
}
x, y = self.samples(100)
self.model.fit(x, y, epochs=10)
with tempfile.NamedTemporaryFile() as file:
self.model.save(file.name, overwrite=True)
model = keras.models.load_model(file.name,
custom_objects=custom_objects)
expected = self.model.evaluate(x, y)[1:]
received = model.evaluate(x, y)[1:]
self.assertEqual(expected, received)
def lstm5(filters,
epochs=15,
table_folder="/",
save_file=None,
input_dim=34,
batch_size=32,
time_from=32,
time_to=8,
downsample_ratio=None,
oversample=None):
timesteps = time_from - time_to
X_train, X_test, y_train, y_test, churn_number, total_number, feature_names = import_and_preprocess_table(
timesteps, time_from, time_to, filters, table_folder, downsample_ratio,
oversample)
print("Creating layers...")
model = Sequential()
model.add(
LSTM(34, input_length=timesteps, input_dim=34, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(input_dim, return_sequences=True))
model.add(Dropout(0.2))
# model.add(LSTM(input_dim, return_sequences=True))
# model.add(Dropout(0.2))
# model.add(LSTM(input_dim, return_sequences=True))
# model.add(Dropout(0.2))
model.add(LSTM(input_dim))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
print("Compiling model...")
model.compile(loss='mean_squared_error',
optimizer='rmsprop',
metrics=[
'accuracy',
keras_metrics.precision(),
keras_metrics.recall(),
keras_metrics.f1_score()
])
print("Fitting model...")
print(model.summary())
callback = [EarlyStopping(monitor='val_loss', patience=5)]
history = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
batch_size=batch_size,
epochs=epochs) #, callbacks=callback)
score = model.evaluate(X_test, y_test, batch_size=batch_size)
y_pred = model.predict(X_test)
log_to_csv("lstm", score, history, filters, table_folder, input_dim,
batch_size, time_from, time_to, model.to_json())
return [score, history, churn_number, total_number, y_pred]
def load_predict(pre_list):
# 清除session，避免重复调用出错
keras.backend.clear_session()
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
model_path = os.path.join(BASE_DIR, 'recommend', 'model.h5')
model = load_model(model_path,
custom_objects={
'binary_precision': km.binary_precision(),
'binary_recall': km.binary_recall(),
'binary_f1_score': km.f1_score()
})
predictions = model.predict(pre_list)
print(predictions)
return predictions
def bert_lstm(train_x,
train_y,
val_x,
val_y,
tokenEmb,
train_posEmb,
target_posEmb,
vocab_size=100000,
embeding_size=64,
hidden_size=64,
batch_size=64,
dropout=0.1):
input = Input(shape=(512, ), batch_size=batch_size, dtype="int32")
input2 = Input(shape=(512, 64), batch_size=batch_size, dtype="float32")
token_embed = Embedding(vocab_size,
embeding_size,
weights=[tokenEmb],
trainable=True)(input)
#pos_embed = PosEmb(posEmb)(input)
embeding = tf.add(token_embed, input2)
embeding = Dropout(dropout)(embeding)
feat = Bidirectional(LSTM(hidden_size, dropout=0.1,
return_sequences=True))(embeding)
att = Attention(512)(feat)
feat1 = GlobalMaxPooling1D()(feat)
feat2 = GlobalAveragePooling1D()(feat)
feat = concatenate([att, feat1, feat2])
feat = Dropout(0.1)(feat)
train_y = to_categorical(train_y, 2)
valy = to_categorical(val_y, 2)
output = Dense(2, activation="softmax")(feat)
model = Model([input, input2], output)
model.compile(optimizer="adam",
loss="binary_crossentropy",
metrics=[km.f1_score(),
km.precision(),
km.recall()])
model.fit([train_x, train_posEmb],
train_y,
batch_size=batch_size,
validation_data=([val_x, target_posEmb], valy),
epochs=5,
verbose=1,
callbacks=[EarlyStopping(patience=5)])
pred = model.predict([val_x, target_posEmb],
batch_size=batch_size).argmax(-1)
return pred
def train_lstm(self):
# build model and compile
embedding_layer = Embedding(len(self.word_index) + 1,
self.GLOVE_DIM,
weights=[self.embedding_matrix],
input_length=self.MAX_PAD,
trainable=False)
model = Sequential()
model.add(embedding_layer)
model.add(SpatialDropout1D(self.dropout))
model.add(
LSTM(self.units,
dropout=self.dropout,
recurrent_dropout=self.dropout,
activation=self.activation))
model.add(Dense(2, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=[
keras_metrics.precision(),
keras_metrics.recall(),
keras_metrics.f1_score(), 'acc'
])
if self.sampler_name != '':
# sample the data
train_resample, train_label_resample = self.sampler.fit_resample(
self.x_train, self.y_train)
train_resample = np.asarray(train_resample)
train_label_resample = to_categorical(train_label_resample)
# training
model.fit(train_resample,
train_label_resample,
batch_size=self.batch_size,
epochs=self.epochs)
else:
x_train = np.asarray(self.x_train)
y_train = to_categorical(self.y_train)
model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs)
self.model = model
print(model.summary())
def train_cnn(self):
# build model and compile
embedding_layer = Embedding(len(self.word_index) + 1,
self.GLOVE_DIM,
weights=[self.embedding_matrix],
input_length=self.MAX_PAD,
trainable=False)
model = Sequential()
model.add(embedding_layer)
model.add(Dropout(self.dropout))
model.add(Conv1D(self.filters, self.kernel_size, activation='relu'))
model.add(GlobalMaxPooling1D())
# model.add(Dropout(self.dropout))
model.add(Dense(10, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[
keras_metrics.precision(),
keras_metrics.recall(),
keras_metrics.f1_score(), 'acc'
])
if self.sampler_name != '':
# sample the data
train_resample, train_label_resample = self.sampler.fit_resample(
self.x_train, self.y_train)
train_resample = np.asarray(train_resample)
train_label_resample = to_categorical(train_label_resample)
# training
model.fit(train_resample,
train_label_resample,
batch_size=self.batch_size,
epochs=self.epochs)
else:
x_train = np.asarray(self.x_train)
y_train = to_categorical(self.y_train)
model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs)
self.model = model
print(model.summary())
def Mobile_Classifier(self,params):
model = load_model(os.path.join(Base_Video_path,'model.h5'),custom_objects={"binary_precision": keras_metrics.precision(),
"binary_recall":keras_metrics.recall(),"binary_f1_score":keras_metrics.f1_score()})
# summarize model.
model.summary()
url = ""
if 'url' in params.keys():
im = self.getimage(params)
elif 'photo_name'in params.keys():
im = cv2.imread(os.path.join(Base_Video_path,params['photo_name']))
IMG_SIZE = 100
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
im = cv2.resize(im, (IMG_SIZE, IMG_SIZE))
im =np.array(im).reshape(-1, IMG_SIZE, IMG_SIZE, 3)
predictions = model.predict_proba([im])
predictions = pd.Series([predictions]).to_json(orient='values')
return predictions
def bilstm_textcnn(trainX, trainy, valX, valy, embedding, vocab):
inputs = Input(shape=(220, ), dtype='int32')
# Embedding
x = Embedding(len(vocab) + 1, 300, weights=[embedding],
trainable=False)(inputs)
# BiLSTM and attention
cnn1 = Conv1D(32, 2, padding="same", strides=1, activation="relu")(x)
cnn1 = MaxPooling1D()(cnn1)
cnn2 = Conv1D(32, 3, padding="same", strides=1, activation="relu")(x)
cnn2 = MaxPooling1D()(cnn2)
cnn3 = Conv1D(32, 4, padding="same", strides=1, activation="relu")(x)
cnn3 = MaxPooling1D()(cnn3)
features = concatenate([cnn1, cnn2, cnn3], axis=-1)
flat = Flatten()(features)
x = Bidirectional(LSTM(128, dropout=0.2, return_sequences=True))(x)
# Attention
x = AttentionLayer(attention_size=128)(x)
#x = MaxPooling1D(pool_size=4)(x)
x = concatenate([x, flat], axis=-1)
x = Dropout(0.2)(x)
output = Dense(2, activation='softmax')(x)
#output = Dense(2, activation="softmax")(outputs)
trainy = to_categorical(trainy, 2)
val_y = to_categorical(valy, 2)
model = Model(inputs=inputs, outputs=output)
model.compile(optimizer=Adam(0.005),
loss="binary_crossentropy",
metrics=[km.recall(),
km.precision(),
km.f1_score()])
model.fit(trainX, trainy, batch_size=64, epochs=5)
pred = model.predict(valX).argmax(-1)
# metrics
f1 = f1_score(valy, pred)
precision = precision_score(valy, pred)
recall = recall_score(valy, pred)
return f1, precision, recall
def build_gru_model(vocab_size, pretrained_weights):
model = Sequential()
model.add(
Embedding(input_dim=vocab_size,
output_dim=hps['embedding_size'],
weights=[pretrained_weights],
trainable=False))
model.add(
Bidirectional(
GRU(units=hps['pad_length'],
dropout=hps['dropout'],
recurrent_dropout=hps['recurrent_dropout'])))
model.add(Dense(1, activation='sigmoid'))
model.compile(
loss='binary_crossentropy',
optimizer=optimizers.Adam(hps['lr']),
metrics=['accuracy',
km.precision(),
km.recall(),
km.f1_score()])
return model
def create_model(args):
input_shape = (params.img_height, params.img_width, 3)
# build the VGG16 network
base_model = applications.VGG16(weights='imagenet',
include_top=False,
input_shape=input_shape)
# build a classifier model to put on top of the convolutional model
top_model = Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(params.drop_rate))
top_model.add(Dense(1, activation='sigmoid'))
# note that it is necessary to start with a fully-trained
# classifier, including the top classifier,
# in order to successfully do fine-tuning
top_model.load_weights(params.top_path)
# add the model on top of the convolutional base
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
# set the first 25 layers (up to the last conv block)
# to non-trainable (weights will not be updated)
for layer in model.layers[1:15]:
layer.trainable = False
# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
model.compile(loss='binary_crossentropy',
optimizer=SGD(lr=params.learning_rate, momentum=0.9),
metrics=[
keras_metrics.precision(),
keras_metrics.recall(),
keras_metrics.f1_score(), 'accuracy'
])
return model
def setUp(self):
tp = keras_metrics.true_positive(sparse=self.sparse)
tn = keras_metrics.true_negative(sparse=self.sparse)
fp = keras_metrics.false_positive(sparse=self.sparse)
fn = keras_metrics.false_negative(sparse=self.sparse)
precision = keras_metrics.precision(sparse=self.sparse)
recall = keras_metrics.recall(sparse=self.sparse)
f1 = keras_metrics.f1_score(sparse=self.sparse)
self.model = keras.models.Sequential()
self.model.add(keras.layers.Activation(keras.backend.sin))
self.model.add(keras.layers.Activation(keras.backend.abs))
if self.sparse:
loss = "sparse_categorical_crossentropy"
else:
loss = "binary_crossentropy"
self.model.compile(optimizer="sgd",
loss=loss,
metrics=[tp, tn, fp, fn, precision, recall, f1])
def textcnn(mode,
train_x,
train_y,
vocab,
val_x,
val_y,
embedding,
maxlen,
tokenEmb=None,
train_posEmb=None,
target_posEmb=None,
trainable=True):
"""
Args:
train_x: training semantics feature
train_sx: training statistical feature
val_x: validation semantics feature
val_sx: validation statistical feature
train_y: label of tr#####################################################################################
#tensorflow 2.0 version of bert modelaining set
val_y: label of validation set
return:[f1 sorce, precision, recall]
"""
#embedding method, if albert, choose albert, else word2vec
if mode == "bert":
# Embedding layer
input = Input(shape=(512, ), batch_size=64, dtype="int32")
input2 = Input(shape=(512, 64), batch_size=64, dtype="float32")
token_embed = Embedding(vocab,
64,
weights=[tokenEmb],
trainable=trainable)(input)
# pos_embed = PosEmb(posEmb)(input)
embeding = tf.add(token_embed, input2)
embed = Dropout(0.1)(embeding)
else:
input = Input(shape=(maxlen, ), dtype="float32")
embed = Embedding(len(vocab) + 1,
300,
weights=[embedding],
trainable=trainable)(input)
#textcnn for text classification in other
cnn1 = Conv1D(32, 2, padding="same", strides=1, activation="relu")(embed)
cnn1 = MaxPooling1D()(cnn1)
cnn2 = Conv1D(32, 3, padding="same", strides=1, activation="relu")(embed)
cnn2 = MaxPooling1D()(cnn2)
cnn3 = Conv1D(32, 4, padding="same", strides=1, activation="relu")(embed)
cnn3 = MaxPooling1D()(cnn3)
features = concatenate([cnn1, cnn2, cnn3], axis=-1)
flat = Flatten()(features)
# statistitical features
if mode == "bert":
drop = Dropout(0.1)(flat)
main_output = Dense(2, activation="softmax")(drop)
model = Model(inputs=[input, input2], outputs=main_output)
model.compile(
loss="categorical_crossentropy",
optimizer="adam",
metrics=[km.f1_score(),
km.binary_recall(),
km.binary_precision()])
one_hot_label = keras.utils.to_categorical(train_y, 2)
one_hot_label1 = keras.utils.to_categorical(val_y, 2)
model.fit([train_x, train_posEmb],
one_hot_label,
batch_size=64,
epochs=5,
validation_data=([val_x, target_posEmb], one_hot_label1))
pred = model.predict([val_x, target_posEmb]).argmax(-1)
else:
drop = Dropout(0.1)(flat)
main_output = Dense(2, activation="softmax")(drop)
model = Model(inputs=input, outputs=main_output)
model.compile(
loss="categorical_crossentropy",
optimizer="adam",
metrics=[km.f1_score(),
km.binary_recall(),
km.binary_precision()])
one_hot_label = keras.utils.to_categorical(train_y, 2)
one_hot_label1 = keras.utils.to_categorical(val_y, 2)
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
early = EarlyStopping()
#check_point = ModelCheckpoint("../data/experiment_results/RQ1/model/")
model.fit(train_x,
one_hot_label,
batch_size=64,
epochs=5,
validation_data=(val_x, one_hot_label1))
pred = model.predict(val_x, batch_size=64).argmax(-1)
# metrics calculation
f1 = f1_score(val_y, pred)
precision = precision_score(val_y, pred)
recall = recall_score(val_y, pred)
return f1, precision, recall
def experiment_RQ2_B(
mode="WPDP",
datatype="ast",
model="textcnn",
):
dataset, dataset_list = dataset_generation(mode=mode, datatype=datatype)
res = {}
for project_name in dataset_list:
if mode == "WPDP":
pre_project_name = dataset[project_name][0][0]
train_seq_feat, train_stat_feat, train_y = dataset[project_name][1][0], dataset[project_name][1][1], \
dataset[project_name][1][2]
target_seq_feat, target_stat_feat, target_y = dataset[project_name][2][0], dataset[project_name][2][1], \
dataset[project_name][2][2]
else:
train_seq_feat, train_stat_feat, train_y = dataset[project_name][0][0], dataset[project_name][0][1], \
dataset[project_name][0][2]
target_seq_feat , target_stat_feat, target_y = dataset[project_name][1][0], dataset[project_name][1][1], \
dataset[project_name][1][2]
train_seq_feat, train_stat_feat, train_y = data_oversampling(
train_seq_feat, train_stat_feat, train_y)
#if augmentation:
# new_data, stat_feat = gen_eda(seq_feat.tolist(), stat_feat, y.tolist(), 0.1, argumentation)
# seq_feat = new_data["seq"]
# y = new_data["bug"]
# del new_data
# del new_data
maxlen = 512
if model != "bert":
tokenizer = Tokenizer(num_words=max_feature, lower=False)
tokenizer.fit_on_texts(
list(train_seq_feat) + list(target_seq_feat))
word_index = tokenizer.word_index
train_seq_feat = tokenizer.texts_to_sequences(list(train_seq_feat))
train_seq_feat = pad_sequences(train_seq_feat, maxlen=maxlen)
target_seq_feat = tokenizer.texts_to_sequences(
list(target_seq_feat))
target_seq_feat = pad_sequences(target_seq_feat, maxlen=maxlen)
#load the embedding index
with open("./data/embedding_index.pkl", "rb") as f:
embedding_index = pickle.load(f)
embedding_matrix = build_matrix(word_index, embedding_index)
if model == "textcnn":
f1, precision, recall = textcnn(train_seq_feat,
train_y,
word_index,
target_seq_feat,
target_y,
embedding=embedding_matrix,
maxlen=maxlen)
else:
f1, precision, recall = bilstm_att_model(
train_seq_feat,
train_y,
target_seq_feat,
target_y,
word_index,
64,
2,
embedding=embedding_matrix)
else:
from transformers import TFAlbertForSequenceClassification
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Model
from tensorflow.keras.utils import to_categorical
train_seq_feat, train_stat_feat, train_y = data_oversampling(
train_seq_feat, train_stat_feat, train_y)
target_rate = len(target_y[target_y == 1]) / len(target_y)
new_data, train_stat_feat = gen_eda(train_seq_feat.tolist(),
train_stat_feat,
train_y.tolist(), 0.1, 3)
# new_data, statics_feat = gen_eda(seq_feat.tolist(), statics_feat, y.tolist(), 0.1, i)
train_seq_feat = new_data["seq"]
train_y = new_data["bug"]
maxlen = 512
del new_data
tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2")
train_seq_feat = train_seq_feat.apply(
lambda x: tokenizer.encode(x, pad_to_max_length=True))
target_seq_feat = target_seq_feat.apply(
lambda x: tokenizer.encode(x, pad_to_max_length=True))
train_seq_feat = list(train_seq_feat)
target_seq_feat = list(target_seq_feat)
valX = []
for i in range(len(target_seq_feat)):
valX.append(target_seq_feat[i][:maxlen])
target_seq_feat = np.array(valX)
del valX
trainX = []
for i in range(len(train_seq_feat)):
trainX.append(train_seq_feat[i][:maxlen])
train_seq_feat = np.array(trainX)
del trainX
#input = Input(shape=(maxlen,),batch_size=64, dtype="int32")
input = Input(shape=(maxlen, ), batch_size=64, dtype="int32")
output = TFAlbertForSequenceClassification.from_pretrained(
"albert-base-v2")(input)[0]
print(output)
model = Model(input, output)
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=[
km.f1_score(),
km.binary_recall(),
km.binary_precision()
])
train_y = to_categorical(train_y, 2)
targety = to_categorical(target_y, 2)
modelcheck = ModelCheckpoint("../data/experiment_results/" +
project_name + ".h5",
save_best_only=True)
callbacks = [early, modelcheck]
model.fit(train_seq_feat,
train_y,
batch_size=64,
validation_data=(target_seq_feat, targety),
epochs=5,
callbacks=callbacks)
pred = model.predict(target_seq_feat, batch_size=64).argmax(-1)
f1 = f1_score(target_y, pred)
print(f1)
print(output)
if mode == "WPDP":
res[pre_project_name] = [f1, precision, recall]
else:
res[project_name] = [f1, precision, recall]
df = pd.DataFrame(res)
df.to_csv("../data/experiment_results/RQ2/b/" + mode + "_" + model + "_" +
".csv",
index=False)
# layer=Dense(256,activation='relu')(layer)
# layer=Dense(128,activation='relu')(layer)
layer = Dense(3, name='out_layer', activation='softmax')(layer)
model = Model(inputs=[inp1], outputs=layer)
return model
model = BidLstm(max_len, max_features=len(tok.word_index) + 1, embed_size=300)
# In[ ]:
model.compile(optimizer='Adam',
loss='categorical_crossentropy',
metrics=['acc', km.f1_score()])
# In[ ]:
model.summary()
# In[ ]:
cp_filepath = root_path + '/checkpoints/bilstm_self_attention.h5'
cp_check_point = keras.callbacks.ModelCheckpoint(cp_filepath,
monitor='val_f1_score',
verbose=0,
save_best_only=True,
save_weights_only=False,
mode='max',
period=1)
encoder = load_model('model_encoder.h5')
encoded_test_x = encoder.predict(test_x[test_index])
model = Sequential()
model.add(Dense(50, activation='relu', input_shape=(ENCODE_DIM, )))
#model.add(Dropout(0.5))
model.add(Dense(20, activation='relu'))
#model.add(Dropout(0.5))
model.add(Dense(5, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.load_weights('new_model_classifier_nodropout.h5')
model.compile(optimizer='adadelta',
loss='binary_crossentropy',
metrics=[km.precision(),
km.recall(), km.f1_score()])
log_filepath = "/home/jira/hackdays/test_log"
tb_cb = TensorBoard(log_dir=log_filepath, histogram_freq=1)
cbks = [tb_cb]
#model.fit(encoded_train_x, train_y[train_index], validation_split=0.1, epochs=50, callbacks=cbks)
score = model.evaluate(encoded_test_x, test_y)
print("eval result")
print("[binary_crossentropy, precision, recall, f1_score] = {}".format(score))
#print(model.predict(encoded_test_x))
#print(test_y)
def train_mlp(x, y, num_epoch, batch_size):
### Cross Validation ###
# Get and compile model
model = mlp()
model.summary()
optimizer = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()])
# Shuffle data
x, y = unison_shuffled_copies(x, y, dual_input=False)
x_split, y_split = split_data(x, y, chunks=True)
# Train model
start_time = time()
history = []
for i in range(5):
x_train, y_train = collect_train_data(x_split, y_split, i)
x_test, y_test = x_split[i], y_split[i]
history.append(model.fit(x=np.array(x_train), y=np.array(y_train), epochs=num_epoch, verbose=1, validation_data=(np.array(x_test), np.array(y_test)), shuffle=True, batch_size=batch_size).history)
model = mlp()
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()])
end_time = time()
time_taken = end_time - start_time
print("Took {} to train over {} epochs and with batch size {}".format(end_time - start_time, num_epoch, batch_size))
# Compute mean metrics
loss_mean, val_loss_mean, loss_stdev, val_loss_stdev = compute_desc_stats(history, 'loss', 'val_loss')
acc_mean, val_acc_mean, acc_stdev, val_acc_stdev = compute_desc_stats(history, 'acc', 'val_acc')
recall_mean, val_recall_mean, recall_stdev, val_recall_stdev = compute_desc_stats(history, 'recall', 'val_recall')
precision_mean, val_precision_mean, precision_stdev, val_precision_stdev = compute_desc_stats(history, 'precision', 'val_precision')
f1_mean, val_f1_mean, f1_stdev, val_f1_stdev = compute_desc_stats(history, 'f1_score', 'val_f1_score')
# Plot graphs
plot_graph(loss_mean, val_loss_mean, 'loss', 'Loss', "loss_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 100)
plot_graph(acc_mean, val_acc_mean, 'accuracy', 'Accuracy', "acc_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 101)
plot_graph(recall_mean, val_recall_mean, 'recall', 'Recall', "rec_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 102)
plot_graph(precision_mean, val_precision_mean, 'precision', 'Precision', "pre_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 103)
plot_graph(f1_mean, val_f1_mean, 'f1', 'F1 Score', "f1_epoch{}_batch{}_mlp.png".format(num_epoch, batch_size), 104)
# Save metrics
save_metrics_ext(acc_mean[-1], recall_mean[-1], precision_mean[-1], f1_mean[-1], val_acc_mean[-1], val_recall_mean[-1], val_precision_mean[-1], val_f1_mean[-1], acc_stdev[-1], recall_stdev[-1], precision_stdev[-1], f1_stdev[-1], val_acc_stdev[-1], val_recall_stdev[-1], val_precision_stdev[-1], val_f1_stdev[-1], time_taken, num_epoch, batch_size, "metrics_epoch{}_batch{}_mlp.txt".format(num_epoch, batch_size))
### Learning Curve ###
model = mlp()
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()])
lc_history = []
i = 1
size = int(len(x)/100*i)
while i < 101 and size < len(x):
lc_history.append(model.fit(x=np.array(x[:size]), y=np.array(y[:size]), epochs=num_epoch, verbose=1, shuffle=True, batch_size=batch_size).history)
model = mlp()
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()])
i = i + 1
size = int(len(x)/100*i)
save_lc(lc_history, "lc_epoch{}_batch{}_mlp.csv".format(num_epoch, batch_size))
# Save model
save_model(model, 'model_epoch{}_batch{}_mlp.h5'.format(num_epoch, batch_size))
def train_cnn(x1, x2, y, num_epoch, batch_size):
# Get and compile model
model = siamese_net()
model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()])
# Shuffle data
x, y = unison_shuffled_copies(np.array([x1, x2]), y)
x_train, x_test, y_train, y_test = split_data(x, y, dual_input=True)
# Train model
start_time = time()
history = model.fit(x=x_train, y=y_train, epochs=num_epoch, verbose=1, validation_data=(x_test, y_test), shuffle=True, batch_size=batch_size, callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=2)])
end_time = time()
time_taken = end_time - start_time
epochs_trained = len(history.history['loss'])
print("Took {} to train over {} epochs with batch size {}".format(end_time - start_time, epochs_trained, batch_size))
# Plot train vs test metric graphs
plot_graph(history.history['loss'], history.history['val_loss'], 'loss', 'Loss', "loss_epoch{}_batch{}.png".format(epochs_trained, batch_size), 200)
plot_graph(history.history['acc'], history.history['val_acc'], 'accuracy', 'Accuracy', "acc_epoch{}_batch{}.png".format(epochs_trained, batch_size), 201)
plot_graph(history.history['recall'], history.history['val_recall'], 'recall', 'Recall', "rec_epoch{}_batch{}.png".format(epochs_trained, batch_size), 202)
plot_graph(history.history['precision'], history.history['val_precision'], 'precision', 'Precision', "pre_epoch{}_batch{}.png".format(epochs_trained, batch_size), 203)
plot_graph(history.history['f1_score'], history.history['val_f1_score'], 'f1', 'F1 Score', "f1_epoch{}_batch{}.png".format(epochs_trained, batch_size), 204)
# Save metrics
save_metrics(history, time_taken, num_epoch, batch_size, "metrics_epoch{}_batch{}.txt".format(epochs_trained, batch_size))
# Save model
save_model(model, 'model_epoch{}_batch{}.h5'.format(epochs_trained, batch_size))
def main(args):
model_name = args.model_name
lr = args.learning_rate
image_url_cols = [args.image_url_cols]
borad_log_dir = str(args.borad_log_dir)
text_sim_metrics = int(args.text_sim_metrics)
if text_sim_metrics == 1:
text_sim_metrics = ['cosine']
if text_sim_metrics == 2:
text_sim_metrics = ['cosine', 'inverse_l1']
if text_sim_metrics == 3:
#text_sim_metrics = ['learnable_l1']
text_sim_metrics = ['concat']
if text_sim_metrics == 4:
text_sim_metrics = ['inverse_l1']
print(text_sim_metrics, '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
text_compositions = [args.text_compositions]
dropout = args.dropout
image_url_cols = [args.image_url_cols]
patience = args.patience
batch_size = args.batch_size
epochs = args.epoches
workers = args.workers
max_queue_size = args.max_queue_size
pd.options.display.max_colwidth = 1000
# rcParams['font.family'] = 'serif'
# rcParams['font.serif'] = 'times new roman'
#%config InlineBackend.figure_format = 'retina'
#%matplotlib inline
# reload(hp)
with open('./embedding/fasttext-300.map', 'rb') as f:
map = pkl.load(f)
print('Data preparation..')
data_dir = os.path.join('.', 'set_transfer')
source_dir = os.path.join(data_dir)
nan_idx = map['word2idx']['NaN']
print('model established... ')
log_dir = './Graph/' + borad_log_dir
import keras
tbCallBack = keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1,
write_graph=True,
write_images=True)
histories = dict(acc=list(),
val_acc=list(),
loss=list(),
val_loss=list(),
precision=list(),
val_precision=list(),
f1_score=list(),
val_f1_score=list(),
auroc=list(),
val_auroc=list())
# from
#with tf.device('/cpu:0'):
if model_name == 'Deeper':
print(model_name, '################')
model = deeper_generator(
embedding_file='./embedding/fasttext-300.matrix.npy',
text_columns=['title_clean'],
numeric_columns=[
'price', 'lat', 'lon', 'categoryID', 'locationID'
],
text_nan_idx=nan_idx,
num_nan_val=0,
text_sim_metrics=text_sim_metrics,
text_compositions=['average'],
numeric_sim_metrics=[
'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio'
],
dense_nodes=[args.dense_node],
document_frequencies=None,
idf_smoothing=2,
make_isna=False,
embedding_trainable=True,
padding_limit=100,
batch_norm=True,
dropout=dropout,
shared_lstm=True,
#lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25),
lstm_args=dict(units=25))
if model_name == 'Deeper_img':
print(model_name, '################')
model = deeper_img_generator(
embedding_file='./embedding/fasttext-300.matrix.npy',
text_columns=['title_clean'],
numeric_columns_1D=['price'],
numeric_columns_2D=['lat', 'lon'],
category_num_cols=['categoryID', 'locationID'],
image_url_cols=['images_array'],
text_nan_idx=nan_idx,
num_nan_val=0,
text_sim_metrics=text_sim_metrics,
text_compositions=['average'],
image_sim_metrics=['cosine', 'inverse_l1', 'inverse_l2'],
numeric_sim_metrics=[
'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio'
],
dense_nodes=[args.dense_node],
document_frequencies=None,
idf_smoothing=2,
make_isna=False,
embedding_trainable=True,
padding_limit=100,
batch_norm=True,
dropout=dropout,
shared_lstm=True,
#lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25),
lstm_args=dict(units=25))
if model_name == 'Deeper_img_decom':
print(model_name, '################')
model = deeper_img_decom_generator(
embedding_file='./embedding/fasttext-300.matrix.npy',
text_columns=['title_clean'],
numeric_columns_1D=['price'],
numeric_columns_2D=['lat', 'lon'],
category_num_cols=['categoryID', 'locationID'],
image_url_cols=['images_array'],
text_nan_idx=nan_idx,
num_nan_val=0,
text_sim_metrics=text_sim_metrics,
text_compositions=['decomposable'],
numeric_sim_metrics=[
'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio'
],
dense_nodes=[args.dense_node],
document_frequencies=None,
idf_smoothing=2,
make_isna=False,
embedding_trainable=True,
padding_limit=100,
batch_norm=True,
dropout=dropout,
shared_lstm=True,
#lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25),
lstm_args=dict(units=25))
if model_name == 'Deeper_img_decom_advanced':
print(model_name, '################')
model = deeper_img_decom_advance_generator(
embedding_file='./embedding/fasttext-300.matrix.npy',
text_columns=['title_clean'],
numeric_columns_1D=['price'],
numeric_columns_2D=['lat', 'lon'],
category_num_cols=['categoryID', 'locationID'],
image_url_cols=['images_array'],
text_nan_idx=nan_idx,
num_nan_val=0,
text_sim_metrics=text_sim_metrics,
text_compositions=['hybrid'],
numeric_sim_metrics=[
'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio'
],
dense_nodes=[args.dense_node],
document_frequencies=None,
idf_smoothing=2,
make_isna=False,
embedding_trainable=True,
padding_limit=100,
batch_norm=True,
dropout=dropout,
shared_lstm=True,
#lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25),
lstm_args=dict(units=25))
if model_name == 'Deeper_img_decom_vbi':
print(model_name, '################')
model = deeper_img_decom_vbi_generator(
embedding_file='./embedding/fasttext-300.matrix.npy',
text_columns=['title_clean'],
numeric_columns_1D=['price'],
numeric_columns_2D=['lat', 'lon'],
category_num_cols=['categoryID', 'locationID'],
image_url_cols=['images_array'],
text_nan_idx=nan_idx,
num_nan_val=0,
text_sim_metrics=text_sim_metrics,
text_compositions=['vbi'],
numeric_sim_metrics=[
'scaled_inverse_lp', 'unscaled_inverse_lp', 'min_max_ratio'
],
dense_nodes=[args.dense_node],
document_frequencies=None,
idf_smoothing=2,
make_isna=False,
embedding_trainable=True,
padding_limit=100,
batch_norm=True,
dropout=dropout,
shared_lstm=True,
#lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25),
lstm_args=dict(units=25))
print('model complie...')
#sgd = keras.optimizers.SGD(lr=0.001, momentum=0.0, decay=0.0, nesterov=False)
import keras_metrics as km
import tensorflow as tf
from sklearn.metrics import roc_auc_score
from keras.utils import multi_gpu_model as gpu
# model = gpu(model, 2)
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)
adam = keras.optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model.compile(optimizer=adam,
loss='binary_crossentropy',
metrics=['acc',
km.binary_precision(),
km.f1_score(), auroc])
print('model fitting...')
import numpy as np
import time
#def load_image(x, y):
# print(x[13])
# x[13] = np.load('./imagess'+x[13][1:]+'.npy').reshape(1,64,2048)
# x[6] = np.load('./imagess'+x[6][1:]+'.npy').reshape(1,64,2048)
# return x,y
#print('loading X_train images.. ')
#t_train = time.time()
#X_train = tf.data.Dataset.from_tensor_slices((X_train, y_train)).map(load_image)
#print(time.time()-t_train, 'loading X_test images..')
#t_test = time.time()
#X_test = tf.data.Dataset.from_tensor_slices((X_test, y_test)).map(load_image)
#print(time.time()-t_test, 'loading X_val inages..')
#X_val = tf.data.Dataset.from_tensor_slices((X_val, y_val)).map(load_image)
#X_train[13] = np.array(list(X_train[13])).reshape(-1,1,64,2048)
#X_train[6] = np.array(list(X_train[6])).reshape(-1,1,64,2048)
#X_test[13] = np.array(list(X_test[13])).reshape(-1,1,64,2048)
#X_test[6] = np.array(list(X_test[6])).reshape(-1,1,64,2048)
#X_val[13] = np.array(list(X_val[13])).reshape(-1,1,64,2048)
#X_val[6] = np.array(list(X_val[6])).reshape(-1,1,64,2048)
with open('./y_fit_trainimg.map', 'rb') as f:
y_train = pkl.load(f)
with open('./y_fit_valimg.map', 'rb') as f:
y_val = pkl.load(f)
with open('./dataset_fit_trainimg.map', 'rb') as f:
X_train = pkl.load(f)
with open('./dataset_fit_valimg.map', 'rb') as f:
X_val = pkl.load(f)
with open('./dataset_fit_testimg.map', 'rb') as f:
X_test = pkl.load(f)
with open('./y_fit_testimg.map', 'rb') as f:
y_test = pkl.load(f)
# import pickle as pkl
# with open('./dataset_fit_trainnoimg.map', 'wb') as f:
# pkl.dump(X_train, f)
# with open('./dataset_fit_testnoimg.map', 'wb') as f:
# pkl.dump(X_test, f)
# with open('./y_fit_testnoimg.map', 'wb') as f:
# pkl.dump(y_test, f)
# with open('./y_fit_trainnoimg.map', 'wb') as f:
# pkl.dump(y_train, f)
# with open('./y_fit_valnoimg.map', 'wb') as f:
# pkl.dump(y_val, f)
# with open('./dataset_fit_valnoimg.map', 'wb') as f:
# pkl.dump(X_val, f)
training_generator = DataGenerator(X_train, y_train, batch_size=batch_size)
validation_generator = DataGenerator(X_val, y_val, batch_size=batch_size)
print('len', len(X_train[1]))
import keras
earlystop = keras.callbacks.EarlyStopping(monitor='val_auroc',
min_delta=0,
patience=2,
verbose=1,
mode='max',
baseline=None,
restore_best_weights=True)
# checkpoint = keras.callbacks.ModelCheckpoint('./checkpoints/'+ model_name + borad_log_dir+'.hdf5', monitor='val_f1_score', verbose=1, save_best_only=True, mode='max')
history = model.fit_generator(generator=training_generator,
steps_per_epoch=len(X_train[1]) //
batch_size,
epochs=3,
validation_data=validation_generator,
validation_steps=len(X_val[1]) // batch_size,
workers=workers,
use_multiprocessing=True,
max_queue_size=max_queue_size,
verbose=1,
shuffle=True,
callbacks=[earlystop])
model_log = './model_log/' + model_name + borad_log_dir + 'h5'
# with open( model_log, 'wb') as f:
# pkl.dump(model, f)
model.save(model_log)
#model.load_weights('./model_log/'+ model_name + borad_log_dir +'.tmod')
generator = DataGenerator(X_test, y_test, batch_size=100)
prediction = model.predict_generator(generator,
steps=len(X_test[1]) // 100,
max_queue_size=10,
workers=7,
use_multiprocessing=True,
verbose=0)
label = y_test
dic = {}
dic['prediction'] = prediction
dic['label'] = label
dic['data'] = X_test
with open('./data_' + model_name + '.map', 'wb') as f:
pkl.dump(dic, f)
#from sklearn.metrics import precision_recall_curve, auc, average_precision_score
#p, r, th = precision_recall_curve(y_test, prediction)
#auc = auc(r, p)
#ap = average_precision_score(y_test, prediction)
#dic['auc'] = auc
#dic['ap'] = ap
#with open('./data_'+model_name + text_sim_metrics[-1]+'.map+auc', 'wb') as f:
# pkl.dump(dic, f)
#history = model.fit(X_train, y_train, epochs=8, batch_size=100,
# validation_data=(X_val, y_val),
# shuffle=True, callbacks = [tbCallBack])
# len(X_train[1])//batch_size
print('history finished.. ')
# model_log = './model_log/'+ model_name + borad_log_dir +'.tmod'
# with open( model_log, 'wb') as f:
# pkl.dump(model, f)
# model.save(model_log)
histories['acc'].extend(history.history['acc'])
histories['val_acc'].extend(history.history['val_acc'])
histories['loss'].extend(history.history['loss'])
histories['val_loss'].extend(history.history['val_loss'])
histories['precision'].extend(history.history['precision'])
histories['f1_score'].extend(history.history['f1_score'])
histories['val_precision'].extend(history.history['val_precision'])
histories['val_f1_score'].extend(history.history['val_f1_score'])
histories['auroc'].extend(history.history['auroc'])
histories['val_auroc'].extend(history.history['val_auroc'])
history_log = './history_log/' + model_name + borad_log_dir + '.tmap'
with open(history_log, 'wb') as f:
pkl.dump(histories, f)
from sklearn.metrics import precision_recall_curve, auc, average_precision_score
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)
adam = keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model.compile(optimizer=adam,
loss='binary_crossentropy',
metrics=['acc',
km.binary_precision(),
km.f1_score(), auroc])
print('model fitting...')
import numpy as np
import time
#def load_image(x, y):
# print(x[13])
# x[13] = np.load('./imagess'+x[13][1:]+'.npy').reshape(1,64,2048)
# x[6] = np.load('./imagess'+x[6][1:]+'.npy').reshape(1,64,2048)
# return x,y
#print('loading X_train images.. ')
#t_train = time.time()
#X_train = tf.data.Dataset.from_tensor_slices((X_train, y_train)).map(load_image)
#print(time.time()-t_train, 'loading X_test images..')
#t_test = time.time()
#X_test = tf.data.Dataset.from_tensor_slices((X_test, y_test)).map(load_image)
def transformer_models(trainX, trainy, valX, valy, embedding, vocab, maxlen,
head_num, encoder_num, hidden_dim, project_name):
#import relevant packages from keras instead of tensorflow.keras to avoid the runtime problem in this model.
from keras.layers import Input, MaxPooling1D, Flatten, Dense, Embedding, SpatialDropout1D, Dropout, Conv1D
from keras.models import Model
from keras.utils import to_categorical
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import keras
input = Input(shape=(maxlen, ), dtype="int32")
# embedding
x = Embedding(len(vocab) + 1, 300, weights=[embedding],
trainable=False)(input)
#x = Dropout(0.2)(x)
print(x.shape)
#postition embedding
x = PositionEmbedding(120, 300, "add")(x)
output = get_encoders(encoder_num=encoder_num,
input_layer=x,
head_num=head_num,
hidden_dim=hidden_dim,
attention_activation="relu",
dropout_rate=0.1)
# three kind of filters size are 2,3,4
cnn1 = Conv1D(32, 2, padding="same", strides=1, activation="relu")(x)
cnn1 = MaxPooling1D()(cnn1)
cnn2 = Conv1D(32, 3, padding="same", strides=1, activation="relu")(x)
cnn2 = MaxPooling1D()(cnn2)
cnn3 = Conv1D(32, 4, padding="same", strides=1, activation="relu")(x)
cnn3 = MaxPooling1D()(cnn3)
features = keras.layers.concatenate([cnn1, cnn2, cnn3], axis=-1)
output = Flatten()(features)
output = Dropout(0.2)(output)
output = Dense(2, activation="softmax")(output)
trainy = to_categorical(trainy, 2)
val_y = to_categorical(valy, 2)
model = Model(inputs=input, outputs=output)
path = "../data/experiment_results/RQ5/model/" + project_name + str(
hidden_dim) + ".h5"
early_stopping = EarlyStopping()
callbacks = [early_stopping, ModelCheckpoint(path, save_best_only=True)]
model.compile(optimizer=Adam(0.005),
loss="binary_crossentropy",
metrics=[km.recall(),
km.precision(),
km.f1_score()])
model.fit(trainX,
trainy,
batch_size=64,
epochs=5,
callbacks=callbacks,
validation_data=(valX, val_y))
pred = model.predict(valX)
pred = pred.argmax(-1)
f1 = f1_score(valy, pred)
precision = precision_score(valy, pred)
recall = recall_score(valy, pred)
return f1, precision, recall
encoder = load_model('model_encoder.h5')
print(train_x.shape)
encoded_train_x = encoder.predict(train_x[train_index])
encoded_test_x = encoder.predict(test_x[test_index])
model = Sequential()
model.add(Dense(50, activation='relu', input_shape=(ENCODE_DIM, )))
#model.add(Dropout(0.5))
model.add(Dense(20, activation='relu'))
#model.add(Dropout(0.5))
model.add(Dense(5, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.compile(optimizer='adadelta', loss='binary_crossentropy', metrics=[km.precision(), km.recall(), km.f1_score()])
early_stopping_monitor = EarlyStopping(patience=15)
log_filepath = "/home/jira/hackdays/train_log"
tb_cb = TensorBoard(log_dir=log_filepath, histogram_freq=1)
cbks = []#[tb_cb]
model.fit(encoded_train_x, train_y[train_index], validation_split=0.1, epochs=50, callbacks=cbks)
model.save('new_model_classifier_nodropout.h5')
test_result = model.evaluate(encoded_test_x, test_y[test_index])
print("eval result")
print(test_result)
#print(model.predict(encoded_test_x))
# XXX benyomok még egy réteget, akkor vajh mi lesz? :) f1=71.5%
model.add(Dense(hidden_dims))
model.add(Dropout(0.2))
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1))
model.add(Activation('sigmoid'))
print(' * Compile...')
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy',
km.precision(),
km.recall(),
km.f1_score()])
# https://stackoverflow.com/questions/43076609
# itt kell beírni, hogy milyen mértékeket szeretnék
# elvileg nem kell km-et használni, de másképp nekem nem megy...
print(model.summary())
print(' * Fit...')
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test))
# Evaluate your performance on test
print(' * Evaluate performance on test:')
from keras_metrics import f1_score, precision, recall all_metrics = [f1_score(label=1), precision(label=1), recall(label=1)]
batch_norm=True,
dropout=0.75,
shared_lstm=True,
#lstm_args = dict(units=50, dropout=0.25, recurrent_dropout=0.25),
lstm_args=dict(units=25)
)
print('model complie...')
#sgd = keras.optimizers.SGD(lr=0.001, momentum=0.0, decay=0.0, nesterov=False)
import keras_metrics as km
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)
adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc', km.binary_precision(),km.f1_score(),auroc])
#adam = keras.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
#import keras_metrics as km
#model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc', km.binary_precision(), km.binary_recall()])
print('model fitting...')
training_generator = DataGenerator(X_train, y_train, batch_size=100)
validation_generator = DataGenerator(X_val, y_val, batch_size = 100)
print('len', len(X_train[1]))
batch_size = 100
epochs = 7
earlystop = keras.callbacks.EarlyStopping(monitor='val_auroc', min_delta=0, patience=2, verbose=0, mode='max', baseline=None, restore_best_weights=True)
#checkpoint = keras.callbacks.ModelCheckpoint('./checkpoints/'+'baseline'+text_sim_metrics[0] +'.hdf5', monitor='val_auroc', verbose=1, save_best_only=True, mode='max')
y_train = keras.utils.to_categorical(y_train, num_classes)
# y_test = keras.utils.to_categorical(y_test, num_classes)
print("********************")
print(y_test.shape)
model = Sequential()
model.add(Dense(512, activation='relu', input_dim=1024))
# model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
# model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy', keras_metrics.precision(), keras_metrics.recall(), keras_metrics.f1_score()])
history = model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
logging = True
if logging:
# list all data in history
print(history.history.keys())
# summarize history for accuracy
plt.plot(history.history['acc'])
# plt.plot(history.history['val_acc'])
def bilstm_att_model(mode,
train_x,
train_y,
val_x,
val_y,
vocab,
hidden_size,
class_nums,
embedding,
tokenEmb=None,
train_posEmb=None,
target_posEmb=None,
trainable=True):
"""
train_x: training semantics feature
train_sx: training statistical feature
val_x: validation semantics feature
val_sx: validation statistical feature
train_y: label of training set
val_y: label of validation set
hidden_size: complete
"""
# input layer
if mode == "bert":
# Embedding layer
input = Input(shape=(512, ), batch_size=64, dtype="int32")
input2 = Input(shape=(512, 64), batch_size=64, dtype="float32")
token_embed = Embedding(vocab,
64,
weights=[tokenEmb],
trainable=trainable)(input)
# pos_embed = PosEmb(posEmb)(input)
embeding = tf.add(token_embed, input2)
x = Dropout(0.1)(embeding)
else:
input = Input(shape=(512, ), dtype="int32")
x = Embedding(len(vocab) + 1,
300,
weights=[embedding],
trainable=trainable)(input)
# BiLSTM layer
x = Bidirectional(LSTM(hidden_size, dropout=0.2, return_sequences=True))(x)
# Attention layer
#x = keras.layers.Attention()([x,x])
#x = Conv1D(16,512,padding="same")(x)
#x = Flatten()(x)
x = Attention(512)(x)
# output layer
if mode == "bert":
outputs = Dense(class_nums, activation='softmax')(x)
# BiLSTM layer
model = Model(inputs=[input, input2], outputs=outputs)
one_hot_label = keras.utils.to_categorical(train_y, class_nums)
one_hot_label1 = keras.utils.to_categorical(val_y, class_nums)
model.compile(
optimizer=Adam(0.005),
loss="binary_crossentropy",
metrics=[km.binary_precision(),
km.binary_recall(),
km.f1_score()])
model.fit([train_x, train_posEmb],
one_hot_label,
batch_size=64,
epochs=5,
validation_data=([val_x, target_posEmb], one_hot_label1),
callbacks=[EarlyStopping(patience=10)])
pred = model.predict([val_x, target_posEmb]).argmax(-1)
else:
outputs = Dense(class_nums, activation='softmax')(x)
# BiLSTM layer
model = Model(inputs=input, outputs=outputs)
one_hot_label = keras.utils.to_categorical(train_y, class_nums)
one_hot_label1 = keras.utils.to_categorical(val_y, class_nums)
model.compile(
optimizer=Adam(0.005),
loss="binary_crossentropy",
metrics=[km.binary_precision(),
km.binary_recall(),
km.f1_score()])
model.fit(train_x,
one_hot_label,
batch_size=64,
epochs=5,
validation_data=(val_x, one_hot_label1))
pred = model.predict(val_x).argmax(-1)
# metrics 计算
f1 = f1_score(val_y, pred)
precision = precision_score(val_y, pred)
recall = recall_score(val_y, pred)
return f1, precision, recall
model.add(Dropout(dropout_fc))
# Output layer
model.add(Dense(num_classes, activation='softmax'))
print(model.summary())
if gpu_count > 1:
model = multi_gpu_model(model, gpus=gpu_count)
model.compile(loss=tf.keras.losses.categorical_crossentropy,
optimizer=tf.keras.optimizers.Adam(),
metrics=[
'accuracy',
keras_metrics.precision(),
keras_metrics.recall(),
keras_metrics.f1_score()
])
# Define callback to save best epoch
chk_name = 'fmnist-cnn-{epoch:04d}'
checkpointer = ModelCheckpoint(filepath=os.path.join(chk_dir, chk_name),
monitor='val_accuracy')
# Define callback for early stopping
early_stopping = EarlyStopping(monitor='val_accuracy',
min_delta=0,
patience=25,
verbose=1,
mode='auto')
def unet(pretrained_weights=None, input_size=(256, 256, 1)):
inputs = Input(input_size)
conv1 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(inputs)
conv1 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool1)
conv2 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool2)
conv3 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool3)
conv4 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv4)
drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(1024,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool4)
conv5 = Conv2D(1024,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv5)
drop5 = Dropout(0.5)(conv5)
up6 = Conv2D(512,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(drop5))
merge6 = concatenate([drop4, up6], axis=3)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge6)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv6)
up7 = Conv2D(256,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv6))
merge7 = concatenate([conv3, up7], axis=3)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge7)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv7)
up8 = Conv2D(128,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv7))
merge8 = concatenate([conv2, up8], axis=3)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge8)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv8)
up9 = Conv2D(64,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv8))
merge9 = concatenate([conv1, up9], axis=3)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge9)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv9 = Conv2D(2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv10 = Conv2D(1, 1, activation='sigmoid')(conv9)
model = Model(input=inputs, output=conv10)
model.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=[
keras_metrics.true_positive(),
keras_metrics.true_negative(),
keras_metrics.false_positive(),
keras_metrics.false_negative(), 'accuracy',
keras_metrics.f1_score(),
keras_metrics.precision(),
keras_metrics.recall()
])
#model.summary()
if (pretrained_weights):
model.load_weights(pretrained_weights)
return model
