def main(data_path, output_path):
X_trainS1, Y_train, X_valS1, Y_val = load_data(data_path)
epochs = 10
batch_size = 256
dropout_rate = 0.15
n_classes = 6
# 三个子模型的输入数据
main_input1 = Input(shape=(128, 3), name='main_input1')
def lstm_cell(main_input):
"""
基于DeepConvLSTM算法, 创建子模型
:param main_input: 输入数据
:return: 子模型
"""
sub_model = TimeDistributed(Dense(384),
input_shape=(128, 3))(main_input)
# sub_model = Flatten()(main_input)
print(sub_model)
sub_model = LSTM(256, return_sequences=True)(sub_model)
sub_model = LSTM(128, return_sequences=True)(sub_model)
sub_model = LSTM(128)(sub_model)
main_output = Dropout(dropout_rate)(sub_model)
return main_output
model = lstm_cell(main_input1)
model = Dropout(0.4)(model)
model = Dense(n_classes)(model)
model = BatchNormalization()(model)
output = Activation('softmax', name="softmax")(model)
model = Model([main_input1], output)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print(model.summary())
# graph_path = os.path.join(output_path, "merged_model.png")
# plot_model(model, to_file=graph_path, show_shapes=True) # 绘制模型图
metrics = Metrics() # 度量FPR
history = model.fit(X_trainS1,
Y_train,
batch_size=batch_size,
validation_data=(X_valS1, Y_val),
epochs=epochs,
callbacks=[metrics]) # 增加FPR输出
model_path = os.path.join(output_path, "merged_dcl.h5")
model.save(model_path) # 存储模型
print(history.history)
def ensemble_cnn_subword(vocabulary_size: int, embedding_size: int,
max_char_length: int, max_seq_length: int,
embedding_matrix: np.array,
y_dictionary: dict) -> Model:
input = Input(shape=(max_char_length, ), name='main_input')
mask = Input(shape=(
max_seq_length,
max_char_length,
), name='mask_input')
embedding = Embedding(
input_dim=vocabulary_size,
output_dim=embedding_size,
weights=[embedding_matrix],
input_length=max_char_length,
trainable=True,
embeddings_regularizer=regularizers.l2(0.000001))(input)
print(mask.shape)
print(embedding.shape)
embedding = Lambda(lambda values: K.batch_dot(values[0], values[1]))(
[mask, embedding])
print(embedding.shape)
# Convolutions
models = []
for i in range(3, 7):
model = Dropout(0.4)(embedding)
model = Conv1D(filters=512,
kernel_size=i,
activation='relu',
padding="same",
kernel_regularizer=regularizers.l2(0.00001))(model)
model = MaxPooling1D(pool_size=5)(model)
model = Flatten()(model)
models.append(model)
# Dense layers
model = GlobalAveragePooling1D()(embedding)
for i in range(1):
model = Dense(4096,
activation='tanh',
kernel_regularizer=regularizers.l2(0.00001))(model)
models.append(model)
model = Average()(models)
model = Dropout(0.4)(model)
model = Dense(len(y_dictionary), activation='softmax')(model)
model = Model([input, mask], model)
optimizer = RMSprop(lr=0.001, decay=0.00005)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
return model
def cnn_static(self):
x_train, y_train, x_test, y_test, vocabolary_dict = data_load_and_preproccess(
self.word_size, self.sequence_length)
embedding_weights = word_to_vector(np.vstack(
(x_train, x_test)), vocabolary_dict, self.embedding_dim,
self.min_word_count,
self.context_window_size)
x_train = np.stack([
np.stack([embedding_weights[word] for word in sentence])
for sentence in x_train
])
x_test = np.stack([
np.stack([embedding_weights[word] for word in sentence])
for sentence in x_test
])
#model building
model_input = Input(shape=(self.sequence_length, self.embedding_dim))
model = Dropout(self.drop_prob[0])(model_input)
multi_cnn_channel = []
for kernal in self.kernal_size:
conv_channel = Convolution1D(filters=self.filters,
kernel_size=kernal,
padding="valid",
activation="relu",
strides=1)(model)
conv_channel = MaxPooling1D(pool_size=2)(conv_channel)
conv_channel = Flatten()(conv_channel)
multi_cnn_channel.append(conv_channel)
model = Concatenate()(multi_cnn_channel)
model = Dropout(self.drop_prob[1])(model)
for dimension in self.hidden_dims:
model = Dense(dimension, activation="relu")(model)
model_output = Dense(1, activation="sigmoid")(model)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
print("Started Training : ")
model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs,
validation_data=(x_test, y_test),
verbose=2)
print("Training Completed")
self.model = model
def create_model(num_hidden1, num_hidden2, rate_drop_1, rate_drop_2):
act = 'relu'
STAMP = 'NN_%d_%d_%.2f_%.2f'%(num_hidden1, num_hidden2, rate_drop_1, \
rate_drop_2)
#model.summary()
print(STAMP)
graph = get_convs()
q1_input = Input(shape=(MAX_SEQUENCE_LENGTH, ))
q2_input = Input(shape=(MAX_SEQUENCE_LENGTH, ))
Q1 = graph(q1_input)
Q2 = graph(q2_input)
q1q2 = concatenate([Q1, Q2])
abhishek_features = Input((train_features.shape[1], ),
name="abhishek_features")
model = concatenate([Q1, Q2, abhishek_features])
model = Dropout(0.2)(model)
model = BatchNormalization()(model)
model = Dense(num_hidden1, activation='relu')(model)
model = Dropout(rate_drop_1)(model)
model = BatchNormalization()(model)
model = Dense(num_hidden2, activation='relu')(model)
model = Dropout(rate_drop_2)(model)
model = BatchNormalization()(model)
model = Dense(1, activation='sigmoid')(model)
model = Model(inputs=[q1_input, q2_input, abhishek_features],
outputs=model)
model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['acc'])
return model, STAMP
def ensemble_cnn(vocabulary_size: int, embedding_size: int,
max_seq_length: int, embedding_matrix: np.array,
y_dictionary: dict) -> Model:
input = Input(shape=(max_seq_length, ))
embedding = Embedding(
input_dim=vocabulary_size,
output_dim=embedding_size,
weights=[embedding_matrix],
input_length=max_seq_length,
trainable=True,
embeddings_regularizer=regularizers.l2(0.00001))(input)
# Convolutions
models = []
for i in range(3, 7):
model = Dropout(0.4)(embedding)
model = Conv1D(filters=512,
kernel_size=i,
activation='relu',
padding="same",
kernel_regularizer=regularizers.l2(0.00001))(model)
model = MaxPooling1D(pool_size=5)(model)
model = Flatten()(model)
models.append(model)
# Dense layers
model = GlobalAveragePooling1D()(embedding)
for i in range(5):
model = Dense(4096,
activation='tanh',
kernel_regularizer=regularizers.l2(0.00001))(model)
models.append(model)
model = Average()(models)
model = Dropout(0.4)(model)
model = Dense(len(y_dictionary), activation='softmax')(model)
model = Model(input, model)
optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, decay=0.00005)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
return model
def get_model():
inp = Input(shape=(MAX_TEXT_LENGTH, 1))
#model = Embedding(MAX_FEATURES, EMBED_SIZE)(inp)
model = Dropout(0.5)(inp)
model = Conv1D(filters=32,
kernel_size=2,
padding='same',
activation='relu')(model)
model = MaxPooling1D(pool_size=2)(model)
model = Flatten()(model)
model = Dense(1024, activation='relu')(model)
model = Dropout(0.5)(model)
model = Dense(NB_CLASS, activation="softmax")(model)
model = Model(inputs=inp, outputs=model)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
return model
top_model = Dropout(0.5, name='drop9')(top_model)
top_model = Flatten()(top_model)
top_model = Dense(600)(top_model)
top_model = BatchNormalization()(top_model)
top_model = Activation('relu', name='relu_conv9')(top_model)
# top_model = Dense(600)(top_model)
# top_model = Activation('relu', name='relu_conv10')(top_model)
top_model = Dense(10)(top_model)
top_model = BatchNormalization()(top_model)
top_model = Activation('softmax', name='loss')(top_model)
top_model = Model(model.input, top_model, name='top_net')
top_model.load_weights(top_model_weights_path)
#設定model參數
top_model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
#設定訓練紀錄
history = LossHistory(model, x_test, y_test)
#開始訓練
top_model.fit_generator(
generator=train_flow,
steps_per_epoch=30000 // 20,
verbose=2,
validation_data=train_flow,
validation_steps=5000 // 32,
epochs=30,
callbacks=[history]
)
#pre=model.predict(test_data)
def main(data_path, output_path):
X_trainS1, Y_train, X_valS1, Y_val = load_data(data_path)
epochs = 10
batch_size = 256
kernel_size = 3
pool_size = 2
dropout_rate = 0.15
n_classes = 6
f_act = 'relu'
# 三个子模型的输入数据
main_input1 = Input(shape=(128, 3), name='main_input1')
def cnn_lstm_cell(main_input):
"""
基于DeepConvLSTM算法, 创建子模型
:param main_input: 输入数据
:return: 子模型
"""
sub_model = Conv1D(512,
kernel_size,
input_shape=(128, 3),
activation=f_act,
padding='same')(main_input)
print('sub_model512:', sub_model)
sub_model = BatchNormalization()(sub_model)
sub_model = MaxPooling1D(pool_size=pool_size)(sub_model)
print('sub_model:', sub_model)
sub_model = Dropout(dropout_rate)(sub_model)
print('sub_model:', sub_model)
sub_model = Conv1D(64, kernel_size, activation=f_act,
padding='same')(sub_model)
print('sub_model64:', sub_model)
sub_model = BatchNormalization()(sub_model)
sub_model = MaxPooling1D(pool_size=pool_size)(sub_model)
print('sub_model:', sub_model)
sub_model = Dropout(dropout_rate)(sub_model)
print('sub_model:', sub_model)
sub_model = Conv1D(32, kernel_size, activation=f_act,
padding='same')(sub_model)
print('sub_model32:', sub_model)
sub_model = BatchNormalization()(sub_model)
sub_model = MaxPooling1D(pool_size=pool_size)(sub_model)
print('sub_model:', sub_model)
sub_model = LSTM(128, return_sequences=True)(sub_model)
print('sub_model128_1:', sub_model)
sub_model = LSTM(128, return_sequences=True)(sub_model)
print('sub_model128_2:', sub_model)
sub_model = LSTM(128)(sub_model)
print('sub_model128_3:', sub_model)
main_output = Dropout(dropout_rate)(sub_model)
print('main_output:', main_output)
return main_output
model = cnn_lstm_cell(main_input1)
model = Dropout(0.4)(model)
model = Dense(n_classes)(model)
model = BatchNormalization()(model)
output = Activation('softmax', name="softmax")(model)
model = Model([main_input1], output)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
# graph_path = os.path.join(output_path, "merged_model.png")
# plot_model(model, to_file=graph_path, show_shapes=True) # 绘制模型图
metrics = Metrics() # 度量FPR
history = model.fit(X_trainS1,
Y_train,
batch_size=batch_size,
validation_data=(X_valS1, Y_val),
epochs=epochs,
callbacks=[metrics]) # 增加FPR输出
model_path = os.path.join(output_path, "merged_dcl.h5")
model.save(model_path) # 存储模型
print(history.history)
tangential_strain_layer = (Conv1D(filters=filters, kernel_size=kernel_size))(tangential_strain_layer)
tangential_strain_layer = Activation('relu')(tangential_strain_layer)
tangential_strain_layer = (Conv1D(filters=filters, kernel_size=kernel_size))(tangential_strain_layer)
tangential_strain_layer = Activation('relu')(tangential_strain_layer)
tangential_strain_layer = (Dense(32, activation='relu'))(tangential_strain_layer)
tangential_strain_layer = Dropout(0.5)(tangential_strain_layer)
tangential_strain_layer = Flatten()(tangential_strain_layer)
merge = concatenate([energy_layer, maximum_amplitude_layer, radial_strain_layer, tangential_strain_layer])
model = (Dense(32, activation='relu'))(merge)
model = Dropout(0.5)(model)
model = (Dense(1, activation='sigmoid'))(model)
model = Model(inputs=[energy_input, maximum_amplitude_input, radial_strain_input, tangential_strain_input],
outputs=model)
model.compile(loss='binary_crossentropy', optimizer="rmsprop", metrics=['accuracy'])
print model.summary()
print "Training....................................................."
model.fit([energies, maximum_amplitudes, radial_strains, tangential_strains], train_labels, epochs=epochs, verbose=1,
batch_size=batch_size, validation_split=0.2)
print("--- train %s seconds ---" % (time.time() - start_time))
original_classified = model.predict([energies, maximum_amplitudes, radial_strains, tangential_strains],
batch_size=batch_size, verbose=0)
best_threshold = find_best_threshold(train_labels, original_classified)
if run_test:
print 'test'
for test in test_sets:
def __init__(self, model_name="Xception", train_class_name=None, training_batch_size=100, existing_weight=None, test_percentage=0.02, learning_rate=0.000, validation_every_X_batch=5):
if train_class_name == None:
print("You must specify train_class_name")
return
self.validation_every_X_batch = validation_every_X_batch
self.Y = []
self.model_file = model_name + "-{date:%Y-%m-%d-%H-%M-%S}".format( date=datetime.datetime.now())
print("model_folder: ", self.model_file)
self.train_class_name = train_class_name
if not os.path.exists(os.path.join("models", train_class_name)):
os.makedirs(os.path.join("models", train_class_name))
self.training_batch_size = training_batch_size
# We know that MNIST images are 28 pixels in each dimension.
img_size = 512
self.img_size_flat = img_size * img_size * 3
self.img_shape_full = (img_size, img_size, 3)
self.test = {}
with open('base/Annotations/label.csv', 'r') as csvfile:
reader = csv.reader(csvfile)
all_class_samples = []
for row in reader:
if row[1] == self.train_class_name:
self.num_classes = len(row[2])
break
# Start construction of the Keras Sequential model.
input_tensor = Input(shape=self.img_shape_full)
if model_name == "Xception":
base_model = Xception(input_tensor=input_tensor, weights='imagenet', include_top=False, classes=self.num_classes)
elif model_name == "MobileNet":
base_model = MobileNet(input_tensor=input_tensor, weights='imagenet', include_top=False, classes=self.num_classes)
elif model_name == "DenseNet121":
base_model = DenseNet121(input_tensor=input_tensor, weights='imagenet', include_top=False, classes=self.num_classes)
x = base_model.output
x = GlobalAveragePooling2D()(x)
model = Dropout(0.2)(x)
predictions = Dense(self.num_classes, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
self.model = model
print(model.summary())
self.optimizer = optimizers.Adam(lr=learning_rate)
model.compile(optimizer=self.optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
with open('base/Annotations/label.csv', 'r') as csvfile:
reader = csv.reader(csvfile)
all_class_samples = []
for row in reader:
if row[1] != self.train_class_name:
continue
all_class_samples.append(row)
self.X = np.zeros((len(all_class_samples), img_size, img_size, 3))
# self.X = np.zeros((10, img_size, img_size, 3))
test_count = int(test_percentage * len(all_class_samples))
index = 0
print("Training " + train_class_name + " with: " + str(int((1 - test_percentage) * len(all_class_samples))) + ", Testing with: " + str(test_count), str(self.num_classes), "Classes")
print("Loading images...")
for row in all_class_samples:
image = Image.open("base/" + row[0])
img_array = np.asarray(image)
if img_array.shape != self.img_shape_full:
image = image.resize((img_size, img_size), Image.ANTIALIAS)
img_array = np.asarray(image)
self.X[index] = img_array
self.Y.append(row[2].index("y"))
if index % 500 == 0:
print(index)
index += 1
self.Y = to_categorical(self.Y, num_classes=self.num_classes)
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(self.X, self.Y, test_size = test_percentage, random_state=42)
del self.X
class_weight = {}
class_count = np.sum(self.y_train, axis=0)
print("Training Sample for each Class", class_count)
for class_index in range(self.num_classes):
class_weight[class_index] = 1 /(class_count[class_index] / np.sum(class_count)) / self.num_classes
self.class_weight = class_weight
print("Class weights: ", self.class_weight)
os.makedirs(os.path.join("models", train_class_name, self.model_file))
model.save(os.path.join("models", train_class_name, self.model_file, train_class_name + "_" + "model.h5"))
def build_model(
self,
output="crf",
word_embeddings=None,
pretrained_embedding="",
word_embedding_size=100,
char_embedding_type="BILSTM",
char_embedding_size=50,
dropout=0,
lstm_hidden=32,
optimizer="rmsprop",
):
"""
Build the bilstm for use in the training and predict methods.
Args:
output(str): One of `"crf"` or `"softmax"`. Sets the prediction
layer of the model.
word_embeddings(str): Path to word embeddings. If `None`, then word
embeddings will not be used.
pretrained_embedding(str): One of: `["", True, False]`
by the model? Setting to `""` will not use any pre-embedding.
Setting to `True` will use a pre-trained embedding to
initialise the weights of an embedding layer which will be
trained. Setting to `False` will use the supplied pre-trained
embedding without any additional training.
word_embedding_size(int): The size of the pre-trained word embedding
to use. One of `[100, 300]`.
char_embedding_type(str): Which architecture to use for the
character embedding. One of `["CNN", "BILSTM"]`.
char_embedding_size(int): Size of the character-level word
representations.
dropout(float): Degree of dropout to use, set to between `0` and
`1`.
lstm_hidden(int): Dimensionality of the hidden layer.
optimizer(str): Which optimizer to use. One of
`["adam", "rmsprop"]`.
"""
inputs = []
embeddings_list = []
nbr_words = len(self.word2ind) + 1
nbr_chars = len(self.char2ind) + 1
out_size = len(self.ind2label) + 1
if word_embeddings:
word_input = Input((self.max_len, ))
inputs.append(word_input)
# TODO: More sensible handling of options for pretrained embedding.
# Currently it uses one of `["", True, False]` where:
# "": Do not use pre-trained embedding
# False: Use pre-trained weights in the embedding layer
# True: Use the pre-trained weights as weight initialisers, and
# train the embedding layer.
if pretrained_embedding == "":
word_embedding = Embedding(nbr_words,
word_embedding_size)(word_input)
else:
embedding_matrix = word2vec_embeddings(
embedding_path=word_embeddings,
word2ind=self.word2ind,
word_embedding_size=word_embedding_size,
)
word_embedding = Embedding(
nbr_words,
word_embedding_size,
weights=[embedding_matrix],
trainable=pretrained_embedding,
mask_zero=False,
)(word_input)
embeddings_list.append(word_embedding)
# Input - Characters Embeddings
if self.max_char != 0:
character_input = Input((
self.max_len,
self.max_char,
))
char_embedding = self.character_embedding_layer(
char_embedding_type=char_embedding_type,
character_input=character_input,
nbr_chars=nbr_chars,
char_embedding_size=char_embedding_size,
)
embeddings_list.append(char_embedding)
inputs.append(character_input)
# Model - Inner Layers - BiLSTM with Dropout
if len(embeddings_list) == 2:
embeddings = concatenate(embeddings_list)
else:
embeddings = embeddings_list[0]
model = Dropout(dropout)(embeddings)
model = Bidirectional(
LSTM(lstm_hidden, return_sequences=True, dropout=dropout))(model)
model = Dropout(dropout)(model)
if output == "crf":
# Output - CRF
crfs = [[CRF(out_size), out_size]
for out_size in [len(x) + 1 for x in self.ind2label]]
outputs = [x[0](Dense(x[1])(model)) for x in crfs]
model_loss = [x[0].loss_function for x in crfs]
model_metrics = [x[0].viterbi_acc for x in crfs]
if output == "softmax":
outputs = [
Dense(out_size, activation="softmax")(model)
for out_size in [len(x) + 1 for x in self.ind2label]
]
model_loss = ["categorical_crossentropy" for x in outputs]
model_metrics = None
# Model
model = Model(inputs=inputs, outputs=outputs)
model.compile(
loss=model_loss,
metrics=model_metrics,
optimizer=self.get_optimizer(optimizer),
)
# NOTE: It's not necessary to save the model as it needs to be recreated
# each time using build_model()
# model_json = model.to_json()
# save_path = os.path.join(self.output_path, "model.json")
# with open(save_path, "w") as json_file:
# json_file.write(model_json)
self.model = model
model = MaxPooling2D(pool_size=(2, 2))(model)
model = Dropout(rate=0.2)(model)
model = Convolution2D(64, kernel_size=3,
activation=activations.relu)(model)
model = Flatten()(model)
dense_layer = Dense(nclass, activation=activations.softmax)(model)
model = models.Model(inputs=input_layer, outputs=dense_layer)
# Compile model
epochs = 10
opt = optimizers.Adam(
) # Default: lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False,
model.compile(optimizer=opt,
loss=losses.binary_crossentropy,
metrics=['accuracy'])
print(get_available_gpus())
# Model Summary
model.summary()
# Start timer
start_time = time.time()
# Fit model
model.fit(x_train,
y_train,
batch_size=16,
validation_data=(x_valid, y_valid),
epochs=epochs,
shuffle=True,
verbose=2,
def BiLSTM_model(filename,
train,
output,
X_train,
X_test,
word2ind,
maxWords,
y_train,
y_test,
ind2label,
validation=False,
X_valid=None,
y_valid=None,
word_embeddings=True,
pretrained_embedding="",
word_embedding_size=100,
maxChar=0,
char_embedding_type="",
char2ind="",
char_embedding_size=50,
lstm_hidden=32,
nbr_epochs=5,
batch_size=32,
dropout=0,
optimizer='rmsprop',
early_stopping_patience=-1,
folder_path="model_results",
gen_confusion_matrix=False):
"""
Build, train and test a BiLSTM Keras model. Works for multi-tasking learning.
The model architecture looks like:
- Words representations:
- Word embeddings
- Character-level representation [Optional]
- Dropout
- Bidirectional LSTM
- Dropout
- Softmax/CRF for predictions
:param filename: File to redirect the printing
:param train: Boolean if the model must be trained or not. If False, the model's wieght are expected to be stored in "folder_path/filename/filename.h5"
:param otput: "crf" or "softmax". Type of prediction layer to use
:param X_train: Data to train the model
:param X_test: Data to test the model
:param word2ind: Dictionary containing all words in the training data and a unique integer per word
:param maxWords: Maximum number of words in a sequence
:param y_train: Labels to train the model for the prediction task
:param y_test: Labels to test the model for the prediction task
:param ind2label: Dictionary where all labels for task 1 are mapped into a unique integer
:param validation: Boolean. If true, the validation score will be computed from 'X_valid' and 'y_valid'
:param X_valid: Optional. Validation dataset
:param y_valid: Optional. Validation dataset labels
:param word_embeddings: Boolean value. Add word embeddings into the model.
:param pretrained_embedding: Use the pretrained word embeddings.
Three values:
- "": Do not use pre-trained word embeddings (Default)
- False: Use the pre-trained embedding vectors as the weights in the Embedding layer
- True: Use the pre-trained embedding vectors as weight initialiers. The Embedding layer will still be trained.
:param word_embedding_size: Size of the pre-trained word embedding to use (100 or 300)
:param maxChar: The maximum numbers of characters in a word. If set to 0, the model will not use character-level representations of the words
:param char_embedding_type: Type of model to use in order to compute the character-level representation of words: Two values: "CNN" or "BILSTM"
:param char2ind: A dictionary where each character is maped into a unique integer
:param char_embedding_size: size of the character-level word representations
:param lstm_hidden: Dimentionality of the LSTM output space
:param nbr_epochs: Number of epochs to train the model
:param batch_size: Size of batches while training the model
:param dropout: Rate to apply for each Dropout layer in the model
:param optimizer: Optimizer to use while compiling the model
:param early_stopping_patience: Number of continuous tolerated epochs without improvement during training.
:param folder_path: Path to the directory storing all to-be-generated files
:param gen_confusion_matrix: Boolean value. Generated confusion matrices or not.
:return: The classification scores for both tasks.
"""
print("====== {0} start ======".format(filename))
end_string = "====== {0} end ======".format(filename)
# Create directory to store results
os.makedirs(folder_path + "/" + filename)
filepath = folder_path + "/" + filename + "/" + filename
# Set print outputs file
file, stdout_original = setPrintToFile("{0}.txt".format(filepath))
# Model params
nbr_words = len(word2ind) + 1
out_size = len(ind2label) + 1
best_results = ""
embeddings_list = []
inputs = []
# Input - Word Embeddings
if word_embeddings:
word_input = Input((maxWords, ))
inputs.append(word_input)
if pretrained_embedding == "":
word_embedding = Embedding(nbr_words,
word_embedding_size)(word_input)
else:
# Retrieve embeddings
embedding_matrix = word2VecEmbeddings(word2ind,
word_embedding_size)
word_embedding = Embedding(nbr_words,
word_embedding_size,
weights=[embedding_matrix],
trainable=pretrained_embedding,
mask_zero=False)(word_input)
embeddings_list.append(word_embedding)
# Input - Characters Embeddings
if maxChar != 0:
character_input = Input((
maxWords,
maxChar,
))
char_embedding = character_embedding_layer(char_embedding_type,
character_input, maxChar,
len(char2ind) + 1,
char_embedding_size)
embeddings_list.append(char_embedding)
inputs.append(character_input)
# Model - Inner Layers - BiLSTM with Dropout
embeddings = concatenate(embeddings_list) if len(
embeddings_list) == 2 else embeddings_list[0]
model = Dropout(dropout)(embeddings)
model = Bidirectional(
LSTM(lstm_hidden, return_sequences=True, dropout=dropout))(model)
model = Dropout(dropout)(model)
if output == "crf":
# Output - CRF
crfs = [[CRF(out_size), out_size]
for out_size in [len(x) + 1 for x in ind2label]]
outputs = [x[0](Dense(x[1])(model)) for x in crfs]
model_loss = [x[0].loss_function for x in crfs]
model_metrics = [x[0].viterbi_acc for x in crfs]
if output == "softmax":
outputs = [
Dense(out_size, activation='softmax')(model)
for out_size in [len(x) + 1 for x in ind2label]
]
model_loss = ['categorical_crossentropy' for x in outputs]
model_metrics = None
# Model
model = Model(inputs=inputs, outputs=outputs)
model.compile(loss=model_loss,
metrics=model_metrics,
optimizer=get_optimizer(optimizer))
print(model.summary(line_length=150), "\n\n\n\n")
# Training Callbacks:
callbacks = []
value_to_monitor = 'val_f1'
best_model_weights_path = "{0}.h5".format(filepath)
# 1) Classifition scores
classification_scores = Classification_Scores([X_train, y_train],
ind2label,
best_model_weights_path)
callbacks.append(classification_scores)
# 2) EarlyStopping
if early_stopping_patience != -1:
early_stopping = EarlyStopping(monitor=value_to_monitor,
patience=early_stopping_patience,
mode='max')
callbacks.append(early_stopping)
# Train
if train:
# Train the model. Keras's method argument 'validation_data' is referred as 'testing data' in this code.
hist = model.fit(X_train,
y_train,
validation_data=[X_test, y_test],
epochs=nbr_epochs,
batch_size=batch_size,
callbacks=callbacks,
verbose=2)
print()
print('-------------------------------------------')
print(
"Best F1 score:", early_stopping.best,
" (epoch number {0})".format(
1 + np.argmax(hist.history[value_to_monitor])))
# Save Training scores
save_model_training_scores("{0}".format(filepath), hist,
classification_scores)
# Print best testing classification report
best_epoch = np.argmax(hist.history[value_to_monitor])
print(classification_scores.test_report[best_epoch])
# Best epoch results
best_results = model_best_scores(classification_scores, best_epoch)
# Load weigths from best training epoch into model
save_load_utils.load_all_weights(model, best_model_weights_path)
# Create confusion matrices
if gen_confusion_matrix:
for i, y_target in enumerate(y_test):
# Compute predictions, flatten
predictions, target = compute_predictions(model, X_test, y_target,
ind2label[i])
# Generate confusion matrices
save_confusion_matrix(
target, predictions, list(ind2label[i].values()),
"{0}_task_{1}_confusion_matrix_test".format(
filepath, str(i + 1)))
# Validation dataset
if validation:
print()
print("Validation dataset")
print("======================")
# Compute classification report
for i, y_target in enumerate(y_valid):
# Compute predictions, flatten
predictions, target = compute_predictions(model,
X_valid,
y_target,
ind2label[i],
nbrTask=i)
# Only for multi-task
if len(y_train) > 1:
print("For task " + str(i + 1) + "\n")
print(
"===================================================================================="
)
print("")
print("With padding into account")
print(
metrics.flat_classification_report([target], [predictions],
digits=4))
print("")
print('----------------------------------------------')
print("")
print("Without the padding:")
print(
metrics.flat_classification_report([target], [predictions],
digits=4,
labels=list(
ind2label[i].values())))
# Generate confusion matrices
save_confusion_matrix(
target, predictions, list(ind2label[i].values()),
"{0}_task_{1}_confusion_matrix_validation".format(
filepath, str(i + 1)))
# Close file
closePrintToFile(file, stdout_original)
print(end_string)
return best_results
rotation_range=35,
width_shift_range=0.22,
height_shift_range=0.22)
inp = Input(shape=(28, 28, 1), dtype='float32')
model = Conv2D(16, (3, 3), padding='same', activation='relu')(inp)
model = MaxPooling2D((2, 2))(model)
model = Dropout(0.3)(model)
model = Conv2D(32, (3, 3), padding='same', activation='relu')(model)
model = MaxPooling2D((2, 2))(model)
model = Conv2D(64, (3, 3), padding='same', activation='relu')(model)
model = MaxPooling2D((2, 2))(model)
model = Flatten()(model)
model = Dropout(0.2)(model)
output = Dense(10, activation='softmax')(model)
model = Model(inputs=inp, outputs=output)
model.summary()
train_generator = train_gen.flow(train[:40000], labels[:40000], batch_size=128)
model.compile(loss='categorical_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
checkpt = ModelCheckpoint(
filepath=r'./models/model.{epoch:02d}-{val_loss:.2f}.hdf5',
monitor='val_loss',
save_best_only=True)
csv_logger = CSVLogger('history.log')
model.fit_generator(train_generator,
steps_per_epoch=int(40000 / 128),
epochs=3000,
callbacks=[csv_logger, checkpt],
validation_data=(train[40000:], labels[40000:]))
def main2(X, X_te, y_te, words=None, tags=None, idx2word=None, idx2tag=None):
from k.models import model_from_json
from k.models import load_model
# load json and create model
# json_file = open('model_architecture.json', 'r')
# loaded_model_json = json_file.read()
# json_file.close()
# loaded_model = model_from_json(loaded_model_json)
# # load weights into new model
# loaded_model.load_weights("model_weights.h5")
# print("Loaded model from disk")
# loaded_model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"])
# # load json and create model
# with open('model.json', 'r') as json_file:
# loaded_model_json = json_file.read()
# loaded_model = model_from_json(loaded_model_json)
# # load weights into new model
# loaded_model.load_weights("model.h5")
# # print("Loaded model from disk")
# # Model reconstruction from JSON file
# with open('model_architecture.json', 'r') as f:
# loaded_model = model_from_json(f.read())
# Load weights into the new model
# loaded_model.load_weights('model_weights.h5')
# evaluate loaded model on test data
# loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
# loaded_model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"])
# loaded_model = load_model("My_Custom_Model3.h5")
from keras_contrib.layers import CRF
max_len = 50
n_tags = len(tags)
n_words = len(words)
word_input = Input(shape=(max_len, ))
word_emb = Embedding(input_dim=n_words + 1,
output_dim=20,
input_length=max_len,
mask_zero=True)(word_input)
model = Dropout(0.1)(word_emb)
model = Bidirectional(
LSTM(50, return_sequences=True, recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
crf = CRF(n_tags)
out = crf(model)
model = Model(inputs=word_input, outputs=out)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.load_weights("model_weights.h5")
"""from keras.models import load_model
loaded_model = load_model("My_Custom_Model.h5")
print("Model Loaded.. Evaluating")
# score = loaded_model.evaluate([X,second_input],Y, verbose=1)
score = loaded_model.evaluate(X,Y, verbose=1)
#print accuracy
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1]*100))
if words is not None and tags is not None:
i = 2318
p = loaded_model.predict(np.array([X[i]]))
# p = loaded_model.predict([np.array(X[i]),second_input[i]])
p = np.argmax(p, axis=-1)
print("{:15} ({:5}): {}".format("Word", "True", "Pred"))
for w, pred in zip(X[i], p[0]): # p[0] = p[[1,3,4,5]]
print("{:15}: {}".format(words[w], tags[pred]))
# print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1]*100))
# for x in score:
# print(x)
"""
# from keras.models import load_model
# loaded_model = load_model("My_Custom_Model3.h5")
# loaded_model = ""
# save_load_utils.load_all_weights(model, "Model_saved_using_contrib.h5")
print("Model Loaded.. Evaluating again")
score = model.evaluate(X_te, np.array(y_te), verbose=1)
print("%s: %.2f%%" % (model.metrics_names[1], score[1] * 100))
# score = model.evaluate(X_te, np.array(y_te), verbose=1)
# score = loaded_model.evaluate(X,Y, verbose=1)
#print accuracy
# print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1]*100))
# if words is not None and tags is not None:
# i = 2318
# p = loaded_model.predict(np.array([X[i]]))
# # p = loaded_model.predict([np.array(X[i]),second_input[i]])
# p = np.argmax(p, axis=-1)
# print("{:15} ({:5}): {}".format("Word", "True", "Pred"))
# for w, pred in zip(X[i], p[0]): # p[0] => p[[1,3,4,5]]
# print("{:15}: {}".format(words[w], tags[pred]))
loaded_model = model
i = 1
for x in X_te[i]:
print(idx2word[x], end=" ")
print(" ")
if words is not None and tags is not None:
p = loaded_model.predict(np.array([X_te[i]]))
p = np.argmax(p, axis=-1)
true = np.argmax(y_te[i], axis=-1)
print("{:15} ({:5}): {}".format("Word", "True", "Pred"))
print("{}th training example:".format(i))
for w, t, pred in zip(X_te[i], true, p[0]):
if w != 0:
print("{:15}: {:10} {}".format(idx2word[w - 1], idx2tag[t],
idx2tag[pred]))
print('*' * 50)
print(p)
print("len(p) = ", len(p))
def runResNet(epoch):
gloveD = 300
embMat = pickle.load(open('embMat' + str(gloveD) + '.pickle', 'rb'))
x_train_seq, y_train, x_val_seq, y_val = pickle.load(
open('trainValData.pickle', 'rb'))
act = 'relu'
inp = Input(shape=(300, ), dtype='int32')
e = Embedding(90461,
gloveD,
weights=[embMat],
input_length=300,
trainable=False)(inp)
#model = Flatten()(e)
model = Conv1D(filters=100,
kernel_size=2,
padding='valid',
activation=act,
strides=1)(e)
model = Dropout(0.5)(model)
model = GlobalMaxPooling1D()(model)
short = model
model = Dense(256, activation=act)(model)
model = Dropout(0.8)(model)
#short2 = model
#model = Add()([model, short])
#model = Activation(act)(model)
model = Dense(256, activation=act)(model)
model = Dropout(0.8)(model)
#short3 = model
#model = Add()([model, short2])
#model = Activation(act)(model)
model = Dense(256, activation=act)(model)
model = Dropout(0.8)(model)
#short4 = model
#model = Add()([model, short3])
#model = Activation(act)(model)
model = Dense(256, activation=act)(model)
model = Dropout(0.8)(model)
#short5 = model
#model = Add()([model,short4])
#model = Activation(act)(model)
model = Dense(100, activation=act)(model)
model = Dropout(0.8)(model)
model = Add()([model, short])
model = Activation(act)(model)
output = Dense(2, activation='softmax')(model)
model = Model(inputs=[inp], outputs=[output])
model.compile(loss="binary_crossentropy",
optimizer='adam',
metrics=['accuracy'])
filepath = "res_best_weights.{epoch:02d}-{val_acc:.4f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
model.fit(x_train_seq,
y_train,
validation_data=(x_val_seq, y_val),
epochs=epoch,
batch_size=128,
verbose=1,
callbacks=[checkpoint])
#CNN_no_dropout\n"
print('Build CNN_no_dropout model...')
embed_dim = 512
input1 = Input(shape=(max_tweet_length,))
Emb = Embedding(max_features, embed_dim)(input1)
#model.add(SpatialDropout1D(0.4))
m = Conv1D(32,(2))(Emb)
#model.add(Conv1D(16,(2)))
m = AveragePooling1D(16,2)(m)
m = Dropout(0.5)(m)
m = Flatten()(m)
m = Dense(64)(m)
m = Dropout(0.5)(m)
output = Dense(3,activation='softmax')(m)
m = Model(inputs=[input1],outputs=[output])
m.compile(loss = 'categorical_crossentropy',
optimizer=rmsprop(lr=1e-4),metrics = ['accuracy'])
rlStop = ReduceLROnPlateau(monitor='val_loss',min_lr=1e-6,patience=4,factor=0.5)
#checkpoint = ModelCheckpoint(filepath='/output/CNN_stacked_w2v.hdfs', save_weights_only=False,
# monitor='val_loss',save_best_only=True)
batch_size = 64
#plot_model(m, to_file='CNN_stacked_w2v.png')
# ### Uncomment the next cell for training
# In[ ]:
#m.fit(X_train, Y_train, epochs = 350,validation_data=(X_valid,Y_valid),callbacks=[checkpoint,rlStop],
# batch_size=batch_size, verbose = 2)
