def _train(self):
data, label = self._load_data()
train_data, train_label, validate_data, validate_label, test_data, test_label = split_data(data, label,
to_categorical=True)
network_input = Input(shape=(100, 3))
network = LSTM(32, return_sequences=True)(network_input)
network = LSTM(32)(network)
network = Dense(5, activation=softmax)(network)
network = Model(inputs=[network_input], outputs=[network])
network.compile(optimizer=RMSprop(lr=0.01), loss=categorical_crossentropy, metrics=[categorical_accuracy])
network.summary()
callback = [
callbacks.ReduceLROnPlateau(monitor="categorical_accuracy", factor=0.1, patience=3)
]
self.train_history = network.fit(train_data, train_label,
validation_data=(validate_data, validate_label), batch_size=self.BATCH_SIZE,
epochs=self.EPOCHS, callbacks=callback)
self.evaluate_history = network.evaluate(test_data, test_label, batch_size=self.BATCH_SIZE)
return network
def run_LSTM(X_train, X_test, y_train, y_test, symbol, price):
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
LSTM = create_LSTM(X_train)
history = LSTM.fit(X_train, y_train, epochs=100, batch_size=32)
print(LSTM.summary())
results = LSTM.evaluate(X_test, y_test, batch_size=32)
print("test loss, test acc:", results)
# summarize history for accuracy
plt.plot(history.history['accuracy'], label='Train')
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
prediction = LSTM.predict(X_test)
y_prediction = prediction[:, -1, :]
y_prediction = np.argmax(prediction, axis=1)
print(y_prediction)
NAV_history = Generate_nav(10000, symbol, price, y_prediction)
# Optimizer = optimizers.Adam(lr = 0.00001)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
callback_list = [
ReduceLROnPlateau(
# Model의 Val_loss를 Monitoring
monitor='val_loss',
# Callback 호출시 Learning rate를 1/10으로 줄임
factor=0.1,
# Val_loss가 5 Epoch동안 개선되지 않을 경우 CallBack 호출
patience=5),
EarlyStopping(
# Monitoring Index
monitor='val_loss',
# 5 Epoch보다 더 길게(즉 6 Epoch 동안) 정확도 향상이 없을 경우 Early Stop
patience=5)
]
# ReduceLR = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=10, verbose=1, mode='auto', min_lr=1e-05)
model.fit(xTrain,
yTrain,
validation_data=(xTest, yTest),
epochs=50,
batch_size=1000,
callbacks=callback_list)
model.save('MSFE.h5')
output_size = GAUSSIAN_MIXTURE_COMPONENTS * 6 + ONE_HOT_LEN
Loss = GaussianMixtureLoss(GAUSSIAN_MIXTURE_COMPONENTS, max_points)
inputs = Input(shape=(max_points, INPUT_VECTOR_LEN))
model = LSTM(LSTM_SIZE, activation='relu', return_sequences=True)(inputs)
for layer in range(REPEAT_DEEP_ARCH):
model = LSTM(LSTM_SIZE, activation='relu', return_sequences=True)(model)
model = Dense(DENSE_SIZE, activation='relu')(model)
model = Dense(output_size)(model)
model = Model(inputs, model)
model.compile(
loss=Loss.geom_gaussian_mixture_loss,
optimizer=OPTIMIZER)
model.summary()
tb_callback = TensorBoard(log_dir='./tensorboard_log/' + TIMESTAMP + ' ' + SCRIPT_NAME, write_graph=False)
decypher = DecypherAll(gmm_size=GAUSSIAN_MIXTURE_COMPONENTS, plot_dir=PLOT_DIR)
history = model.fit(
x=training_vectors,
y=target_vectors,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_split=TRAIN_VALIDATE_SPLIT,
callbacks=[decypher, tb_callback]).history
notify(TIMESTAMP, SCRIPT_NAME, 'validation loss of ' + str(history['val_loss'][-1]))
print(SCRIPT_NAME, 'finished successfully')
input_2d = np.repeat([[[0.2, 15, 0, 0, 0]]], 11, axis=1)
input_2d = np.repeat(input_2d, TRAINING_SIZE, axis=0)
(data_points, max_points, vector_len) = input_2d.shape
inputs = Input(name='Input', shape=(max_points, vector_len))
model = LSTM(vector_len, return_sequences=True)(inputs)
# The dense layer is required for input values exceeding 1e0
model = Dense(vector_len)(model)
model = Model(inputs, model)
model.compile(
loss=bivariate_gaussian_loss,
optimizer=Adam(lr=0.01))
model.summary()
callbacks = [
TensorBoard(log_dir='./tensorboard_log/' + TIMESTAMP, write_graph=False),
DecypherAll(lambda x: str(x)),
EarlyStopping(patience=20)
]
history = model.fit(x=input_2d,
y=input_2d,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_split=TRAIN_VALIDATE_SPLIT,
callbacks=callbacks).history
notify(TIMESTAMP, SCRIPT_NAME, 'validation loss of ' + str(history['val_loss'][-1]))
print('Done!')
def CNN_model(filename, train, X_train, X_test, word2ind, maxWords,
y_train, y_test, ind2label, maxChar, char2ind, validation=False, X_valid=None, y_valid=None,
pretrained_embedding="", word_embedding_size=100, char_embedding_size=50,
lstm_hidden=32, nbr_epochs=25, batch_size=128, dropout=0.5, optimizer='rmsprop', early_stopping_patience=-1,
folder_path="CNN_results", print_to_file = True, gen_confusion_matrix=False, return_model = False
):
"""
Build, train and test the CNN-CNN-LSTM Keras model. Works for multi-tasking learning.
The model architecture looks like:
- CNN character-level representation
- CNN word-level + character representation
- LSTM
- Softmax for prediction
:param filename: File to redirect the printing
:param train: Boolean if the model must be trained or not. If False, the model's wieght are expected to be stored in "folder_path/filename/filename.h5"
:param X_train: Data to train the model. It must be a list of word and character indices.
:param X_test: Data to test the model. It must be a list of word and character indices.
:param word2ind: Dictionary containing all words in the training data and a unique integer per word
:param maxWords: Maximum number of words in a sequence
:param y_train: Labels to train the model for the prediction task
:param y_test: Labels to test the model for the prediction task
:param ind2label: Dictionary where all labels are mapped into a unique integer
:param maxChar: The maximum numbers of characters in a word. If set to 0, the model will not use character-level representations of the words
:param char2ind: A dictionary where each character is mapped into a unique integer
:param validation: Boolean. If true, the validation score will be computed from 'X_valid' and 'y_valid'
:param X_valid: Optional. Validation dataset
:param y_valid: Optional. Validation dataset labels
:param pretrained_embedding: Use the pretrained word embeddings. Pretrained vectors must be located here: "dataset/pretrained_vectors"
Three values:
- "": Do not use pre-trained word embeddings (Default)
- False: Use the pre-trained embedding vectors as the weights in the Embedding layer
- True: Use the pre-trained embedding vectors as weight initialiers. The Embedding layer will still be trained.
:param word_embedding_size: Size of the pre-trained word embedding to use (100 or 300)
:param char_embedding_size: size of the character-level word representations
:param lstm_hidden: Dimentionality of the LSTM output space
:param nbr_epochs: Number of epochs to train the model
:param batch_size: Size of batches while training the model
:param dropout: Rate to apply for each Dropout layer in the model
:param optimizer: Optimizer to use while compiling the model
:param early_stopping_patience: Number of continuous tolerated epochs without improvement during training.
:param folder_path: Path to the directory storing all to-be-generated files
:param print_to_file: if True redirects the printings to a file (given in filename), if False std_out is kept
:param gen_confusion_matrix: Boolean value. Generated confusion matrices or not.
:parm return_model: if True returns the Keras model object, otherwise return best results
:return: The classification scores for both tasks (default for compatibility). If returnModel = True, returns model object for further computation
"""
print("====== {0} start ======".format(filename))
end_string = "====== {0} end ======".format(filename)
os.makedirs(folder_path+"/"+filename, exist_ok=True)
filepath = folder_path+"/"+filename+"/"+filename
# Set print outputs file
if print_to_file:
file, stdout_original = setPrintToFile("{0}.txt".format(filepath))
nbr_words = len(word2ind)+1
out_size = len(ind2label)+1
nbr_chars = len(char2ind)+1
best_results = None
# Embedding - Characters
character_input = Input((maxWords,maxChar,))
embed_char_out = TimeDistributed(Embedding(nbr_chars, char_embedding_size))(character_input)
conv1d_out = TimeDistributed(Convolution1D(filters = 50, kernel_size = 3, strides=1, activation="relu", padding='same'))(embed_char_out)
pool_out = TimeDistributed(MaxPooling1D(pool_size=2, strides=1, padding='same'))(conv1d_out)
char_enc = TimeDistributed(Flatten())(pool_out)
# Embedding - Words
word_input = Input((maxWords,))
if pretrained_embedding=="":
word_emb = Embedding(nbr_words, word_embedding_size)(word_input)
else:
# Retrieve embeddings from word2vec
embedding_matrix = word2VecEmbeddings(word2ind, word_embedding_size)
word_emb = Embedding(nbr_words, word_embedding_size, weights=[embedding_matrix], trainable=pretrained_embedding, mask_zero=False)(word_input)
# Model inputs
inputs = [word_input, character_input]
# Full word representation
word_full = concatenate([char_enc, word_emb])
# encode words w_full = w_char + w_embed
conv1d_w1_out = Convolution1D(filters = 800, kernel_size = 5, strides=1, activation="relu", padding='same')(word_full)
drop_w1_layer = Dropout(dropout, noise_shape=None, seed=None)(conv1d_w1_out)
conv1d_w2_out = Convolution1D(filters = 800, kernel_size = 5, strides=1, activation="relu", padding='same')(drop_w1_layer)
# pool size out?
word_enc = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv1d_w2_out)
# LSTM layer
model = LSTM(lstm_hidden, return_sequences=True, dropout=dropout)(word_enc)
# Output - Softmax
outputs = [Dense(out_size, activation='softmax')(model) for out_size in [len(x)+1 for x in ind2label]]
model_loss = ['categorical_crossentropy' for x in outputs]
model_metrics = None
# Model
model = Model(inputs=inputs, outputs=outputs)
model.compile(loss=model_loss, metrics=model_metrics, optimizer=get_optimizer(optimizer))
print(model.summary(line_length=150),"\n\n\n\n")
# Training Callbacks:
callbacks = []
value_to_monitor = 'val_f1'
best_model_weights_path = "{0}.h5".format(filepath)
# 1) Classifition scores
classification_scores = Classification_Scores([X_train, y_train], ind2label, best_model_weights_path)
callbacks.append(classification_scores)
# 2) EarlyStopping
if early_stopping_patience != -1:
early_stopping = EarlyStopping(monitor=value_to_monitor, patience=early_stopping_patience, mode='max')
callbacks.append(early_stopping)
else:
early_stopping = EarlyStopping(monitor=value_to_monitor, patience=nbr_epochs, mode='max')
callbacks.append(early_stopping)
# Train
if train:
# Train the model. Keras's method argument 'validation_data' is referred as 'testing data' in this code.
hist = model.fit(X_train, y_train, validation_data=[X_test, y_test], epochs=nbr_epochs, batch_size=batch_size, callbacks=callbacks, verbose=2)
print()
print('-------------------------------------------')
print("Best F1 score:", early_stopping.best, " (epoch number {0})".format(1+np.argmax(hist.history[value_to_monitor])))
# Save Training scores
save_model_training_scores("{0}".format(filepath), hist, classification_scores)
# Print best testing classification report
best_epoch = np.argmax(hist.history[value_to_monitor])
print(classification_scores.test_report[best_epoch])
# Best epoch results
best_results = model_best_scores(classification_scores, best_epoch)
# HACK: optmizer weight length issue
# https://github.com/keras-team/keras/issues/4044
import h5py
with h5py.File(best_model_weights_path, 'a') as f:
if 'optimizer_weights' in f.keys():
del f['optimizer_weights']
# Load weigths from best training epoch into model
save_load_utils.load_all_weights(model, best_model_weights_path)
# Create confusion matrices
if gen_confusion_matrix:
for i, y_target in enumerate(y_test):
# Compute predictions, flatten
predictions, target = compute_predictions(model, X_test, y_target, ind2label[i])
# Generate confusion matrices
save_confusion_matrix(target, predictions, list(ind2label[i].values()), "{0}_task_{1}_confusion_matrix_test".format(filepath,str(i+1)))
# Validation dataset
if validation:
print()
print("Validation dataset")
print("======================")
# Compute classification report
for i, y_target in enumerate(y_valid):
# Compute predictions, flatten
predictions, target = compute_predictions(model, X_valid, y_target, ind2label[i], nbrTask=i)
# Only for multi-task
if len(y_train) > 1:
print("For task "+str(i+1)+"\n")
print("====================================================================================")
print("")
print("With padding into account")
print(metrics.flat_classification_report([target], [predictions], digits=4))
print("")
print('----------------------------------------------')
print("")
print("Without the padding:")
print(metrics.flat_classification_report([target], [predictions], digits=4, labels=list(ind2label[i].values())))
# Generate confusion matrices
save_confusion_matrix(target, predictions, list(ind2label[i].values()), "{0}_task_{1}_confusion_matrix_validation".format(filepath,str(i+1)))
# Close file
if print_to_file:
closePrintToFile(file, stdout_original)
print(end_string)
# Returns model itself for further computation, otherwise best results
if return_model:
return model
else:
return best_results
merge = Dense(8)(merge) output = Dense(1)(merge) output_1 = Dense(4)(output) output_1 = Dense(1)(output_1) output_2 = Dense(4)(output) output_2 = Dense(1)(output_2) model = Model(inputs=[input, input2], outputs=[output_1, output_2]) model.summary() # quit() model.compile(loss='mse', optimizer='adam', metrics=['mse']) model.fit([x, x2], [y, y2], epochs=100, batch_size=1) res = model.evaluate([x, x2], [y, y2], batch_size=1) from keras.callbacks import EarlyStopping # callback = EarlyStopping(monitor='loss', patience=20, mode='auto') # # callback = EarlyStopping(monitor='acc', patience=20, mode='max') # model2.compile(loss='mse', optimizer='adam', metrics=['mse']) # model2.fit(x_, y, epochs=1000, batch_size=1, callbacks=[callback]) # loss2, mse2 = model2.evaluate(x_, y, batch_size=1, verbose=1) # x_input = np.array([[6.5, 7.5, 8.5], [50, 60, 70], [70, 80, 90], [100, 110, 120]]) # x = x_input.reshape(-1, 3, 1) # y_pred = model.predict(x) print(res)
