def fit_model(model,
train_batches,
epochs=1,
lr=None,
val_batches=None,
save_path='',
callbacks=None):
fit_callbacks = [
TQDMNotebookCallback(),
UpdateHistoryCallback(),
LearningRateSetter(epochs, lr)
]
if save_path != '':
fit_callbacks.append(ModelSave(save_path))
if callbacks is not None:
fit_callbacks.append(callbacks)
if val_batches is None:
model.fit_generator(train_batches,
train_batches.num_batches,
epochs=np.sum(epochs),
verbose=0,
callbacks=fit_callbacks)
else:
model.fit_generator(train_batches,
train_batches.num_batches,
epochs=np.sum(epochs),
verbose=0,
callbacks=fit_callbacks,
validation_data=val_batches,
validation_steps=val_batches.num_batches)
def get_callbacks(MODEL_PATH):
"""
Saves model if validation IoU increased,
applies Cyclic Learning Rate,
shows TQDM based progress bar.
"""
checkpoint = ModelCheckpoint(MODEL_PATH,
monitor='val_iou_score',
verbose=1,
save_best_only=True,
mode='max')
clr = CyclicLR(base_lr=0.000001,
max_lr=0.001,
step_size=3000,
mode='exp_range')
callbacks = [checkpoint,
clr,
TQDMNotebookCallback(leave_inner=True,
leave_outer=True)]
return callbacks
def fit(self, train_data, batch_size, epochs, learning_rate=1e-3):
'''
Function to fit the style transfer model.
Args:
train_data (ndarray): Array of symmetrical train images in
(288 x 288) size.
batch_size (int): Batch size for training
epochs (int): Number of epochs to run
Returns: None
'''
# Dictionary to use TQDMN to visualise training
params = {
'verbose': 0,
'callbacks': [TQDMNotebookCallback(leave_inner=True)]
}
# Our target is 0 loss of the MSE (final model output)
target = np.zeros((train_data.shape[0], 1))
# Set learning rate
K.set_value(self.m_style.optimizer.lr, learning_rate)
# Fit model
self.m_style.fit(x=[train_data, train_data],
y=target,
batch_size=batch_size,
epochs=epochs,
**params)
def getModel(n, num_epochs=Epochs):
model = Sequential()
model.add(Dense(n, activation='relu', input_dim=len(feature_columns)))
model.add(Dropout(0.5))
model.add(Dense(int(n / 2), activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nitems, activation='softmax'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
callbacks = [
EarlyStopping(monitor='acc', patience=10, verbose=0),
TQDMNotebookCallback()
]
model.fit(X_train,
y_train,
epochs=num_epochs,
batch_size=128,
verbose=1,
callbacks=callbacks)
return model
def train_model(input_to_softmax,
pickle_path,
save_model_path,
train_json='train_corpus.json',
valid_json='valid_corpus.json',
minibatch_size=20,
spectrogram=True,
mfcc_dim=13,
optimizer=SGD(lr=0.02,
decay=1e-6,
momentum=0.9,
nesterov=True,
clipnorm=5),
epochs=20,
verbose=1,
sort_by_duration=False,
max_duration=10.0):
# create a class instance for obtaining batches of data
audio_gen = AudioGenerator(minibatch_size=minibatch_size,
spectrogram=spectrogram,
mfcc_dim=mfcc_dim,
max_duration=max_duration,
sort_by_duration=sort_by_duration)
# add the training data to the generator
audio_gen.load_train_data(train_json)
audio_gen.load_validation_data(valid_json)
# calculate steps_per_epoch
num_train_examples = len(audio_gen.train_audio_paths)
steps_per_epoch = num_train_examples // minibatch_size
# calculate validation_steps
num_valid_samples = len(audio_gen.valid_audio_paths)
validation_steps = num_valid_samples // minibatch_size
# add CTC loss to the NN specified in input_to_softmax
model = add_ctc_loss(input_to_softmax)
# CTC loss is implemented elsewhere, so use a dummy lambda function for the loss
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=optimizer)
# make results directory, if necessary
if not os.path.exists('results'):
os.makedirs('results')
# add checkpointer
checkpointer = ModelCheckpoint(filepath=os.path.join(
'results', save_model_path),
verbose=0)
# train the model
callbacks = [TQDMNotebookCallback(), checkpointer
] if verbose < 0 else [checkpointer]
hist = model.fit_generator(generator=audio_gen.next_train(),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=audio_gen.next_valid(),
validation_steps=validation_steps,
callbacks=callbacks,
verbose=verbose)
# save model loss
with open(os.path.join('results', pickle_path), 'wb') as f:
pickle.dump(hist.history, f)
def fit(self, batches, val_batches, nb_epoch=1):
"""
Fits the model on data yielded batch-by-batch by a Python generator.
See Keras documentation: https://keras.io/models/model/
"""
self.model.fit_generator(batches, samples_per_epoch=batches.nb_sample, nb_epoch=nb_epoch,
validation_data=val_batches, nb_val_samples=val_batches.nb_sample, verbose=1,
callbacks=[TQDMNotebookCallback(leave_inner=True, leave_outer=True)])
def train(self, epochs=1):
"""
Train the model and save afterwards.
If the shared model is not yet frozen, the network corresponding the first problem statement will be trained.
Note that it is trained on all data, which isn't how it should be! When done properly, one should ALWAYS
separate the full dataset in a training and it is a test set. It is also highly encouraged to incorporate a
validation set as well. But again, to simplify the model in its whole, we will ignore this best practice.
If the shared model is indeed frozen, only manually curated samples will be used to train the model on.
:param epochs: Number of epochs the model will be trained on
"""
prep('Predicting...', key='training', silent=True)
# Create TensorBoard instantiation
tensorboard = TensorBoard(log_dir='logs/{}'.format(time()))
# Train the suitable network
if self.is_frozen:
self.network_2.fit(
x=self.train_images,
y=self.train_labels,
batch_size=config.BATCH_SIZE,
epochs=epochs,
verbose=0,
callbacks=[
TQDMNotebookCallback(leave_inner=True), tensorboard
],
)
self.current_epoch += epochs # Epoch count only considered for second network
else:
self.network_1.fit(
x=self.evaluation_images,
y=self.evaluation_labels,
batch_size=config.BATCH_SIZE,
epochs=epochs,
verbose=0,
callbacks=[
TQDMNotebookCallback(leave_inner=True), tensorboard
],
)
drop(key='training', silent=True)
self.save_model()
def fit_model_nb(model, X_train, train_t, X_test, test_t):
# for fitting the model in a jupyter notebook
history = History()
model.fit(X_train,
train_t,
epochs=EPOCHS,
batch_size=BATCH,
validation_data=[X_test, test_t],
verbose=0,
callbacks=[TQDMNotebookCallback(), history])
return history
def fit(self,
train_x: Union[Dict, List, np.ndarray],
train_y: Union[Dict, List, np.ndarray],
test_x: Union[Dict, List, np.ndarray] = None,
test_y: Union[Dict, List, np.ndarray] = None) -> pd.DataFrame:
"""Fit the model using given training parameters.
Parameters
--------------------------
train: Tuple[Union[Dict, List, np.ndarray]],
Either a tuple of list, np.ndarrays or dictionaries,
containing the training data.
test: Tuple[Union[Dict, List, np.ndarray]] = None
Either a tuple of list, np.ndarrays or dictionaries,
containing the validation data data.
These data are optional, but they are required if
the given monitor metric starts with "val_"
Raises
--------------------------
ValueError,
If no test data are given but the monitor metric
starts with the word "val_", meaning it has to be
computed on the test data.
Returns
---------------------------
The training history as a pandas dataframe.
"""
test = None
if all(d is not None for d in (test_x, test_y)):
test = (test_x, test_y)
if test is None and self._monitor.startswith("val_"):
raise ValueError(
"No test set was given, "
"but a validation metric was required for the early stopping.")
return pd.DataFrame(
self._model.fit(
train_x,
train_y,
epochs=self._max_epochs,
batch_size=self._batch_size,
validation_data=test,
verbose=False,
shuffle=True,
callbacks=[
EarlyStopping(self._monitor, patience=self._patience),
# We show the correct kind of callback depending if this
# is running in a CLI or jupyter notebook-like environment.
TQDMNotebookCallback()
if is_notebook() else TQDMCallback()
]).history)
def main():
print('Loading train')
train = load_train()
X_train, y_train = get_X_y(train)
print('Oversampling')
X_train, y_train = oversample(X_train, y_train)
print('Creating an embedding matrix')
embedding_matrix, x1, x2, word_index = get_embedding_matrix(X_train)
print('Training')
model = get_model(word_index, embedding_matrix, X_train)
model.compile(loss='binary_crossentropy',
optimizer=RMSprop(0.001),
metrics=['accuracy'])
X_train, X_val, x1_train, x1_val, x2_train, x2_val, y_train, y_val = train_test_split(
X_train, x1, x2, y_train, test_size=0.1, random_state=SEED)
train_data = [
x1_train, x2_train, x1_train, x2_train, x1_train, x2_train, x1_train,
x2_train, X_train
]
val_data = [
x1_val, x2_val, x1_val, x2_val, x1_val, x2_val, x1_val, x2_val, X_val
]
model.fit(train_data,
y_train,
batch_size=64 * 16,
epochs=4,
verbose=0,
validation_data=(val_data, y_val),
callbacks=[TQDMNotebookCallback()])
print('Loading test')
test = load_test()
X_test = test[features]
del test
gc.collect()
print('Generating predictions without correction')
predictions = model.predict_proba(X_test)
generate_submission(predictions, 'xgb3_yO_nC')
print('Generating predictions with correction')
predictions = prediction_correction(predictions)
generate_submission(predictions, 'xgb3_yO_yC')
def make_steps(step, ampl):
"""
Perform training epochs
@param step Number of epochs to perform
@param ampl the K, the randomized component of the score matrix.
"""
global w2ts, t2i, steps, features, score, histories
# shuffle the training pictures
random.shuffle(train)
# Map whale id to the list of associated training picture hash value
w2ts = {}
for w, hs in w2hs.items():
for h in hs:
if h in train_set:
if w not in w2ts: w2ts[w] = []
if h not in w2ts[w]: w2ts[w].append(h)
for w, ts in w2ts.items():
w2ts[w] = np.array(ts)
# Map training picture hash value to index in 'train' array
t2i = {}
for i, t in enumerate(train):
t2i[t] = i
# Compute the match score for each picture pair
features, score = compute_score()
# Train the model for 'step' epochs
history = model.fit_generator(
TrainingData(score + ampl * np.random.random_sample(size=score.shape),
steps=step,
batch_size=32),
initial_epoch=steps,
epochs=steps + step,
max_queue_size=12,
workers=6,
verbose=0,
callbacks=[
TQDMNotebookCallback(leave_inner=True,
metric_format='{value:0.3f}')
]).history
steps += step
# Collect history data
history['epochs'] = steps
history['ms'] = np.mean(score)
history['lr'] = get_lr(model)
print(history['epochs'], history['lr'], history['ms'])
histories.append(history)
def evaluate_model(runs=5):
scores = []
for i in range(runs):
print('Executing run %d' % (i+1))
model = mymodel()
history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs,
callbacks=[TQDMNotebookCallback()], verbose=0)
scores.append(model.evaluate(x_test, y_test, verbose=0))
print(' * Test set Loss: %.4f, Accuracy: %.4f' % (scores[-1][0], scores[-1][1]))
accuracies = [score[1] for score in scores]
return np.mean(accuracies), np.std(accuracies)
def _default_callbacks(self):
return [
ModelCheckpoint(self._checkpoint_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True),
ReduceLROnPlateau(monitor='val_loss',
patience=3,
verbose=1,
factor=0.65,
min_lr=0.00001),
EarlyStopping(monitor='val_loss', patience=8),
TQDMNotebookCallback()
]
def lstm_trainer(train_tfrecord, train_lr, train_epochs):
# Building the model
lstm_model = Sequential()
lstm_model.add(BatchNormalization(input_shape=(None, 128)))
lstm_model.add(Dropout(0.5))
lstm_model.add(
LSTM(128,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l2(0.01)))
lstm_model.add(Dense(1, activation='sigmoid'))
# try using different optimizers and different optimizer configs
lstm_model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Fitting the LSTM model
batch_size = 32
CV_frac = 0.1
train_gen = data_generator(batch_size, train_tfrecord, 0, 1 - CV_frac)
val_gen = data_generator(20, train_tfrecord, 1 - CV_frac, 1)
rec_len = 17662
lstm_h = lstm_model.fit_generator(
train_gen,
steps_per_epoch=int(rec_len * (1 - CV_frac)) // batch_size,
epochs=train_epochs,
validation_data=val_gen,
validation_steps=int(rec_len * CV_frac) // 20,
verbose=1,
callbacks=[TQDMNotebookCallback()])
# Plot the model architecture
plot_model(
lstm_model,
to_file='../results/LSTM/LSTM_Loss=BinCE_Epochs{}_lt={}.png'.format(
train_epochs, train_lr))
# Save the lstm model
lstm_model.save(
'../models/1LayerLSTM__Loss=BinCE_lr={}_Epochs={}.h5'.format(
train_lr, train_epochs))
return lstm_h
def trainer(train_tfrecord, train_lr, train_epochs):
#Logistic Regression Model
# Adam optimizer with Loss = Binary Cross Entropy
lr_model = Sequential()
lr_model.add(BatchNormalization(input_shape=(10, 128)))
lr_model.add(Flatten())
lr_model.add(Dense(1, activation='sigmoid'))
adam = optimizers.Adam(lr=train_lr)
lr_model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
#Training the model
# Hyperparameters : Epochs = train_epochs, batch_size = 40, learning rate = train_lr
batch_size = 40
CV_frac = 0.1
train_gen = data_generator(batch_size, train_tfrecord, 0, 1 - CV_frac)
val_gen = data_generator(20, train_tfrecord, 1 - CV_frac, 1)
rec_len = 17662
lr_h = lr_model.fit_generator(
train_gen,
steps_per_epoch=int(rec_len * (1 - CV_frac)) // batch_size,
epochs=train_epochs,
validation_data=val_gen,
validation_steps=int(rec_len * CV_frac) // 20,
verbose=1,
callbacks=[TQDMNotebookCallback()])
#Save the model architecture image always at same path
plot_model(
lr_model,
to_file=
'../results/LogisticReg/model_LogisticRegression_BinaryCE_Adam_lr={}_Epochs{}.png'
.format(train_lr, train_epochs))
#Save the model at the same path
lr_model.save(
'../models/LogisticRegression_BinCE_Adam_lr={}_Epochs={}.h5'.format(
train_lr, train_epochs))
return lr_h
def evaluate_model(runs=5):
scores = []
for i in range(runs):
print('Executing run %d' % (i + 1))
model = mymodel()
model.fit_generator(train_generator,
callbacks=[TQDMNotebookCallback()],
steps_per_epoch=num_train_samples // batch_size,
epochs=epochs,
verbose=0)
print(' * Evaluating model on test set')
scores.append(
model.evaluate_generator(test_generator,
steps=num_test_samples // batch_size,
verbose=0))
print(' * Test set Loss: %.4f, Accuracy: %.4f' %
(scores[-1][0], scores[-1][1]))
accuracies = [score[1] for score in scores]
return np.mean(accuracies), np.std(accuracies)
def train_fixed(self, x_trn, x_val, y_trn, y_val):
self.models.append(model1())
#self.models.append(model1())
self.models.append(model1())
self.models.append(model1())
self.models.append(model1())
self.models.append(model1())
self.models.append(model1())
self.models.append(model1())
self.models.append(model1())
self.models.append(model1())
losses = np.zeros(len(self.models))
for i_model, model in enumerate(self.models):
es = EarlyStopping(monitor='val_loss', mode='min', patience=10)
mc = ModelCheckpoint('best_model.h5',
monitor='val_loss',
mode='min',
save_best_only=True)
model.fit(x=x_trn,
y=y_trn,
validation_data=(x_val, y_val),
callbacks=[es, mc, TQDMNotebookCallback()],
batch_size=25,
verbose=0,
epochs=150) #
model = load_model('best_model.h5')
losses[i_model] = model.evaluate(x_val, y_val)[0]
best_model = self.models[np.argmin(losses)]
best_model.save('fixed_model.h5')
# Add model to fixed models
#K.clear_session()
self.fixed_models.append(load_model('fixed_model.h5'))
self.models = []
return losses, np.argmin(losses)
def model_training_logging(self,autoencoder, train_x, params):
"""
Model training and Logging. Below is where we set up the actual run including checkpoints and the tensorboard.
"""
nb_epoch = 50
batch_size = 128
input_dim = train_x.shape[1] # num of columns, 30
encoding_dim = 14
hidden_dim = int(encoding_dim / 2) # i.e. 7
learning_rate = 1e-7
path_to_pickle = 'gdrive/My Drive/DataSet_Colab_NoteBooks/pickle-machine/'
file = path_to_pickle + "autoencoder_husky50.h5"
autoencoder.compile(metrics=['accuracy'],
loss='mean_squared_error',
optimizer='adam')
cp = ModelCheckpoint(filepath=file,
save_best_only=True,
verbose=0)
logs = '/data/tensorflow_logs'
tb = TensorBoard(log_dir=logs,
histogram_freq=0,
write_graph=True,
write_images=True)
history = autoencoder.fit(train_x, train_x,
epochs=nb_epoch,
batch_size=batch_size,
shuffle=True,
# validation_data=(test_x, test_x),
validation_split=0.2,
verbose=1,
callbacks=[TQDMNotebookCallback(), cp, tb]).history
return history
def train(model, dataset_arrays, labels, config):
"""All the training without any of the extra printing."""
earlystop = callbacks.EarlyStopping(
monitor="val_loss",
patience=config["patience"],
restore_best_weights=True,
min_delta=1.0e-4,
)
callback_list = [earlystop]
if config.get("progbar"):
tqdmcb = TQDMNotebookCallback()
tqdmcb.on_train_batch_begin = tqdmcb.on_batch_begin
tqdmcb.on_train_batch_end = tqdmcb.on_batch_end
tqdmcb.on_test_batch_begin = tqdmcb.on_batch_begin
tqdmcb.on_test_batch_end = tqdmcb.on_batch_end
setattr(tqdmcb, "on_test_begin", lambda x: None)
setattr(tqdmcb, "on_test_end", lambda x: None)
callback_list.append(tqdmcb)
if config.get("logdir"):
tensorboard_callback = callbacks.TensorBoard(log_dir=config["logdir"])
callback_list.append(tensorboard_callback)
model.compile(
optimizer=config["optimizer"],
loss=config["loss"],
metrics=config.get("metrics", ["accuracy"]),
weighted_metrics=config.get("weighted_metrics"),
)
history = model.fit(
dataset_arrays["train"],
labels["train"],
class_weight=config.get("class_weights"),
verbose=config["verbose"],
batch_size=config["batch_size"],
epochs=config["epochs"],
callbacks=callback_list,
validation_split=config["val_split"],
)
return history
history = LossHistory()
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=2,
verbose=0,
mode='auto')
fitted_model = model.fit_generator(train_generator,
steps_per_epoch=int(7394 * 1 - 0.5) //
batch,
epochs=epochs,
validation_data=valid_generator,
validation_steps=int(7394 * 0.25) // batch,
callbacks=[
TQDMNotebookCallback(leave_inner=True,
leave_outer=True),
early_stopping, history
])
model.save('./models/new_model_cla_mode.h5')
# model_json = model.to_json()
# with open('./data_model2.json', 'w') as jf:
# jf.write(model_json)
#
losses, val_losses = history.losses, history.val_losses
fig = plt.figure(figsize=(15, 5))
plt.plot(fitted_model.history['loss'], 'g', label="train losses")
plt.plot(fitted_model.history['val_loss'], 'r', label="valid losses")
plt.grid(True)
plt.title('Training loss vs. Validation loss')
write_graph=True)
modelCallback = ModelCheckpoint(
'model_checkpoint_advbig_sos_eos_unk.{epoch:03d}-{loss:.3f}.hdf5',
monitor='val_loss',
verbose=1,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=1)
# creating different sets of Params to easily import into the model at train time
params = {
'verbose':
1,
'callbacks': [
TQDMNotebookCallback(), reduce_LR, early_stopping, tbCallBack,
modelCallback
]
}
params2 = {
'verbose': 1,
'callbacks': [LRDrop,
TQDMNotebookCallback(), reduce_LR, early_stopping]
}
params3 = {
'verbose': 1,
'callbacks': [LRDrop,
TQDMNotebookCallback(), reduce_LR, early_stopping]
}
# In[ ]:
def train_tlnn(regions, start, end, deep):
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import Adam
from IPython.core.display import HTML
from keras_tqdm import TQDMNotebookCallback
display(HTML("""
<style>
.p-Widget.jp-OutputPrompt.jp-OutputArea-prompt:empty {
padding: 0;
border: 0;
}
</style>
"""))
X_test, y_test, _ = gen_data(regions, start, end)
mod = [Dense(128, kernel_initializer='uniform', input_dim=len(X_test[0]), activation='relu'),
Dropout(0.1)]
if deep:
mod.extend([Dense(32, kernel_initializer='uniform', activation='relu'),
Dropout(0.1)])
model = Sequential(mod + [Dense(1, kernel_initializer='uniform', activation='linear')])
model.compile(optimizer=Adam(), loss='mse')
batch_size = int(len(X_test)/64)
model.fit(X_test, y_test, epochs=100, batch_size=batch_size, verbose=0, callbacks=[TQDMNotebookCallback()])
return model
def train_neural_network(x_train,
y_train,
x_val,
y_val,
configs,
partial_model_architecture,
batch_size=32,
nb_epoch=5,
normalize=True,
checkpoint_dir=None,
neural_network_name='my_neural_network',
training_log_file='training.log',
early_stopping=False):
"""Train a neural network to classify crystal structures represented as two-dimensional diffraction fingerprints.
This model was introduced in [1]_.
x_train: np.array, [batch, width, height, channels]
.. [1] A. Ziletti, D. Kumar, M. Scheffler, and L. M. Ghiringhelli,
“Insightful classification of crystal structures using deep learning”,
Nature Communications, vol. 9, pp. 2775 (2018)
.. codeauthor:: Angelo Ziletti <*****@*****.**>
"""
if checkpoint_dir is None:
checkpoint_dir = configs['io']['results_folder']
filename_no_ext = os.path.abspath(
os.path.normpath(os.path.join(checkpoint_dir, neural_network_name)))
training_log_file_path = os.path.abspath(
os.path.normpath(os.path.join(checkpoint_dir, training_log_file)))
# reshape to follow the image conventions
# - TensorFlow backend: [batch, width, height, channels]
# - Theano backend: [batch, channels, width, height]
x_train = reshape_images_to_theano(x_train)
x_val = reshape_images_to_theano(x_val)
assert x_train.shape[1] == x_val.shape[1]
assert x_train.shape[2] == x_val.shape[2]
assert x_train.shape[3] == x_val.shape[3]
img_channels = x_train.shape[1]
img_width = x_train.shape[2]
img_height = x_train.shape[3]
logger.info('Loading datasets.')
logger.debug('x_train shape: {0}'.format(x_train.shape))
logger.debug('y_train shape: {0}'.format(y_train.shape))
logger.debug('x_val shape: {0}'.format(x_val.shape))
logger.debug('y_val shape: {0}'.format(y_val.shape))
logger.debug('Training samples: {0}'.format(x_train.shape[0]))
logger.debug('Validation samples: {0}'.format(x_val.shape[0]))
logger.debug("Img channels: {}".format(x_train.shape[1]))
logger.debug("Img width: {}".format(x_train.shape[2]))
logger.debug("Img height: {}".format(x_train.shape[3]))
x_train = x_train.astype('float32')
x_val = x_val.astype('float32')
# normalize each image separately
if normalize:
for idx in range(x_train.shape[0]):
x_train[idx, :, :, :] = (x_train[idx, :, :, :] - np.amin(
x_train[idx, :, :, :])) / (np.amax(x_train[idx, :, :, :]) -
np.amin(x_train[idx, :, :, :]))
for idx in range(x_val.shape[0]):
x_val[idx, :, :, :] = (x_val[idx, :, :, :] - np.amin(
x_val[idx, :, :, :])) / (np.amax(x_val[idx, :, :, :]) -
np.amin(x_val[idx, :, :, :]))
# check if the image is already normalized
logger.info(
'Maximum value in x_train for the 1st image (to check normalization): {0}'
.format(np.amax(x_train[0, :, :, :])))
logger.info(
'Maximum value in x_val for the 1st image (to check normalization): {0}'
.format(np.amax(x_val[0, :, :, :])))
# convert class vectors to binary class matrices
nb_classes = len(set(y_train))
nb_classes_val = len(set(y_val))
if nb_classes_val != nb_classes:
raise ValueError(
"Different number of unique classes in training and validation set: {} vs {}."
"Training set unique classes: {}"
"Validation set unique classes: {}".format(nb_classes,
nb_classes_val,
set(y_train),
set(y_val)))
y_train = np_utils.to_categorical(y_train, nb_classes)
y_val = np_utils.to_categorical(y_val, nb_classes)
logger.info('Loading and formatting of data completed.')
# return the Keras model
model = partial_model_architecture(n_rows=img_width,
n_columns=img_height,
img_channels=img_channels,
nb_classes=nb_classes)
model.summary()
# serialize model to JSON
model_json = model.to_json()
with open(filename_no_ext + ".json", "w") as json_file:
json_file.write(model_json)
callbacks = []
csv_logger = CSVLogger(training_log_file_path, separator=',', append=False)
save_model_per_epoch = ModelCheckpoint(filename_no_ext + ".h5",
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max',
period=1)
callbacks.append(csv_logger)
callbacks.append(save_model_per_epoch)
# if you are running on Notebook
if configs['runtime']['isBeaker']:
callbacks.append(
TQDMNotebookCallback(leave_inner=True, leave_outer=True))
else:
callbacks.append(TQDMCallback(leave_inner=True, leave_outer=True))
if early_stopping:
EarlyStopping(monitor='val_loss',
min_delta=0.001,
patience=1,
verbose=0,
mode='auto')
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, decay=0.0)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(x_val, y_val),
shuffle=True,
verbose=0,
callbacks=callbacks)
# serialize weights to HDF5
model.save(filename_no_ext + ".h5")
logger.info("Model saved to disk.")
logger.info("Filename: {0}".format(filename_no_ext))
del model
model.summary()
## NO SE PUEDE USAR TENSORBOARD CON DATOS DE VALIDACION QUE SEAN GENERADORES
tbCallBack = TensorBoard(log_dir='./gdrive/My Drive/TFM/notebooks/logs_lstmsimple/',
update_freq='batch', batch_size=1, histogram_freq=0,
write_graph=True, write_images=True,
write_grads=False)
checkpoints = ModelCheckpoint('./checkpoints_recurrente_fully/recurrente_weights.{epoch:02d}-{val_loss:.2f}.hdf5', monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
history = model.fit_generator(frame_generator_train(batch_size, path_imagenes, train_data),
validation_data = frame_generator_validation(batch_size, path_imagenes, validation_data),
validation_steps=int(len(validation_data)/batch_size), max_queue_size=1, verbose=2,
use_multiprocessing=True, workers=1, epochs=10, steps_per_epoch=int(len(train_data)/batch_size),
shuffle=False, callbacks=[TQDMNotebookCallback(), checkpoints])#, checkpoints, tbCallBack])
"""# TESTEO"""
prediction = model.predict_generator(frame_generator_test(batch_size, path_imagenes, test_data), steps = len(test_data))
mean_squared_error(ytest, prediction)
alv = ytest.astype(float)
alv2 = np.expand_dims(alv, 1)
from scipy.stats.stats import pearsonr
pearsonr(alv2, prediction)
pyplot.figure(figsize=(20,10))
# Instantiate the model
model = PConvUnet(vgg_weights='./data/pytorch_vgg16.h5')
FOLDER = './data/logs/C2/'
# Run training for certain amount of epochs
model.fit_generator(
train_generator,
steps_per_epoch=500,
validation_data=val_generator,
validation_steps=50,
epochs=50,
verbose=0,
callbacks=[
TensorBoard(
log_dir=FOLDER,
write_graph=True
),
ModelCheckpoint(
FOLDER+'weights.{epoch:02d}-{loss:.2f}.h5',
monitor='val_loss',
save_best_only=True,
save_weights_only=True
),
LambdaCallback(
on_epoch_end=lambda epoch, logs: plot_callback(model)
),
TQDMNotebookCallback()
]
)
def train_model(dataset_directory: str,
model_name: str,
width: int,
height: int,
output_name: str = None):
print("Loading configuration and data-readers...")
start_time = time()
number_of_classes = len(
os.listdir(os.path.join(dataset_directory, "training")))
training_configuration = ConfigurationFactory.get_configuration_by_name(
model_name, width, height, number_of_classes)
train_generator = ImageDataGenerator(
rotation_range=training_configuration.rotation_range,
zoom_range=training_configuration.zoom_range)
training_data_generator = DirectoryIterator(
directory=os.path.join(dataset_directory, "training"),
image_data_generator=train_generator,
target_size=(training_configuration.input_image_rows,
training_configuration.input_image_columns),
batch_size=training_configuration.training_minibatch_size,
)
training_steps_per_epoch = np.math.ceil(training_data_generator.samples /
training_data_generator.batch_size)
validation_generator = ImageDataGenerator()
validation_data_generator = DirectoryIterator(
directory=os.path.join(dataset_directory, "validation"),
image_data_generator=validation_generator,
target_size=(training_configuration.input_image_rows,
training_configuration.input_image_columns),
batch_size=training_configuration.training_minibatch_size)
validation_steps_per_epoch = np.math.ceil(
validation_data_generator.samples /
validation_data_generator.batch_size)
test_generator = ImageDataGenerator()
test_data_generator = DirectoryIterator(
directory=os.path.join(dataset_directory, "test"),
image_data_generator=test_generator,
target_size=(training_configuration.input_image_rows,
training_configuration.input_image_columns),
batch_size=training_configuration.training_minibatch_size,
shuffle=False)
test_steps_per_epoch = np.math.ceil(test_data_generator.samples /
test_data_generator.batch_size)
model = training_configuration.classifier()
model.summary()
print("Model {0} loaded.".format(training_configuration.name()))
print(training_configuration.summary())
start_of_training = datetime.date.today()
if output_name is None:
best_model_path = "{0}_{1}.h5".format(start_of_training,
training_configuration.name())
else:
best_model_path = "{0}.h5".format(output_name)
monitor_variable = 'val_acc'
model_checkpoint = ModelCheckpoint(best_model_path,
monitor=monitor_variable,
save_best_only=True,
verbose=1)
early_stop = EarlyStopping(
monitor=monitor_variable,
patience=training_configuration.number_of_epochs_before_early_stopping,
verbose=1)
learning_rate_reduction = ReduceLROnPlateau(
monitor=monitor_variable,
patience=training_configuration.
number_of_epochs_before_reducing_learning_rate,
verbose=1,
factor=training_configuration.learning_rate_reduction_factor,
min_lr=training_configuration.minimum_learning_rate)
if output_name is None:
log_directory = "./logs/{0}_{1}/".format(start_of_training,
training_configuration.name())
else:
log_directory = "./logs/{0}/".format(output_name)
tensorboard_callback = TensorBoard(
log_dir=log_directory,
batch_size=training_configuration.training_minibatch_size)
if running_inside_jupyter_notebook():
callbacks = [
model_checkpoint, early_stop, tensorboard_callback,
learning_rate_reduction,
TQDMNotebookCallback()
]
else:
callbacks = [
model_checkpoint, early_stop, tensorboard_callback,
learning_rate_reduction,
TQDMCallback()
]
class_weight_calculator = ClassWeightCalculator()
balancing_method = "skBalance"
class_weights = class_weight_calculator.calculate_class_weights(
dataset_directory,
method=balancing_method,
class_indices=training_data_generator.class_indices)
if balancing_method is not None:
print(
"Using {0} method for obtaining class weights to compensate for an unbalanced dataset."
.format(balancing_method))
print("Training on dataset...")
history = model.fit_generator(
generator=training_data_generator,
steps_per_epoch=training_steps_per_epoch,
epochs=training_configuration.number_of_epochs,
callbacks=callbacks,
validation_data=validation_data_generator,
validation_steps=validation_steps_per_epoch,
class_weight=class_weights,
verbose=0)
print("Loading best model from check-point and testing...")
best_model = keras.models.load_model(best_model_path)
test_data_generator.reset()
file_names = test_data_generator.filenames
class_labels = os.listdir(os.path.join(dataset_directory, "test"))
# Notice that some classes have so few elements, that they are not present in the test-set and do not
# appear in the final report. To obtain the correct classes, we have to enumerate all non-empty class
# directories inside the test-folder and use them as labels
names_of_classes_with_test_data = [
class_name for class_name in class_labels
if os.listdir(os.path.join(dataset_directory, "test", class_name))
]
true_classes = test_data_generator.classes
predictions = best_model.predict_generator(test_data_generator,
steps=test_steps_per_epoch)
predicted_classes = numpy.argmax(predictions, axis=1)
test_data_generator.reset()
evaluation = best_model.evaluate_generator(test_data_generator,
steps=test_steps_per_epoch)
classification_accuracy = 0
print("Reporting classification statistics with micro average")
report = sklearn_reporting.classification_report(
true_classes,
predicted_classes,
digits=3,
target_names=names_of_classes_with_test_data,
average='micro')
print(report)
print("Reporting classification statistics with macro average")
report = sklearn_reporting.classification_report(
true_classes,
predicted_classes,
digits=3,
target_names=names_of_classes_with_test_data,
average='macro')
print(report)
print("Reporting classification statistics with weighted average")
report = sklearn_reporting.classification_report(
true_classes,
predicted_classes,
digits=3,
target_names=names_of_classes_with_test_data,
average='weighted')
print(report)
indices_of_misclassified_files = [
i for i, e in enumerate(true_classes - predicted_classes) if e != 0
]
misclassified_files = [
file_names[i] for i in indices_of_misclassified_files
]
misclassified_files_actual_prediction_indices = [
predicted_classes[i] for i in indices_of_misclassified_files
]
misclassified_files_actual_prediction_classes = [
class_labels[i] for i in misclassified_files_actual_prediction_indices
]
print("Misclassified files:")
for i in range(len(misclassified_files)):
print("\t{0} is incorrectly classified as {1}".format(
misclassified_files[i],
misclassified_files_actual_prediction_classes[i]))
for i in range(len(best_model.metrics_names)):
current_metric = best_model.metrics_names[i]
print("{0}: {1:.5f}".format(current_metric, evaluation[i]))
if current_metric == 'acc' or current_metric == 'output_class_acc':
classification_accuracy = evaluation[i]
print("Total Accuracy: {0:0.5f}%".format(classification_accuracy * 100))
print("Total Error: {0:0.5f}%".format((1 - classification_accuracy) * 100))
end_time = time()
print("Execution time: {0:.1f}s".format(end_time - start_time))
training_result_image = "{1}_{0}_{2:.1f}p.png".format(
training_configuration.name(), start_of_training,
classification_accuracy * 100)
TrainingHistoryPlotter.plot_history(history, training_result_image)
def load_bi_gru_model(x_train, y_train, x_train_dev, y_train_dev, params):
"""
The callback method that will be used from the Talos API in order to
generate a configurable Keras RNN Model with Bidirection GRUs.
:return (tuple): A tuple with the history object of the model train and the ,
generated Keras model itself.
"""
inputs = Input((MAX_SEQUENCE_LENGTH, ))
EMBEDDING_DIM = params.get('embedding_dim')
GRU_SIZE = params.get('gru_size', 200)
DENSE = params.get('dense', 300)
N_CLASSES = y_train.shape[1]
embeddings_matrix_path = os.path.join(DATA_DIR,
params['embeddings_matrix_path'])
with open(embeddings_matrix_path, 'rb') as embeddings_matrix_pickle:
embeddings_matrix = pickle.load(embeddings_matrix_pickle)
visualize_process = params.get('visualize_process', False)
visualize_process = (visualize_process if isinstance(
visualize_process, bool) else visualize_process == 'True')
with_early_stoping = params.get('with_early_stoping', True)
with_early_stoping = (with_early_stoping if isinstance(
with_early_stoping, bool) else with_early_stoping == 'True')
multistack_run = params.get('multistack_run', False)
multistack_run = (multistack_run if isinstance(multistack_run, bool) else
multistack_run == 'True')
# Apply default Callback methods
if with_early_stoping:
stopper = EarlyStopping(
monitor=params.get('early_stopping_config__monitor', 'val_f1'),
min_delta=params.get('early_stopping_config__min_delta', 0),
patience=params.get('early_stopping_config__patience', 5),
mode=params.get('early_stopping_config__mode', 'auto'),
)
default_callbacks = [stopper]
else:
default_callbacks = []
# Apply extra monitoring Callback methods
if visualize_process:
checkpoint = ModelCheckpoint(params['model_type'],
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
checkpoint2 = ModelCheckpoint(params['model_type'],
monitor='val_f1',
verbose=1,
save_best_only=True,
mode='max')
extra_callbacks = [checkpoint, TQDMNotebookCallback(), checkpoint2]
else:
extra_callbacks = []
# Add an embedding layer with 0.2 dropout probability
embeddings = Embedding(MAX_WORDS + 2,
EMBEDDING_DIM,
weights=[embeddings_matrix],
input_length=MAX_SEQUENCE_LENGTH,
mask_zero=True,
trainable=False)(inputs)
drop_emb = Dropout(params['embeddings_dropout'])(embeddings)
# add a bidirectional gru layer with variational (recurrent) dropout
bi_gru = Bidirectional(
GRU(GRU_SIZE,
return_sequences=True,
recurrent_dropout=params['var_dropout']))(drop_emb)
if multistack_run:
# In Case of multistack RNN
deep_attention_entry = GRU(
GRU_SIZE,
return_sequences=True,
recurrent_dropout=params['var_dropout'])(bi_gru)
else:
deep_attention_entry = bi_gru
# add a deep self attention layer
x, attn = DeepAttention(return_attention=True)(deep_attention_entry)
# add a hidden MLP layer
drop = Dropout(params["mlp_dropout"])(x)
out = Dense(DENSE, activation=params['rnn_activation'])(x)
# add the output MLP layer
out = Dense(N_CLASSES, activation=params['mlp_activation'])(out)
model = Model(inputs, out)
if visualize_process:
print(model.summary())
model.compile(
loss='categorical_crossentropy',
optimizer=params['optimizer'],
metrics=[precision, recall, f1, accuracy, 'categorical_accuracy'])
history = model.fit(x_train,
y_train,
batch_size=params.get('batch_size', 32),
epochs=params.get('epochs', 10),
verbose=0,
callbacks=default_callbacks + extra_callbacks,
validation_data=(x_train_dev, y_train_dev),
shuffle=True)
return history, model
# In[20]:
modelCallback = ModelCheckpoint(
'model_checkpoint_advsmall_sos_eos_unk_att.{epoch:03d}-{loss:.3f}.hdf5',
monitor='val_loss',
verbose=1,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=1)
# creating different sets of Params to easily import into the model at train time
params = {
'verbose': 1,
'callbacks':
[TQDMNotebookCallback(), reduce_LR, early_stopping, modelCallback]
}
params2 = {
'verbose': 1,
'callbacks': [LRDrop,
TQDMNotebookCallback(), reduce_LR, early_stopping]
}
params3 = {
'verbose': 1,
'callbacks': [LRDrop,
TQDMNotebookCallback(), reduce_LR, early_stopping]
}
# #### Set some parameters for the model
# In[13]:
from keras_tqdm import TQDMNotebookCallback
from keras.optimizers import SGD, Adam, Nadam, RMSprop
### Start Here with pre-loaded images
X = np.load('mdata_X.npy')
y = np.load('mdata_y.npy')
### base VGG-16 model
model_base = VGG16(include_top=False,
weights='imagenet',
input_shape=(224, 224, 3))
### Add additional custom-designed top-layers
x = model_base.layers[-1].output
x = Flatten()(x)
x = Dense(512, init='orthogonal', activation='relu')(x)
x = Dense(4, init='orthogonal', activation='softmax')(x)
### Train, Validate, and Test the Model
model2 = Model(model_base.inputs, x)
model2.compile(loss='sparse_categorical_crossentropy',
metrics=['accuracy'],
optimizer=Adam(lr=0.0001))
model2.fit(X,
y,
batch_size=64,
epochs=10,
validation_split=0.2,
verbose=0,
callbacks=[TQDMNotebookCallback(leave_inner=True)])
y_pred = model2.predict(X)
#
# * **ReduceLrOnPlateau** - If the model is near a minimum and the learning rate is too high, then the model will circle around that minimum without ever reaching it. This callback will allow us to reduce the learning rate when the validation loss stops improving, allowing us to reach the optimal point.
# * **CsvLogger** - This lets us log the output of the model after each epoch, which will allow us to track the progress without needing to use the console.
# * **ModelCheckpoint** - Generally, we will want to use the model that has the lowest loss on the validation set. This callback will save the model each time the validation loss improves.
# * **EarlyStopping** - We will want to stop training when the validation loss stops improving. Otherwise, we risk overfitting. This monitor will detect when the validation loss stops improving, and will stop the training process when that occurs.
# In[7]:
plateau_callback = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=0.0001, verbose=1)
checkpoint_filepath = os.path.join(MODEL_OUTPUT_DIR, 'models', '{0}_model.{1}-{2}.h5'.format('model', '{epoch:02d}', '{val_loss:.7f}'))
checkAndCreateDir(checkpoint_filepath)
checkpoint_callback = ModelCheckpoint(checkpoint_filepath, save_best_only=True, verbose=1)
csv_callback = CSVLogger(os.path.join(MODEL_OUTPUT_DIR, 'training_log.csv'))
early_stopping_callback = EarlyStopping(monitor='val_loss', patience=10, verbose=1)
callbacks=[plateau_callback, csv_callback, checkpoint_callback, early_stopping_callback, TQDMNotebookCallback()]
# It's time to train the model! With the default setup, this model takes ~45 min to train on an NVidia GTX970 GPU. Note: sometimes the model will get stuck with a constant validation loss for up to 7 epochs. If left to run, the model should terminate with a validation loss of approximately .0003.
# In[8]:
history = model.fit_generator(train_generator, steps_per_epoch=num_train_examples//batch_size, epochs=500, callbacks=callbacks, validation_data=eval_generator, validation_steps=num_eval_examples//batch_size, verbose=2)
# Let's do a quick sanity check. We'll load a few training images and compare the labels and the predictions. These should be very close in value if our model has learned properly.
# In[9]: