def find_lr(data_index):
D = AutoSpeechDataset(
os.path.join(
r"/home/chengfeng/autospeech/data/data0{}".format(data_index),
'data0{}.data'.format(data_index)))
D.read_dataset()
metadata = D.get_metadata()
x_train, y_train = D.get_train()
my_model = CrnnModel()
x_train = my_model.preprocess_data(x_train)
log(f'x_train shape: {x_train.shape}; y_train shape: {y_train.shape}')
y_train = ohe2cat(y_train)
my_model.init_model(input_shape=x_train.shape[1:],
num_classes=metadata[CLASS_NUM])
lr_finder = LRFinder(my_model._model)
lr_finder.find(x_train,
y_train,
start_lr=0.0001,
end_lr=1,
batch_size=64,
epochs=200)
# Plot the loss, ignore 20 batches in the beginning and 5 in the end
lr_finder.plot_loss(n_skip_beginning=20, n_skip_end=5)
lr_finder.plot_loss_change(sma=20,
n_skip_beginning=20,
n_skip_end=5,
y_lim=(-0.01, 0.01))
def plot_lr(model, X_train, y_train):
"""
Plots learning rate vs loss for model on training set.
:param model: Keras model to find learning rate of.
:param X_train: Training data.
:param y_train: Training labels.
"""
lr_finder = LRFinder(model)
# Adam default LR = 0.001
lr_finder.find(X_train, y_train, start_lr=0.0001, end_lr=1)
lr_finder.plot_loss()
plt.show()
'similarity_pred': 'binary_crossentropy',
'final_pred': 'categorical_crossentropy'
}
metrics = {'similarity': 'accuracy', 'final_pred': 'accuracy'}
siamese_model_full.compile(optimizer="adagrad", loss=losses, metrics=metrics)
# In[ ]:
from keras_lr_finder import LRFinder
from math import ceil
training_steps = ceil(4825 / 8)
lr_finder = LRFinder(siamese_model_full)
lr_finder.find_generator(train_gen(),
0.00001,
1,
epochs=5,
steps_per_epoch=training_steps)
# In[ ]:
from keras.callbacks import ModelCheckpoint, EarlyStopping
from math import ceil
training_steps = ceil(4825 / 8)
validation_steps = ceil(603 / 8)
pre_file_path = "siamese-classifier-"
post_file_path = "--{epoch:02d}-{val_loss:.2f}.hdf5"
# 'x' is the array of input variables, 'y' is the q/pT
x_train, x_test, y_train, y_test, w_train, w_test, x_mask_train, x_mask_test, x_road_train, x_road_test = \
muon_data_split(infile_muon, reg_pt_scale=reg_pt_scale, test_size=0.31)
## Use ShuffleSplit as the CV iterator
#from nn_data import muon_data
#from sklearn.model_selection import ShuffleSplit
#x, y, w, x_mask, x_road = muon_data(infile_muon, reg_pt_scale=reg_pt_scale, correct_for_eta=False)
#cv = ShuffleSplit(n_splits=1, test_size=0.31)
# ______________________________________________________________________________
# Create LRFinder
from keras_lr_finder import LRFinder
epochs = 20
batch_size = 4096
model = create_model_sequential_bn2(nvariables=nvariables, lr=learning_rate, clipnorm=gradient_clip_norm,
l1_reg=l1_reg, l2_reg=l2_reg,
nodes1=30, nodes2=25, nodes3=20)
lr_finder = LRFinder(model)
lr_finder.find(x_train, y_train, 0.0001, 1, batch_size=batch_size, epochs=epochs)
print len(lr_finder.lrs)
#lr_finder.plot_loss(n_skip_beginning=100, n_skip_end=50)
#lr_finder.plot_loss_change(sma=100, n_skip_beginning=100, n_skip_end=50, y_lim=(-0.1, 0.1))
logger.info('DONE')
]
# Set containers for tests
test_learning_rates = []
test_losses = []
test_loss_changes = []
labels = []
# Perform each test
for test_optimizer, label in tests:
# Compile the model
model.compile(loss=loss_function,
optimizer=test_optimizer,
metrics=['accuracy'])
# Instantiate the Learning Rate Range Test / LR Finder
lr_finder = LRFinder(model)
# Perform the Learning Rate Range Test
outputs = lr_finder.find(X_train, y_train, start_lr=start_lr, end_lr=end_lr, batch_size=batch_size, epochs=no_epochs)
# Get values
learning_rates = lr_finder.lrs
losses = lr_finder.losses
loss_changes = []
# Compute smoothed loss changes
# Inspired by Keras LR Finder: https://github.com/surmenok/keras_lr_finder/blob/master/keras_lr_finder/lr_finder.py
for i in range(moving_average, len(learning_rates)):
loss_changes.append((losses[i] - losses[i - moving_average]) / moving_average)
# Append values to container
model.add(Dense(1))
model.compile(loss="mean_squared_error",
optimizer="adam",
metrics=["mean_absolute_error"])
#model.fit(x_train, y_train, epochs=0)
#custom callback to print learning rate
class MyCallback(callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
print(K.eval(self.model.optimizer.lr))
#learning rate search
# model is a Keras model
lr_finder = LRFinder(model)
#searches learning rates from 0.00001 to 1 and plots loss
lr_finder.find(x_train,
y_train,
start_lr=0.00001,
end_lr=1,
batch_size=64,
epochs=20)
lr_finder.plot_loss()
#gets a region to zoom in on
min_lr = float(input("Enter the lowest learning rate to show "))
max_lr = float(input("Enter the highest learning rate to show "))
#same again but with the new region
decay=0.0,
nesterov=False),
loss="MSE",
metrics=["MAE", "MSE"])
lrate = keras.callbacks.LearningRateScheduler(step_decay, verbose=1)
callbacks = [check_point, early_stopping, tbCallBack, lrate]
elif config.optimizer == "Adam" or config.optimizer == "adam":
model.compile(optimizer="Adam", loss="MSE", metrics=["MAE", "MSE"])
callbacks = [check_point, early_stopping, tbCallBack]
else:
raise
while config.batch_size > 1:
# To find a largest batch size that can be fit into GPU
try:
lr_finder = LRFinder(model)
lr_finder.find_generator(
generate_pair(x_train, config.batch_size, status="train"),
start_lr=0.00001,
end_lr=2,
epochs=5,
steps_per_epoch=x_train.shape[0] // config.batch_size,
)
print("\n\nBest learning rate = {}".format(
lr_finder.get_best_lr(sma=20)))
K.set_value(model.optimizer.lr, lr_finder.get_best_lr(sma=20))
model.fit_generator(
generate_pair(x_train, config.batch_size, status="train"),
validation_data=generate_pair(x_valid,
config.batch_size,
def train(self, train_data, devtest_data):
train_data.extend(devtest_data)
# x_train = [np.asarray(x[1].split()) for x in train_data]
x_train = [x[1] for x in train_data]
y_train = [x[0] for x in train_data]
y_train = np.asarray(y_train)
# finally, vectorize the text samples into a 2D integer tensor
self.tokenizer = Tokenizer(num_words=MAX_NUM_WORDS, char_level=False)
self.tokenizer.fit_on_texts(x_train)
sequences = self.tokenizer.texts_to_sequences(x_train)
word_index = self.tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = y_train[indices]
num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])
x_train = data[:-num_validation_samples]
y_train = labels[:-num_validation_samples]
x_val = data[-num_validation_samples:]
y_val = labels[-num_validation_samples:]
# prepare embedding matrix
num_words = min(MAX_NUM_WORDS, len(word_index) + 1)
embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))
for word, i in word_index.items():
# print(i)
# print(word)
if i >= MAX_NUM_WORDS:
continue
embedding_vector = self.embeddings_index.get(word)
# print(embedding_vector)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
print('Training model.')
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(256, 5, activation='relu')(embedded_sequences)
x = GlobalMaxPooling1D()(x)
x = Dense(256, activation='relu')(x)
x = Dropout(rate=0.5)(x)
preds = Dense(1, activation='sigmoid')(x)
self.model = Model(sequence_input, preds)
self.model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
plot_model(self.model, to_file='model.png')
self.model.summary()
self.model_output_dir = os.path.join(
'models', datetime.strftime(datetime.now(), "%Y-%m-%d_%H-%M-%S"))
filepath = os.path.join(
self.model_output_dir,
"weights-improvement-{epoch:02d}-{val_loss:.4f}.hdf5")
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0.0001,
patience=3,
verbose=0,
mode='auto')
prtensorboard_dir = os.path.join(self.model_output_dir,
'pr_tensorboard_logs')
lr_finder = LRFinder(min_lr=1e-5,
max_lr=1e-2,
steps_per_epoch=np.ceil(x_train.shape[0] /
BATCH_SIZE),
epochs=10)
self.model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=10,
callbacks=[
checkpoint, early_stopping,
PRTensorBoard(log_dir=prtensorboard_dir), lr_finder
],
validation_data=(x_val, y_val))
lr_finder.plot_loss()