def class_avg_chainthaw(model,
nb_classes,
train,
val,
test,
batch_size,
loss,
epoch_size,
nb_epochs,
checkpoint_weight_path,
f1_init_weight_path,
patience=5,
initial_lr=0.001,
next_lr=0.0001,
seed=None,
verbose=True):
""" Finetunes given model using chain-thaw and evaluates using F1.
For a dataset with multiple classes, the model is trained once for
each class, relabeling those classes into a binary classification task.
The result is an average of all F1 scores for each class.
# Arguments:
model: Model to be finetuned.
nb_classes: Number of classes in the given dataset.
train: Training data, given as a tuple of (inputs, outputs)
val: Validation data, given as a tuple of (inputs, outputs)
test: Testing data, given as a tuple of (inputs, outputs)
batch_size: Batch size.
loss: Loss function to be used during training.
epoch_size: Number of samples in an epoch.
nb_epochs: Number of epochs.
checkpoint_weight_path: Filepath where weights will be checkpointed to
during training. This file will be rewritten by the function.
f1_init_weight_path: Filepath where weights will be saved to and
reloaded from before training each class. This ensures that
each class is trained independently. This file will be rewritten.
initial_lr: Initial learning rate. Will only be used for the first
training step (i.e. the softmax layer)
next_lr: Learning rate for every subsequent step.
seed: Random number generator seed.
verbose: Verbosity flag.
# Returns:
Averaged F1 score.
"""
# Unpack args
X_train, y_train = train
X_val, y_val = val
X_test, y_test = test
total_f1 = 0
nb_iter = nb_classes if nb_classes > 2 else 1
model.save_weights(f1_init_weight_path)
for i in range(nb_iter):
if verbose:
print('Iteration number {}/{}'.format(i + 1, nb_iter))
model.load_weights(f1_init_weight_path, by_name=False)
y_train_new, y_val_new, y_test_new = prepare_labels(
y_train, y_val, y_test, i, nb_classes)
train_gen, X_val_resamp, y_val_resamp = \
prepare_generators(X_train, y_train_new, X_val, y_val_new,
batch_size, epoch_size)
if verbose:
print("Training..")
callbacks = finetuning_callbacks(checkpoint_weight_path,
patience=patience)
# Train using chain-thaw
train_by_chain_thaw(model=model,
train_gen=train_gen,
val_data=(X_val_resamp, y_val_resamp),
loss=loss,
callbacks=callbacks,
epoch_size=epoch_size,
nb_epochs=nb_epochs,
checkpoint_weight_path=checkpoint_weight_path,
initial_lr=initial_lr,
next_lr=next_lr,
batch_size=batch_size,
verbose=verbose)
# Evaluate
y_pred_val = np.array(model.predict(X_val, batch_size=batch_size))
y_pred_test = np.array(model.predict(X_test, batch_size=batch_size))
f1_test, best_t = find_f1_threshold(y_val_new, y_pred_val, y_test_new,
y_pred_test)
if verbose:
print('f1_test: {}'.format(f1_test))
print('best_t: {}'.format(best_t))
total_f1 += f1_test
return total_f1 / nb_iter
def class_avg_tune_trainable(model,
nb_classes,
train,
val,
test,
epoch_size,
nb_epochs,
batch_size,
init_weight_path,
checkpoint_weight_path,
patience=5,
verbose=True):
""" Finetunes the given model using the F1 measure.
# Arguments:
model: Model to be finetuned.
nb_classes: Number of classes in the given dataset.
train: Training data, given as a tuple of (inputs, outputs)
val: Validation data, given as a tuple of (inputs, outputs)
test: Testing data, given as a tuple of (inputs, outputs)
epoch_size: Number of samples in an epoch.
nb_epochs: Number of epochs.
batch_size: Batch size.
init_weight_path: Filepath where weights will be initially saved before
training each class. This file will be rewritten by the function.
checkpoint_weight_path: Filepath where weights will be checkpointed to
during training. This file will be rewritten by the function.
verbose: Verbosity flag.
# Returns:
F1 score of the trained model
"""
total_f1 = 0
nb_iter = nb_classes if nb_classes > 2 else 1
# Unpack args
X_train, y_train = train
X_val, y_val = val
X_test, y_test = test
# Save and reload initial weights after running for
# each class to avoid learning across classes
model.save_weights(init_weight_path)
for i in range(nb_iter):
if verbose:
print('Iteration number {}/{}'.format(i + 1, nb_iter))
model.load_weights(init_weight_path, by_name=False)
y_train_new, y_val_new, y_test_new = prepare_labels(
y_train, y_val, y_test, i, nb_classes)
train_gen, X_val_resamp, y_val_resamp = \
prepare_generators(X_train, y_train_new, X_val, y_val_new,
batch_size, epoch_size)
if verbose:
print("Training..")
callbacks = finetuning_callbacks(checkpoint_weight_path, patience)
steps = int(epoch_size / batch_size)
model.fit_generator(train_gen,
steps_per_epoch=steps,
max_q_size=2,
epochs=nb_epochs,
validation_data=(X_val_resamp, y_val_resamp),
callbacks=callbacks,
verbose=0)
# Reload the best weights found to avoid overfitting
# Wait a bit to allow proper closing of weights file
sleep(1)
model.load_weights(checkpoint_weight_path, by_name=False)
# Evaluate
y_pred_val = np.array(model.predict(X_val, batch_size=batch_size))
y_pred_test = np.array(model.predict(X_test, batch_size=batch_size))
f1_test, best_t = find_f1_threshold(y_val_new, y_pred_val, y_test_new,
y_pred_test)
if verbose:
print('f1_test: {}'.format(f1_test))
print('best_t: {}'.format(best_t))
total_f1 += f1_test
return total_f1 / nb_iter
def class_avg_chainthaw(model, nb_classes, train, val, test, batch_size,
loss, epoch_size, nb_epochs, checkpoint_weight_path,
f1_init_weight_path, patience=5,
initial_lr=0.001, next_lr=0.0001,
seed=None, verbose=True):
""" Finetunes given model using chain-thaw and evaluates using F1.
For a dataset with multiple classes, the model is trained once for
each class, relabeling those classes into a binary classification task.
The result is an average of all F1 scores for each class.
# Arguments:
model: Model to be finetuned.
nb_classes: Number of classes in the given dataset.
train: Training data, given as a tuple of (inputs, outputs)
val: Validation data, given as a tuple of (inputs, outputs)
test: Testing data, given as a tuple of (inputs, outputs)
batch_size: Batch size.
loss: Loss function to be used during training.
epoch_size: Number of samples in an epoch.
nb_epochs: Number of epochs.
checkpoint_weight_path: Filepath where weights will be checkpointed to
during training. This file will be rewritten by the function.
f1_init_weight_path: Filepath where weights will be saved to and
reloaded from before training each class. This ensures that
each class is trained independently. This file will be rewritten.
initial_lr: Initial learning rate. Will only be used for the first
training step (i.e. the softmax layer)
next_lr: Learning rate for every subsequent step.
seed: Random number generator seed.
verbose: Verbosity flag.
# Returns:
Averaged F1 score.
"""
# Unpack args
X_train, y_train = train
X_val, y_val = val
X_test, y_test = test
total_f1 = 0
nb_iter = nb_classes if nb_classes > 2 else 1
model.save_weights(f1_init_weight_path)
for i in range(nb_iter):
if verbose:
print('Iteration number {}/{}'.format(i + 1, nb_iter))
model.load_weights(f1_init_weight_path, by_name=False)
y_train_new, y_val_new, y_test_new = prepare_labels(y_train, y_val,
y_test, i, nb_classes)
train_gen, X_val_resamp, y_val_resamp = \
prepare_generators(X_train, y_train_new, X_val, y_val_new,
batch_size, epoch_size)
if verbose:
print("Training..")
callbacks = finetuning_callbacks(checkpoint_weight_path, patience=patience)
# Train using chain-thaw
train_by_chain_thaw(model=model, train_gen=train_gen,
val_data=(X_val_resamp, y_val_resamp),
loss=loss, callbacks=callbacks,
epoch_size=epoch_size, nb_epochs=nb_epochs,
checkpoint_weight_path=checkpoint_weight_path,
initial_lr=initial_lr, next_lr=next_lr,
batch_size=batch_size, verbose=verbose)
# Evaluate
y_pred_val = np.array(model.predict(X_val, batch_size=batch_size))
y_pred_test = np.array(model.predict(X_test, batch_size=batch_size))
f1_test, best_t = find_f1_threshold(y_val_new, y_pred_val,
y_test_new, y_pred_test)
if verbose:
print('f1_test: {}'.format(f1_test))
print('best_t: {}'.format(best_t))
total_f1 += f1_test
return total_f1 / nb_iter
#please modify the checkpoint_weight_path
checkpoint_weight_path = '/storage1/user/ss/tmoji_ori/weight/tmoji-lstm-checkpoint-%s-h-1.hdf5' % cur_lan
idx_shuffle = list(range(input_len))
np.random.shuffle(idx_shuffle)
idx_train, idx_val, idx_test = idx_shuffle[ :int(input_len*0.7) ], idx_shuffle[int(input_len*0.7):int(input_len*0.9)], idx_shuffle[int(input_len*0.9):]
(X_train, y_train) = (input_vec[idx_train], input_label[idx_train])
(X_val, y_val) = (input_vec[idx_val], input_label[idx_val])
(X_test, y_test) = (input_vec[idx_test], input_label[idx_test])
LSTM_hidden = config['LSTM_hidden']
LSTM_drop = config['LSTM_dropout']
final_dropout_rate = config['final_dropout_rate']
embed_dropout_rate = config['embed_dropout_rate']
high = config['high']
load_embedding = config['load_embedding']
embed_dim = config['embed_dim']
model = elsa_architecture(nb_classes=64, nb_tokens=nb_tokens, maxlen=maxlen, final_dropout_rate=final_dropout_rate, embed_dropout_rate=embed_dropout_rate,
load_embedding=True, pre_embedding=word_vec, high=high, embed_dim=embed_dim)
model.summary()
if optim == 'adam':
adam = Adam(clipnorm=1, lr=lr)
model.compile(loss=loss, optimizer=adam, metrics=['accuracy'])
elif optim == 'rmsprop':
model.compile(loss=loss, optimizer='rmsprop', metrics=['accuracy'])
callbacks = finetuning_callbacks(checkpoint_weight_path, patience, verbose=1)
for i in range(2):
train_gen = sampling_generator(X_train, y_train, batch_size, upsample=False, epoch_size=epoch_size)
model.fit_generator(train_gen, steps_per_epoch=steps, epochs=nb_epochs,validation_data=(X_val, y_val),validation_steps=steps, callbacks=callbacks, verbose=True)
_, acc = model.evaluate(X_test, y_test, batch_size=batch_size, verbose=0)
print(acc)
def class_avg_tune_trainable(model, nb_classes, train, val, test, epoch_size,
nb_epochs, batch_size, init_weight_path,
checkpoint_weight_path, patience=5,
verbose=True):
""" Finetunes the given model using the F1 measure.
# Arguments:
model: Model to be finetuned.
nb_classes: Number of classes in the given dataset.
train: Training data, given as a tuple of (inputs, outputs)
val: Validation data, given as a tuple of (inputs, outputs)
test: Testing data, given as a tuple of (inputs, outputs)
epoch_size: Number of samples in an epoch.
nb_epochs: Number of epochs.
batch_size: Batch size.
init_weight_path: Filepath where weights will be initially saved before
training each class. This file will be rewritten by the function.
checkpoint_weight_path: Filepath where weights will be checkpointed to
during training. This file will be rewritten by the function.
verbose: Verbosity flag.
# Returns:
F1 score of the trained model
"""
total_f1 = 0
nb_iter = nb_classes if nb_classes > 2 else 1
# Unpack args
X_train, y_train = train
X_val, y_val = val
X_test, y_test = test
# Save and reload initial weights after running for
# each class to avoid learning across classes
model.save_weights(init_weight_path)
for i in range(nb_iter):
if verbose:
print('Iteration number {}/{}'.format(i + 1, nb_iter))
model.load_weights(init_weight_path, by_name=False)
y_train_new, y_val_new, y_test_new = prepare_labels(y_train, y_val,
y_test, i, nb_classes)
train_gen, X_val_resamp, y_val_resamp = \
prepare_generators(X_train, y_train_new, X_val, y_val_new,
batch_size, epoch_size)
if verbose:
print("Training..")
callbacks = finetuning_callbacks(checkpoint_weight_path, patience)
steps = int(epoch_size / batch_size)
model.fit_generator(train_gen, steps_per_epoch=steps,
max_q_size=2, epochs=nb_epochs,
validation_data=(X_val_resamp, y_val_resamp),
callbacks=callbacks, verbose=0)
# Reload the best weights found to avoid overfitting
# Wait a bit to allow proper closing of weights file
sleep(1)
model.load_weights(checkpoint_weight_path, by_name=False)
# Evaluate
y_pred_val = np.array(model.predict(X_val, batch_size=batch_size))
y_pred_test = np.array(model.predict(X_test, batch_size=batch_size))
f1_test, best_t = find_f1_threshold(y_val_new, y_pred_val,
y_test_new, y_pred_test)
if verbose:
print('f1_test: {}'.format(f1_test))
print('best_t: {}'.format(best_t))
total_f1 += f1_test
return total_f1 / nb_iter
