train_model.fit(
pretrain_generator.generator(),
steps_per_epoch=len(pretrain_generator),
epochs=pretrain_epochs,
callbacks=[checkpoint, csv_logger],
)
# build task fine-tune model
# reload weights without mlm
# bert_without_mlm = build_transformer_model(checkpoint_path=model_saved_path,
# config_path=config_path, with_mlm=False)
idx = 11
feed_forward_name = 'Transformer-%d-FeedForward' % idx
bert_without_mlm = bert.layers[feed_forward_name]
output = Lambda(lambda x: x[:, 0])(bert_without_mlm.output)
output = Dense(num_classes, activation='softmax')(output)
model = Model(bert.inputs, output)
model.summary()
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(fine_tune_lr),
metrics=['acc'])
evaluator = Evaluator()
model.fit_generator(train_generator.generator(),
steps_per_epoch=len(train_generator),
epochs=fine_tune_epochs,
callbacks=[evaluator])
dense = ScaleDense(lr_multiplier=5,
units=num_classes,
activation='softmax',
weights=predecessor_model.layers[-1].get_weights())
output = dense(output)
predecessor_3_model = Model(predecessor_model.inputs, output)
predecessor_3_model.compile(
loss='sparse_categorical_crossentropy',
optimizer=Adam(1e-5), # 用足够小的学习率
metrics=['sparse_categorical_accuracy'],
)
predecessor_3_model.summary()
if __name__ == '__main__':
epochs = 5
# 训练predecessor
predecessor_evaluator = Evaluator('best_predecessor.weights')
predecessor_model.fit_generator(train_generator.generator(),
steps_per_epoch=len(train_generator),
epochs=epochs,
callbacks=[predecessor_evaluator])
# 训练predecessor_3_model
predecessor_model.load_weights('best_predecessor.weights')
predecessor_3_evaluator = Evaluator('best_predecessor_3.weights')
predecessor_3_model.fit_generator(train_generator.generator(),
steps_per_epoch=len(train_generator),
epochs=10,
callbacks=[predecessor_3_evaluator])
print('test: f1: %.5f, precision: %.5f, recall: %.5f\n' %
(f1, precision, recall))
if __name__ == '__main__':
train_generator = data_generator(train_data, batch_size)
predecessor_model_name = 'predecessor_best.weights'
predecessor_evaluator = Evaluator(predecessor_model_name)
predecessor_model.fit(train_generator.generator(),
steps_per_epoch=len(train_generator),
epochs=epochs,
callbacks=[predecessor_evaluator])
theseus_model_name = 'theseus_best.weights'
theseus_evaluator = Evaluator(theseus_model_name)
theseus_model.fit_generator(train_generator.generator(),
steps_per_epoch=len(train_generator),
epochs=epochs * 2,
callbacks=[theseus_evaluator])
theseus_model.load_weights(theseus_model_name)
successor_model_name = 'successor_best.weights'
successor_evaluator = Evaluator(successor_model_name)
successor_model.fit_generator(train_generator.generator(),
steps_per_epoch=len(train_generator),
epochs=epochs,
callbacks=[successor_evaluator])
total += len(y_true)
right += (y_true == y_pred).sum()
print(total, right)
return right / total
class Evaluator(keras.callbacks.Callback):
def __init__(self, save_path='best_model.weights'):
self.best_val_acc = 0.
self.save_path = save_path
def on_epoch_end(self, epoch, logs=None):
val_acc = evaluate()
if val_acc > self.best_val_acc:
self.best_val_acc = val_acc
self.model.save_weights(self.save_path)
print('current acc :{}, best val acc: {}'.format(
val_acc, self.best_val_acc))
if __name__ == '__main__':
evaluator = Evaluator()
train_model.fit_generator(train_generator.generator(),
steps_per_epoch=len(train_generator),
epochs=5,
callbacks=[evaluator])
else:
classifier.load_weights('best_model.weights')
