train_model.fit( pretrain_generator.generator(), steps_per_epoch=len(pretrain_generator), epochs=pretrain_epochs, callbacks=[checkpoint, csv_logger], ) # build task fine-tune model # reload weights without mlm # bert_without_mlm = build_transformer_model(checkpoint_path=model_saved_path, # config_path=config_path, with_mlm=False) idx = 11 feed_forward_name = 'Transformer-%d-FeedForward' % idx bert_without_mlm = bert.layers[feed_forward_name] output = Lambda(lambda x: x[:, 0])(bert_without_mlm.output) output = Dense(num_classes, activation='softmax')(output) model = Model(bert.inputs, output) model.summary() model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam(fine_tune_lr), metrics=['acc']) evaluator = Evaluator() model.fit_generator(train_generator.generator(), steps_per_epoch=len(train_generator), epochs=fine_tune_epochs, callbacks=[evaluator])
dense = ScaleDense(lr_multiplier=5, units=num_classes, activation='softmax', weights=predecessor_model.layers[-1].get_weights()) output = dense(output) predecessor_3_model = Model(predecessor_model.inputs, output) predecessor_3_model.compile( loss='sparse_categorical_crossentropy', optimizer=Adam(1e-5), # 用足够小的学习率 metrics=['sparse_categorical_accuracy'], ) predecessor_3_model.summary() if __name__ == '__main__': epochs = 5 # 训练predecessor predecessor_evaluator = Evaluator('best_predecessor.weights') predecessor_model.fit_generator(train_generator.generator(), steps_per_epoch=len(train_generator), epochs=epochs, callbacks=[predecessor_evaluator]) # 训练predecessor_3_model predecessor_model.load_weights('best_predecessor.weights') predecessor_3_evaluator = Evaluator('best_predecessor_3.weights') predecessor_3_model.fit_generator(train_generator.generator(), steps_per_epoch=len(train_generator), epochs=10, callbacks=[predecessor_3_evaluator])
print('test: f1: %.5f, precision: %.5f, recall: %.5f\n' % (f1, precision, recall)) if __name__ == '__main__': train_generator = data_generator(train_data, batch_size) predecessor_model_name = 'predecessor_best.weights' predecessor_evaluator = Evaluator(predecessor_model_name) predecessor_model.fit(train_generator.generator(), steps_per_epoch=len(train_generator), epochs=epochs, callbacks=[predecessor_evaluator]) theseus_model_name = 'theseus_best.weights' theseus_evaluator = Evaluator(theseus_model_name) theseus_model.fit_generator(train_generator.generator(), steps_per_epoch=len(train_generator), epochs=epochs * 2, callbacks=[theseus_evaluator]) theseus_model.load_weights(theseus_model_name) successor_model_name = 'successor_best.weights' successor_evaluator = Evaluator(successor_model_name) successor_model.fit_generator(train_generator.generator(), steps_per_epoch=len(train_generator), epochs=epochs, callbacks=[successor_evaluator])
total += len(y_true) right += (y_true == y_pred).sum() print(total, right) return right / total class Evaluator(keras.callbacks.Callback): def __init__(self, save_path='best_model.weights'): self.best_val_acc = 0. self.save_path = save_path def on_epoch_end(self, epoch, logs=None): val_acc = evaluate() if val_acc > self.best_val_acc: self.best_val_acc = val_acc self.model.save_weights(self.save_path) print('current acc :{}, best val acc: {}'.format( val_acc, self.best_val_acc)) if __name__ == '__main__': evaluator = Evaluator() train_model.fit_generator(train_generator.generator(), steps_per_epoch=len(train_generator), epochs=5, callbacks=[evaluator]) else: classifier.load_weights('best_model.weights')