def objective(trial: optuna.trial.Trial) -> float: learning_rate = trial.suggest_float("learning_rate", 1e-5, 2e-5) model = BertForRace( pretrained_model='./model/bert-large-uncased', learning_rate=learning_rate, num_train_epochs=20, train_batch_size=4, train_all=True, ) dm = RACEDataModule( model_name_or_path='./model/bert-large-uncased', datasets_loader='./data/RACELocalLoader.py', train_batch_size=32, max_seq_length=128, num_workers=8, num_preprocess_processes=48, use_sentence_selection=True, ) trainer = pl.Trainer( gpus=-1 if torch.cuda.is_available() else None, # callbacks=[checkpoint_callback], amp_backend='native', amp_level='O2', precision=16, accelerator='ddp', gradient_clip_val=1.0, max_epochs=3, plugins='ddp_sharded', callbacks=[PyTorchLightningPruningCallback(trial, monitor="val_acc")], # limit_train_batches=0.1, # limit_val_batches=0.1, # accumulate_grad_batches=2, ) trainer.fit(model, dm) return trainer.callback_metrics["val_acc"].item()
if __name__ == '__main__': tb_logger = pl_loggers.TensorBoardLogger('./result/asc01/') pl.seed_everything(42) model = DUMAForRace( pretrained_model='./model/bert-large-uncased', learning_rate=2e-5, num_train_epochs=20, train_batch_size=4, train_all=True, use_bert_adam=True, ) dm = RACEDataModule( model_name_or_path='./model/bert-large-uncased', datasets_loader='./data/RACELocalLoader.py', train_batch_size=4, max_seq_length=512, num_workers=8, num_preprocess_processes=48, use_sentence_selection=False, best_k_sentences=5, ) # checkpoint_callback = ModelCheckpoint( # dirpath='./result/checkpoints/', # filename='epoch{epoch:02d}', # save_top_k=-1, # ) trainer = pl.Trainer( logger=tb_logger, gpus=-1 if torch.cuda.is_available() else None, # callbacks=[checkpoint_callback], amp_backend='native', amp_level='O2',
self.log('val_loss', val_loss) def test_step(self, batch, batch_idx): loss, correct_count = self.compute(batch) return { "test_loss": loss, "correct_count": correct_count, "batch_size": len(batch['label']) } def test_epoch_end(self, outputs: List[Any]) -> None: test_acc = sum([out["correct_count"] for out in outputs]).float() / sum(out["batch_size"] for out in outputs) test_loss = sum([out["test_loss"] for out in outputs]) / len(outputs) self.log('test_acc', test_acc) self.log('test_loss', test_loss) if __name__ == '__main__': model = ALBERTForRace() dm = RACEDataModule() trainer = pl.Trainer( gpus=-1 if torch.cuda.is_available() else None, # amp_level='O2', # precision=16, gradient_clip_val=1.0, max_epochs=5 ) trainer.fit(model, dm) trainer.test(datamodule=dm)
from model.DUMAForRace import DUMAForRace if __name__ == '__main__': model = DUMAForRace( pretrained_model='bert-large-uncased', learning_rate=2e-5, num_train_epochs=20, train_batch_size=16, train_all=True, use_bert_adam=True, ) dm = RACEDataModule( model_name_or_path='bert-large-uncased', datasets_loader='race', train_batch_size=16, max_seq_length=128, num_workers=8, num_preprocess_processes=48, ) trainer = pl.Trainer( gpus=-1 if torch.cuda.is_available() else None, amp_backend='native', amp_level='O2', precision=16, accelerator='ddp', gradient_clip_val=1.0, max_epochs=1, plugins='ddp_sharded', val_check_interval=0.2, ) trainer.fit(model, dm)