def objective(trial: optuna.trial.Trial) -> float:
learning_rate = trial.suggest_float("learning_rate", 1e-5, 2e-5)
model = BertForRace(
pretrained_model='./model/bert-large-uncased',
learning_rate=learning_rate,
num_train_epochs=20,
train_batch_size=4,
train_all=True,
)
dm = RACEDataModule(
model_name_or_path='./model/bert-large-uncased',
datasets_loader='./data/RACELocalLoader.py',
train_batch_size=32,
max_seq_length=128,
num_workers=8,
num_preprocess_processes=48,
use_sentence_selection=True,
)
trainer = pl.Trainer(
gpus=-1 if torch.cuda.is_available() else None,
# callbacks=[checkpoint_callback],
amp_backend='native',
amp_level='O2',
precision=16,
accelerator='ddp',
gradient_clip_val=1.0,
max_epochs=3,
plugins='ddp_sharded',
callbacks=[PyTorchLightningPruningCallback(trial, monitor="val_acc")],
# limit_train_batches=0.1,
# limit_val_batches=0.1,
# accumulate_grad_batches=2,
)
trainer.fit(model, dm)
return trainer.callback_metrics["val_acc"].item()
if __name__ == '__main__':
tb_logger = pl_loggers.TensorBoardLogger('./result/asc01/')
pl.seed_everything(42)
model = DUMAForRace(
pretrained_model='./model/bert-large-uncased',
learning_rate=2e-5,
num_train_epochs=20,
train_batch_size=4,
train_all=True,
use_bert_adam=True,
)
dm = RACEDataModule(
model_name_or_path='./model/bert-large-uncased',
datasets_loader='./data/RACELocalLoader.py',
train_batch_size=4,
max_seq_length=512,
num_workers=8,
num_preprocess_processes=48,
use_sentence_selection=False,
best_k_sentences=5,
)
# checkpoint_callback = ModelCheckpoint(
# dirpath='./result/checkpoints/',
# filename='epoch{epoch:02d}',
# save_top_k=-1,
# )
trainer = pl.Trainer(
logger=tb_logger,
gpus=-1 if torch.cuda.is_available() else None,
# callbacks=[checkpoint_callback],
amp_backend='native',
amp_level='O2',
self.log('val_loss', val_loss)
def test_step(self, batch, batch_idx):
loss, correct_count = self.compute(batch)
return {
"test_loss": loss,
"correct_count": correct_count,
"batch_size": len(batch['label'])
}
def test_epoch_end(self, outputs: List[Any]) -> None:
test_acc = sum([out["correct_count"] for out in outputs]).float() / sum(out["batch_size"] for out in outputs)
test_loss = sum([out["test_loss"] for out in outputs]) / len(outputs)
self.log('test_acc', test_acc)
self.log('test_loss', test_loss)
if __name__ == '__main__':
model = ALBERTForRace()
dm = RACEDataModule()
trainer = pl.Trainer(
gpus=-1 if torch.cuda.is_available() else None,
# amp_level='O2',
# precision=16,
gradient_clip_val=1.0,
max_epochs=5
)
trainer.fit(model, dm)
trainer.test(datamodule=dm)
from model.DUMAForRace import DUMAForRace
if __name__ == '__main__':
model = DUMAForRace(
pretrained_model='bert-large-uncased',
learning_rate=2e-5,
num_train_epochs=20,
train_batch_size=16,
train_all=True,
use_bert_adam=True,
)
dm = RACEDataModule(
model_name_or_path='bert-large-uncased',
datasets_loader='race',
train_batch_size=16,
max_seq_length=128,
num_workers=8,
num_preprocess_processes=48,
)
trainer = pl.Trainer(
gpus=-1 if torch.cuda.is_available() else None,
amp_backend='native',
amp_level='O2',
precision=16,
accelerator='ddp',
gradient_clip_val=1.0,
max_epochs=1,
plugins='ddp_sharded',
val_check_interval=0.2,
)
trainer.fit(model, dm)