def train_eval_lightning( train_dataset, eval_dataset, trainer_module, num_epochs, use_gpu, batch_preprocessor=None, reader_options: Optional[ReaderOptions] = None, checkpoint_path: Optional[str] = None, ) -> pl.Trainer: reader_options = reader_options or ReaderOptions() datamodule = PetastormLightningDataModule(train_dataset, eval_dataset, batch_preprocessor, reader_options) # pyre-fixme[16]: Module `pl` has no attribute `Trainer`. # pyre-fixme[16]: Module `pl` has no attribute `Trainer`. trainer = pl.Trainer( max_epochs=num_epochs * 1000, gpus=int(use_gpu), reload_dataloaders_every_epoch=True, resume_from_checkpoint=checkpoint_path, callbacks=[StoppingEpochCallback(num_epochs)], ) trainer.fit(trainer_module, datamodule=datamodule) # TODO: evaluate return trainer
def train_eval_lightning( train_dataset, eval_dataset, test_dataset, trainer_module, data_module, num_epochs, logger_name: str, batch_preprocessor=None, reader_options: Optional[ReaderOptions] = None, checkpoint_path: Optional[str] = None, resource_options: Optional[ResourceOptions] = None, ) -> pl.Trainer: resource_options = resource_options or ResourceOptions() use_gpu = resource_options.use_gpu reader_options = reader_options or ReaderOptions() datamodule = data_module or PetastormLightningDataModule( train_dataset, eval_dataset, batch_preprocessor, reader_options) trainer = pl.Trainer( logger=OssTensorboardLogger(save_dir="pl_log_tensorboard", name=logger_name), max_epochs=num_epochs * 1000, gpus=int(use_gpu), reload_dataloaders_every_epoch=True, resume_from_checkpoint=checkpoint_path, callbacks=[StoppingEpochCallback(num_epochs)], ) trainer.fit(trainer_module, datamodule=datamodule) trainer.test() if checkpoint_path is not None: # Overwrite the warmstart path with the new model trainer_module.set_clean_stop(True) trainer.save_checkpoint(checkpoint_path) return trainer
def train_eval_lightning( train_dataset, eval_dataset, test_dataset, trainer_module, data_module, num_epochs, use_gpu, logger_name: str, batch_preprocessor=None, reader_options: Optional[ReaderOptions] = None, checkpoint_path: Optional[str] = None, resource_options: Optional[ResourceOptions] = None, ) -> pl.Trainer: reader_options = reader_options or ReaderOptions() datamodule = data_module or PetastormLightningDataModule( train_dataset, eval_dataset, batch_preprocessor, reader_options) # pyre-fixme[16]: Module `pl` has no attribute `Trainer`. # pyre-fixme[16]: Module `pl` has no attribute `Trainer`. trainer = pl.Trainer( logger=TensorBoardLogger(save_dir="pl_log_tensorboard", name=logger_name), max_epochs=num_epochs * 1000, gpus=int(use_gpu), reload_dataloaders_every_epoch=True, resume_from_checkpoint=checkpoint_path, callbacks=[StoppingEpochCallback(num_epochs)], ) trainer.fit(trainer_module, datamodule=datamodule) trainer.test() return trainer