def run_model_test_without_loggers(trainer_options: dict,
model: LightningModule,
data: LightningDataModule = None,
min_acc: float = 0.50):
reset_seed()
# fit model
trainer = Trainer(**trainer_options)
trainer.fit(model, datamodule=data)
# correct result and ok accuracy
assert trainer.state.finished, f"Training failed with {trainer.state}"
model2 = load_model_from_checkpoint(
trainer.logger, trainer.checkpoint_callback.best_model_path,
type(model))
# test new model accuracy
test_loaders = model2.test_dataloader(
) if not data else data.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
if not isinstance(model2, BoringModel):
for dataloader in test_loaders:
run_prediction_eval_model_template(model2,
dataloader,
min_acc=min_acc)
def run_model_test_without_loggers(trainer_options,
model,
min_acc: float = 0.50):
reset_seed()
# fit model
trainer = Trainer(**trainer_options)
trainer.fit(model)
# correct result and ok accuracy
assert trainer.state == TrainerState.FINISHED, f"Training failed with {trainer.state}"
pretrained_model = load_model_from_checkpoint(
trainer.logger, trainer.checkpoint_callback.best_model_path,
type(model))
# test new model accuracy
test_loaders = model.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
for dataloader in test_loaders:
run_prediction(pretrained_model, dataloader, min_acc=min_acc)
if trainer._distrib_type in (DistributedType.DDP,
DistributedType.DDP_SPAWN):
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.model = pretrained_model
trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers(
)
def run_model_test(
trainer_options,
model: LightningModule,
data: LightningDataModule = None,
on_gpu: bool = True,
version=None,
with_hpc: bool = True,
min_acc: float = 0.25,
):
reset_seed()
save_dir = trainer_options["default_root_dir"]
# logger file to get meta
logger = get_default_logger(save_dir, version=version)
trainer_options.update(logger=logger)
trainer = Trainer(**trainer_options)
initial_values = torch.tensor(
[torch.sum(torch.abs(x)) for x in model.parameters()])
trainer.fit(model, datamodule=data)
post_train_values = torch.tensor(
[torch.sum(torch.abs(x)) for x in model.parameters()])
assert trainer.state.finished, f"Training failed with {trainer.state}"
# Check that the model is actually changed post-training
change_ratio = torch.norm(initial_values - post_train_values)
assert change_ratio > 0.1, f"the model is changed of {change_ratio}"
# test model loading
pretrained_model = load_model_from_checkpoint(
logger, trainer.checkpoint_callback.best_model_path, type(model))
# test new model accuracy
test_loaders = model.test_dataloader(
) if not data else data.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
if not isinstance(model, BoringModel):
for dataloader in test_loaders:
run_prediction_eval_model_template(model,
dataloader,
min_acc=min_acc)
if with_hpc:
if trainer._distrib_type in (DistributedType.DDP,
DistributedType.DDP_SPAWN,
DistributedType.DDP2):
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.optimizers, trainer.lr_schedulers, trainer.optimizer_frequencies = trainer.init_optimizers(
pretrained_model)
# test HPC saving
trainer.checkpoint_connector.hpc_save(save_dir, logger)
# test HPC loading
checkpoint_path = trainer.checkpoint_connector.get_max_ckpt_path_from_folder(
save_dir)
trainer.checkpoint_connector.restore(checkpoint_path)
def run_model_test(
trainer_options,
model: LightningModule,
data: LightningDataModule = None,
on_gpu: bool = True,
version=None,
with_hpc: bool = True,
min_acc: float = 0.25,
):
reset_seed()
save_dir = trainer_options["default_root_dir"]
# logger file to get meta
logger = get_default_logger(save_dir, version=version)
trainer_options.update(logger=logger)
trainer = Trainer(**trainer_options)
initial_values = torch.tensor([torch.sum(torch.abs(x)) for x in model.parameters()])
trainer.fit(model, datamodule=data)
post_train_values = torch.tensor([torch.sum(torch.abs(x)) for x in model.parameters()])
assert trainer.state.finished, f"Training failed with {trainer.state}"
# Check that the model is actually changed post-training
change_ratio = torch.norm(initial_values - post_train_values)
assert change_ratio > 0.03, f"the model is changed of {change_ratio}"
# test model loading
_ = load_model_from_checkpoint(logger, trainer.checkpoint_callback.best_model_path, type(model))
# test new model accuracy
test_loaders = model.test_dataloader() if not data else data.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
if not isinstance(model, BoringModel):
for dataloader in test_loaders:
run_model_prediction(model, dataloader, min_acc=min_acc)
if with_hpc:
# test HPC saving
# save logger to make sure we get all the metrics
if logger:
logger.finalize("finished")
hpc_save_path = trainer.checkpoint_connector.hpc_save_path(save_dir)
trainer.save_checkpoint(hpc_save_path)
# test HPC loading
checkpoint_path = trainer.checkpoint_connector._CheckpointConnector__get_max_ckpt_path_from_folder(save_dir)
trainer.checkpoint_connector.restore(checkpoint_path)
def run_model_test_without_loggers(trainer_options,
model,
min_acc: float = 0.50):
reset_seed()
# fit model
trainer = Trainer(**trainer_options)
trainer.fit(model)
# correct result and ok accuracy
assert trainer.state == TrainerState.FINISHED, f"Training failed with {trainer.state}"
pretrained_model = load_model_from_checkpoint(
trainer.logger, trainer.checkpoint_callback.best_model_path,
type(model))
# test new model accuracy
test_loaders = model.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
for dataloader in test_loaders:
run_prediction(pretrained_model, dataloader, min_acc=min_acc)